diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..da94080f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/nvocdr"] + path = src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/nvocdr + url = https://github.com/NVIDIA-AI-IOT/NVIDIA-Optical-Character-Detection-and-Recognition-Solution.git +[submodule "src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/modules/Lidar_AI_Solution"] + path = src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/modules/Lidar_AI_Solution + url = https://github.com/NVIDIA-AI-IOT/Lidar_AI_Solution.git diff --git a/src/apps/reference_apps/.gitattributes b/src/apps/reference_apps/.gitattributes new file mode 100644 index 00000000..cf69bee7 --- /dev/null +++ b/src/apps/reference_apps/.gitattributes @@ -0,0 +1,4 @@ +deepstream_parallel_inference_app/tritonserver/models/bodypose2d/1/model.onnx filter=lfs diff=lfs merge=lfs -text +deepstream_parallel_inference_app/tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx filter=lfs diff=lfs merge=lfs -text +deepstream-tracker-3d-multi-view/assets/datasets.zip filter=lfs diff=lfs merge=lfs -text +deepstream-bodypose-3d/streams/bodypose.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/src/apps/reference_apps/LICENSE b/src/apps/reference_apps/LICENSE new file mode 100644 index 00000000..971af825 --- /dev/null +++ b/src/apps/reference_apps/LICENSE @@ -0,0 +1,16 @@ +Apache2.0 License + +Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/src/apps/reference_apps/README.md b/src/apps/reference_apps/README.md new file mode 100644 index 00000000..def361dd --- /dev/null +++ b/src/apps/reference_apps/README.md @@ -0,0 +1,48 @@ +# Reference Apps using DeepStream 9.0 + +This repository contains the reference applications for video analytics tasks using TensorRT and DeepSTream SDK 9.0. + +## Getting Started ## +We currently provide three different reference applications: + +Preferably copy all the apps from reference_apps dir to +`/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +To clone the project in the above directory, sudo permission may be required. + +For further details, please see each project's README. + +### Anomaly Detection : [README](anomaly/README.md) ### + The project contains auxiliary dsdirection plugin to show the capability of DeepstreamSDK in anomaly detection. + ![sample anomaly output](anomaly/.opticalflow.png) +### Runtime Source Addition Deletion: [README](runtime_source_add_delete/README.md) ### + The project demonstrates addition and deletion of video sources in a live Deepstream pipeline. +### MaskTracker: [README](deepstream-masktracker/README.md) ### + This sample app demonstrates DeepStream MaskTracker for multi-object tracking and segmentation using SAM2. + ![sample MaskTracker output](deepstream-masktracker/figures/.retail_osd.gif) +### Single-View 3D Tracking: [README](deepstream-tracker-3d/README.md) ### + The sample app demonstrates single-view 3D tracking with DeepStream multi-object tracking to reconstruct 3D human model in world coordinates under occlusion. + ![sample 3D tracking output](deepstream-tracker-3d/figures/.retail_viz.png) +### Multi-View 3D Tracking: [README](deepstream-tracker-3d-multi-view) ### + The samples demonstrate multi-view 3D tracking in DeepStream, a distributed, real-time framework designed for large-scale, calibrated camera networks. + + sample mulit-view 3D tracking output sample BEV output from multi-view 3D tracking +### Parallel Multiple Models Inferencing: [README](deepstream_parallel_inference_app/README.md) ### + The project demonstrate how to implement multiple models inferencing in parallel with DeepStream APIs. +### Bodypose 3D Model Inferencing: [README](deepstream-bodypose-3d/README.md) ### + The sample demonstrate how to customize the multiple input layers model preprocessing and the customization of the bodypose 3D model postprocessing. + ![Bodypose 3D sample output](deepstream-bodypose-3d/sources/.screenshot.png) +### Video Buffers sharing between pipelines through IPC: [README](deepstream-ipc-test-sr/README.md) ### + This sample demonstrates how to share video buffers over IPC and how to change output video buffers. +### Multiple Dynamic Sources with Single Decoder:[README](deepstream-dynamicsrcbin-test/README) ### + The sample demonstrates the usage of nvdsdynamicsrcbin in DeepStream pipeline.It helps to construct the application of multiple dynamic sources with a single video decoder to adapt to the high decoder throughput scenarios. +### Custom Video Tiling Config: [README](deepstream-custom-tile-config/README.md) ### + This sample demonstrates the usage of "custom-tile-config" of nvmultistreamtiler to customize the tiling positions and sizes of multiple videos in batch. +### DeepStream VLLM Plugin: [README](deepstream-vllm-plugin/README.md) ### + A GStreamer plugin for NVIDIA DeepStream that integrates Vision-Language Models (VLM) using VLLM for real-time video understanding and analysis. + +## Pyservicemaker Sample apps +The apps in pyservicemaker_sample_apps are additional samples demonstrating usage of the Python API for DeepStream Service Maker, either by flow API or by pipeline API. See the [Python Service Maker documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_service_maker_python.html) for details. + +## Legacy DeepStream Reference Samples +Some old samples are not supported any more for different reasons. The legacy samples are moved to the legacy folder /legacy_apps. diff --git a/src/apps/reference_apps/anomaly/.dsdirection_pipeline.png b/src/apps/reference_apps/anomaly/.dsdirection_pipeline.png new file mode 100644 index 00000000..d4b84341 Binary files /dev/null and b/src/apps/reference_apps/anomaly/.dsdirection_pipeline.png differ diff --git a/src/apps/reference_apps/anomaly/.opticalflow.png b/src/apps/reference_apps/anomaly/.opticalflow.png new file mode 100644 index 00000000..8ee9c871 Binary files /dev/null and b/src/apps/reference_apps/anomaly/.opticalflow.png differ diff --git a/src/apps/reference_apps/anomaly/Makefile b/src/apps/reference_apps/anomaly/Makefile new file mode 100644 index 00000000..f269ba9c --- /dev/null +++ b/src/apps/reference_apps/anomaly/Makefile @@ -0,0 +1,6 @@ +all: + make -C apps/deepstream-anomaly-detection-test + make -C plugins/gst-dsdirection +%: + make -C apps/deepstream-anomaly-detection-test $@ + make -C plugins/gst-dsdirection $@ diff --git a/src/apps/reference_apps/anomaly/README.md b/src/apps/reference_apps/anomaly/README.md new file mode 100644 index 00000000..63f49560 --- /dev/null +++ b/src/apps/reference_apps/anomaly/README.md @@ -0,0 +1,80 @@ +# ANOMALY DETECTION REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains anomaly detection application and auxiliary plug-ins to show the +capability of Deepstream SDK. + +## Prequisites: +DeepStream SDK installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the `apps/sample_apps/deepstream-app/README` on how +to install the prequisites for Deepstream SDK apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit the `dsanomaly_pgie_config.txt` or `dsanomaly_pgie_nvinferserver_config.txt` according to the location of the models to be used + + +## Compilation Steps for dsdirection plugin +``` + $ cd plugins/gst-dsdirection/ + $ sudo make && sudo make install +``` + +1. Test direction calculation on one video input, on dGPU, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location=samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=1 width=1920 height=1080 ! nvinfer config-file-path=samples/configs/deepstream-app/config_infer_primary.txt \ +! nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=1080 ! nveglglessink t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! nveglglessink +``` +2. Test direction calculation on one video input, on Jetson, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location=samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=1 width=1280 height=720 ! nvinfer config-file-path=samples/configs/deepstream-app/config_infer_primary.txt \ +! nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=1080 ! nv3dsink sync=0 t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! nv3dsink sync=0 +``` + +3. Test direction calculation using optical flow on two video inputs on dGPU, run following commands +``` +cd /opt/nvidia/deepstream/deepstream/ +gst-launch-1.0 filesrc location=samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 \ +nvstreammux name=m batch-size=2 width=1920 height=1080 ! nvinfer config-file-path=samples/configs/deepstream-app/config_infer_primary.txt ! \ +nvof ! tee name=t ! queue ! nvofvisual ! nvmultistreamtiler width=1920 height=540 ! nveglglessink t. ! queue ! dsdirection ! \ +nvmultistreamtiler width=1920 height=540 ! nvvideoconvert ! nvdsosd ! nveglglessink filesrc location=samples/streams/sample_1080p_h264.mp4 ! \ +qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 --gst-debug=3 + +``` +Anomaly detection app pipeline: +![DS Anomaly Detection Pipeline](.dsdirection_pipeline.png) + +## Compilation Steps and Execution: +``` + $ cd sources/apps/sample_apps/deepstream_reference_apps/anomaly/ + $ cd apps/deepstream-anomaly-detection-test/ + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For jetson, CUDA_VER=13.0 + $ sudo make + + $ ./deepstream-anomaly-detection-app [uri2] ... [uriN] + Ex.: ./deepstream-anomaly-detection-app file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 + +Use option "-t inferserver" to select nvinferserver as the inference plugin + $ ./deepstream-anomaly-detection-app -t inferserver [uri2] ... [uriN] + Ex.: ./deepstream-anomaly-detection-app -t inferserver file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +``` + The result should be like below: + ![DS Anomaly Detection Screenshot](.opticalflow.png) + +## NOTE: +- Minimum supported resolution: DGPU - 160 x 64, Jetson - 256 x 96 +- Due to an issue in nvofvisual plugin, when using nvofvisual along with nvof + plugin, the width of input to nvof should be multiple of 32 on DGPU and multiple + of 256 on Jetson. This will be fixed in the nvofvisual plugin in the next DeepStream + release. diff --git a/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/Makefile b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/Makefile new file mode 100644 index 00000000..b73b5dc3 --- /dev/null +++ b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/Makefile @@ -0,0 +1,62 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-anomaly-detection-app + + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ +APP_INSTALL_DIR?=$(DS_SDK_ROOT)/bin/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lm \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) diff --git a/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c new file mode 100644 index 00000000..158ebef2 --- /dev/null +++ b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/deepstream_anomaly_detection_test.c @@ -0,0 +1,502 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "gst-nvmessage.h" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 33 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 33000 + +#define TILED_OUTPUT_WIDTH_INFER 1280 +#define TILED_OUTPUT_HEIGHT_INFER 720 + +#define TILED_OUTPUT_WIDTH_OF 640 +#define TILED_OUTPUT_HEIGHT_OF 360 + +#define NVINFER_PLUGIN "nvinfer" +#define NVINFERSERVER_PLUGIN "nvinferserver" + +#define PGIE_CONFIG_FILE "dsanomaly_pgie_config.txt" +#define PGIE_NVINFERSERVER_CONFIG_FILE "dsanomaly_pgie_nvinferserver_config.txt" + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing NvBufSurface. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + } + } + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvv4l2decoder. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } +} + +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin"); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + +static void +usage(const char *bin) +{ + g_printerr ("Usage: %s [uri2] ... [uriN]\n", bin); + g_printerr ("For nvinferserver, Usage: %s -t inferserver [uri2] ... [uriN]\n", bin); +} + +int +main (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *streammux = NULL, *streammux_queue = NULL, + *sink_of = NULL, *pgie_queue = NULL, *dsdirection_queue = NULL, + *nvvidconv_queue = NULL, *nvosd_queue = NULL, *tiler_infer_queue = NULL, + *tiler_of = NULL, *nvof = NULL, *nvofvisual = NULL, *dsdirection = NULL, + *of_queue = NULL, *ofvisual_queue = NULL, *sink_infer = NULL, + *tiler_infer = NULL, *pgie = NULL, *nvvidconv = NULL, + *nvosd = NULL, *tee = NULL, *of_branch_queue = NULL, *infer_branch_queue = + NULL; + + GstBus *bus = NULL; + guint bus_watch_id; + guint i, num_sources; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + gboolean is_nvinfer_server = FALSE; + + GstPad *tee_of_pad, *tee_infer_pad; + GstPad *queue_of_pad, *queue_infer_pad; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Check input arguments */ + if (argc < 2) { + usage(argv[0]); + return -1; + } + + if (argc >=2 && !strcmp("-t", argv[1])) { + if (!strcmp("inferserver", argv[2])) { + is_nvinfer_server = TRUE; + } else { + usage(argv[0]); + return -1; + } + g_print ("Using nvinferserver as the inference plugin\n"); + } + + if (is_nvinfer_server) { + num_sources = argc - 3; + } else { + num_sources = argc - 1; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("anomaly-detection-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + streammux_queue = gst_element_factory_make ("queue", "streammux-queue"); + + if (!pipeline || !streammux || !streammux_queue) { + g_printerr ("(Line=%d) One element could not be created. Exiting.\n", + __LINE__); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + + for (i = 0; i < num_sources; i++) { + GstPad *sinkpad, *srcpad; + GstElement *source_bin; + gchar pad_name[16] = { }; + if (is_nvinfer_server) { + source_bin = create_source_bin (i, argv[i + 3]); + } else { + source_bin = create_source_bin (i, argv[i + 1]); + } + + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_bin); + + g_snprintf (pad_name, 15, "sink_%u", i); + sinkpad = gst_element_get_request_pad (streammux, pad_name); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_bin, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Create a tee for two sinks. */ + tee = gst_element_factory_make ("tee", "tee"); + + /* Use nvinfer/nvinferserver to infer on batched frame. */ + pgie = gst_element_factory_make ( + is_nvinfer_server ? NVINFERSERVER_PLUGIN : NVINFER_PLUGIN, + "primary-nvinference-engine"); + pgie_queue = gst_element_factory_make ("queue", "nvinfer-queue"); + + /* For Optical Flow output */ + tiler_of = gst_element_factory_make ("nvmultistreamtiler", "nvtiler-of"); + /* Use nvtiler to composite the batched frames into a 2D tiled array based + * on the source of the frames. */ + tiler_infer = + gst_element_factory_make ("nvmultistreamtiler", "nvtiler-infer"); + tiler_infer_queue = + gst_element_factory_make ("queue", "nvtiler-infer-queue"); + + /* create nv optical flow element */ + nvof = gst_element_factory_make ("nvof", "nvopticalflow"); + + /* create nv ds direction element */ + dsdirection = gst_element_factory_make ("dsdirection", "dsdirection"); + dsdirection_queue = gst_element_factory_make ("queue", "dsdirection-queue"); + + /* create nv optical flow visualisation element */ + nvofvisual = gst_element_factory_make ("nvofvisual", "nvopticalflowvisual"); + + /* create queue element */ + of_queue = gst_element_factory_make ("queue", "q_after_of"); + + /* create queue element */ + ofvisual_queue = gst_element_factory_make ("queue", "q_after_ofvisual"); + + /* create queue element */ + of_branch_queue = gst_element_factory_make ("queue", "q_of"); + + /* create queue element */ + infer_branch_queue = gst_element_factory_make ("queue", "q_infer"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + nvvidconv_queue = gst_element_factory_make ("queue", "nvvideoconvert-queue"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + nvosd_queue = gst_element_factory_make ("queue", "nvdsosd-queue"); + + /* Finally render the osd output */ + if(prop.integrated) { + sink_of = gst_element_factory_make ("nv3dsink", "nv3dsink-of"); + sink_infer = + gst_element_factory_make ("nv3dsink", "nv3dsink-infer"); + + } else { +#ifdef __aarch64__ + sink_of = gst_element_factory_make ("nv3dsink", "nv3dsink-of"); + sink_infer = + gst_element_factory_make ("nv3dsink", "nv3dsink-infer"); +#else + sink_of = gst_element_factory_make ("nveglglessink", "nvelgglessink-of"); + sink_infer = + gst_element_factory_make ("nveglglessink", "nvelgglessink-infer"); +#endif + } + + if (!tee) { + g_printerr ("Tee could not be created. Exiting.\n"); + return -1; + } + + if (!nvof || !dsdirection || !nvofvisual || !tiler_of || !sink_of) { + g_printerr ("One OF element could not be created. Exiting.\n"); + return -1; + } + + if (!pgie || !tiler_infer || !nvvidconv || !nvosd || !sink_infer) { + g_printerr ("One Infer element could not be created. Exiting.\n"); + return -1; + } + + if (!pgie_queue || !tiler_infer_queue || !nvvidconv_queue || !nvosd_queue) { + g_printerr ("One Queue element could not be created. Exiting.\n"); + return -1; + } + + /* We set the sync value of both sink elements */ + g_object_set (G_OBJECT (sink_of), "sync", 1, NULL); + g_object_set (G_OBJECT (sink_infer), "sync", 1, NULL); + + g_object_set (G_OBJECT (streammux), "sync-inputs", TRUE, NULL); + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", num_sources, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Configure the nvinfer/nvinferserver element using the config file. */ + if (is_nvinfer_server) { + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_NVINFERSERVER_CONFIG_FILE, NULL); + } else { + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + } + + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL); + } + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler_of), "rows", tiler_rows, "columns", + tiler_columns, "width", TILED_OUTPUT_WIDTH_OF, "height", + TILED_OUTPUT_HEIGHT_OF, NULL); + g_object_set (G_OBJECT (tiler_infer), "rows", tiler_rows, "columns", + tiler_columns, "width", TILED_OUTPUT_WIDTH_INFER, "height", + TILED_OUTPUT_HEIGHT_INFER, NULL); + + /* We set the sink properties here */ + g_object_set (G_OBJECT (sink_of), "window-x", 0, "window-y", 0, NULL); + g_object_set (G_OBJECT (sink_infer), "window-x", TILED_OUTPUT_WIDTH_OF, + "window-y", TILED_OUTPUT_HEIGHT_OF, NULL); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), streammux_queue, pgie, pgie_queue, + nvof, of_queue, dsdirection, dsdirection_queue, tee, + of_branch_queue, nvofvisual, ofvisual_queue, tiler_of, sink_of, + infer_branch_queue, tiler_infer, tiler_infer_queue, nvvidconv, + nvvidconv_queue, nvosd, nvosd_queue, sink_infer, NULL); + + + if ((!gst_element_link_many (streammux, streammux_queue, pgie, pgie_queue, + nvof, of_queue, dsdirection, dsdirection_queue, tee, NULL)) + || (!gst_element_link_many (of_branch_queue, nvofvisual, ofvisual_queue, + tiler_of, NULL)) + || (!gst_element_link_many (infer_branch_queue, tiler_infer, + tiler_infer_queue, nvvidconv, nvvidconv_queue, nvosd, + nvosd_queue, NULL)) || + (!gst_element_link_many (tiler_of, sink_of, NULL)) || + (!gst_element_link_many (nvosd_queue, sink_infer, NULL))) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + + /* Manually link the Tee, which has "Request" pads */ + tee_of_pad = gst_element_get_request_pad (tee, "src_%u"); + g_print ("Obtained request pad %s for OF branch.\n", + gst_pad_get_name (tee_of_pad)); + queue_of_pad = gst_element_get_static_pad (of_branch_queue, "sink"); + tee_infer_pad = gst_element_get_request_pad (tee, "src_%u"); + g_print ("Obtained request pad %s for infer branch.\n", + gst_pad_get_name (tee_infer_pad)); + queue_infer_pad = gst_element_get_static_pad (infer_branch_queue, "sink"); + if (gst_pad_link (tee_of_pad, queue_of_pad) != GST_PAD_LINK_OK || + gst_pad_link (tee_infer_pad, queue_infer_pad) != GST_PAD_LINK_OK) { + g_printerr ("Tee could not be linked.\n"); + gst_object_unref (pipeline); + return -1; + } + gst_object_unref (queue_of_pad); + gst_object_unref (queue_infer_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing...\n"); + + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipeline), + GST_DEBUG_GRAPH_SHOW_ALL, "nvof_test_playing"); + + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + gst_deinit (); + return 0; +} diff --git a/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt new file mode 100644 index 00000000..ba2006b0 --- /dev/null +++ b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_config.txt @@ -0,0 +1,70 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# enable-dbscan(Default=false), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +model-engine-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt +batch-size=1 +process-mode=1 +model-color-format=0 +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt new file mode 100644 index 00000000..dc1d1081 --- /dev/null +++ b/src/apps/reference_apps/anomaly/apps/deepstream-anomaly-detection-test/dsanomaly_pgie_nvinferserver_config.txt @@ -0,0 +1,74 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_cov/Sigmoid:0"}, + {name: "output_bbox/BiasAdd:0"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "/opt/nvidia/deepstream/deepstream/samples/triton_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/Makefile b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/Makefile new file mode 100644 index 00000000..9145ab6b --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/Makefile @@ -0,0 +1,63 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +CXX:= g++ +SRCS:= gstdsdirection.cpp +INCS:= $(wildcard *.h) +LIB:=libnvdsgst_dsdirection.so + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +DEP:=dsdirection_lib/libdsdirection.a +DEP_FILES:=$(wildcard dsdirection_lib/dsdirection_lib.* ) +DEP_FILES-=$(DEP) + +CFLAGS+= -fPIC -DDS_VERSION=\"6.0.0\" \ + -I $(DS_SDK_ROOT)/sources/includes + +GST_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/gst-plugins/ +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +LIBS := -shared -Wl,-no-undefined \ + -L dsdirection_lib -ldsdirection \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.cpp=.o) + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) $(DEP) + +$(DEP): $(DEP_FILES) + $(MAKE) -C dsdirection_lib/ + +install: $(LIB) + cp -rv $(LIB) $(GST_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile new file mode 100644 index 00000000..f524c418 --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/Makefile @@ -0,0 +1,26 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################# + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream +CFLAGS+=-I $(DS_SDK_ROOT)/sources/includes +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) + +all: + gcc -ggdb $(CFLAGS) -c -o dsdirection_lib.o -fPIC dsdirection_lib.cpp + ar rcs libdsdirection.a dsdirection_lib.o diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp new file mode 100644 index 00000000..f7fc198b --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.cpp @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dsdirection_lib.h" +#include +#include +#include +using namespace std; + +#define PI 3.141592 +#define PI_IN_DEGREES 180 +//NVOF gives quarter-pixel output using OFSDK +#define FACTOR_QPEL 4.0 + + +struct DsDirectionLabel +{ + //Start of interval in degrees + float start; + //End of interval in degrees + float end; + //Direction in UTF-8 Format + const char *dirName; +}; + +static const DsDirectionLabel label[8] = { + {157.5, -157.5, "\u21D0"}, //left + {-67.5, -22.5, "\u21D8"}, //bottom-right + {-112.5, -67.5, "\u21D3"}, //bottom + {-157.5, -112.5, "\u21D9"}, //bottom-left + {-22.5, 22.5, "\u21D2"}, //right + {112.5, 157.5, "\u21D6"}, //top left + {67.5, 112.5, "\u21D1"}, //top + {22.5, 67.5, "\u21D7"} //top-right +}; + +DsDirectionOutput * +DsDirectionProcess (NvOFFlowVector * in_flow, int flow_cols, int flow_rows, + int flow_bsize, NvOSD_RectParams * rect_param) +{ + DsDirectionOutput *out = + (DsDirectionOutput *) calloc (1, sizeof (DsDirectionOutput)); + //Mean for the whole object Bounding Box + float x_mean = 0, y_mean = 0, max_radius = 0; + //Frequency or no of pixels in the bounding box + int freq_pix = 0; + //Sum of Optical Flow value of pixels + float x_mean_sum = 0, y_mean_sum = 0; + + //Get the motion inside the bbox. Sum it in x and y direction. + for (unsigned int j = rect_param->top; + j <= rect_param->top + rect_param->height; j++) { + unsigned int block_j = j / flow_bsize; + for (unsigned int i = rect_param->left; + i <= rect_param->left + rect_param->width; i++) { + // To get the mapping to the optical flow data (in_flow) + unsigned int block_i = i / flow_bsize; + unsigned int pos = block_j * flow_cols + block_i; + + //IF condition removes zero-motion pixels. thus getting a better estimate + if ((in_flow[pos].flowx != 0) || in_flow[pos].flowy != 0) { + x_mean_sum += (in_flow[pos].flowx) / FACTOR_QPEL; + y_mean_sum += (in_flow[pos].flowy) / FACTOR_QPEL; + freq_pix++; + } + } + } + + //Final Mean + x_mean = ((float) x_mean_sum) / freq_pix; + y_mean = ((float) y_mean_sum) / freq_pix; + out->object.flowx = x_mean; + out->object.flowy = -y_mean; + max_radius = sqrt (x_mean * x_mean + y_mean * y_mean); + + int dir = -1; + //Assigning the direction based on optical flow data + if (max_radius > 2) { + //dir initialized to 0 to take care of special case for left direction. + dir = 0; + float angle = (atan2 (-y_mean, x_mean) * PI_IN_DEGREES) / PI; + for (int i = 0; i <= 7; i++) { + if (angle > label[i].start && angle <= label[i].end) { + dir = i; + break; + } + } + } + + if (dir >= 0 && dir < 8) { + snprintf (out->object.direction, 128, "%s", label[dir].dirName); + } else { + //Blank output when threshold is not crossed. + snprintf (out->object.direction, 128, "%s", ""); + } + + return out; +} diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h new file mode 100644 index 00000000..8b691274 --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/dsdirection_lib/dsdirection_lib.h @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __DSDIRECTION_LIB__ +#define __DSDIRECTION_LIB__ +#include "nvll_osd_struct.h" +#include "nvds_opticalflow_meta.h" + +#define MAX_LABEL_SIZE 128 +#ifdef __cplusplus +extern "C" +{ +#endif + + +// Detected/Labelled object structure, stores bounding box info along with label +typedef struct +{ + float flowx; + float flowy; + char direction[MAX_LABEL_SIZE]; +} DsDirectionObject; + +// Output data returned after processing +typedef struct +{ + DsDirectionObject object; +} DsDirectionOutput; + +// Dequeue processed output +DsDirectionOutput *DsDirectionProcess (NvOFFlowVector * in_flow, + int flow_cols, int flow_rows, int flow_bsize, + NvOSD_RectParams * rect_param); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp new file mode 100644 index 00000000..7ee49851 --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.cpp @@ -0,0 +1,339 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "gstdsdirection.h" +#include "nvds_opticalflow_meta.h" +GST_DEBUG_CATEGORY_STATIC (gst_dsdirection_debug); +#define GST_CAT_DEFAULT gst_dsdirection_debug + +static GQuark _dsmeta_quark = 0; + +/* Enum to identify properties */ +enum +{ + PROP_0, + PROP_UNIQUE_ID +}; +// Block size used in NVOF application +#define NVOF_BLK_SIZE 4 +/*set the user metadata type*/ +#define NVDS_DIRECTION_USER_META (nvds_get_user_meta_type(((gchar *)"NVIDIA.NVDSDIRECTION.DIR_META"))) +/* Default values for properties */ +#define DEFAULT_UNIQUE_ID 15 + +/* By default NVIDIA Hardware allocated memory flows through the pipeline. We + * will be processing on this type of memory only. */ +#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM" +static GstStaticPadTemplate gst_dsdirection_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_NVMM, + "{ NV12, RGBA, I420 }"))); + +static GstStaticPadTemplate gst_dsdirection_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_NVMM, + "{ NV12, RGBA, I420 }"))); + +/* Define our element type. Standard GObject/GStreamer boilerplate stuff */ +#define gst_dsdirection_parent_class parent_class +G_DEFINE_TYPE (GstDsDirection, gst_dsdirection, GST_TYPE_BASE_TRANSFORM); + +static void gst_dsdirection_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_dsdirection_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_dsdirection_transform_ip (GstBaseTransform * + btrans, GstBuffer * inbuf); + + +static void attach_metadata_object (GstDsDirection * dsdirection, + NvDsObjectMeta * obj_meta, DsDirectionOutput * output); + +/* Install properties, set sink and src pad capabilities, override the required + * functions of the base class, These are common to all instances of the + * element. + */ +static void +gst_dsdirection_class_init (GstDsDirectionClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *gstbasetransform_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasetransform_class = (GstBaseTransformClass *) klass; + + /* Overide base class functions */ + gobject_class->set_property = + GST_DEBUG_FUNCPTR (gst_dsdirection_set_property); + gobject_class->get_property = + GST_DEBUG_FUNCPTR (gst_dsdirection_get_property); + + gstbasetransform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_dsdirection_transform_ip); + + /* Install properties */ + g_object_class_install_property (gobject_class, PROP_UNIQUE_ID, + g_param_spec_uint ("unique-id", + "Unique ID", + "Unique ID for the element. Can be used to identify output of the" + " element", 0, G_MAXUINT, DEFAULT_UNIQUE_ID, (GParamFlags) + (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /* Set sink and src pad capabilities */ + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_dsdirection_src_template)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_dsdirection_sink_template)); + + /* Set metadata describing the element */ + gst_element_class_set_details_simple (gstelement_class, + "DsDirection plugin", + "DsDirection Plugin", + "Estimate direction in which object is moving", + "NVIDIA Corporation. Post on Deepstream for Tesla forum for any queries " + "@ https://devtalk.nvidia.com/default/board/209/"); +} + +static void +gst_dsdirection_init (GstDsDirection * dsdirection) +{ + GstBaseTransform *btrans = GST_BASE_TRANSFORM (dsdirection); + + /* We will not be generating a new buffer. Just adding / updating + * metadata. */ + gst_base_transform_set_in_place (GST_BASE_TRANSFORM (btrans), TRUE); + /* We do not want to change the input caps. Set to passthrough. transform_ip + * is still called. */ + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (btrans), TRUE); + + /* Initialize all property variables to default values */ + dsdirection->unique_id = DEFAULT_UNIQUE_ID; + /* This quark is required to identify NvDsMeta when iterating through + * the buffer metadatas */ + if (!_dsmeta_quark) + _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING); +} + +/* Function called when a property of the element is set. Standard boilerplate. +*/ +static void +gst_dsdirection_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (object); + switch (prop_id) { + case PROP_UNIQUE_ID: + dsdirection->unique_id = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* Function called when a property of the element is requested. Standard + * boilerplate. + */ +static void +gst_dsdirection_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (object); + switch (prop_id) { + case PROP_UNIQUE_ID: + g_value_set_uint (value, dsdirection->unique_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/** + * Called when element recieves an input buffer from upstream element. + */ +static GstFlowReturn +gst_dsdirection_transform_ip (GstBaseTransform * btrans, GstBuffer * inbuf) +{ + GstDsDirection *dsdirection = GST_DSDIRECTION (btrans); + DsDirectionOutput *output; + NvDsBatchMeta *batch_meta = NULL; + NvDsFrameMeta *frame_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsObjectMeta *obj_meta = NULL; + dsdirection->frame_num++; + + GST_DEBUG_OBJECT (dsdirection, + "Processing Frame %lu", dsdirection->frame_num); + + batch_meta = gst_buffer_get_nvds_batch_meta (inbuf); + if (batch_meta == nullptr) { + GST_ELEMENT_ERROR (dsdirection, STREAM, FAILED, + ("NvDsBatchMeta not found for input buffer."), (NULL)); + return GST_FLOW_ERROR; + } + //Iterating through frames in batched meta from diff. sources + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + + frame_meta = (NvDsFrameMeta *) (l_frame->data); + NvDsFrameMetaList *fmeta_list = NULL; + //Iterating through each frame of the batched meta + for (fmeta_list = frame_meta->frame_user_meta_list; fmeta_list != NULL; + fmeta_list = fmeta_list->next) { + NvDsUserMeta *of_user_meta = NULL; + //previous meta + optical flow meta + of_user_meta = (NvDsUserMeta *) fmeta_list->data; + + if (of_user_meta + && of_user_meta->base_meta.meta_type == NVDS_OPTICAL_FLOW_META) { + //optical flow meta for each frame + NvDsOpticalFlowMeta *ofmeta = + (NvDsOpticalFlowMeta *) (of_user_meta->user_meta_data); + if (ofmeta) { + //Iterating through each object in the frame + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + + //processing the meta data for direction detection + output = + DsDirectionProcess ((NvOFFlowVector *) ofmeta->data, + ofmeta->cols, ofmeta->rows, NVOF_BLK_SIZE, + &obj_meta->rect_params); + // Attach direction to the object + attach_metadata_object (dsdirection, obj_meta, output); + + } + } + } + } + } + return GST_FLOW_OK; +} + +/* copy function set by user. "data" holds a pointer to NvDsUserMeta*/ +static gpointer +copy_ds_direction_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + DsDirectionOutput *src_user_metadata + = (DsDirectionOutput *) user_meta->user_meta_data; + DsDirectionOutput *dst_user_metadata + = (DsDirectionOutput *) calloc (1, sizeof (DsDirectionOutput)); + memcpy (dst_user_metadata, src_user_metadata, sizeof (DsDirectionOutput)); + return (gpointer) dst_user_metadata; +} + +/* release function set by user. "data" holds a pointer to NvDsUserMeta*/ +static void +release_ds_direction_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + if (user_meta->user_meta_data) { + free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; + } +} + +/** + * Only update string label in an existing object metadata. No bounding boxes. + * We assume only one label per object is generated + */ +static void +attach_metadata_object (GstDsDirection * dsdirection, NvDsObjectMeta * obj_meta, + DsDirectionOutput * output) +{ + NvDsBatchMeta *batch_meta = obj_meta->base_meta.batch_meta; + + // Attach - DsDirection MetaData + NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool (batch_meta); + NvDsMetaType user_meta_type = NVDS_DIRECTION_USER_META; + + user_meta->user_meta_data = output; + user_meta->base_meta.meta_type = user_meta_type; + user_meta->base_meta.copy_func = copy_ds_direction_meta; + user_meta->base_meta.release_func = release_ds_direction_meta; + + nvds_add_user_meta_to_obj (obj_meta, user_meta); + + nvds_acquire_meta_lock (batch_meta); + NvOSD_TextParams & text_params = obj_meta->text_params; + NvOSD_RectParams & rect_params = obj_meta->rect_params; + + /* Below code to display the result */ + // Set black background for the text + // display_text required heap allocated memory + if (text_params.display_text) { + gchar *conc_string = g_strconcat (text_params.display_text, " ", + output->object.direction, NULL); + g_free (text_params.display_text); + text_params.display_text = conc_string; + text_params.font_params.font_size = 12; + } else { + // Display text above the left top corner of the object + text_params.x_offset = rect_params.left; + text_params.y_offset = rect_params.top - 10; + text_params.display_text = g_strdup (output->object.direction); + // Font face, size and color + text_params.font_params.font_name = (char *) "Serif"; + text_params.font_params.font_size = 15; + text_params.font_params.font_color = (NvOSD_ColorParams) { + 1, 1, 1, 1}; + // Set black background for the text + text_params.set_bg_clr = 1; + text_params.text_bg_clr = (NvOSD_ColorParams) { + 0, 0, 0, 1}; + } + nvds_release_meta_lock (batch_meta); +} + +/** + * Boiler plate for registering a plugin and an element. + */ +static gboolean +dsdirection_plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_dsdirection_debug, "dsdirection", 0, + "dsdirection plugin"); + + return gst_element_register (plugin, "dsdirection", GST_RANK_PRIMARY, + GST_TYPE_DSDIRECTION); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + nvdsgst_dsdirection, + DESCRIPTION, dsdirection_plugin_init, DS_VERSION, LICENSE, BINARY_PACKAGE, + URL) diff --git a/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.h b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.h new file mode 100644 index 00000000..a17fedaf --- /dev/null +++ b/src/apps/reference_apps/anomaly/plugins/gst-dsdirection/gstdsdirection.h @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __GST_DSDIRECTION_H__ +#define __GST_DSDIRECTION_H__ + +#include +#include + +#include "gstnvdsmeta.h" +#include "dsdirection_lib/dsdirection_lib.h" + +/* Package and library details required for plugin_init */ +#define PACKAGE "dsdirection" +#define VERSION "1.0" +#define LICENSE "Proprietary" +#define DESCRIPTION "NVIDIA dsdirection plugin for integration with DeepStream" +#define BINARY_PACKAGE "NVIDIA DeepStream 3rdparty IP integration dsdirection plugin" +#define URL "http://nvidia.com/" + + +G_BEGIN_DECLS +/* Standard boilerplate stuff */ +typedef struct _GstDsDirection GstDsDirection; +typedef struct _GstDsDirectionClass GstDsDirectionClass; + +/* Standard boilerplate stuff */ +#define GST_TYPE_DSDIRECTION (gst_dsdirection_get_type()) +#define GST_DSDIRECTION(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_DSDIRECTION,GstDsDirection)) +#define GST_DSDIRECTION_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_DSDIRECTION,GstDsDirectionClass)) +#define GST_DSDIRECTION_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_DSDIRECTION, GstDsDirectionClass)) +#define GST_IS_DSDIRECTION(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_DSDIRECTION)) +#define GST_IS_DSDIRECTION_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_DSDIRECTION)) +#define GST_DSDIRECTION_CAST(obj) ((GstDsDirection *)(obj)) + +struct _GstDsDirection +{ + GstBaseTransform base_trans; + + // Unique ID of the element. The labels generated by the element will be + // updated at index `unique_id` of attr_info array in NvDsObjectParams. + guint unique_id; + + // Frame number of the current input buffer + guint64 frame_num; + + // Input video info (resolution, color format, framerate, etc) + GstVideoInfo video_info; + + // Amount of objects processed in single call to algorithm + guint batch_size; + + // GPU ID on which we expect to execute the task + guint gpu_id; + +}; + +// Boiler plate stuff +struct _GstDsDirectionClass +{ + GstBaseTransformClass parent_class; +}; + +GType gst_dsdirection_get_type (void); + +G_END_DECLS +#endif /* __GST_DSDIRECTION_H__ */ diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/LICENSE b/src/apps/reference_apps/deepstream-bodypose-3d/LICENSE new file mode 100644 index 00000000..07a5f423 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/LICENSE @@ -0,0 +1,16 @@ +SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/Makefile b/src/apps/reference_apps/deepstream-bodypose-3d/Makefile new file mode 100644 index 00000000..ffb1708c --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/Makefile @@ -0,0 +1,6 @@ +all: + make -C sources + make -C sources/nvdsinfer_custom_impl_BodyPose3DNet +%: + make -C sources $@ + make -C sources/nvdsinfer_custom_impl_BodyPose3DNet $@ diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/README.md b/src/apps/reference_apps/deepstream-bodypose-3d/README.md new file mode 100644 index 00000000..ae746b09 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/README.md @@ -0,0 +1,133 @@ +# 3d-bodypose-deepstream + +## Introduction +The project contains 3D Body Pose application built using Deepstream SDK. + +This application is built for [KAMA: 3D Keypoint Aware Body Mesh Articulation](https://arxiv.org/abs/2104.13502). +![sample pose output](./sources/.screenshot.png) +## Prerequisites: +DeepStream SDK 9.0 installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/README` on how +to install the prequisites for building Deepstream SDK apps. + +The pretrained TAO models [PeopleNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet) and [BodyPose3DNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/bodypose3dnet) from NGC. + +## Installation +Follow https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Quickstart.html to setup the DeepStream SDK + +1. Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` +and define project home as `export BODYPOSE3D_HOME=/deepstream-bodypose-3d`. + +2. Install Eigen development packages +``` + sudo apt install libeigen3-dev + cd /usr/include + sudo ln -sf eigen3/Eigen Eigen +``` + +3. For Deepstream SDK version older than 6.2, copy and build custom `NvDsEventMsgMeta` into Deepstream SDK installation path. Copy and build custom `NvDsEventMsgMeta` into Deepstream SDK installation path. +The custom `NvDsEventMsgMeta` structure handles pose3d and pose25d meta data. +```bash +# Copy deepstream sources +cp $BODYPOSE3D_HOME/sources/deepstream-sdk/eventmsg_payload.cpp /opt/nvidia/deepstream/deepstream/sources/libs/nvmsgconv/deepstream_schema +# Build new nvmsgconv library for custom Product metadata +cd /opt/nvidia/deepstream/deepstream/sources/libs/nvmsgconv +make; make install +``` +Please note that this step is not necessary for Deepstream SDK version 6.2 or newer. + +## Build the applications +```bash +# Build custom nvinfer parser of BodyPose3DNet +cd $BODYPOSE3D_HOME/sources/nvdsinfer_custom_impl_BodyPose3DNet +make +# Build deepstream-pose-estimation-app +cd $BODYPOSE3D_HOME/sources +make +``` +If the above steps are successful, `deepstream-pose-estimation-app` shall be built in the same directory. Under `$BODYPOSE3D_HOME/sources/nvdsinfer_custom_impl_BodyPose3DNet`, `libnvdsinfer_custom_impl_BodyPose3DNet.so` should be present as well. + +## Run the applications +### `deepstream-pose-estimation-app` +Download the pretrained TAO models with the download script +``` +cd $BODYPOSE3D_HOME +bash ./download_models.sh +``` + +The command line options of this application are listed below: +```bash +$ ./deepstream-pose-estimation-app -h +Usage: + deepstream-pose-estimation-app [OPTION?] Deepstream BodyPose3DNet App + +Help Options: + -h, --help Show help options + --help-all Show all help options + --help-gst Show GStreamer Options + +Application Options: + -v, --version Print DeepStreamSDK version. + --version-all Print DeepStreamSDK and dependencies version. + --input [Required] Input video address in URI format by starting with "rtsp://" or "file://". + --output Output video address. Either "rtsp://" or a file path is acceptable. If the value is "rtsp://", then the result video is published at "rtsp://localhost:8554/ds-test". + --save-pose The file path to save both the pose25d and the recovered pose3d in JSON format. + --conn-str Connection string for Gst-nvmsgbroker, e.g. ;;. + --publish-pose Specify the type of pose to publish. Acceptable value is either "pose3d" or "pose25d". If not specified, both "pose3d" and "pose25d" are published to the message broker. + --tracker Specify the NvDCF tracker mode. The acceptable value is either "accuracy" or "perf". The default value is "accuracy". + --fps Print FPS in the format of current_fps (averaged_fps). + --width Input video width in pixels. The default value is 1280. + --height Input video height in pixels. The default value is 720. + --focal Camera focal length in millimeters. The default value is 800.79041. +``` + +Here are examples running this application: +1. Below command processes an input video in URI format and renders the overlaid pose estimation in a window. +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 +``` +Please provide the absolute path to the source video file. + +2. When the data source is a video file, below command saves the output video with the skeleton overlay to `$BODYPOSE3D_HOME/streams/bodypose_3dbp.mp4` and save the skeleton's keypoints to `$BODYPOSE3D_HOME/streams/bodypose_3dbp.json`. +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 --output $BODYPOSE3D_HOME/streams/bodypose_3dbp.mp4 --focal 800.0 --width 1280 --height 720 --fps --save-pose $BODYPOSE3D_HOME/streams/bodypose_3dbp.json +``` +`bodypose_3dbp.json` contains the predicted 34 keypoints in both `pose25d` and `pose3d` space: +```bash +[{ + "num_frames_in_batch": 1, + "batches": [{ + "batch_id": 0, + "frame_num": 3, + "ntp_timestamp": 1639431716322229000, + "num_obj_meta": 6, + "objects": [{ + "object_id": 3, + "pose25d": [707.645203, 338.592499, -0.000448, 0.867188, ...], + "pose3d": [297.649933, -94.196518, 3520.129883, 0.867188, ...] + },{ + ... + }] + }] +}, { +``` +`pose25d` contains `34x4` floats. A four-item group represents a keypoint's `[x, y, zRel, conf]` +values. `x` and `y` are the keypoint's position in the image coordinate; `zRel` is the relative +depth value from the skeleton's root keypoint, i.e. pelvis. `x, y, zRel` values are in millimeters. +`conf` is the confidence value of the prediction. + +`pose3d` also contains `34x4` floats. A four-item group represents a keypoint's `[x, y, z, conf]` +values. `x`, `y`, `z` are the keypoint's 3D position in the world coordinate whose origin is the +camera. `x, y, z` values are in millimeters. `conf` is the confidence value of the prediction. + +3. When the data source is an RTSP stream and the result is published to RTSP stream `rtsp://localhost:8554/ds-test`, +```bash +$ ./deepstream-pose-estimation-app --input rtsp://:/ --output rtsp:// +``` + +4. In order to publish both pose3D and pose25D metadata to a message broker, please do +```bash +$ ./deepstream-pose-estimation-app --input file://$BODYPOSE3D_HOME/streams/bodypose.mp4 --conn-str "localhost;9092;test" +``` +where `\"localhost;9092;test\"` is the connection string to the message broker `localhost`, port number `9092`, and topic name `test`. Please apply double quotes around the connection string since `;` is a reserved character in shell. diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt b/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt new file mode 100644 index 00000000..dae4dcb5 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_primary_peoplenet.txt @@ -0,0 +1,47 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=../models/peoplenet/resnet34_peoplenet_int8.onnx +labelfile-path=../models/peoplenet/labels.txt +model-engine-file=../models/peoplenet/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine +infer-dims=3;544;960 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +cluster-mode=2 +interval=0 +gie-unique-id=1 +maintain-aspect-ratio=0 + +## Use the config params below for NMS clustering mode +[class-attrs-all] +topk=8 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt b/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt new file mode 100644 index 00000000..0f04b402 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/configs/config_infer_secondary_bodypose3dnet.txt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.00392156 +model-engine-file=../models/bodypose3dnet/bodypose3dnet_accuracy.onnx_b8_gpu0_fp16.engine +onnx-file=../models/bodypose3dnet/bodypose3dnet_accuracy.onnx +#model-engine-file=../models/bodypose3dnet/bodypose3dnet_performance.onnx_b8_gpu0_fp16.engine +#onnx-file=../models/bodypose3dnet/bodypose3dnet_performance.onnx +infer-dims=3;256;192 +batch-size=8 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +## 0=Detection 1=Classifier 2=Segmentation 100=other +network-type=100 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +output-blob-names=pose2d;pose2d_org_img;pose25d;pose3d +classifier-threshold=0.7 +operate-on-class-ids=0 +## Integer 0:NCHW 1:NHWC +network-input-order=0 +# Enable tensor metadata output +output-tensor-meta=1 +## 1-Primary 2-Secondary +process-mode=2 +## 0=RGB 1=BGR 2=GRAY +model-color-format=1 +maintain-aspect-ratio=0 +symmetric-padding=0 +scaling-filter=1 +custom-lib-path=../sources/nvdsinfer_custom_impl_BodyPose3DNet/libnvdsinfer_custom_impl_BodyPose3DNet.so + + diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/download_models.sh b/src/apps/reference_apps/deepstream-bodypose-3d/download_models.sh new file mode 100755 index 00000000..bf5d6817 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/download_models.sh @@ -0,0 +1,36 @@ +#!/bin/sh +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +echo "===================================================================" +echo "begin downloading PeopleNet model " +echo "===================================================================" +mkdir -p ./models/peoplenet +cd ./models/peoplenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' -O labels.txt + +echo "===================================================================" +echo "begin downloading BodyPose3DNet model " +echo "===================================================================" +cd - +mkdir -p ./models/bodypose3dnet +cd ./models/bodypose3dnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O bodypose3dnet_accuracy.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_performance_onnx_v1.0/files?redirect=true&path=bodypose3dnet_performance.onnx' -O bodypose3dnet_performance.onnx + diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/.screenshot.png b/src/apps/reference_apps/deepstream-bodypose-3d/sources/.screenshot.png new file mode 100644 index 00000000..b291037c Binary files /dev/null and b/src/apps/reference_apps/deepstream-bodypose-3d/sources/.screenshot.png differ diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/Makefile b/src/apps/reference_apps/deepstream-bodypose-3d/sources/Makefile new file mode 100644 index 00000000..9d0f87a3 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/sources/Makefile @@ -0,0 +1,77 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX=g++ -std=c++14 + +APP:= deepstream-pose-estimation-app + +CUDA_HOME:= /usr/local/cuda-$(CUDA_VER) +DEEPSTREAM_HOME:= /opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DEEPSTREAM_HOME)/lib/ +APP_INSTALL_DIR?=$(DEEPSTREAM_HOME)/bin/ + +SRCS:= deepstream_pose_estimation_app.cpp + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(patsubst %.c,%.o, $(patsubst %.cpp,%.o, $(SRCS))) + +CFLAGS+= \ + -I$(CUDA_HOME)/include \ + -I$(DEEPSTREAM_HOME)/sources/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/apps-common/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/sample_apps/deepstream-app \ + -I../eigen + +LIBS+= \ + -L$(CUDA_HOME)/lib64 -lcudart -lcuda \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvds_utils -lgstapp-1.0 \ + -lpthread -lm -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + +debug: CXXFLAGS += -DDEBUG -ggdb +debug: CCFLAGS += -DDEBUG -ggdb +debug: CFLAGS += -DDEBUG -ggdb +debug: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp b/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp new file mode 100644 index 00000000..d165bda0 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream-sdk/eventmsg_payload.cpp @@ -0,0 +1,1051 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +constexpr int NVDS_OBJECT_TYPE_PERSON_EXT_POSE = 0x103; //NVDS_OBJECT_TYPE_UNKNOWN = 102 +static std::vector> _joint_maps = { + {// Type 0 - pose2D + std::string("nose"), + std::string("neck"), + std::string("right-shoulder"), + std::string("right-elbow"), + std::string("right-hand"), + std::string("left-shoulder"), + std::string("left-elbow"), + std::string("left-hand"), + std::string("right-hip"), + std::string("right-knee"), + std::string("right-foot"), + std::string("left-hip"), + std::string("left-knee"), + std::string("left-foot"), + std::string("right-eye"), + std::string("left-eye"), + std::string("right-ear"), + std::string("left-ear"), + }, + { + std::string("pelvis"), + std::string("left-hip"), + std::string("right-hip"), + std::string("torso"), + std::string("left-knee"), + std::string("right-knee"), + std::string("neck"), + std::string("left-ankle"), + std::string("right-ankle"), + std::string("left-big-toe"), + std::string("right-big-toe"), + std::string("left-small-toe"), + std::string("right-small-toe"), + std::string("left-heel"), + std::string("right-heel"), + std::string("nose"), + std::string("left-eye"), + std::string("right-eye"), + std::string("left-ear"), + std::string("right-ear"), + std::string("left-shoulder"), + std::string("right-shoulder"), + std::string("left-elbow"), + std::string("right-elbow"), + std::string("left-wrist"), + std::string("right-wrist"), + std::string("left-pinky-knuckle"), + std::string("right-pinky-knuckle"), + std::string("left-middle-tip"), + std::string("right-middle-tip"), + std::string("left-index-knuckle"), + std::string("right-index-knuckle"), + std::string("left-thumb-tip"), + std::string("right-thumb-tip"), + }, + { + std::string("pelvis"), + std::string("left-hip"), + std::string("right-hip"), + std::string("torso"), + std::string("left-knee"), + std::string("right-knee"), + std::string("neck"), + std::string("left-ankle"), + std::string("right-ankle"), + std::string("left-big-toe"), + std::string("right-big-toe"), + std::string("left-small-toe"), + std::string("right-small-toe"), + std::string("left-heel"), + std::string("right-heel"), + std::string("nose"), + std::string("left-eye"), + std::string("right-eye"), + std::string("left-ear"), + std::string("right-ear"), + std::string("left-shoulder"), + std::string("right-shoulder"), + std::string("left-elbow"), + std::string("right-elbow"), + std::string("left-wrist"), + std::string("right-wrist"), + std::string("left-pinky-knuckle"), + std::string("right-pinky-knuckle"), + std::string("left-middle-tip"), + std::string("right-middle-tip"), + std::string("left-index-knuckle"), + std::string("right-index-knuckle"), + std::string("left-thumb-tip"), + std::string("right-thumb-tip"), + } +}; + +typedef struct NvDsJoint { + gdouble confidence; + gdouble x; + gdouble y; + gdouble z; +}NvDsJoint; + +typedef struct NvDsJoints { + + gint pose_type; + gint num_joints; + NvDsJoint *joints; + +}NvDsJoints; + +typedef struct NvDsPersonPoseExt { + gint num_poses; + NvDsJoints *poses; +}NvDsPersonPoseExt; + +static JsonObject* +generate_place_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (meta->placeId); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << meta->placeId + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // parkingSpot / aisle /entrance sub object + jobject = json_object_new (); + + switch (meta->type) { + case NVDS_EVENT_MOVING: + case NVDS_EVENT_STOPPED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + break; + case NVDS_EVENT_EMPTY: + case NVDS_EVENT_PARKED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "type", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "parkingSpot", jobject); + break; + case NVDS_EVENT_ENTRY: + case NVDS_EVENT_EXIT: + if (meta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + } else { + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "lane", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "entrance", jobject); + } + break; + default: + cout << "Event type not implemented " << endl; + break; + } + + // coordinate sub sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* +generate_sensor_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (meta->sensorId); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << meta->sensorId + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* +generate_analytics_module_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (meta->moduleId); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << meta->moduleId + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + JsonObject *pobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE_EXT: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: + { + if (meta->extMsgSize){ + NvDsPersonPoseExt *pose_meta = (NvDsPersonPoseExt*)meta->extMsg; + + // // DEBUG + // g_message("generate_object_object(): pose_meta->num_poses = %d", pose_meta->num_poses); + + for (int i=0; inum_poses; i++) { + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + pobject = json_object_new (); + int joint_index = 0; + + for (joint_index = 0;joint_index < pose_meta->poses[i].num_joints; joint_index++) + { + if (pose_meta->poses[i].joints[joint_index].confidence > 0.0) + { + jobject = json_object_new (); + std::string s = _joint_maps[pose_meta->poses[i].pose_type][joint_index]; + char *json_name = const_cast(s.c_str()); + + if (pose_meta->poses[i].pose_type==0) { + json_object_set_int_member (jobject, "x", pose_meta->poses[i].joints[joint_index].x); + json_object_set_int_member (jobject, "y", pose_meta->poses[i].joints[joint_index].y); + json_object_set_int_member (jobject, "confidence", pose_meta->poses[i].joints[joint_index].confidence); + json_object_set_object_member (pobject, reinterpret_cast(json_name), jobject); + } + else if ((pose_meta->poses[i].pose_type==1)||(pose_meta->poses[i].pose_type==2)) { + // pose3d or pose25d from BodyPose3DNet + json_object_set_int_member (jobject, "x", pose_meta->poses[i].joints[joint_index].x); + json_object_set_int_member (jobject, "y", pose_meta->poses[i].joints[joint_index].y); + json_object_set_int_member (jobject, "z", pose_meta->poses[i].joints[joint_index].z); + json_object_set_int_member (jobject, "confidence", pose_meta->poses[i].joints[joint_index].confidence); + json_object_set_object_member (pobject, reinterpret_cast(json_name), jobject); + } + + } + } + + if (pose_meta->poses[i].pose_type==0) + json_object_set_object_member (objectObj, "pose2D", pobject); + else if (pose_meta->poses[i].pose_type==1) + json_object_set_object_member (objectObj, "pose3D", pobject); + else if (pose_meta->poses[i].pose_type==2) + json_object_set_object_member (objectObj, "pose25D", pobject); + } + } + } + break; + default: + cout << "Object type not implemented" << endl; + break; + } + + + if (objectMask) { + GList *l; + JsonArray *maskArray = json_array_sized_new (g_list_length(objectMask)); + + for (l = objectMask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + JsonArray *polygonArray = json_array_sized_new (polygon->len); + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + + json_array_add_double_element (polygonArray, value); + } + + json_array_add_array_element (maskArray, polygonArray); + } + + json_object_set_array_member (objectObj, "maskoutline", maskArray); + } + + // signature sub array + if (meta->objSignature.size) { + JsonArray *jArray = json_array_sized_new (meta->objSignature.size); + + for (i = 0; i < meta->objSignature.size; i++) { + json_array_add_double_element (jArray, meta->objSignature.signature[i]); + } + json_object_set_array_member (objectObj, "signature", jArray); + } + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", meta->location.lat); + json_object_set_double_member (jobject, "lon", meta->location.lon); + json_object_set_double_member (jobject, "alt", meta->location.alt); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", meta->coordinate.x); + json_object_set_double_member (jobject, "y", meta->coordinate.y); + json_object_set_double_member (jobject, "z", meta->coordinate.z); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, meta); + + // sensor object + sensorObj = generate_sensor_object (privData, meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, meta); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", meta->ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case (NvDsObjectType)NVDS_OBJECT_TYPE_PERSON_EXT_POSE: { + NvDsPersonPoseExt *pose_meta = (NvDsPersonPoseExt *) meta->extMsg; + if (pose_meta) { + // STUB person + ss << "|#||||||" << meta->confidence; + + for (int i = 0; i < pose_meta->num_poses; i++) { + int type = pose_meta->poses[i].pose_type; + + if (type == 0) { + ss << "|#|pose2D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + else if (type == 1) {// BodyPose3DNet + ss << "|#|pose3D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].z) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + else if (type == 2) {// BodyPose3DNet + ss << "|#|pose25D|"; + for (int joint_index = 0; joint_index < pose_meta->poses[i].num_joints; joint_index++) { + std::string s = _joint_maps[type][joint_index] + + "," + std::to_string(pose_meta->poses[i].joints[joint_index].x) + + "," + to_string(pose_meta->poses[i].joints[joint_index].y) + + "," + to_string(pose_meta->poses[i].joints[joint_index].z) + + "," + to_string(pose_meta->poses[i].joints[joint_index].confidence); + ss << s << "|"; + } + } + } + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_int_member (jobject, "id", events[0].metadata->frameId); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} + diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp b/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp new file mode 100644 index 00000000..00ad914f --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/sources/deepstream_pose_estimation_app.cpp @@ -0,0 +1,2045 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "cuda_runtime_api.h" +#include "gstnvdsinfer.h" +#include "gstnvdsmeta.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" +#include "nvdsmeta_schema.h" +#include "deepstream_common.h" +#include "deepstream_perf.h" +#include "deepstream_app_version.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC (NVDS_APP); // define category (statically) +#define GST_CAT_DEFAULT NVDS_APP // set as default + +#define EPS 1e-6 +#define MAX_TIME_STAMP_LEN 32 +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 +#define PGIE_CLASS_ID_PRODUCT 4 + +// Default camera attributes +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 +#define FOCAL_LENGTH 800.79041f + +/* Padding due to AR SDK model requires bigger bboxes*/ +#define PAD_DIM 128 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" +#define CONFIG_GPU_ID "gpu-id" + +#define PGIE_CONFIG_FILE "../configs/config_infer_primary_peoplenet.txt" +#define SGIE_CONFIG_FILE "../configs/config_infer_secondary_bodypose3dnet.txt" +#define TRACKER_CONFIG_FILE "../configs/config_tracker.txt" +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" +#define MAX_TRACKING_ID_LEN 16 + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr ("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +//---Global variables derived from program arguments--- +static gboolean _print_version = FALSE; +static gboolean _print_dependencies_version = FALSE; +static gboolean _print_fps = FALSE; +static gchar *_input = NULL; +static gchar *_output = NULL; +static gchar *_nvmsgbroker_conn_str = NULL; +static gchar *_pose_filename = NULL; +static gchar *_tracker = NULL; +static gchar *_publish_pose = NULL; +static guint _cintr = FALSE; +static gboolean _quit = FALSE; +FILE *_pose_file = NULL; +double _focal_length_dbl = FOCAL_LENGTH; +float _focal_length = (float)_focal_length_dbl; +int _image_width = MUXER_OUTPUT_WIDTH; +int _image_height = MUXER_OUTPUT_HEIGHT; +int _pad_dim = PAD_DIM;// A scaled version of PAD_DIM +Eigen::Matrix3f _K;// Camera intrinsic matrix +//---Global variables derived from program arguments--- + +static GstElement *pipeline = NULL; +static gint _fps_interval=1; +static gint _osd_process_mode = 0; + +gint frame_number = 0; + +typedef struct NvDsPersonPoseExt { + gint num_poses; + NvDsJoints *poses; +}NvDsPersonPoseExt; + + +class OneEuroFilter { +public: + /// Default constructor + OneEuroFilter() { + reset(30.0f /* Hz */, 0.1f /* Hz */, 0.09f /* ??? */, 0.5f /* Hz */); + } + /// Constructor + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + OneEuroFilter(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(dataUpdateRate, minCutoffFreq, cutoffSlope, derivCutoffFreq); + } + /// Reset all parameters of the filter. + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + void reset(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(); _rate = dataUpdateRate; _minCutoff = minCutoffFreq; _beta = cutoffSlope; _dCutoff = derivCutoffFreq; + } + /// Reset only the initial condition of the filter, leaving parameters the same. + void reset() { _firstTime = true; _xFilt.reset(); _dxFilt.reset(); } + /// Apply the one euro filter to the given input. + /// @param x the unfiltered input value. + /// @return the filtered output value. + float filter(float x) + { + float dx, edx, cutoff; + if (_firstTime) { + _firstTime = false; + dx = 0; + } else { + dx = (x - _xFilt.hatXPrev()) * _rate; + } + edx = _dxFilt.filter(dx, alpha(_rate, _dCutoff)); + cutoff = _minCutoff + _beta * fabsf(edx); + return _xFilt.filter(x, alpha(_rate, cutoff)); + } + + +private: + class LowPassFilter { + public: + LowPassFilter() { reset(); } + void reset() { _firstTime = true; } + float hatXPrev() const { return _hatXPrev; } + float filter(float x, float alpha){ + if (_firstTime) { + _firstTime = false; + _hatXPrev = x; + } + float hatX = alpha * x + (1.f - alpha) * _hatXPrev; + _hatXPrev = hatX; + return hatX; + + } + private: + float _hatXPrev; + bool _firstTime; + }; + inline float alpha(float rate, float cutoff) { + const float kOneOverTwoPi = 0.15915494309189533577f; // 1 / (2 * pi) + // The paper has 4 divisions, but we only use one + // float tau = kOneOverTwoPi / cutoff, te = 1.f / rate; + // return 1.f / (1.f + tau / te); + return cutoff / (rate * kOneOverTwoPi + cutoff); +} + bool _firstTime; + float _rate, _minCutoff, _dCutoff, _beta; + LowPassFilter _xFilt, _dxFilt; +}; + +//===Global variables=== +std::unordered_map> g_filter_pose25d; +OneEuroFilter m_filterRootDepth; // Root node in pose25d. + +fpos_t g_fp_25_pos; + +//===Global variables=== + +#define ACQUIRE_DISP_META(dmeta) \ + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_labels == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) \ + { \ + dmeta = nvds_acquire_display_meta_from_pool(bmeta);\ + nvds_add_display_meta_to_frame(frame_meta, dmeta);\ + }\ + +#define GET_LINE(lparams) \ + ACQUIRE_DISP_META(dmeta)\ + lparams = &dmeta->line_params[dmeta->num_lines];\ + dmeta->num_lines++;\ + +static float _sgie_classifier_threshold = FLT_MIN; + +typedef struct NvAR_Point3f { + float x, y, z; +} NvAR_Point3f; + +static Eigen::Matrix3f m_K_inv_transpose; +const float m_scale_ll[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; +const float m_mean_ll[] = { + 246.3427f, 246.3427f, 492.6854f, 402.4380f, 487.0321f, 128.6856f, 391.6295f, + 281.9928f, 249.9478f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 169.1832f, 402.2611f, 488.1824f, 128.6848f, 407.5836f, + 281.9897f, 249.9489f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 168.6137f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f}; + +/* Given 2D and ZRel, we need to find the depth of the root to reconstruct the scale normalized 3D Pose. + While there exists many 3D poses that can have the same 2D projection, given the 2.5D pose and intrinsic camera parameters, + there exists a unique 3D pose that satisfies (Xˆn − Xˆm)**2 + (Yˆn − Yˆm)**2 + (Zˆn − Zˆm)**2 = C**2. + Refer Section 3.3 of https://arxiv.org/pdf/1804.09534.pdf for more details. +*/ +std::vector calculateZRoots(const std::vector& X0, const std::vector& X1, + const std::vector& Y0, const std::vector& Y1, + const std::vector& Zrel0, + const std::vector& Zrel1, const std::vector& C) { + std::vector zRoots(X0.size()); + for (int i = 0; i < X0.size(); i++) { + double x0 = (double)X0[i], x1 = (double)X1[i], y0 = (double)Y0[i], y1 = (double)Y1[i], + z0 = (double)Zrel0[i], z1 = (double)Zrel1[i]; + double a = ((x1 - x0) * (x1 - x0)) + ((y1 - y0) * (y1 - y0)); + double b = 2 * (z1 * ((x1 * x1) + (y1 * y1) - x1 * x0 - y1 * y0) + + z0 * ((x0 * x0) + (y0 * y0) - x1 * x0 - y1 * y0)); + double c = ((x1 * z1 - x0 * z0) * (x1 * z1 - x0 * z0)) + + ((y1 * z1 - y0 * z0) * (y1 * z1 - y0 * z0)) + + ((z1 - z0) * (z1 - z0)) - (C[i] * C[i]); + double d = (b * b) - (4 * a * c); + + // make sure the solutions are valid + a = fmax(DBL_EPSILON, a); + d = fmax(DBL_EPSILON, d); + zRoots[i] = (float) ((-b + sqrt(d)) / (2 * a + 1e-8)); + } + return zRoots; +} + +float median(std::vector& v) { + size_t n = v.size() / 2; + nth_element(v.begin(), v.begin() + n, v.end()); + return v[n]; +} + +/* Given 2D keypoints and the relative depth of each keypoint w.r.t the root, we find the depth of the root + to reconstruct the scale normalized 3D pose. +*/ +std::vector liftKeypoints25DTo3D(const float* p2d, + const float* pZRel, + const int numKeypoints, + const Eigen::Matrix3f& KInv, + const float limbLengths[]) { + + const int ROOT = 0; + + // Contains the relative depth values of each keypoints + std::vector zRel(numKeypoints, 0.f); + + // Matrix containing the 2D keypoints. + Eigen::MatrixXf XY1 = Eigen::MatrixXf(numKeypoints, 3); + + // Mean distance between a specific pair and its parent. + std::vector C; + + // Indices representing keypoints and its parents for limb lengths > 0. + // In our dataset, we only have limb length information for few keypoints. + std::vector idx0 = { 0, 3, 6, 8, 5, 2, 2, 21, 23, 21, 7, 4, 1, 1, 20, 22, 20 }; + std::vector idx1 = { 3, 6, 0, 5, 2, 0, 21, 23, 25, 6, 4, 1, 0, 20, 22, 24, 6 }; + + std::vector X0(idx0.size(), 0.f), Y0(idx0.size(), 0.f), X1(idx0.size(), 0.f), Y1(idx0.size(), 0.f), + zRel0(idx0.size(), 0.f), zRel1(idx0.size(), 0.f); + + for (int i = 0; i < numKeypoints; i++) { + zRel[i] = pZRel[i]; + + XY1.row(i) << p2d[i * 2], p2d[(i * 2) + 1], 1.f; + + if (limbLengths[i] > 0.f) C.push_back(limbLengths[i]); + } + + // Set relative depth of root to be 0 as the relative depth is measure w.r.t the root. + zRel[ROOT] = 0.f; + + for (int i = 0; i < XY1.rows(); i++) { + float x = XY1(i, 0); + float y = XY1(i, 1); + float z = XY1(i, 2); + XY1.row(i) << x, y, z; + } + + XY1 = XY1 * KInv; + + for (int i = 0; i < idx0.size(); i++) { + X0[i] = XY1(idx0[i], 0); + Y0[i] = XY1(idx0[i], 1); + X1[i] = XY1(idx1[i], 0); + Y1[i] = XY1(idx1[i], 1); + zRel0[i] = zRel[idx0[i]]; + zRel1[i] = zRel[idx1[i]]; + } + + std::vector zRoots = calculateZRoots(X0, X1, Y0, Y1, zRel0, zRel1, C); + + float zRootsMedian = median(zRoots); + + zRootsMedian = m_filterRootDepth.filter(zRootsMedian); + + std::vector p3d(numKeypoints, { 0.f, 0.f, 0.f }); + + for (int i = 0; i < numKeypoints; i++) { + p3d[i].x = XY1(i, 0) * (zRel[i] + zRootsMedian); + p3d[i].y = XY1(i, 1) * (zRel[i] + zRootsMedian); + p3d[i].z = XY1(i, 2) * (zRel[i] + zRootsMedian); + } + + return p3d; +} + +/* Once we have obtained the scale normalized 3D pose, we use the mean limb lengths of keypoint-keypointParent pairs +* to find the scale of the whole body. We solve for +* s^ = argmin sum((s * L2_norm(P_k - P_l) - meanLimbLength_k_l)**2), solve for s. +* meanLimbLength_k_l = mean length of the bone between keypoints k and l in the training data +* P_k and P_l are the keypoint location of k and l. +* +* We have a least squares minimization, where we are trying to minimize the magnitude of the error: + (target - scale * unit_length). Thus, we're minimizing T - sL. By the normal equations, the optimal solution is: + s = inv([L'L]) * L'T + +*/ +float recoverScale(const std::vector& p3d, const float* scores, + const float targetLengths[]) { + std::vector validIdx; + + // Indices of keypoints for which we have the length information. + for (int i = 0; i < p3d.size(); i++) { + if (targetLengths[i] > 0.f) validIdx.push_back(i); + } + + Eigen::MatrixXf targetLenMatrix = Eigen::MatrixXf(validIdx.size(), 1); + + for (int i = 0; i < validIdx.size(); i++) { + targetLenMatrix(i, 0) = targetLengths[validIdx[i]]; + } + + // Indices representing keypoints and its parents for limb lengths > 0. + // In our dataset, we have only have limb length information for few keypoints. + std::vector idx0 = { 0, 3, 6, 8, 5, 2, 2, 21, 23, 21, 7, 4, 1, 1, 20, 22, 20 }; + std::vector idx1 = { 3, 6, 0, 5, 2, 0, 21, 23, 25, 6, 4, 1, 0, 20, 22, 24, 6 }; + + Eigen::MatrixXf unitLength = Eigen::MatrixXf(idx0.size(), 1); + Eigen::VectorXf limbScores(unitLength.size()); + float squareNorms = 0.f; + float limbScoresSum = 0.f; + for (int i = 0; i < idx0.size(); i++) { + unitLength(i, 0) = sqrtf((p3d[idx0[i]].x - p3d[idx1[i]].x) * (p3d[idx0[i]].x - p3d[idx1[i]].x) + + (p3d[idx0[i]].y - p3d[idx1[i]].y) * (p3d[idx0[i]].y - p3d[idx1[i]].y) + + (p3d[idx0[i]].z - p3d[idx1[i]].z) * (p3d[idx0[i]].z - p3d[idx1[i]].z)); + + limbScores[i] = scores[idx0[i]] * scores[idx1[i]]; + limbScoresSum += limbScores[i]; + } + + for (int i = 0; i < limbScores.size(); i++) { + limbScores[i] /= limbScoresSum; + squareNorms += ((unitLength(i, 0) * unitLength(i, 0)) * limbScores[i]); + } + + auto limbScoreDiag = limbScores.asDiagonal(); + + //Eigen::MatrixXf numerator1 = ; + Eigen::MatrixXf numerator = (unitLength.transpose() * limbScoreDiag) * targetLenMatrix; + + return numerator(0, 0) / squareNorms; +} + +void generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime(CLOCK_REALTIME, &ts); + memcpy(&tloc, (void *)(&ts.tv_sec), sizeof(time_t)); + gmtime_r(&tloc, &tm_log); + strftime(buf, buf_size,"%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec/1000000; + g_snprintf(strmsec, sizeof(strmsec),".%.3dZ", ms); + strncat(buf, strmsec, buf_size); +} + +static +gpointer copy_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + NvDsPersonObject *srcExt = NULL; + NvDsPersonObject *dstExt = NULL; + + dstMeta = (NvDsEventMsgMeta *)g_memdup ((gpointer)srcMeta, sizeof(NvDsEventMsgMeta)); + + // pose + dstMeta->pose.num_joints = srcMeta->pose.num_joints; + dstMeta->pose.pose_type = srcMeta->pose.pose_type; + dstMeta->pose.joints = (NvDsJoint *)g_memdup ((gpointer)srcMeta->pose.joints, + sizeof(NvDsJoint)*srcMeta->pose.num_joints); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = (gdouble *)g_memdup ((gpointer)srcMeta->objSignature.signature, + sizeof(gdouble)*srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if(srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsg){ + dstMeta->extMsg = g_memdup(srcMeta->extMsg, srcMeta->extMsgSize); + dstMeta->extMsgSize = srcMeta->extMsgSize; + srcExt = (NvDsPersonObject *)srcMeta->extMsg; + dstExt = (NvDsPersonObject *)dstMeta->extMsg; + dstExt->gender = g_strdup(srcExt->gender); + dstExt->hair = g_strdup(srcExt->hair); + dstExt->cap = g_strdup(srcExt->cap); + dstExt->apparel = g_strdup(srcExt->apparel); + dstExt->age = srcExt->age; + } + return dstMeta; +} + +static void +release_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + // pose + g_free (srcMeta->pose.joints); + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if(srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + srcMeta->extMsg = NULL; + + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +void build_msg_meta(NvDsFrameMeta *frame_meta, + NvDsObjectMeta *obj_meta, + const int numKeyPoints, + const float *keypoints, + const float *keypointsZRel, + const float *keypoints_confidence, + const std::vector &p3dLifted) +{ + NvDsEventMsgMeta *msg_meta = (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + NvDsPersonObject *msg_meta_ext = (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + msg_meta->type = NVDS_EVENT_ENTRY; //Should this be ENTRY + msg_meta->objType = (NvDsObjectType) NVDS_OBJECT_TYPE_PERSON; + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->extMsg = msg_meta_ext; + msg_meta->extMsgSize = sizeof(NvDsPersonObject); + msg_meta_ext->gender = g_strdup(""); + msg_meta_ext->hair = g_strdup(""); + msg_meta_ext->cap = g_strdup(""); + msg_meta_ext->apparel = g_strdup(""); + msg_meta_ext->age = 0; + + //---msg_meta->poses--- + if (1) { + int pose_types[8]; + + if (_publish_pose) { + if (!strcmp(_publish_pose, "pose3d")) + msg_meta->pose.pose_type = 2;// pdatapose3D + else if (!strcmp(_publish_pose, "pose25d")) + msg_meta->pose.pose_type = 1;// pdatapose3D + } + else { + msg_meta->pose.pose_type = 2;// pdatapose3D + } + + msg_meta->pose.num_joints = numKeyPoints; + msg_meta->pose.joints = (NvDsJoint *)g_malloc0(sizeof(NvDsJoint) * numKeyPoints); + + if (msg_meta->pose.pose_type == 0) {// pose25d without zRel + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = keypoints[2*i ]; + msg_meta->pose.joints[i].y = keypoints[2*i+1]; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + else if (msg_meta->pose.pose_type == 2) {// pose3d + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = p3dLifted[i].x; + msg_meta->pose.joints[i].y = p3dLifted[i].y; + msg_meta->pose.joints[i].z = p3dLifted[i].z; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + else if (msg_meta->pose.pose_type == 1) {// pose25d + for(int i = 0; i < msg_meta->pose.num_joints; i++){ + msg_meta->pose.joints[i].x = keypoints[2*i ]; + msg_meta->pose.joints[i].y = keypoints[2*i+1]; + msg_meta->pose.joints[i].z = keypointsZRel[i]; + msg_meta->pose.joints[i].confidence = keypoints_confidence[i]; + } + } + } + // // DEBUG + // g_message("Metadata poses are built.\n"); + //---msg_meta->poses--- + // msg_meta->embedding = + // msg_meta->location = + // msg_meta->coordinate = + // msg_meta->objSignature = + //msg_meta->objClassId = PGIE_CLASS_ID_PERSON; + msg_meta->objClassId = obj_meta->class_id; + // msg_meta->sensorId = + // msg_meta->moduleId = + // msg_meta->placeId = + // msg_meta->componentId = + msg_meta->frameId = frame_meta->frame_num; + msg_meta->confidence = obj_meta->confidence; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + generate_ts_rfc3339(msg_meta->ts, MAX_TIME_STAMP_LEN); + msg_meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + strncpy(msg_meta->objectId, obj_meta->obj_label, MAX_LABEL_SIZE); + // msg_meta->sensorStr = + // msg_meta->otherAttr = + // msg_meta->videoPath = + // // DEBUG + // g_message("Metadata is built."); + + NvDsBatchMeta *batch_meta = frame_meta->base_meta.batch_meta; + NvDsUserMeta *user_event_meta = nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = (NvDsMetaCopyFunc) copy_bodypose_meta; + user_event_meta->base_meta.release_func = (NvDsMetaReleaseFunc)release_bodypose_meta; + nvds_add_user_meta_to_frame(frame_meta, user_event_meta); + } else { + g_printerr("Error in attaching event meta to buffer\n"); + } +} + +void osd_upper_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 24; + const int idx_joints[] = { 0, 1, 2, 3, 6, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}; + const int num_bones = 25; + const int idx_bones[] = { 21, 6, 20, 6, 21, 23, 20, 22, 24, 22, 23, 25, + 27, 25, 31, 25, 33, 25, 29, 25, 24, 30, 24, 26, + 24, 32, 24, 28, 2, 21, 1, 20, 3, 6, 6, 15, + 15, 16, 15, 17, 19, 17, 18, 16, 0, 1, 0, 2, + 0, 3}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void osd_lower_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 10; + const int idx_joints[] = { 4, 5, 7, 8, 9, 10, 11, 12, 13, 14}; + const int num_bones = 10; + const int idx_bones[] = { 2, 5, 5, 8, 1, 4, 4, 7, 7, 13, + 8, 14, 8, 10, 7, 9, 11, 9, 12, 10}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void parse_25dpose_from_tensor_meta(NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + // const int pelvis = 0; + // const int left_hip = 1; + // const int right_hip = 2; + // const int torso = 3; + // const int left_knee = 4; + // const int right_knee = 5; + // const int neck = 6; + // const int left_ankle = 7; + // const int right_ankle = 8; + // const int left_big_toe = 9; + // const int right_big_toe = 10; + // const int left_small_toe = 11; + // const int right_small_toe = 12; + // const int left_heel = 13; + // const int right_heel = 14; + // const int nose = 15; + // const int left_eye = 16; + // const int right_eye = 17; + // const int left_ear = 18; + // const int right_ear = 19; + // const int left_shoulder = 20; + // const int right_shoulder = 21; + // const int left_elbow = 22; + // const int right_elbow = 23; + // const int left_wrist = 24; + // const int right_wrist = 25; + // const int left_pinky_knuckle = 26; + // const int right_pinky_knuckle = 27; + // const int left_middle_tip = 28; + // const int right_middle_tip = 29; + // const int left_index_knuckle = 30; + // const int right_index_knuckle = 31; + // const int left_thumb_tip = 32; + // const int right_thumb_tip = 33; + + float threshold = 0.1; + int window_size = 5; + int max_num_parts = 20; + int num_integral_samples = 7; + float link_threshold = 0.1; + int max_num_objects = 100; + + void *cmap_data = tensor_meta->out_buf_ptrs_host[0]; + NvDsInferDims &cmap_dims = tensor_meta->output_layers_info[0].inferDims; + void *paf_data = tensor_meta->out_buf_ptrs_host[1]; + NvDsInferDims &paf_dims = tensor_meta->output_layers_info[1].inferDims; + + const int numKeyPoints = 34; + // std::vector keypoints(68) ; + // std::vector keypointsZRel(34); + // std::vector keypoints_confidence(34); + float keypoints[2 * numKeyPoints]; + float keypointsZRel[numKeyPoints]; + float keypoints_confidence[numKeyPoints]; + + m_K_inv_transpose = _K.inverse().eval(); + m_K_inv_transpose = m_K_inv_transpose.transpose().eval(); + + NvDsBatchMeta *bmeta = frame_meta->base_meta.batch_meta; + NvDsDisplayMeta *dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + + for (unsigned int m=0; m < tensor_meta->num_output_layers;m++){ + NvDsInferLayerInfo *info = &tensor_meta->output_layers_info[m]; + + if (!strcmp(info->layerName, "pose25d")) { + float *data = (float *)tensor_meta->out_buf_ptrs_host[m]; + /* for (int j =0 ; j < 34; j++){ + printf ("a=%f b=%f c=%f d=%f\n",data[j*4],data[j*4+1],data[j*4+2], data[j*4+3]); + }*/ + + // Initialize + if (g_filter_pose25d.find(obj_meta->object_id) == g_filter_pose25d.end()) { + const float m_oneEuroSampleRate = 30.0f; + // const float m_oneEuroMinCutoffFreq = 0.1f; + // const float m_oneEuroCutoffSlope = 0.05f; + const float m_oneEuroDerivCutoffFreq = 1.0f;// Hz + + //std::vector filter_vec; + std::vector filter_vec; + + for (int j=0; j < numKeyPoints*3; j++) { + //TODO:Pending delete especially when object goes out of view, or ID switch + //will cause memleak, cleanup required wrap into class + // filter_vec.push_back(SF1eFilterCreate(30, 1.0, 0.0, 1.0)); + + // filters for x and y + // for (auto& fil : m_filterKeypoints2D) fil.reset(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + + // filters for z (depth) + // for (auto& fil : m_filterKeypointsRelDepth) fil.reset(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq)); + } + g_filter_pose25d[obj_meta->object_id] = filter_vec; + + // Filters depth of root keypoint + m_filterRootDepth.reset(m_oneEuroSampleRate, 0.1f, 0.05f, m_oneEuroDerivCutoffFreq); + } + + int batchSize_offset = 0; + + //std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + + // x,y,z,c + for (int i = 0; i < numKeyPoints; i++) { + int index = batchSize_offset + i * 4; + + // Update with filtered results + keypoints[2 * i ] = filt_val[3 * i ].filter(data[index ] * + (obj_meta->rect_params.width / 192.0) + obj_meta->rect_params.left); + keypoints[2 * i + 1] = filt_val[3 * i + 1].filter(data[index + 1] * + (obj_meta->rect_params.height / 256.0) + obj_meta->rect_params.top); + keypointsZRel[i] = filt_val[3 * i + 2].filter(data[index + 2]); + + keypoints_confidence[i] = data[index + 3]; + } + + // Since we have cropped and resized the image buffer provided to the SDK from the app, + // we scale and offset the points back to the original resolution + float scaleOffsetXY[] = {1.0f, 0.0f, 1.0f, 0.0f}; + + // Render upper body + if (1) { + osd_upper_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + // Render lower body + if (1) { + osd_lower_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + + // SGIE operates on an enlarged/padded image buffer. + // const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + // const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + // Before outputting result, the image frame with overlay is cropped by removing _pad_dim. + // The final pose estimation result should counter the padding before deriving 3D keypoints. + for (int i = 0; i < numKeyPoints; i++) { + keypoints[2 * i ]-= _pad_dim; + keypoints[2 * i + 1]-= _pad_dim; + } + + // Recover pose 3D + std::vector p3dLifted; + p3dLifted = liftKeypoints25DTo3D(keypoints, keypointsZRel, numKeyPoints, m_K_inv_transpose, m_scale_ll); + float scale = recoverScale(p3dLifted, keypoints_confidence, m_mean_ll); + // printf("scale = %f\n", scale); + for (auto i = 0; i < p3dLifted.size(); i++) { + p3dLifted[i].x *= scale; + p3dLifted[i].y *= scale; + p3dLifted[i].z *= scale; + } + + if (_nvmsgbroker_conn_str) {// Prepare metadata to message broker + build_msg_meta(frame_meta, obj_meta, + numKeyPoints, keypoints, keypointsZRel, + keypoints_confidence, p3dLifted); + g_debug("Sent metadata of frame %6d to message broker.", frame_meta->frame_num); + } + + // Output pose25d and pose3d tensors + if (_pose_file) { + fprintf(_pose_file, + "{\n" + " \"object_id\": %lu,\n", + obj_meta->object_id); + + // Write pose25d + fprintf(_pose_file, + " \"pose25d\": ["); + for (int i = 0; i < p3dLifted.size(); i++) { + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, "%f, %f, %f, %f", keypoints[2*i], keypoints[2*i+1], + keypointsZRel[i], keypoints_confidence[i]); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "],\n"); + + // Write the recovered pose3d + fprintf(_pose_file, + " \"pose3d\": ["); + for (int i = 0; i < p3dLifted.size(); i++) { + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, "%f, %f, %f, %f", p3dLifted[i].x, p3dLifted[i].y, p3dLifted[i].z, keypoints_confidence[i]); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n"); + + // Remember the position of "," so that we can remove it on the last entry. + fprintf(_pose_file, " }"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } +} + +/* pgie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +pgie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + float sizex = obj_meta->rect_params.width * .5f; + float sizey = obj_meta->rect_params.height * .5f; + float centrx = obj_meta->rect_params.left + sizex; + float centry = obj_meta->rect_params.top + sizey; + sizex *= (1.25f); + sizey *= (1.25f); + if (sizex < sizey) + sizex = sizey; + else + sizey = sizex; + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + + sizex= obj_meta->rect_params.width * .5f, sizey = obj_meta->rect_params.height * .5f; + centrx = obj_meta->rect_params.left + sizex, centry = obj_meta->rect_params.top + sizey; + // Make sure box has same aspect ratio as 3D Body Pose model's input dimensions + // (e.g 192x256 -> 0.75 aspect ratio) by enlarging in the appropriate dimension. + float xScale = (float)192.0 / (float)sizex, yScale = (float)256.0 / (float)sizey; + if (xScale < yScale) { // expand on height + sizey = (float)256.0/ xScale; + } + else { // expand on width + sizex = (float)192.0 / yScale; + } + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + if (obj_meta->rect_params.left < 0.0) { + obj_meta->rect_params.left = 0.0; + } + if (obj_meta->rect_params.top < 0.0) { + obj_meta->rect_params.top = 0.0; + } + if (obj_meta->rect_params.left + obj_meta->rect_params.width > muxer_output_width_pad -1){ + obj_meta->rect_params.width = muxer_output_width_pad - 1 - obj_meta->rect_params.left; + } + if (obj_meta->rect_params.top + obj_meta->rect_params.height > muxer_output_height_pad -1){ + obj_meta->rect_params.height = muxer_output_height_pad - 1 - obj_meta->rect_params.top; + } + + } + } + return GST_PAD_PROBE_OK; +} + +/* sgie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +sgie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + if (_pose_file) {// Write batch header + if (batch_meta->frame_meta_list) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(batch_meta->frame_meta_list->data); + if (frame_meta->obj_meta_list) { + fprintf(_pose_file, + "{\n" + " \"num_frames_in_batch\": %d,\n" + " \"batches\": [", + batch_meta->num_frames_in_batch); + } + } + } + + // g_mutex_lock(&str->struct_lock); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + if (_pose_file) {// Write frame header + if (frame_meta->obj_meta_list) { + fprintf(_pose_file, + "{\n" + " \"batch_id\": %d,\n" + " \"frame_num\": %d,\n" + " \"ntp_timestamp\": %ld,\n" + " \"num_obj_meta\": %d,\n" + " \"objects\": [", + frame_meta->batch_id, frame_meta->frame_num, frame_meta->ntp_timestamp, frame_meta->num_obj_meta); + fgetpos(_pose_file, &g_fp_25_pos); + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + + // Set below values to 0 in order to disable bbox and text output + obj_meta->rect_params.border_width = 0;//2; + obj_meta->text_params.font_params.font_size = 0;//10; + + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + parse_25dpose_from_tensor_meta(tensor_meta, frame_meta, obj_meta) ; + } + } + } + + if (_pose_file) {// closing off "objects" key. + if (frame_meta->obj_meta_list) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n");// closing off "objects" key. + + fprintf(_pose_file, " }"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } + + if (_pose_file) {// closing off "batches" key. + if (batch_meta->frame_meta_list) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(batch_meta->frame_meta_list->data); + if (frame_meta->obj_meta_list) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n");// closing off "batches" key. + + fprintf(_pose_file, "}"); + fgetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, ", "); + } + } + } + // g_mutex_unlock (&str->struct_lock); + + return GST_PAD_PROBE_OK; +} + +/* osd_sink_pad_buffer_probe will extract metadata received from OSD + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) + { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + + /* Parameters to draw text onto the On-Screen-Display */ + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Frame Number = %d", frame_number); + offset = snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, " "); + + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + char font_name[] = "Mono"; + txt_params->font_params.font_name = font_name; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_EOS: + g_print("End of Stream\n"); + g_main_loop_quit(loop); + break; + + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(loop); + break; + } + + default: + break; + } + return TRUE; +} + +gboolean +link_element_to_tee_src_pad(GstElement *tee, GstElement *sinkelem) +{ + gboolean ret = FALSE; + GstPad *tee_src_pad = NULL; + GstPad *sinkpad = NULL; + GstPadTemplate *padtemplate = NULL; + + padtemplate = (GstPadTemplate *)gst_element_class_get_pad_template(GST_ELEMENT_GET_CLASS(tee), "src_%u"); + tee_src_pad = gst_element_request_pad(tee, padtemplate, NULL, NULL); + + if (!tee_src_pad) + { + g_printerr("Failed to get src pad from tee"); + goto done; + } + + sinkpad = gst_element_get_static_pad(sinkelem, "sink"); + if (!sinkpad) + { + g_printerr("Failed to get sink pad from '%s'", + GST_ELEMENT_NAME(sinkelem)); + goto done; + } + + if (gst_pad_link(tee_src_pad, sinkpad) != GST_PAD_LINK_OK) + { + g_printerr("Failed to link '%s' and '%s'", GST_ELEMENT_NAME(tee), + GST_ELEMENT_NAME(sinkelem)); + goto done; + } + ret = TRUE; + +done: + if (tee_src_pad) + { + gst_object_unref(tee_src_pad); + } + if (sinkpad) + { + gst_object_unref(sinkpad); + } + return ret; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvdec_*. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "urisourcebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + /* Commented to suppress warnings*/ + /*if (!(g_strcmp0 (source_type, "rtmp"))) { + g_object_set (G_OBJECT (object), "do-timestamp", 1, NULL); + g_object_set (G_OBJECT (object), "timeout", 10000, NULL); + }*/ +} + +// Imported from deepstream_test3_app.c +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("uridecodebin", "uri-decode-bin"); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + _cintr = TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (_quit) { + return FALSE; + } + + if (_cintr) { + _cintr = FALSE; + + _quit = TRUE; + GMainLoop *loop = (GMainLoop *) data; + g_main_loop_quit (loop); + + return FALSE; + } + return TRUE; +} +//===from deepstream_test5_app_main.c=== + +bool verify_arguments() +{ + if (!_input) { + g_printerr("--input option is not specified. Exiting...\n"); + return false; + } + else { + if (strncmp(_input, "rtsp://", 7) && strncmp(_input, "file://", 7)) { + g_printerr("--input value is not a valid URI address. Exiting...\n"); + return false; + } + } + + if (_tracker) { + if (strcmp(_tracker, "accuracy") && strcmp(_tracker, "perf")) { + g_printerr("--tracker value is neither \"accuracy\", nor \"perf\". Exiting...\n"); + return false; + } + } + else {// default value + _tracker = (gchar *)g_malloc0(64); + strcpy(_tracker, "perf"); + } + + if (_pose_filename) { + _pose_file = fopen(_pose_filename, "wt"); + if (!_pose_file) { + g_printerr("Cannot open file %s. Exiting...\n", _pose_filename); + return false; + } + fprintf(_pose_file, "["); + fgetpos(_pose_file, &g_fp_25_pos); + } + + if (_publish_pose) { + if (strcmp(_publish_pose, "pose3d") && strcmp(_publish_pose, "pose25d")) { + g_printerr("--publish-pose value is neither \"pose3d\", nor \"pose25d\". Exiting...\n"); + return false; + } + } + + if (_image_width <= 0) { + g_printerr("--width value %d is non-positive. Exiting...\n", _image_width); + return false; + } + if (_image_height <= 0) { + g_printerr("--height value %d is non-positive. Exiting...\n", _image_height); + return false; + } + _focal_length = (float)_focal_length_dbl; + if (_focal_length <= 0) { + g_printerr("--focal value %f is non-positive. Exiting...\n", _focal_length); + return false; + } + + _K.row(0) << _focal_length, 0, _image_width / 2.f; + _K.row(1) << 0, _focal_length, _image_height / 2.f; + _K.row(2) << 0, 0, 1.f; + + _pad_dim = PAD_DIM * _image_width / MUXER_OUTPUT_WIDTH; + + return true; +} + +int main(int argc, char *argv[]) +{ + const guint num_sources = 1; + + GMainLoop *loop = NULL; + GstCaps *caps = NULL; + GstElement *source = NULL, *streammux_pgie = NULL; + GstElement *sink = NULL, *pgie = NULL; + // Padding the image and removing the padding + GstElement *nvvideoconvert_enlarge = NULL, *nvvideoconvert_reduce = NULL, + *capsFilter_enlarge = NULL, *capsFilter_reduce = NULL; + GstElement *nvvidconv = NULL, *nvosd = NULL, *tracker = NULL, *nvdslogger = NULL, + *filesink = NULL, *nvvideoencfilesinkbin = NULL, + *nvrtspoutsinkbin = NULL; + GstElement *tee = NULL, *msgbroker = NULL, *msgconv = NULL;// msg broker and converter + GstBus *bus = NULL; + guint bus_watch_id; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + //---Parse command line options--- + { + const GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &_print_version, + "Print DeepStreamSDK version.", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &_print_dependencies_version, + "Print DeepStreamSDK and dependencies version.", NULL} + , + {"input", 0, 0, G_OPTION_ARG_STRING, &_input, + "[Required] Input video address in URI format by starting \ +with \"rtsp://\" or \"file://\".", + NULL} + , + {"output", 0, 0, G_OPTION_ARG_STRING, &_output, + "Output video address. Either \"rtsp://\" or a file path or \"fakesink\" is \ +acceptable. If the value is \"rtsp://\", then the result video is \ +published at \"rtsp://localhost:8554/ds-test\".", + NULL} + , + {"save-pose", 0, 0, G_OPTION_ARG_STRING, &_pose_filename, + "The file path to save both the pose25d and the recovered \ +pose3d in JSON format.", + NULL} + , + {"conn-str", 0, 0, G_OPTION_ARG_STRING, &_nvmsgbroker_conn_str, + "Connection string for Gst-nvmsgbroker, e.g. ;;.", + NULL} + , + {"publish-pose", 0, 0, G_OPTION_ARG_STRING, &_publish_pose, + "Specify the type of pose to publish. Acceptable \ +value is either \"pose3d\" or \"pose25d\". If not specified, both \"pose3d\" and \"pose25d\" \ +are published to the message broker.", + NULL} + , + {"tracker", 0, 0, G_OPTION_ARG_STRING, &_tracker, + "Specify the NvDCF tracker mode. The acceptable value is either \ +\"accuracy\" or \"perf\". The default value is \"perf\" \"accuracy\" mode"\ +" requires DeepSORT model to be installed. Please refer to [Setup Official Re-ID Model]"\ +"(https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html) section for details.", + NULL} + , + {"fps", 0, 0, G_OPTION_ARG_NONE, &_print_fps, + "Print FPS in the format of current_fps (averaged_fps).", + NULL} + , + {"fps-interval", 0, 0, G_OPTION_ARG_INT, &_fps_interval, + "Interval in seconds to print the fps, applicable only with --fps flag.", + NULL} + , + {"width", 0, 0, G_OPTION_ARG_INT, &_image_width, + "Input video width in pixels. The default value is 1280.",//MUXER_OUTPUT_WIDTH + NULL} + , + {"height", 0, 0, G_OPTION_ARG_INT, &_image_height, + "Input video height in pixels. The default value is 720.",//MUXER_OUTPUT_HEIGHT + NULL} + , + {"focal", 0, 0, G_OPTION_ARG_DOUBLE, &_focal_length_dbl, + "Camera focal length in millimeters. The default value is 800.79041.",//FOCAL_LENGTH + NULL} + , + {"osd-process-mode", 0, 0, G_OPTION_ARG_INT, &_osd_process_mode, + "OSD process mode CPU - 0 or GPU 1.", + NULL} + , + {NULL} + , + }; + + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + guint i; + + ctx = g_option_context_new ("Deepstream BodyPose3DNet App"); + group = g_option_group_new ("arguments", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "Deepstream BodyPose3DNet App", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + g_printerr ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + + if (_print_version) { + g_print ("deepstream-bodypose3dnet-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (_print_dependencies_version) { + g_print ("deepstream-bodypose3dnet-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (!verify_arguments()) { + g_printerr ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + } + //---Parse command line options--- + + + /* Standard GStreamer initialization */ + // signal(SIGINT, sigintHandler); + gst_init(&argc, &argv); + loop = g_main_loop_new(NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, loop); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new("deepstream-bodypose3dnet"); + if (!pipeline) { + g_printerr ("Pipeline could not be created. Exiting.\n"); + return -1; + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux_pgie = gst_element_factory_make ("nvstreammux", "streammux-pgie"); + if (!streammux_pgie) { + g_printerr ("PGIE streammux could not be created. Exiting.\n"); + return -1; + } + //---Set properties of streammux_pgie--- + g_object_set(G_OBJECT(streammux_pgie), "batch-size", num_sources, NULL); + g_object_set(G_OBJECT(streammux_pgie), "width", _image_width, "height", + _image_height, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + gst_bin_add(GST_BIN(pipeline), streammux_pgie); + //---Set properties of streammux_pgie--- + + // !!!TODO: support >1 input streams!!! + /* Source element for reading from the file/uri */ + { + GstPad *sinkpad, *srcpad; + gchar pad_name[16] = { }; + + source = create_source_bin(0, const_cast(_input)); + if (!source) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + gst_bin_add(GST_BIN(pipeline), source); + + g_snprintf (pad_name, 15, "sink_%u", 0); + sinkpad = gst_element_get_request_pad(streammux_pgie, pad_name); + if (!sinkpad) { + g_printerr ("Source Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad(source, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + pgie = gst_element_factory_make("nvinfer", "primary-nvinference-engine"); + if (!pgie) { + g_printerr ("PGIE element could not be created. Exiting.\n"); + return -1; + } + //---Set pgie properties--- + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set(G_OBJECT(pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + + /* Override the batch-size set in the config file with the number of sources. */ + guint pgie_batch_size = 0; + g_object_get(G_OBJECT(pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + + g_object_set(G_OBJECT(pgie), "batch-size", num_sources, NULL); + } + + //---Set pgie properties--- + + /* We need to have a tracker to track the identified objects */ + tracker = gst_element_factory_make ("nvtracker", "tracker"); + if (!tracker) { + g_printerr ("Nvtracker could not be created. Exiting.\n"); + return -1; + } + g_object_set (G_OBJECT(tracker), "ll-lib-file", + "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so", + NULL); + + if (!strcmp(_tracker, "accuracy")) { + g_object_set(G_OBJECT(tracker), "ll-config-file", + "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml", + NULL); + } + else if (!strcmp(_tracker, "perf")) { + g_object_set(G_OBJECT(tracker), "ll-config-file", + "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml", + NULL); + } + + if (_print_fps){ + nvdslogger = gst_element_factory_make ("nvdslogger", "nvdslogger"); + if (!nvdslogger) { + g_printerr ("Nvdslogger could not be created. Exiting.\n"); + return -1; + } + if (_fps_interval){ + g_object_set (G_OBJECT(nvdslogger), "fps-measurement-interval-sec", + _fps_interval, + NULL); + } + } + else { + nvdslogger = gst_element_factory_make ("queue", NULL); + if (!nvdslogger) { + g_printerr ("queue could not be created. Exiting.\n"); + return -1; + } + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* pgie_src_pad = gst_element_get_static_pad(tracker, "src"); + if (!pgie_src_pad) + g_printerr ("Unable to get src pad for pgie\n"); + else + gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + pgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (pgie_src_pad); + + /* 3d bodypose secondary gie */ + GstElement* sgie = gst_element_factory_make("nvinfer", "secondary-nvinference-engine"); + if (!sgie) { + g_printerr ("Secondary nvinfer could not be created. Exiting.\n"); + return -1; + } + //---Set sgie properties--- + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set(G_OBJECT(sgie), + "output-tensor-meta", TRUE, + "config-file-path", SGIE_CONFIG_FILE, + NULL); + + /* Override the batch-size set in the config file with the number of sources. */ + guint sgie_batch_size = 0; + g_object_get(G_OBJECT(sgie), "batch-size", &sgie_batch_size, NULL); + if (sgie_batch_size < num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + sgie_batch_size, num_sources); + + g_object_set(G_OBJECT(sgie), "batch-size", num_sources, NULL); + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* sgie_src_pad = gst_element_get_static_pad(sgie, "src"); + if (!sgie_src_pad) + g_printerr("Unable to get src pad for sgie\n"); + else + gst_pad_add_probe(sgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + sgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref(sgie_src_pad); + //---Set sgie properties--- + + /* Create tee to render buffer and send message simultaneously*/ + tee = gst_element_factory_make ("tee", "nvsink-tee"); + + /* Add queue elements between every two elements */ + GstElement* queue_nvvidconv = NULL; + queue_nvvidconv = gst_element_factory_make("queue", "queue_nvvidconv"); + if (!queue_nvvidconv) { + g_printerr ("queue_nvvidconv could not be created. Exiting.\n"); + return -1; + } + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter"); + if (!nvvidconv) { + g_printerr ("nvvidconv could not be created. Exiting.\n"); + return -1; + } + + //---Manipulate image size so that PGIE bbox is large enough--- + // Enlarge image so that PeopleNet detected bbox is larger which would fully cover the + // detected object in the original sized image. + nvvideoconvert_enlarge = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_enlarge"); + if (!nvvideoconvert_enlarge) { + g_printerr ("nvvideoconvert_enlarge could not be created. Exiting.\n"); + return -1; + } + capsFilter_enlarge = gst_element_factory_make("capsfilter", "capsFilter_enlarge"); + if (!capsFilter_enlarge) { + g_printerr ("capsFilter_enlarge could not be created. Exiting.\n"); + return -1; + } + + // Reduce the previously enlarged image frame so that the final output video retains the + // same dimension as the pipeline's input video dimension. + nvvideoconvert_reduce = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_reduce"); + if (!nvvideoconvert_reduce) { + g_printerr ("nvvideoconvert_reduce could not be created. Exiting.\n"); + return -1; + } + capsFilter_reduce = gst_element_factory_make("capsfilter", "capsFilter_reduce"); + if (!capsFilter_reduce) { + g_printerr ("capsFilter_reduce could not be created. Exiting.\n"); + return -1; + } + + gchar *string1 = NULL; + asprintf (&string1, "%d:%d:%d:%d", _pad_dim, _pad_dim, _image_width, _image_height); + // "dest-crop" - input size < output size + g_object_set(G_OBJECT(nvvideoconvert_enlarge), "dest-crop", string1,"interpolation-method",1 ,NULL); + // "src-crop" - input size > output size + g_object_set(G_OBJECT(nvvideoconvert_reduce), "src-crop", string1,"interpolation-method",1 ,NULL); + free(string1); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + muxer_output_width_pad, muxer_output_height_pad); + GstCaps *caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_enlarge),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + _image_width, _image_height); + caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_reduce),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + //---Manipulate image size so that PGIE bbox is large enough--- + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + if (!nvosd) { + g_printerr ("Nvdsosd could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT(nvosd), "process-mode", _osd_process_mode, NULL); + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* osd_sink_pad = gst_element_get_static_pad(nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, (gpointer)sink, NULL); + gst_object_unref(osd_sink_pad); + + /* Set output file location */ + if (_output) { + if (!strcmp(_output, "rtsp://")) { + filesink = gst_element_factory_make("nvrtspoutsinkbin", "nv-filesink"); + } + else if (!strcmp(_output,"fakesink")){ + filesink = gst_element_factory_make("fakesink", "nv-sink"); + } + else { + filesink = gst_element_factory_make("nvvideoencfilesinkbin", "nv-filesink"); + } + if (!filesink) { + g_printerr ("Filesink could not be created. Exiting.\n"); + return -1; + } + + if (strcmp(_output,"fakesink")){ + g_object_set(G_OBJECT(filesink), "output-file", _output, NULL); + g_object_set(G_OBJECT(filesink), "bitrate", 4000000, NULL); + //g_object_set(G_OBJECT(filesink), "profile", 3, NULL); + g_object_set(G_OBJECT(filesink), "codec", 2, NULL);//hevc + // g_object_set(G_OBJECT(filesink), "control-rate", 0, NULL);//hevc + } + } + else { +#ifdef __aarch64__ + filesink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + filesink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + /* Add all elements to the pipeline */ + // streammux_pgie has been added into pipeline already. + gst_bin_add_many(GST_BIN(pipeline), + nvvideoconvert_enlarge, capsFilter_enlarge, + pgie, tracker, sgie, tee, + queue_nvvidconv, nvvidconv, nvosd, filesink, nvdslogger, + nvvideoconvert_reduce, capsFilter_reduce, NULL); + + // Link elements + if (!gst_element_link_many(streammux_pgie, + nvvideoconvert_enlarge, capsFilter_enlarge, + pgie, tracker, sgie, nvdslogger, tee, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + if (prop.integrated && _output) { // Jetson + if (!gst_element_link_many(queue_nvvidconv, nvvidconv, nvosd, + nvvideoconvert_reduce, capsFilter_reduce, + filesink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + else { // dGPU & Jetson Display + if (!gst_element_link_many(queue_nvvidconv, nvvidconv, nvosd, + nvvideoconvert_reduce, capsFilter_reduce, + filesink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + + // Link tee and queue_nvvidconv + { + GstPad *sinkpad, *srcpad; + + srcpad = gst_element_get_request_pad (tee, "src_%u"); + sinkpad = gst_element_get_static_pad (queue_nvvidconv, "sink"); + if (!srcpad || !sinkpad) { + g_printerr ("Unable to get request pads\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Unable to link tee and queue_nvvidconv.\n"); + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + if (_nvmsgbroker_conn_str) {// Publish metadata to a broker as well + GstElement* queue_msgconv = gst_element_factory_make("queue", "queue_msgconv"); + if (!queue_msgconv) { + g_printerr ("queue_msgconv could not be created. Exiting.\n"); + return -1; + } + + /* Set up message broker */ + /* Create msg converter to generate payload from buffer metadata */ + msgconv = gst_element_factory_make ("nvmsgconv", "nvmsg-converter"); + // g_object_set (G_OBJECT(msgconv), "config", MSGCONV_CONFIG_FILE, NULL); + g_object_set (G_OBJECT(msgconv), "payload-type", 1, NULL);// Minimal schema + g_object_set (G_OBJECT(msgconv), "msg2p-newapi", 0, NULL);// Event Msg meta + + /* Create msg broker to send payload to server */ + msgbroker = gst_element_factory_make ("nvmsgbroker", "nvmsg-broker"); + g_object_set (G_OBJECT(msgbroker), + "proto-lib", "/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so", + "conn-str", _nvmsgbroker_conn_str, + "sync", FALSE, + NULL); + + gst_bin_add_many(GST_BIN(pipeline), queue_msgconv, msgconv, msgbroker, NULL); + + if (!gst_element_link_many(queue_msgconv, msgconv, msgbroker, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + // Link tee with queue_msgconv + GstPad *sinkpad, *srcpad; + + srcpad = gst_element_get_request_pad (tee, "src_%u"); + sinkpad = gst_element_get_static_pad (queue_msgconv, "sink"); + if (!srcpad || !sinkpad) { + g_printerr ("Unable to get request pads\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Unable to link tee and queue_msgconv.\n"); + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + /* Set the pipeline to "playing" state */ + g_print("Now playing: %s\n", _input); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(loop); + + /* Out of the main loop, clean up nicely */ + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline, GST_STATE_NULL); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + if (_pose_file) { + fsetpos(_pose_file, &g_fp_25_pos); + fprintf(_pose_file, "]\n"); + + fclose(_pose_file); + } + + + return 0; +} diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile b/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile new file mode 100644 index 00000000..73404ebb --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/Makefile @@ -0,0 +1,46 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CC:= g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I/usr/local/cuda-$(CUDA_VER)/include + + +LIBS:= -lnvinfer -lnvinfer_plugin +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinitinputlayers_BodyPose3DNet.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_BodyPose3DNet.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) + +install: + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp b/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp new file mode 100644 index 00000000..8f89d811 --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/sources/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nvdsinfer_custom_impl.h" +#include +/* Assumes only one input layer "im_info" needs to be initialized */ +bool NvDsInferInitializeInputLayers (std::vector const &inputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + unsigned int maxBatchSize) +{ + float scale_normalized_mean_limb_lengths[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; + float mean_limb_lengths[] = { + 246.3427, 246.3427, 492.6854, 402.4380, 487.0321, 128.6856, 391.6295, + 281.9928, 249.9478, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 169.1832, 402.2611, 488.1824, 128.6848, 407.5836, + 281.9897, 249.9489, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 168.6137, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000}; + + //k_inv would change for camera parameters + float k_inv[] = {0.00124876620338, 0, -0.119881555525, + 0, 0.00124876620338, -0.159842074033, + 0, 0, 1}; + + float t_form_inv[] = {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0}; + for (auto v : inputLayersInfo){ + if (!strcmp(v.layerName, "scale_normalized_mean_limb_lengths")){ + memcpy(v.buffer,scale_normalized_mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "mean_limb_lengths")){ + memcpy(v.buffer,mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "k_inv")){ + memcpy(v.buffer,k_inv,sizeof(float)*9); + } + if (!strcmp(v.layerName, "t_form_inv")){ + memcpy(v.buffer,t_form_inv,sizeof(float)*9); + } + } + + return true; +} + diff --git a/src/apps/reference_apps/deepstream-bodypose-3d/streams/bodypose.mp4 b/src/apps/reference_apps/deepstream-bodypose-3d/streams/bodypose.mp4 new file mode 100644 index 00000000..c3a533eb --- /dev/null +++ b/src/apps/reference_apps/deepstream-bodypose-3d/streams/bodypose.mp4 @@ -0,0 +1,3 @@ +version https://git.lfs.github.com/spec/v1 +oid sha256:46f38d405ef54ea9bee381bb794d32499e12911bd68f06d999372490ee64ca5a +size 3934589 diff --git a/src/apps/reference_apps/deepstream-custom-tile-config/Makefile b/src/apps/reference_apps/deepstream-custom-tile-config/Makefile new file mode 100644 index 00000000..01a3b0ba --- /dev/null +++ b/src/apps/reference_apps/deepstream-custom-tile-config/Makefile @@ -0,0 +1,35 @@ + +APP:= deepstream-custom-tile-config + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 glib-2.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -lm -L/usr/local/cuda/lib64/ -lcudart +CFLAGS+= -I/usr/local/cuda/include + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +all: $(APP) + +.o: .c $(INCS) Makefile + $(CC) -c $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/src/apps/reference_apps/deepstream-custom-tile-config/README.md b/src/apps/reference_apps/deepstream-custom-tile-config/README.md new file mode 100644 index 00000000..04e3b7de --- /dev/null +++ b/src/apps/reference_apps/deepstream-custom-tile-config/README.md @@ -0,0 +1,36 @@ +# DeepStream Custom Tiler Configuration Sample + +## Introduction +This sample demonstrates the usage of "custom-tile-config" of nvmultistreamtiler to customize the tiling positions and sizes of multiple videos within the display window. The rectangle display areas of every video can be configured by the "custom-tile-config" property of nvmultistreamtiler for CustomTileConfig struct. + +## Prerequisites +The sample works with DeepStream 9.0 GA or above version. Please follow [DeepStream SDK installation instruction](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Installation.html) or use [DeepStream docker container](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html) to prepare the DeepStream environment. + +## Running the Application +Download the source code and build + +``` +make +``` + +Run the sample with four video files and generate mp4 video for the output + +```bash +./deepstream-custom-tile-config -i file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + --no-display +``` + +Run the sample with four video files and dislay on the screen + +```bash +./deepstream-custom-tile-config -i file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 \ +``` + +## Known Issue +The video blending is not supported now. If there are overlapping parts of the videos, the overlapping parts will flicker. \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-custom-tile-config/custom_tiler_cfg.c b/src/apps/reference_apps/deepstream-custom-tile-config/custom_tiler_cfg.c new file mode 100644 index 00000000..d9ae31a2 --- /dev/null +++ b/src/apps/reference_apps/deepstream-custom-tile-config/custom_tiler_cfg.c @@ -0,0 +1,373 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvdstilerconfig.h" + +typedef struct { + GstElement *pipeline; + GstElement *streammux; + GstElement *tiler; + gboolean no_display; + guint num_sources; + /* Keep these alive for the lifetime of the pipeline */ + CustomTile *tiles_arr; + CustomTileConfig *cfg; +} AppCtx; + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + g_printerr("End-of-stream\n"); + g_main_loop_quit(loop); + break; + case GST_MESSAGE_ERROR: { + GError *err = NULL; + gchar *debug = NULL; + gst_message_parse_error(msg, &err, &debug); + g_printerr("Error: %s\n", err ? err->message : "(unknown)"); + if (debug) g_printerr("Debug details: %s\n", debug); + g_clear_error(&err); + g_free(debug); + g_main_loop_quit(loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad(GstElement *decodebin, GstPad *pad, gpointer user_data) +{ + GstCaps *caps = NULL; + const gchar *name = NULL; + GstStructure *str = NULL; + GstBin *source_bin = GST_BIN(user_data); + + caps = gst_pad_get_current_caps(pad); + if (!caps) caps = gst_pad_query_caps(pad, NULL); + if (!caps) return; + str = gst_caps_get_structure(caps, 0); + name = gst_structure_get_name(str); + + if (name && g_str_has_prefix(name, "video")) { + /* Only link NVMM pads */ + GstCapsFeatures *features = gst_caps_get_features(caps, 0); + if (features && gst_caps_features_contains(features, "memory:NVMM")) { + GstPad *ghost = gst_element_get_static_pad(GST_ELEMENT(source_bin), "src"); + if (ghost) { + if (!gst_ghost_pad_set_target(GST_GHOST_PAD(ghost), pad)) { + g_printerr("Failed to set ghost pad target\n"); + } + gst_object_unref(ghost); + } + } else { + g_printerr("Decodebin did not pick NVMM memory\n"); + } + } + if (caps) gst_caps_unref(caps); +} + +static void +decodebin_child_added(GstChildProxy *child_proxy, GObject *object, gchar *name, gpointer user_data) +{ + if (g_strrstr(name, "decodebin")) { + g_signal_connect(object, "child-added", G_CALLBACK(decodebin_child_added), user_data); + } +} + +static GstElement * +create_source_bin(guint index, const gchar *uri) +{ + gchar bin_name[32]; + g_snprintf(bin_name, sizeof(bin_name), "source-bin-%02u", index); + GstElement *bin = gst_bin_new(bin_name); + if (!bin) return NULL; + + GstElement *uri_decode_bin = gst_element_factory_make("uridecodebin", "uri-decode-bin"); + if (!uri_decode_bin) { + g_printerr("Unable to create uridecodebin\n"); + gst_object_unref(bin); + return NULL; + } + g_object_set(G_OBJECT(uri_decode_bin), "uri", uri, NULL); + g_signal_connect(uri_decode_bin, "pad-added", G_CALLBACK(cb_newpad), bin); + g_signal_connect(uri_decode_bin, "child-added", G_CALLBACK(decodebin_child_added), bin); + + gst_bin_add(GST_BIN(bin), uri_decode_bin); + + /* Create an initially no-target ghost pad that will be set in cb_newpad */ + GstPad *ghost = gst_ghost_pad_new_no_target("src", GST_PAD_SRC); + if (!ghost) { + g_printerr("Failed to add ghost pad\n"); + gst_object_unref(bin); + return NULL; + } + gst_element_add_pad(bin, ghost); + return bin; +} + +static void +build_custom_layout(AppCtx *app) +{ + /* Example asymmetric layout similar to the Python sample */ + guint n = app->num_sources; + if (n == 0) return; + + guint length = n; + CustomTile *tiles = g_new0(CustomTile, length); + + /* Default: 2x2 style base with one big-left if >=3 */ + if (length >= 1) { tiles[0].sourceId = 0; tiles[0].x = 0.00f; tiles[0].y = 0.00f; tiles[0].width = (n >= 3 ? 0.66f : 1.0f); tiles[0].height = 1.00f; } + if (length >= 2) { tiles[1].sourceId = 1; tiles[1].x = (n >= 3 ? 0.66f : 0.00f); tiles[1].y = 0.00f; tiles[1].width = (n >= 3 ? 0.34f : 1.0f - tiles[0].x); tiles[1].height = (n >= 3 ? 0.50f : 1.0f); } + if (length >= 3) { tiles[2].sourceId = 2; tiles[2].x = 0.66f; tiles[2].y = 0.50f; tiles[2].width = 0.34f; tiles[2].height = 0.50f; } + if (length >= 4) { tiles[3].sourceId = 3; tiles[3].x = 0.33f; tiles[3].y = 0.33f; tiles[3].width = 0.33f; tiles[3].height = 0.33f; } + + app->tiles_arr = tiles; + app->cfg = g_new0(CustomTileConfig, 1); + app->cfg->tiles = app->tiles_arr; + app->cfg->length = length; +} + +static gboolean +update_layout_cb(gpointer user_data) +{ + AppCtx *app = (AppCtx *)user_data; + if (!app || !app->cfg || !app->cfg->tiles || app->cfg->length == 0) return G_SOURCE_REMOVE; + + /* Make the first tile full screen as a demo update */ + app->cfg->tiles[0].x = 0.0f; + app->cfg->tiles[0].y = 0.0f; + app->cfg->tiles[0].width = 1.0f; + app->cfg->tiles[0].height = 1.0f; + + g_object_set(G_OBJECT(app->tiler), "custom-tile-config", (gpointer)app->cfg, NULL); + g_printerr("Custom layout updated at runtime\n"); + return G_SOURCE_REMOVE; /* one-shot */ +} + +int +main(int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + AppCtx app; + int current_device = -1; + gboolean enc_hw_support = TRUE; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (prop.integrated) { + FILE* ptr; + char device_name[50]; + ptr = fopen("/proc/device-tree/model", "r"); + + if(ptr){ + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } + + memset(&app, 0, sizeof(app)); + + gst_init(&argc, &argv); + + /* Simple arg handling: program -i uri1 uri2 ... [--no-display] */ + GPtrArray *uris = g_ptr_array_new_with_free_func(g_free); + for (int i = 1; i < argc; ++i) { + if (g_strcmp0(argv[i], "-i") == 0 || g_strcmp0(argv[i], "--input") == 0) { + for (int j = i + 1; j < argc && argv[j][0] != '-'; ++j) { + g_ptr_array_add(uris, g_strdup(argv[j])); + i = j; + } + } else if (g_strcmp0(argv[i], "--no-display") == 0) { + app.no_display = TRUE; + } + } + + if (uris->len == 0) { + g_printerr("Usage: %s -i [uri2 ...] [--no-display]\n", argv[0]); + g_ptr_array_unref(uris); + return -1; + } + + app.num_sources = uris->len; + + app.pipeline = gst_pipeline_new("ds-custom-tiler"); + if (!app.pipeline) { + g_printerr("Failed to create pipeline\n"); + g_ptr_array_unref(uris); + return -1; + } + + app.streammux = gst_element_factory_make("nvstreammux", "stream-muxer"); + if (!app.streammux) { + g_printerr("Failed to create nvstreammux\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + g_object_set(G_OBJECT(app.streammux), + "width", 1920, + "height", 1080, + "batch-size", app.num_sources, + "batched-push-timeout", 33000, + NULL); + + gst_bin_add(GST_BIN(app.pipeline), app.streammux); + + for (guint i = 0; i < app.num_sources; ++i) { + GstElement *src_bin = create_source_bin(i, (const gchar *)g_ptr_array_index(uris, i)); + if (!src_bin) { + g_printerr("Failed to create source bin %u\n", i); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + gst_bin_add(GST_BIN(app.pipeline), src_bin); + + gchar pad_name[32]; + g_snprintf(pad_name, sizeof(pad_name), "sink_%u", i); + GstPad *sinkpad = gst_element_request_pad_simple(app.streammux, pad_name); + GstPad *srcpad = gst_element_get_static_pad(src_bin, "src"); + if (!sinkpad || !srcpad) { + g_printerr("Failed to get pads for source %u\n", i); + if (sinkpad) gst_object_unref(sinkpad); + if (srcpad) gst_object_unref(srcpad); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr("Failed to link source bin to streammux for %u\n", i); + gst_object_unref(sinkpad); + gst_object_unref(srcpad); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + gst_object_unref(sinkpad); + gst_object_unref(srcpad); + } + + GstElement *queue1 = gst_element_factory_make("queue", "queue1"); + GstElement *queue2 = gst_element_factory_make("queue", "queue2"); + GstElement *queue3 = gst_element_factory_make("queue", "queue3"); + GstElement *queue4 = gst_element_factory_make("queue", "queue4"); + GstElement *queue5 = gst_element_factory_make("queue", "queue5"); + + GstElement *pgie = gst_element_factory_make("queue", "primary-inference"); /* placeholder */ + app.tiler = gst_element_factory_make("nvmultistreamtiler", "nvtiler"); + GstElement *nvvidconv = gst_element_factory_make("nvvideoconvert", "convertor"); + GstElement *nvosd = gst_element_factory_make("nvdsosd", "onscreendisplay"); + GstElement *sink = NULL; + if(app.no_display) { + sink = gst_element_factory_make("nvvideoencfilesinkbin", "fakesink"); + } else { + if (prop.integrated) { + sink = gst_element_factory_make("nv3dsink", "nvvideo-renderer"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); +#endif + } + } + + if (!queue1 || !queue2 || !queue3 || !queue4 || !queue5 || !pgie || !app.tiler || !nvvidconv || !nvosd || !sink) { + g_printerr("Failed to create one or more elements\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + + g_object_set(G_OBJECT(nvosd), "process-mode", 0, "display-text", 1, NULL); + g_object_set(G_OBJECT(app.tiler), "square-seq-grid", FALSE, NULL); + g_object_set(G_OBJECT(app.tiler), "compute-hw", 1, NULL); + if (app.no_display) { + g_object_set(G_OBJECT(sink), "output-file", "/tmp/tile_out.mp4", "enc-type", 0, NULL); + if (!enc_hw_support) { + // fallback to software encoding + g_object_set(G_OBJECT(sink), "enc-type", 1, NULL); + } + } else { + g_object_set(G_OBJECT(sink), "qos", FALSE, NULL); + } + + /* Initial custom asymmetric layout */ + build_custom_layout(&app); + if (app.cfg) { + g_object_set(G_OBJECT(app.tiler), "custom-tile-config", (gpointer)app.cfg, NULL); + //g_timeout_add_seconds(5, update_layout_cb, &app); + } + + gst_bin_add_many(GST_BIN(app.pipeline), + queue1, pgie, queue2, app.tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL); + + if (!gst_element_link_many(app.streammux, queue1, pgie, queue2, app.tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL)) { + g_printerr("Failed to link elements\n"); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return -1; + } + + /* Rows/Columns computed like the Python sample */ + guint rows = (guint)floor(sqrt((double)app.num_sources)); + if (rows == 0) rows = 1; + guint cols = (guint)ceil(((double)app.num_sources) / (double)rows); + g_object_set(G_OBJECT(app.tiler), "rows", rows, "columns", cols, "width", 1280, "height", 720, NULL); + + loop = g_main_loop_new(NULL, FALSE); + GstBus *bus = gst_element_get_bus(app.pipeline); + gst_bus_add_signal_watch(bus); + g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(bus_call), loop); + gst_object_unref(bus); + + g_print("Now playing...\n"); + for (guint i = 0; i < uris->len; ++i) { + g_print("%u: %s\n", i, (gchar *)g_ptr_array_index(uris, i)); + } + + gst_element_set_state(app.pipeline, GST_STATE_PLAYING); + g_main_loop_run(loop); + + gst_element_set_state(app.pipeline, GST_STATE_NULL); + g_main_loop_unref(loop); + + /* Cleanup */ + if (app.tiles_arr) g_free(app.tiles_arr); + if (app.cfg) g_free(app.cfg); + gst_object_unref(app.pipeline); + g_ptr_array_unref(uris); + return 0; +} diff --git a/src/apps/reference_apps/deepstream-custom-tile-config/nvdstilerconfig.h b/src/apps/reference_apps/deepstream-custom-tile-config/nvdstilerconfig.h new file mode 100644 index 00000000..b9e21f5d --- /dev/null +++ b/src/apps/reference_apps/deepstream-custom-tile-config/nvdstilerconfig.h @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _NVDSTILER_CONFIG_H_ +#define _NVDSTILER_CONFIG_H_ + +/** + * @defgroup ds_nvtiler_custom_api NvTiler Custom canvas API module + * NvTiler CustomTile configuration structures + */ + +#include + +/** + * Holds information about an individual tile in the custom canvas. + * One tile is the space on canvas where nvmultistreamtiler + * render an individual source + */ +typedef struct +{ + /** sourceId identifying the video source in a DeepStream pipeline */ + uint32_t sourceId; + /** offset from left in the unit - 1/100 or + * percentage of canvas-width */ + float x; + /** offset from top in the unit - 1/100 or + * percentage of canvas-height */ + float y; + /** tile width in the unit - 1/100 or + * percentage of canvas-width */ + float width; + /** tile height in the unit - 1/100 or + * percentage of canvas-height */ + float height; +}CustomTile; + +/** + * Holds information about the custom tile canvas + * A pointer to this memory (transfer-none) shall be + * set on custom-tile-config property on nvmultistreamtiler plugin. + * Please check `gst-inspect-1.0 nvmultistreamtiler` for more info. + * Note 1: This data structure shall be filled with + * individual tile resolution for all involved sources. + * Note 2: custom-tile-config property can be configured dynamically + * while the pipeline is running. + * Note 3: To remove sources from the canvas (example: EOS), user shall set the + * custom-tile-config property again by removing entry in + * the array CustomTileConfig->tiles and update CustomTileConfig->length + */ +typedef struct +{ + /** custom tile-level config array + * NOTE: nullable + * Used to customize the tile resolution per source + * If used, user shall pass configuration for all the + * individual tiles in [rows X columns] canvas + */ + CustomTile* tiles; + uint32_t length; /**< length of tiles config array */ +}CustomTileConfig; + +/** + * Holds the Tiler Canvas Configuration + * Note: The user cannot directly set this on nvmultistreamtiler + * User shall leverage properties on nvmultistreamtiler plugin + * to configure the canvas. + * Please check `gst-inspect-1.0 nvmultistreamtiler` for more info. + */ +typedef struct +{ + uint32_t width; /**< canvas width */ + uint32_t height; /**< canvas height */ + uint32_t columns; /**< #columns of tiles in canvas */ + uint32_t rows; /**< #rows of tiles in canvas */ + uint32_t gpuId; /**< #gpuId to be used for stream creation */ + CustomTileConfig customTileConfig; +}TilerConfig; + + +#endif /**< _NVDSTILER_CONFIG_H_ */ diff --git a/src/apps/reference_apps/deepstream-dynamicsrcbin-test/Makefile b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/Makefile new file mode 100644 index 00000000..68c74421 --- /dev/null +++ b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/Makefile @@ -0,0 +1,56 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-dynamicsrcbin-test + + + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I../../../includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + + + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/src/apps/reference_apps/deepstream-dynamicsrcbin-test/README b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/README new file mode 100644 index 00000000..63aa602e --- /dev/null +++ b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/README @@ -0,0 +1,177 @@ +***************************************************************************** +* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: LicenseRef-NvidiaProprietary +* +* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +* property and proprietary rights in and to this material, related +* documentation and any modifications thereto. Any use, reproduction, +* disclosure or distribution of this material and related documentation +* without an express license agreement from NVIDIA CORPORATION or +* its affiliates is strictly prohibited. +***************************************************************************** + +***************************************************************************** + deepstream-dynamicsrcbin-test-app + README +***************************************************************************** + +=============================================================================== +1. Prerequisites: +=============================================================================== +Please follow instructions in the apps/sample_apps/deepstream-app/README on how +to install the prerequisites for Deepstream SDK, the DeepStream SDK itself and the +apps. + +You must have the following development packages installed: + GStreamer-1.0 + GStreamer-1.0 Base Plugins + GStreamer-1.0 gstrtspserver + X11 client-side library + +To install these packages, execute the following command: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ + libgstrtspserver-1.0-dev libx11-dev + +=============================================================================== +2. Purpose: +=============================================================================== + +This application demonstrates the capabilities of the **nvdsdynamicsrcbin** element, +specifically designed for high decoder throughput scenarios. The key innovation +of this element is that it creates a single pipeline that keeps the decoder +active at all times, avoiding decoder initialization overhead for every new stream. + +**Key Benefits:** +- **Eliminates Decoder Initialization Overhead**: When you add N streams to a + traditional pipeline, each stream typically requires its own decoder instance + with initialization time. The nvdsdynamicsrcbin element maintains a single + active decoder, significantly reducing latency when switching between streams. + +- **Improved Throughput**: By keeping the decoder warm and ready, the element + can handle rapid stream switching without the performance penalty of repeated + decoder initialization. + +- **Dynamic Source Management**: The element supports real-time addition and + removal of video sources without pipeline reconstruction. + +=============================================================================== +3. Element Architecture: +=============================================================================== + +The **nvdsdynamicsrcbin** element is a custom GStreamer bin that contains: +- **filesrc**: Source element for reading video files +- **queue_filesrc**: Buffering queue for source data +- **parsebin**: Media format detection and parsing +- **queue_parsebin**: Buffering queue for parsed data +- **nvv4l2decoder**: Hardware decoder (kept active throughout) + +The element maintains internal queues and maps to track active sources and their +corresponding file paths, enabling seamless switching between multiple video streams. + +=============================================================================== +4. Supported Signals: +=============================================================================== + +The nvdsdynamicsrcbin element supports three key signals for dynamic source management: + +### 4.1 add-source Signal +**Purpose**: Adds a new video source to the dynamic source bin +**Parameters**: +- `file_path` (string): Path to the video file +- `source_id` (integer): Unique identifier for the source + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "add-source", "/path/to/video.mp4", source_id); +``` + +**Behavior**: +- Validates the file path exists +- Prevents duplicate source IDs +- Stores source_id -> file_path mapping +- Adds source to internal tracking queues +- For the first source, links elements to the pipeline +- Subsequent sources are queued for processing + +### 4.2 remove-source Signal +**Purpose**: Removes a video source from the dynamic source bin +**Parameters**: +- `source_id` (integer): ID of the source to remove + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "remove-source", source_id); +``` + +**Behavior**: +- If the source is currently active (head of queue), sends EOS event +- If the source is queued but not active, removes it from tracking +- Cleans up internal data structures +- Posts file change message to notify downstream elements + +### 4.3 terminate Signal +**Purpose**: Terminates the entire pipeline gracefully +**Parameters**: None + +**Usage**: +```c +g_signal_emit_by_name(dynamicsrcbin, "terminate"); +``` + +**Behavior**: +- Sends EOS event from the decoder element +- Posts termination message to exit the pipeline +- Cleans up all resources and stops processing + +=============================================================================== +5. Custom Events and Metadata: +=============================================================================== + +The element generates custom events to track source changes: + +### 5.1 Custom Stream Start Event +- **Event Name**: "custom-stream-start-event" +- **Contains**: source-id parameter +- **Purpose**: Notifies downstream elements when a new source begins processing + +### 5.2 Custom EOS Event +- **Event Name**: "custom-eos-event" +- **Contains**: source-id parameter +- **Purpose**: Notifies when a source completes processing + +### 5.3 Source Metadata +The element attaches custom metadata to each buffer containing: +- **chunk_id**: The source ID that generated the buffer +- **frame_id**: Sequential frame counter for the current source + +=============================================================================== +6. To compile: +=============================================================================== + + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For jetson, CUDA_VER=13.0 + $ sudo make (sudo not required in case of docker containers) + +=============================================================================== +7. Usage: +=============================================================================== + +The application demonstrates dynamic source management with the following workflow: + +1. **Pipeline Setup**: Creates a pipeline with nvdsdynamicsrcbin and fakesink +2. **Source Addition**: Adds multiple video sources with unique IDs +3. **Dynamic Processing**: Processes sources sequentially with active decoder +4. **Monitoring**: Tracks frame processing and source changes +5. **Cleanup**: Terminates pipeline gracefully + +**Running the Application**: +```bash +$ ./deepstream-dynamicsrcbin-test +``` + +**Configuration**: +- Modify `N_CHUNKS` in the source code to change the number of sources +- Update `VIDEO_FILE` path to use different video files +- Adjust timing parameters in the signal thread for different test scenarios + diff --git a/src/apps/reference_apps/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c new file mode 100644 index 00000000..783d6ba6 --- /dev/null +++ b/src/apps/reference_apps/deepstream-dynamicsrcbin-test/deepstream_dynamicsrcbin_test.c @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include + +/* Configuration constants */ +#define N_CHUNKS 2 +#define VIDEO_FILE "/opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4" + +/** + * @brief Application data structure to hold pipeline components and state + */ +typedef struct { + GstElement *pipeline; /* Main GStreamer pipeline */ + GstElement *src; /* nvdsdynamicsrcbin source element */ + GstElement *sink; /* fakesink element for testing */ + GMainLoop *loop; /* Main event loop */ + pthread_t signal_thread; /* Thread for sending dynamic source signals */ + gboolean thread_running; /* Flag to track thread state */ +} AppData; + +/* Global variables for tracking frame processing */ +guint frame_count = 0; +gint current_source_id = -1; + +/** + * @brief Pad probe callback for sink input to monitor frame processing + * + * This callback is attached to the sink pad to: + * - Count processed frames + * - Handle custom stream start events from nvdsdynamicsrcbin + * - Track source ID changes + * + * @param pad The sink pad being probed + * @param info Probe information containing buffer or event data + * @param user_data User data (unused in this implementation) + * @return GST_PAD_PROBE_OK to continue processing + */ +static GstPadProbeReturn sink_input_buffer_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { + if (GST_PAD_PROBE_INFO_TYPE(info) & GST_PAD_PROBE_TYPE_BUFFER) { + frame_count++; + g_print("Frame Number: %d, Current Source ID: %d\n", frame_count, current_source_id); + } else if (GST_PAD_PROBE_INFO_TYPE(info) & GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM) { + GstEvent *event = GST_PAD_PROBE_INFO_EVENT(info); + /* Check for custom stream start event from nvdsdynamicsrcbin */ + if (GST_EVENT_TYPE(event) == GST_EVENT_CUSTOM_DOWNSTREAM) { + const GstStructure *structure = gst_event_get_structure(event); + if (structure && gst_structure_has_name(structure, "custom-stream-start-event")) { + gint source_id; + if (gst_structure_get_int(structure, "source-id", &source_id)) { + g_print("Source change event received for source ID: %d\n", source_id); + current_source_id = source_id; + /* Reset frame count for new source */ + frame_count = 0; + } + } + } + } + return GST_PAD_PROBE_OK; +} + +/** + * @brief Utility function to find an element by factory name within a bin + * + * @param bin The GStreamer bin to search in + * @param factory_name The factory name of the element to find + * @return Pointer to the found element (with increased ref count) or NULL if not found + */ +GstElement* get_element_by_factory_name(GstBin *bin, const gchar *factory_name) { + GstElement *element = NULL; + GstIterator *iterator = gst_bin_iterate_all_by_element_factory_name(GST_BIN(bin), factory_name); + GValue item = G_VALUE_INIT; + + /* Get first matching element */ + if (gst_iterator_next(iterator, &item) == GST_ITERATOR_OK) { + element = GST_ELEMENT(g_value_get_object(&item)); + gst_object_ref(element); /* Increase ref count since we'll be using it */ + g_value_reset(&item); + } + + gst_iterator_free(iterator); + return element; +} + +/** + * @brief Load a new file into the dynamic source bin + * + * Creates and adds filesrc, queue, and parsebin elements to the nvdsdynamicsrcbin + * and sets them to PLAYING state. + * + * @param dynamicsrcbin The nvdsdynamicsrcbin element + * @param current_file Path to the file to load + * @return TRUE if successful, FALSE otherwise + */ +static gboolean load_file(GstElement *dynamicsrcbin, gchar *current_file) { + /* Create new elements for the file */ + GstElement *filesrc = gst_element_factory_make("filesrc", "source"); + GstElement *queue = gst_element_factory_make("queue", "queue"); + GstElement *parsebin = gst_element_factory_make("parsebin", "parsebin"); + + if (!filesrc || !queue || !parsebin) { + g_print("One of the elements not successfully created\n"); + return FALSE; + } + + /* Update filesrc location */ + g_object_set(filesrc, "location", current_file, NULL); + g_print("Playing next file: %s\n", current_file); + + /* Add the elements to dynamicsrcbin */ + gst_bin_add_many(GST_BIN(dynamicsrcbin), filesrc, queue, parsebin, NULL); + + /* Link elements */ + gst_element_link_many(filesrc, queue, parsebin, NULL); + + /* Set elements to PLAYING state */ + gst_element_set_state(parsebin, GST_STATE_PLAYING); + gst_element_set_state(queue, GST_STATE_PLAYING); + gst_element_set_state(filesrc, GST_STATE_PLAYING); + + return TRUE; +} + +/** + * @brief Timeout callback to attempt loading a file + * + * This function is called periodically to check if a file is ready to be loaded + * into the nvdsdynamicsrcbin. + * + * @param data Pointer to the nvdsdynamicsrcbin element + * @return TRUE to continue calling, FALSE to stop + */ +static gboolean try_load_file(gpointer data) { + GstElement *dynamicsrcbin = GST_ELEMENT(data); + int current_id = -1; + gchar *current_file = NULL; + + g_object_get(G_OBJECT(dynamicsrcbin), "current-id", ¤t_id, NULL); + g_object_get(G_OBJECT(dynamicsrcbin), "current-file", ¤t_file, NULL); + + if (!current_file || !current_id) { + g_print("Waiting for file to come...\n"); + return TRUE; + } + + if (!load_file(dynamicsrcbin, current_file)) { + g_print("Failed to load file: %s\n", current_file); + } + + g_free(current_file); + /* File loaded, no need to try again */ + return FALSE; +} + +/** + * @brief Handle dynamic source message from nvdsdynamicsrcbin + * + * This function is called when the nvdsdynamicsrcbin sends a file change message. + * It removes the old elements and loads the new file. + * + * @param dynamicsrcbin The nvdsdynamicsrcbin element + * @return TRUE if successful, FALSE otherwise + */ +static gboolean handle_dynamic_source_message(GstElement *dynamicsrcbin) { + int current_id = -1; + gchar *current_file = NULL; + GstElement *filesrc = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "filesrc"); + GstElement *queue = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "queue"); + GstElement *parsebin = get_element_by_factory_name(GST_BIN(dynamicsrcbin), "parsebin"); + + if (!filesrc || !queue || !parsebin) { + g_print("One of the elements not found\n"); + } else { + /* Stop and remove old elements */ + gst_element_set_state(parsebin, GST_STATE_NULL); + gst_element_set_state(queue, GST_STATE_NULL); + gst_element_set_state(filesrc, GST_STATE_NULL); + + gst_bin_remove(GST_BIN(dynamicsrcbin), filesrc); + gst_bin_remove(GST_BIN(dynamicsrcbin), queue); + gst_bin_remove(GST_BIN(dynamicsrcbin), parsebin); + + /* Get current file and ID from nvdsdynamicsrcbin */ + g_object_get(G_OBJECT(dynamicsrcbin), "current-id", ¤t_id, NULL); + g_object_get(G_OBJECT(dynamicsrcbin), "current-file", ¤t_file, NULL); + g_print("Current ID: %d, Current File: %s\n", current_id, current_file); + + if (current_file && current_id) { + gboolean ret = load_file(dynamicsrcbin, current_file); + g_free(current_file); + return ret; + } + + /* Schedule retry if file not ready */ + g_timeout_add(1000, try_load_file, dynamicsrcbin); + } + return TRUE; +} + +/** + * @brief Bus callback to handle GStreamer messages + * + * Handles various GStreamer bus messages including: + * - EOS (End of Stream) + * - Warnings and Errors + * - Application messages for dynamic source changes + * - Latency messages + * + * @param bus The GStreamer bus + * @param msg The message to handle + * @param data Pointer to AppData structure + * @return TRUE to continue receiving messages + */ +static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) { + AppData *app = (AppData *)data; + + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: { + g_print("End of stream\n"); + g_main_loop_quit(app->loop); + break; + } + case GST_MESSAGE_WARNING: { + gchar *debug = NULL; + GError *error = NULL; + gst_message_parse_warning(msg, &error, &debug); + g_printerr("WARNING from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + g_free(debug); + g_printerr("Warning: %s\n", error->message); + g_error_free(error); + break; + } + case GST_MESSAGE_ERROR: { + gchar *debug = NULL; + GError *error = NULL; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(app->loop); + break; + } + case GST_MESSAGE_APPLICATION: { + /* Handle dynamic source change messages from nvdsdynamicsrcbin */ + const GstStructure *str = gst_message_get_structure(msg); + if (gst_structure_has_name(str, "dynamic-src-bin-file-change")) { + GstElement *dynamicsrcbin = get_element_by_factory_name(GST_BIN(app->pipeline), "nvdsdynamicsrcbin"); + if (!dynamicsrcbin) { + g_printerr("Failed to get dynamicsrcbin\n"); + return FALSE; + } + handle_dynamic_source_message(dynamicsrcbin); + gst_object_unref(dynamicsrcbin); + } + break; + } + case GST_MESSAGE_LATENCY: { + /* Recalculate latency when receiving latency message */ + gst_bin_recalculate_latency(GST_BIN(app->pipeline)); + break; + } + default: + break; + } + return TRUE; +} + +/** + * @brief Test function to add and immediately remove a source + * + * Demonstrates how to add a source and then remove it before completion. + * + * @param data Pointer to AppData structure + * @param source_id The source ID to add and remove + */ +void add_remove_source_test(void *data, gint source_id) { + AppData *app = (AppData *)data; + g_signal_emit_by_name(app->src, "add-source", VIDEO_FILE, source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); + g_usleep(500000); /* 0.5 second delay */ + g_signal_emit_by_name(app->src, "remove-source", source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); + g_usleep(500000); /* 0.5 second delay */ +} + +/** + * @brief Test function to add a source + * + * @param data Pointer to AppData structure + * @param source_id The source ID to add + */ +void add_source_test(void *data, gint source_id) { + AppData *app = (AppData *)data; + g_signal_emit_by_name(app->src, "add-source", VIDEO_FILE, source_id); + g_print("Thread: Emitted signal for ID: %u, File: %s\n", source_id, VIDEO_FILE); +} + +/** + * @brief Thread function to send dynamic source signals + * + * This thread demonstrates how to dynamically add sources to the nvdsdynamicsrcbin + * element. It adds sources sequentially and then terminates the pipeline. + * + * @param data Pointer to AppData structure + * @return NULL + */ +static void* signal_thread_func(void *data) { + AppData *app = (AppData *)data; + + /* Wait for the pipeline to be ready */ + g_usleep(1000000); /* 1 second delay */ + + /* Add sources sequentially */ + for (gint source_id = 0; source_id < N_CHUNKS; source_id++) { + add_source_test(app, source_id); + /* Uncomment the line below to see how to remove source before it completes */ + /* add_remove_source_test(app, source_id); */ + } + + g_usleep(10000000); /* 10 second delay */ + /* Terminate the pipeline - for demonstration purposes only */ + /* Use when you don't need to add more sources and don't want to wait for completion */ + g_signal_emit_by_name(app->src, "terminate"); + + g_print("Signal thread completed\n"); + app->thread_running = FALSE; + return NULL; +} + +/** + * @brief Main function + * + * Sets up the GStreamer pipeline with nvdsdynamicsrcbin and fakesink, + * creates a signal thread for dynamic source management, and runs the main loop. + * + * @param argc Number of command line arguments + * @param argv Array of command line arguments + * @return 0 on success, -1 on failure + */ +int main(int argc, char *argv[]) { + AppData app; + guint bus_watch_id; + GstBus *bus; + + /* Initialize GStreamer */ + gst_init(NULL, NULL); + app.loop = g_main_loop_new(NULL, FALSE); + + /* Create pipeline elements */ + app.pipeline = gst_pipeline_new("dynamicsrcbin-test"); + app.src = gst_element_factory_make("nvdsdynamicsrcbin", "src"); + app.sink = gst_element_factory_make("fakesink", "sink"); + + if (!app.src || !app.sink) { + g_printerr("Failed to create elements\n"); + return -1; + } + + /* Configure sink element properties */ + g_object_set(app.sink, "sync", 0, NULL); + g_object_set(app.sink, "qos", 0, NULL); + + /* Set up bus monitoring */ + bus = gst_pipeline_get_bus(GST_PIPELINE(app.pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, &app); + + /* Add elements to pipeline and link them */ + gst_bin_add_many(GST_BIN(app.pipeline), app.src, app.sink, NULL); + + if (!gst_element_link_many(app.src, app.sink, NULL)) { + g_printerr("Failed to link nvdsdynamicsrcbin to fakesink\n"); + return -1; + } + + /* Add probe to monitor frame processing */ + GstPad *sink_pad = gst_element_get_static_pad(app.sink, "sink"); + gst_pad_add_probe(sink_pad, + GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, + (GstPadProbeCallback)sink_input_buffer_callback, + NULL, + NULL); + gst_object_unref(sink_pad); + + /* Set pipeline to PLAYING state */ + gst_element_set_state(app.pipeline, GST_STATE_PLAYING); + + /* Create and start the signal thread */ + app.thread_running = TRUE; + if (pthread_create(&app.signal_thread, NULL, signal_thread_func, &app) != 0) { + g_printerr("Failed to create signal thread\n"); + return -1; + } + g_print("Signal thread created and started\n"); + + /* Add initial sources to the nvdsdynamicsrcbin */ + for (gint source_id = 0; source_id < N_CHUNKS; source_id++) { + g_signal_emit_by_name(app.src, "add-source", VIDEO_FILE, source_id); + g_print("Current ID: %u\n", source_id); + } + + g_print("Running main loop...\n"); + g_main_loop_run(app.loop); + + /* Wait for signal thread to complete if it's still running */ + if (app.thread_running) { + g_print("Waiting for signal thread to complete...\n"); + pthread_join(app.signal_thread, NULL); + } + + /* Cleanup */ + g_print("Stopping playback\n"); + gst_element_set_state(app.pipeline, GST_STATE_NULL); + gst_object_unref(GST_OBJECT(app.pipeline)); + gst_object_unref(bus); + g_source_remove(bus_watch_id); + g_main_loop_unref(app.loop); + + return 0; +} \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/Makefile b/src/apps/reference_apps/deepstream-ipc-test-sr/Makefile new file mode 100644 index 00000000..b5612ed3 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/Makefile @@ -0,0 +1,74 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-ipc-test-app + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= $(wildcard *.c) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -lnvdsgst_helper -lm \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvds_yml_parser -lgstapp-1.0 \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) + $(CC) -o $(APP) $(OBJS) $(LIBS) + make -C video_template_impl + make -C latency_serialization + @echo "Build complete." + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + make -C video_template_impl install + make -C latency_serialization install + @echo "Install complete." + +clean: + make -C video_template_impl clean + make -C latency_serialization clean + rm -rf $(OBJS) $(APP) + @echo "Clean complete." + +.PHONY: all diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/README.md b/src/apps/reference_apps/deepstream-ipc-test-sr/README.md new file mode 100644 index 00000000..0daaa064 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/README.md @@ -0,0 +1,164 @@ +# IPC Test Supper Resolution +## Introduction +This sample demonstrates how to Zero-copy share decoded buffers over IPC and how to integarte Super-Resolution +model. This sample can support Jetson and DGPU platform. +The client pipeline looks like "......-> nvstreammux -> pgie ->nvvideoconvert -> capsfilter -> nvvideotemplate + ......". + +## Prerequisites + +Please follow instructions in the /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/README on how +to install the prerequisites for the Deepstream SDK, the DeepStream SDK itself, and the apps. + +You must have the following development packages installed + GStreamer-1.0 + GStreamer-1.0 Base Plugins + GStreamer-1.0 gstrtspserver + X11 client-side library + +To install these packages, execute the following command: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ + libgstrtspserver-1.0-dev libx11-dev + +## Build + +```bash + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make +``` + +NOTE: To compile the sources, run make with "sudo" or root permission. + To improve performance on specific GPUs, please add "-gencode=arch=compute_xx,code=sm_xx" in Makefile. Computing capability can be found in this link https://developer.nvidia.com/zh-cn/cuda-gpus#compute. + +## Generate super resolution model + +The model is from pytorch [code](https://github.com/pytorch/tutorials/blob/5efa2e52aafdd94ef9ae6fbfa8c63fe888a15374/advanced_source/super_resolution_with_onnxruntime.py). +Here are the steps to generate the model. + +```bash + $ pip install onnx onnxruntime torch torchvision onnxscript + $ git clone --shallow-since=2025-07-1 https://github.com/pytorch/tutorials.git + $ cd tutorials && git reset --hard `git rev-list --max-parents=0 HEAD` + #update opset_version to 18 in advanced_source/super_resolution_with_onnxruntime.py + $ python3 advanced_source/super_resolution_with_onnxruntime.py + # copy the generated super_resolution.onnx and super_resolution.onnx.data to deepstream-ipc-test-sr. + $ cp super_resolution.onnx* /path/to/your/deepstream-ipc-test-sr +``` + +## Run + +Run with the command line. This sample act as either server or client based on command line arguments. + +```shell + # server + $ ./deepstream-ipc-test-app server + # client + $ ./deepstream-ipc-test-app client +``` +e.g. + +- Server generates a url using a local file. Multiple clients play the url. + +```shell + $ ./deepstream-ipc-test-app server file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 +``` + +- Server generates a url using RTSP. Client plays the url. + +```shell + $ ./deepstream-ipc-test-app server rtsp://127.0.0.1/video1 /tmp/test1 + $ ./deepstream-ipc-test-app client /tmp/test1 +``` + +- Server generates two urls. Client plays these two urls. + +```shell + $ ./deepstream-ipc-test-app server rtsp://127.0.0.1/video1 /tmp/test1 rtsp://127.0.0.1/video2 /tmp/test2 + $ ./deepstream-ipc-test-app client /tmp/test1 /tmp/test2 +``` + +The server accepts H.264/H.265 video stream RTSP URL and IPC socket path +as input. It does the decoding of the stream and listens for the connection +on the IPC socket path. It sends decoded data over IPC to the connected client. + +The client accepts IPC socket path as input. It sends connection request to the +server. Once server accepts the request, it starts receiving the decoded data +over IPC which is further pushed to deepstream pipeline. The rest of the pipeline +is similar to the deepstream-test3 sample. + +## Performance +### FPS Measurement +The client supports FPS statistic. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app server \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test2 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test3 \ + file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test4 + +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app client \ + /tmp/test1 /tmp/test2 \ + /tmp/test3 /tmp/test4 | grep FPS +``` + +The client will output FPS in the terminal log, such as +``` +AVG FPS (num_sources 4 * 65.370): 261.481 +AVG FPS (num_sources 4 * 65.631): 262.525 +AVG FPS (num_sources 4 * 65.758): 263.031 +AVG FPS (num_sources 4 * 65.880): 263.518 +``` + +FPS statistic of the client: +|Device | FPS(batch_size:4) | +| ---------------- | ----- | +|A40 | 263.780 | +|Thor | 306.331 | + +### Latency Measurement +On the server side, Add `GstReferenceTimestampMeta` in the probe function of `nvunixfdsink` sink pad and serialize it through `serialize_meta`. +On the client side, Deserialize through `deserialize_meta`, and get the sending time in the probe function of the src pad of `nvunixfdsrc` + +Start sever with the following command. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app server file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 /tmp/test1 +``` +Open another terminal, start client with the following command. + +```shell +$ IPC_SR_PERF_MODE=1 ./deepstream-ipc-test-app client /tmp/test1 | grep latency +``` + +The client will output latency in the terminal log, such as + +``` +IPC latency 0.406 +IPC latency 0.423 +IPC latency 0.429 +IPC latency 0.467 +IPC latency 0.402 +IPC latency 0.403 +IPC latency 0.402 +IPC latency 0.385 +``` +The latency of IPC is related to both CPU and GPU. + +|Device | Latency | +| ---------------- | --------- | +|A40 & AMD 7232P | ~0.4ms | +|Thor | ~0.117ms | + +NOTE: +- On Thor, it is a known issue that `Latency Measurement` is unavailable. +- To reuse engine files generated in previous runs, update the +model-engine-file parameter in the nvinfer config file to an existing +engine file. +- The engine model should be present to run IPC use-case. If it is not +present, the IPC test will timeout in first run as it takes some time +to generate the model. +- This example only support nvinfer. diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/config_videotemplate.yml b/src/apps/reference_apps/deepstream-ipc-test-sr/config_videotemplate.yml new file mode 100644 index 00000000..7473067a --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/config_videotemplate.yml @@ -0,0 +1,29 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + #sr model output tensor width + width: 672 + #sr model output tensor height + height: 672 + diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/deepstream_ipc_test_app.c b/src/apps/reference_apps/deepstream-ipc-test-sr/deepstream_ipc_test_app.c new file mode 100644 index 00000000..736034fa --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/deepstream_ipc_test_app.c @@ -0,0 +1,854 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvds_yml_parser.h" +#include "gst-nvmessage.h" + +#include +#include +#include +#include +#include +#include +#include + +#define MAX_SOURCE_BINS 8 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +/* By default, OSD process-mode is set to GPU_MODE. To change mode, set as: + * 0: CPU mode + * 1: GPU mode + */ +#define OSD_PROCESS_MODE 1 + +/* By default, OSD will not display text. To display text, change this to 1 */ +#define OSD_DISPLAY_TEXT 0 + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 224 +#define MUXER_OUTPUT_HEIGHT 224 + +/* width and height of model output tensor*/ +#define MODEL_OUTPUT_WIDTH 672 +#define MODEL_OUTPUT_HEIGHT 672 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +#define TILED_OUTPUT_WIDTH 1280 +#define TILED_OUTPUT_HEIGHT 720 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +typedef struct +{ + gchar *uri; + gchar *socket_path; + guint bus_id; + GstElement *pipeline; +} NvIpcServerPipeline; + +typedef struct +{ + gchar *socket_path[MAX_SOURCE_BINS]; + guint bus_id; + GstElement *pipeline; +} NvIpcClientPipeline; + +typedef struct +{ + GMainLoop *loop; + NvIpcServerPipeline ipcserver[MAX_SOURCE_BINS]; + NvIpcClientPipeline ipcclient; +} AppCtx; + +#define FPS_INTERVAL 300 + +static AppCtx gAppCtx = {0}; +static guint cintr = FALSE; +static gboolean g_perf_mode = FALSE; + +static gdouble get_current_timestamp() +{ + struct timeval t1; + double elapsed_time = 0; + gettimeofday(&t1, NULL); + elapsed_time = (t1.tv_sec) * 1000.0; + elapsed_time += (t1.tv_usec) / 1000.0; + return elapsed_time; +} + +static GstPadProbeReturn +server_sink_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + if (g_perf_mode) { + GstBuffer *buf = (GstBuffer *) info->data; + if (gst_buffer_is_writable (buf)) { + GstCaps *caps = gst_caps_new_simple ("video/x-raw", + "server_send_time", G_TYPE_DOUBLE, get_current_timestamp(), NULL); + gst_buffer_add_reference_timestamp_meta (buf, caps, 0, 0); + gst_caps_unref(caps); + // g_print ("server_send_time %.3f %s\n", get_current_timestamp(), gst_caps_to_string (caps)); + } + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +client_source_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + if (g_perf_mode) { + GstBuffer *buf = (GstBuffer *) info->data; + GstReferenceTimestampMeta *meta = + gst_buffer_get_reference_timestamp_meta (buf, NULL); + if (meta == NULL) { + // g_print ("%s: no reference timestamp meta\n", __FUNCTION__); + return GST_PAD_PROBE_OK; + } + GstCaps *caps = meta->reference; + double server_send_time = 0.0; + const GstStructure *str = gst_caps_get_structure (caps, 0); + gst_structure_get_double (str, "server_send_time", &server_send_time); + g_print ("IPC latency %.3f\n", get_current_timestamp() - server_send_time); + } + return GST_PAD_PROBE_OK; +} + +/* client_sgie_src_pad_buffer_probe will extract metadata received on OSD sink pad + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +client_sgie_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *) info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + guint vehicle_count = 0; + guint person_count = 0; + NvDsMetaList * l_frame = NULL; + NvDsMetaList * l_obj = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + num_rects = vehicle_count = person_count = 0; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { + vehicle_count++; + num_rects++; + } + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { + person_count++; + num_rects++; + } + } + g_print ("Frame Number = %d Number of objects = %d " + "Vehicle Count = %d Person Count = %d\n", + frame_meta->frame_num, num_rects, vehicle_count, person_count); + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +client_osd_src_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + static double start_time = 0.0f; + static int frame_count = 0; + if (frame_count == 0) { + start_time = get_current_timestamp(); + } else if (frame_count % FPS_INTERVAL == 0) { + int num_sources = *(guint *) u_data; + double current_time = get_current_timestamp(); + double fps = frame_count / (current_time - start_time) * 1000.0; + g_print ("AVG FPS (num_sources %d * %.3f): %.3f\n", + num_sources, fps, num_sources * fps); + } + frame_count++; + return GST_PAD_PROBE_OK; +} + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler(int signum) { + struct sigaction action; + + g_print("User Interrupted.. \n"); + + memset(&action, 0, sizeof(action)); + action.sa_handler = SIG_DFL; + + sigaction(SIGINT, &action, NULL); + + cintr = TRUE; +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (cintr) { + cintr = FALSE; + g_main_loop_quit (gAppCtx.loop); + return FALSE; + } + return TRUE; +} + +/* +* Function to install custom handler for program interrupt signal. +*/ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + } + } + break; + } + case GST_MESSAGE_STATE_CHANGED: + { + GstState oldstate, newstate; + gst_message_parse_state_changed (msg, &oldstate, &newstate, NULL); + switch (newstate) { + case GST_STATE_PLAYING: + //g_print ("Pipeline running\n"); + break; + case GST_STATE_PAUSED: + if (oldstate == GST_STATE_PLAYING) { + //g_print ("Pipeline paused\n"); + } + break; + case GST_STATE_READY: + if (oldstate == GST_STATE_NULL) { + //g_print ("Pipeline ready\n"); + } else { + //g_print ("Pipeline stopped\n"); + } + break; + case GST_STATE_NULL: + //g_print ("Pipeline Null\n"); + g_main_loop_quit (loop); + return FALSE; + break; + default: + break; + } + break; + } + default: + break; + } + return TRUE; +} + +/* delete the pipeline */ +void +destroy_pipeline(AppCtx* appCtx) { + for(int i = 0; i < MAX_SOURCE_BINS; i++){ + if(appCtx->ipcserver[i].pipeline) { + gst_element_set_state (appCtx->ipcserver[i].pipeline, GST_STATE_NULL); + gst_object_unref (GST_OBJECT (appCtx->ipcserver[i].pipeline)); + g_source_remove (appCtx->ipcserver[i].bus_id); + g_print("server is closed uri: %s path: %s\n", + appCtx->ipcserver[i].uri, appCtx->ipcserver[i].socket_path); + g_free(appCtx->ipcserver[i].uri); + g_free(appCtx->ipcserver[i].socket_path); + } + } + if(appCtx->ipcclient.pipeline) { + GstBus *bus = NULL; + bus = gst_pipeline_get_bus (GST_PIPELINE (appCtx->ipcclient.pipeline)); + while (TRUE) { + GstMessage *message = gst_bus_pop (bus); + if (message == NULL) + break; + else if (GST_MESSAGE_TYPE (message) == GST_MESSAGE_ERROR) + bus_call (bus, message, appCtx->loop); + else + gst_message_unref (message); + } + gst_object_unref (bus); + gst_element_set_state (appCtx->ipcclient.pipeline, GST_STATE_NULL); + gst_object_unref (GST_OBJECT (appCtx->ipcclient.pipeline)); + g_source_remove (appCtx->ipcclient.bus_id); + for(int i = 0; i < MAX_SOURCE_BINS; i++){ + if(appCtx->ipcclient.socket_path[i]) { + g_print("client is closed path: %s\n", appCtx->ipcclient.socket_path[i]); + g_free(appCtx->ipcclient.socket_path[i]); + } + } + } + g_main_loop_unref (appCtx->loop); + g_print("destroy_pipeline end\n"); +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + if (!caps) { + caps = gst_pad_query_caps (decoder_src_pad, NULL); + } + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + GstElement *source_bin = (GstElement *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad only if decodebin has picked nvidia + * decoder plugin nvdec_*. We do this by checking if the pad caps contain + * NVMM memory features. */ + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + /* Get the source bin ghost pad */ + GstPad *bin_ghost_pad = gst_element_get_static_pad (source_bin, "src"); + if (!gst_ghost_pad_set_target (GST_GHOST_PAD (bin_ghost_pad), + decoder_src_pad)) { + g_printerr ("Failed to link decoder src pad to source bin ghost pad\n"); + } + gst_object_unref (bin_ghost_pad); + } else { + g_printerr ("Error: Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + if (g_strrstr (name, "source") == name) { + g_object_set(G_OBJECT(object),"drop-on-latency",true,NULL); + } +} + +static GstElement * +create_source_bin (guint index, gchar * uri) +{ + GstElement *bin = NULL, *uri_decode_bin = NULL; + gchar bin_name[16] = { 0 }; + + g_snprintf (bin_name, 15, "source-bin-%02d", index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + uri_decode_bin = gst_element_factory_make ("nvurisrcbin", NULL); + g_object_set (G_OBJECT (uri_decode_bin), "file-loop", TRUE, NULL); + g_object_set (G_OBJECT (uri_decode_bin), "cudadec-memtype", 0, NULL); + + if (!bin || !uri_decode_bin) { + g_printerr ("One element in source bin could not be created.\n"); + return NULL; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), bin); + g_signal_connect (G_OBJECT (uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), bin); + + gst_bin_add (GST_BIN (bin), uri_decode_bin); + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + if (!gst_element_add_pad (bin, gst_ghost_pad_new_no_target ("src", + GST_PAD_SRC))) { + g_printerr ("Failed to add ghost pad in source bin\n"); + return NULL; + } + + return bin; +} + +static int +create_client_pipeline (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *sgie = NULL, + *sr_conv = NULL, *sr_capsfilter = NULL, *sr_video_template = NULL, + *queue1, *queue2, *queue3, *queue4, *queue5, *nvvidconv = NULL, + *nvosd = NULL, *tiler = NULL; + GstCaps *caps = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *src_pad = NULL; + guint i = 0, num_sources = 0; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + gchar tmp_buf[256] = {0}; + gint tmp_buf_len = 255; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + gAppCtx.loop = loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + gAppCtx.ipcclient.pipeline = pipeline = gst_pipeline_new ("ipc-client-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + + num_sources = argc - 2; + for (i = 0; i < num_sources; i++) { + GstElement *source = NULL, *caps_filter = NULL, *queue = NULL; + GstPad *sinkpad, *srcpad; + gchar pad_name[16] = { 0 }; + + const char *socket_path = argv[i + 2]; + gAppCtx.ipcclient.socket_path[i] = strdup(socket_path); + g_print("client is connected path: %s\n", gAppCtx.ipcclient.socket_path[i]); + + g_snprintf (tmp_buf, tmp_buf_len, "nvunixfdsrc_%u", i); + source = gst_element_factory_make ("nvunixfdsrc", tmp_buf); + gst_bin_add (GST_BIN (pipeline), source); + g_object_set (G_OBJECT(source), "socket-path", gAppCtx.ipcclient.socket_path[i], + "buffer_timestamp_copy", TRUE, NULL); + if (g_perf_mode) { + g_object_set (G_OBJECT(source), "meta-deserialization-lib", + "latency_serialization/liblatency_serialization.so", NULL); + } + + g_snprintf (tmp_buf, tmp_buf_len, "capsfilter_src_%u", i); + caps_filter = gst_element_factory_make ("capsfilter", NULL); + if (!caps_filter) { + g_printerr ("Failed to create caps_filter. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), caps_filter); + + caps = gst_caps_from_string ("video/x-raw(memory:NVMM),format=NV12"); + g_object_set (G_OBJECT(caps_filter), "caps", caps, NULL); + gst_caps_unref (caps); + + g_snprintf (tmp_buf, tmp_buf_len, "queue_src_%u", i); + queue = gst_element_factory_make ("queue", tmp_buf); + if (!queue) { + g_printerr ("Failed to create queue. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), queue); + /* link the elements together */ + if (!gst_element_link_many (source, caps_filter, queue, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + src_pad = gst_element_get_static_pad (source, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_source_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + + g_snprintf (pad_name, 15, "sink_%u", i); + sinkpad = gst_element_request_pad_simple (streammux, pad_name); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (queue, "src"); + if (!srcpad) { + g_printerr ("Failed to get src pad of source bin. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link source bin to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + } + + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + sgie = gst_element_factory_make ("nvinfer", "second-nvinference-engine"); + + /* Add queue elements between every two elements */ + queue1 = gst_element_factory_make ("queue", "queue1"); + queue2 = gst_element_factory_make ("queue", "queue2"); + queue3 = gst_element_factory_make ("queue", "queue3"); + queue4 = gst_element_factory_make ("queue", "queue4"); + queue5 = gst_element_factory_make ("queue", "queue5"); + sr_conv = gst_element_factory_make ("nvvideoconvert", "sr_conv"); + sr_capsfilter = gst_element_factory_make ("capsfilter", "sr_capsfilter"); + g_snprintf (tmp_buf, tmp_buf_len, "video/x-raw(memory:NVMM),format=NV12, width=%d, height=%d", + MODEL_OUTPUT_WIDTH, MODEL_OUTPUT_HEIGHT); + caps = gst_caps_from_string (tmp_buf); + g_object_set (G_OBJECT(sr_capsfilter), "caps", caps, NULL); + gst_caps_unref (caps); + sr_video_template = gst_element_factory_make("nvdsvideotemplate", "nvdsvideotemplate"); + g_object_set(G_OBJECT(sr_video_template), "customlib-name", "./video_template_impl/libnvds_vt_impl.so", NULL); + g_object_set(G_OBJECT(sr_video_template), "customlib-props", "config-file:config_videotemplate.yml", NULL); + + + /* Use nvtiler to composite the batched frames into a 2D tiled array based + * on the source of the frames. */ + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + if (g_perf_mode) { + sink = gst_element_factory_make ("fakesink", "nvvideo-renderer"); + } else if(prop.integrated) { + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!pgie || !sgie || !tiler || !nvvidconv || !nvosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (streammux), "batch-size", num_sources, NULL); + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Configure the nvinfer element using the nvinfer config file. */ + g_object_set (G_OBJECT (pgie), + "config-file-path", "dsipctest_pgie_config.yml", "output-tensor-meta", TRUE, NULL); + g_object_set (G_OBJECT (sgie), + "config-file-path", "dsipctest_sgie_config.yml", NULL); + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", num_sources, NULL); + g_object_set (G_OBJECT (sgie), "batch-size", num_sources, NULL); + } + +#ifdef PLATFORM_TEGRA + /* for NvBufSurfaceMap in nvvideotemplate */ + g_object_set (G_OBJECT(sr_conv), "nvbuf-memory-type", 2, NULL); + g_object_set (G_OBJECT(sr_conv), "compute-hw", 1, NULL); + g_object_set (G_OBJECT(tiler), "compute-hw", 1, NULL); +#endif + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + + g_object_set (G_OBJECT (nvosd), "process-mode", OSD_PROCESS_MODE, + "display-text", OSD_DISPLAY_TEXT, NULL); + + g_object_set (G_OBJECT (sink), "qos", 0, NULL); + // g_object_set (G_OBJECT (sink), "sync", FALSE, NULL); + g_object_set (G_OBJECT (streammux), "nvbuf-memory-type", 0, NULL); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + gAppCtx.ipcclient.bus_id = bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), queue1, pgie, + sr_conv, sr_capsfilter, sr_video_template, queue2, sgie, tiler, + queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL); + + /* link the elements together */ + if (!gst_element_link_many (streammux, queue1, pgie, sr_conv, sr_capsfilter, sr_video_template, + queue2, sgie, tiler, queue3, nvvidconv, queue4, nvosd, queue5, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + src_pad = gst_element_get_static_pad (sgie, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_sgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + + src_pad = gst_element_get_static_pad (nvosd, "src"); + if (!src_pad) + g_print ("Unable to get src pad\n"); + else + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + client_osd_src_pad_buffer_probe, (gpointer)&num_sources, NULL); + gst_object_unref (src_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing:"); + for (i = 0; i < num_sources; i++) { + g_print (" %s,", argv[i + 2]); + } + g_print ("\n"); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + g_print ("Deleting pipeline\n"); + destroy_pipeline(&gAppCtx); + return 0; +} + +static int +create_server_pipeline (int argc, char *argv[]) +{ + guint i =0, num_sources = 0; + GstPad *sink_pad = NULL; + num_sources = (argc - 2)/2; + GMainLoop *loop = NULL; + gAppCtx.loop = loop = g_main_loop_new (NULL, FALSE); + + for (i = 0; i < num_sources; i++) { + GstElement *pipeline = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstElement *source_bin=NULL, *queue, *sink= NULL; + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + gAppCtx.ipcserver[i].pipeline = pipeline = gst_pipeline_new ("ipc-server-pipeline"); + if (!pipeline) { + g_printerr ("Failed to create pipeline. Exiting.\n"); + return -1; + } + + source_bin = create_source_bin (i, argv[(i*2) + 2]); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), source_bin); + gAppCtx.ipcserver[i].uri = strdup(argv[(i*2) + 2]); + + queue = gst_element_factory_make ("queue", NULL); + if (!queue) { + g_printerr ("Failed to create queue. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), queue); + + sink = gst_element_factory_make ("nvunixfdsink", NULL); + if (!sink) { + g_printerr ("Failed to create nvunixfdsink. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), sink); + + gAppCtx.ipcserver[i].socket_path = strdup(argv[(i*2) + 3]); + g_print("server is started uri: %s path: %s\n", + gAppCtx.ipcserver[i].uri, gAppCtx.ipcserver[i].socket_path); + g_object_set (G_OBJECT(sink), "socket-path", gAppCtx.ipcserver[i].socket_path, + "buffer_timestamp_copy", TRUE, NULL); + if (g_perf_mode) { + g_object_set (G_OBJECT (sink), "sync", FALSE, + "meta-serialization-lib", "latency_serialization/liblatency_serialization.so", NULL); + } + + /* link the elements together */ + if (!gst_element_link_many (source_bin, queue, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + sink_pad = gst_element_get_static_pad (sink, "sink"); + if (!sink_pad) + g_print ("Unable to get sink pad\n"); + else + gst_pad_add_probe (sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + server_sink_sink_pad_buffer_probe, NULL, NULL); + gst_object_unref (sink_pad); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + gAppCtx.ipcserver[i].bus_id = bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + gst_element_set_state (gAppCtx.ipcserver[i].pipeline, GST_STATE_PLAYING); + } + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + g_print ("Deleting pipeline\n"); + destroy_pipeline(&gAppCtx); + return 0; +} + +int +main (int argc, char *argv[]) +{ + int ret = 0; + + /* Check input arguments */ + if (argc < 3) { + g_printerr ("Usage: %s \n", argv[0]); + g_printerr ("OR: %s \n", argv[0]); + return -1; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + + /* setup signal handler */ + _intr_setup(); + g_timeout_add(400, check_for_interrupt, NULL); + + g_perf_mode = g_getenv("IPC_SR_PERF_MODE") && + !g_strcmp0(g_getenv("IPC_SR_PERF_MODE"), "1"); + g_print ("g_perf_mode: %d\n", g_perf_mode); + + if (strcmp(argv[1], "client") == 0 || strcmp(argv[1], "c") == 0) { + signal(SIGPIPE, SIG_IGN); + ret = create_client_pipeline(argc, argv); + } else if (strcmp(argv[1], "server") == 0 || strcmp(argv[1], "s") == 0) { + signal(SIGPIPE, SIG_IGN); + ret = create_server_pipeline(argc, argv); + } else { + g_printerr ("Invalid argument %s. Exiting.\n", argv[1]); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr ("OR: %s \n", argv[0]); + return -1; + } + + return ret; +} diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_pgie_config.yml b/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_pgie_config.yml new file mode 100644 index 00000000..a22c0e96 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_pgie_config.yml @@ -0,0 +1,55 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + onnx-file: super_resolution.onnx + model-engine-file: super_resolution.onnx_b2_gpu0_fp16.engine + #model-engine-file: 1.engine + batch-size: 2 + network-mode: 2 + process-mode: 1 + model-color-format: 2 + interval: 0 + gie-unique-id: 1 + output-blob-names: output + cluster-mode: 2 + infer-dims: 1;224;224 + offsets: 0.0 + maintain-aspect-ratio: 0 + scaling-compute-hw: 1 + disable-output-host-copy: 1 + + ## 0=Detector, 1=Classifier, 2=Segmentation, 100=Other + network-type: 100 + # Enable tensor metadata output + output-tensor-meta: 1 + + #scaling-filter: 0 + #scaling-compute-hw: 0 + +class-attrs-all: + pre-cluster-threshold: 0.2 + topk: 20 + nms-iou-threshold: 0.5 diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_sgie_config.yml b/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_sgie_config.yml new file mode 100644 index 00000000..75d31674 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/dsipctest_sgie_config.yml @@ -0,0 +1,44 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.00392156862745098 + onnx-file: /opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx + model-engine-file: /opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b2_gpu0_fp16.engine + labelfile-path: /opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/labels.txt + int8-calib-file: /opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/cal_trt.bin + batch-size: 2 + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + #scaling-filter=0 + scaling-compute-hw: 1 + cluster-mode: 2 + gie-unique-id: 2 + +class-attrs-all: + pre-cluster-threshold: 0.2 + topk: 20 + nms-iou-threshold: 0.5 diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/Makefile b/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/Makefile new file mode 100644 index 00000000..4d2cab62 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/Makefile @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +CC:= gcc + +SRCS:= latency_serialization.c +LIB:=liblatency_serialization.so + +CFLAGS+= -fPIC + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +LIBS += -shared -Wl,-no-undefined \ + -L$(LIB_INSTALL_DIR) \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.c=.o) +PKGS:= gstreamer-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.c $(INCS) + @echo $(CFLAGS) + $(CC) -c -o $@ $(CFLAGS) $< + +$(LIB): $(OBJS) + @echo $(CFLAGS) + $(CC) -o $@ $(OBJS) $(LIBS) + +install: $(LIB) + cp -rv $(LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c b/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c new file mode 100644 index 00000000..176e234c --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/latency_serialization/latency_serialization.c @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +void serialize_meta(GstBuffer *buf, guint8 **data, guint *len) { + if (buf == NULL || data == NULL || len == NULL) { + g_print("Invalid arguments\n"); + return; + } + guint out_len = 0; + GstReferenceTimestampMeta *meta = + gst_buffer_get_reference_timestamp_meta(buf, NULL); + if (meta == NULL) { + // g_print("serialize_meta: no reference timestamp meta\n"); + return; + } + GstCaps *ref = meta->reference; + if (ref) { + gchar *caps_str = gst_caps_to_string (ref); + out_len = strlen(caps_str) + 1; + // g_print("caps_str %s\n", caps_str); + *len = out_len; + // Allocate memory for the serialized data, free it after use + *data = g_malloc0(*len); + memcpy(*data, caps_str, out_len); + } else { + *data = NULL; + *len = 0; + } +} + +void deserialize_meta(GstBuffer *buf, guint8 *data, guint len) { + if (buf == NULL || data == NULL || len == 0) { + g_print("Invalid arguments\n"); + return; + } + GstCaps *caps = gst_caps_from_string((const gchar *)data); + if (caps) { + gst_buffer_add_reference_timestamp_meta(buf, caps, 0, 0); + gst_caps_unref(caps); + } + // g_print("deserialize_meta data: %s\n", (const gchar *)data); +} diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/Makefile b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/Makefile new file mode 100644 index 00000000..324e2017 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/Makefile @@ -0,0 +1,75 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CXX:= g++ +NVCC:=/usr/local/cuda-$(CUDA_VER)/bin/nvcc +SRCS:= vt_impl.cpp yaml_parser.cpp data_conversion.cu + +INCS:= $(wildcard *.h) +LIB:=libnvds_vt_impl.so + +CFLAGS+= -I /usr/local/cuda/include \ + -I /opt/nvidia/deepstream/deepstream/sources/includes \ + -I /opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsvideotemplate/includes \ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +LIBS := -shared -Wl,-no-undefined -lnvds_yml_parser \ + -L/usr/local/cuda/lib64/ -lcudart -ldl -lpthread -lyaml-cpp \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lnvdsgst_meta -lnvds_meta \ + -lnvbufsurface -lnvbufsurftransform -lnvdsbufferpool -lnvds_infer\ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +GENCODE_FLAGS := $(GENCODE_FLAGS) + +OBJS:= $(SRCS:.cpp=.o) +OBJS:= $(OBJS:.cu=.o) + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 gstreamer-video-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) +CFLAGS += -Wno-deprecated-declarations -fPIC + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + @echo $(CFLAGS) + $(NVCC) -c -o $@ $(GENCODE_FLAGS) --compiler-options '-fPIC' $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) + +$(DEP): $(DEP_FILES) + $(MAKE) -C customlib_impl/ + +install: $(LIB) + cp -rv $(LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu new file mode 100644 index 00000000..f63838ac --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.cu @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#define THREADS_PER_BLOCK 32 +#define THREADS_PER_BLOCK_1 (THREADS_PER_BLOCK - 1) + +__global__ void +Convert_FtFTensorKernel( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height) +{ + unsigned int row = blockIdx.y * blockDim.y + threadIdx.y; + unsigned int col = blockIdx.x * blockDim.x + threadIdx.x; + + if (col < width && row < height) + { + int v = 255 * inBuffer[row * width + col]; + if(v < 0) { + v = 0; + } else if(v > 255) { + v = 255; + } + outBuffer[row * width + col] = v; + } +} + +void +Convert_FtFTensor( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height) +{ + dim3 threadsPerBlock(THREADS_PER_BLOCK, THREADS_PER_BLOCK); + dim3 blocks((width+THREADS_PER_BLOCK_1)/threadsPerBlock.x, (height+THREADS_PER_BLOCK_1)/threadsPerBlock.y); + + Convert_FtFTensorKernel <<>> + (inBuffer, outBuffer, width, height); +} diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.h b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.h new file mode 100644 index 00000000..db505800 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/data_conversion.h @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef __DATA_CONVERSION_H__ +#define __DATA_CONVERSION_H__ + +void +Convert_FtFTensor( + float *inBuffer, + unsigned char *outBuffer, + unsigned int width, + unsigned int height); + +#endif \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp new file mode 100644 index 00000000..ed84e5a4 --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/vt_impl.cpp @@ -0,0 +1,409 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" +#include "gst-nvquery.h" +#include "gstnvdsmeta.h" +#include "gst-nvevent.h" + +#include "nvdscustomlib_base.hpp" +#include +#include +#include +#include "yaml_parser.h" +#include "data_conversion.h" + +using namespace std; +using std::string; + +#define FORMAT_NV12 "NV12" +#define FORMAT_RGBA "RGBA" +#define HARDWARE_ACCELERATION 1 + +/* Strcture used to share between the threads */ +struct PacketInfo { + GstBuffer *inbuf; + guint frame_num; +}; + +class Algorithm : public DSCustomLibraryBase +{ +public: + Algorithm() { + outputthread_stopped = false; + m_cfgParams.m_tensor_width = 640; + m_cfgParams.m_tensor_height = 360; + m_pYbuf_cuda = NULL; + } + + /* Set Init Parameters */ + virtual bool SetInitParams(DSCustom_CreateParams *params); + + /* Set Custom Properties of the library */ + virtual bool SetProperty(Property &prop); + + /* Pass GST events to the library */ + virtual bool HandleEvent(GstEvent *event); + + virtual char *QueryProperties (); + + /* Process Incoming Buffer */ + virtual BufferResult ProcessBuffer(GstBuffer *inbuf); + + /* Retrun Compatible Caps */ + virtual GstCaps * GetCompatibleCaps (GstPadDirection direction, + GstCaps* in_caps, GstCaps* othercaps); + + /* Deinit members */ + ~Algorithm(); + +private: + /* Output Processing Thread, push buffer to downstream */ + void OutputThread(void); + +public: + guint source_id = 0; + guint m_frameNum = 0; + bool outputthread_stopped = false; + + /* Output Thread Pointer */ + std::thread *m_outputThread = NULL; + + /* Queue and Lock Management */ + std::queue m_processQ; + std::mutex m_processLock; + std::condition_variable m_processCV; + + /* Aysnc Stop Handling */ + gboolean m_stop = FALSE; + /*sr tensor width*/ + int m_tensor_width; + /*sr tensor height*/ + int m_tensor_height; + std::string m_config_file_path; + cfg_params m_cfgParams; + unsigned char* m_pYbuf_cuda; +}; + +// Create Custom Algorithm / Library Context +extern "C" IDSCustomLibrary *CreateCustomAlgoCtx(DSCustom_CreateParams *params) +{ + GST_DEBUG(" %d %s", __LINE__, __func__); + return new Algorithm(); +} + +// Set Init Parameters +bool Algorithm::SetInitParams(DSCustom_CreateParams *params) +{ + DSCustomLibraryBase::SetInitParams(params); + m_outputThread = new std::thread(&Algorithm::OutputThread, this); + GST_DEBUG(" %d %s", __LINE__, __func__); + + return true; +} + +// Return Compatible Output Caps based on input caps +GstCaps* Algorithm::GetCompatibleCaps (GstPadDirection direction, + GstCaps* in_caps, GstCaps* othercaps) +{ + GstCaps* result = NULL; + GstStructure *s1, *s2; + gint width, height; + gint i, num, denom; + const gchar *inputFmt = NULL; + + printf ("\n----------\ndirection = %d (1=Src, 2=Sink) -> %s:\nCAPS =" + " %s\n", direction, __func__, gst_caps_to_string(in_caps)); + printf ("%s : OTHERCAPS = %s\n", __func__, gst_caps_to_string(othercaps)); + + othercaps = gst_caps_truncate(othercaps); + othercaps = gst_caps_make_writable(othercaps); + + int num_output_caps = gst_caps_get_size (othercaps); + printf("num_output_caps:%d\n", num_output_caps); + num_output_caps = gst_caps_get_size (in_caps); + printf("in_caps, num_output_caps:%d\n", num_output_caps); + + // TODO: Currently it only takes first caps + s1 = gst_caps_get_structure(in_caps, 0); + for (i=0; i lk(m_processLock); + m_processCV.wait(lk, [&]{return m_processQ.empty();}); + m_stop = TRUE; + m_processCV.notify_all(); + lk.unlock(); + + /* Wait for OutputThread to complete */ + if (m_outputThread) { + m_outputThread->join(); + } + cudaFree(m_pYbuf_cuda); +} + +/* Process Buffer */ +BufferResult Algorithm::ProcessBuffer (GstBuffer *inbuf) +{ + GstMapInfo in_map_info; + + GST_DEBUG ("CustomLib: ---> Inside %s frame_num = %d\n", __func__, + m_frameNum++); + + // Push buffer to process thread for further processing + PacketInfo packetInfo; + packetInfo.inbuf = inbuf; + packetInfo.frame_num = m_frameNum; + + // Add custom preprocessing logic if required, here + // Pass the buffer to output_loop for further processing and pusing to next component + // Currently its just dumping few decoded video frames + + m_processLock.lock(); + m_processQ.push(packetInfo); + m_processCV.notify_all(); + m_processLock.unlock(); + + return BufferResult::Buffer_Async; +} + +void PostProcess_cuda(NvDsInferTensorMeta *meta, unsigned char* pY) { + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + if (meta->out_buf_ptrs_dev[i]) { + int h = info->inferDims.d[1]; + int w = info->inferDims.d[2]; + Convert_FtFTensor((float*)meta->out_buf_ptrs_dev[i], pY, w, h); + cudaDeviceSynchronize(); + } + } +} + +/* replace lumin part */ +void replace_Y(unsigned char* pY, NvBufSurfaceParams *surParam) { + unsigned char* pSrc = pY; + int height = surParam->height; + int width = surParam->width; + int pitch = surParam->pitch; + unsigned char * dataPtr = (unsigned char *)surParam->dataPtr; + for (int i = 0; i < height; i++) { + cudaMemcpy (dataPtr, pSrc, width, cudaMemcpyDeviceToDevice); + pSrc += width; + dataPtr += pitch; + } +} + +/* Output Processing Thread */ +void Algorithm::OutputThread(void) +{ + GstFlowReturn flow_ret; + GstBuffer *outBuffer = NULL; + NvBufSurface *outSurf = NULL; + NvDsBatchMeta *batch_meta = NULL; + GstMapInfo in_map_info; + std::unique_lock lk(m_processLock); + printf("in OutputThread\n"); + while(1){ + /* Wait if processing queue is empty. */ + if (m_processQ.empty()) { + if (m_stop == TRUE) { + break; + } + m_processCV.wait(lk); + continue; + } + + PacketInfo packetInfo = m_processQ.front(); + m_processQ.pop(); + + m_processCV.notify_all(); + lk.unlock(); + + NvBufSurface *in_surf = getNvBufSurface (packetInfo.inbuf); + batch_meta = gst_buffer_get_nvds_batch_meta (packetInfo.inbuf); + + NvDsMetaList * l_frame = NULL; + nvds_acquire_meta_lock (batch_meta); + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + for (NvDsMetaList * l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + NvDsInferTensorMeta *meta = (NvDsInferTensorMeta *) user_meta->user_meta_data; + PostProcess_cuda(meta, m_pYbuf_cuda); + NvBufSurfaceParams *surParam = &(in_surf->surfaceList[frame_meta->batch_id]); + if (surParam->colorFormat == NVBUF_COLOR_FORMAT_NV12 || + surParam->colorFormat == NVBUF_COLOR_FORMAT_NV12_709 ) { + if(in_surf->memType == NVBUF_MEM_CUDA_DEVICE) + replace_Y(m_pYbuf_cuda, surParam); + } + } + } + + nvds_release_meta_lock (batch_meta); + + // Transform IP case + outSurf = in_surf; + outBuffer = packetInfo.inbuf; + + // Output buffer parameters checking + if (outSurf->numFilled != 0) + { + g_assert ((guint)m_outVideoInfo.width == outSurf->surfaceList->width); + g_assert ((guint)m_outVideoInfo.height == outSurf->surfaceList->height); + } + + flow_ret = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (m_element), + outBuffer); + GST_DEBUG ("CustomLib: %s in_surf=%p, Pushing Frame %d to downstream..." + " flow_ret = %d TS=%" GST_TIME_FORMAT " \n", __func__, in_surf, + packetInfo.frame_num, flow_ret, + GST_TIME_ARGS(GST_BUFFER_PTS(outBuffer))); + + lk.lock(); + } + outputthread_stopped = true; + printf("exit OutputThread\n"); +} diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp new file mode 100644 index 00000000..5da7b31e --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.cpp @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "yaml_parser.h" +#include +#include +#include +#include +#include "cuda_runtime_api.h" + #include + +using std::endl; +using std::cout; + +static gboolean +gst_parse_props_yaml (const gchar * cfg_file_path, cfg_params& cfg_params) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + if(!(configyml.size() > 0)) { + cout << "Can't open config file (" << cfg_file_path << ")" << endl; + } + for(YAML::const_iterator itr = configyml["property"].begin(); itr != configyml["property"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "width") { + cfg_params.m_tensor_width = itr->second.as(); + } else if (paramKey == "height") { + cfg_params.m_tensor_height = itr->second.as(); + } else { + std::string paramVal = itr->second.as(); + printf("not need %s\n", paramVal.c_str()); + } + } + + ret = TRUE; +done: + return ret; +} + +/* Parse nvinfer config file for context params. Returns FALSE in case of an error. */ +gboolean +gst_parse_context_params_yaml (const gchar * cfg_file_path, cfg_params& cfg_params) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + if(!(configyml.size() > 0)) { + cout << "Can't open config file (" << cfg_file_path << ")" << endl; + } + /* 'property' group is mandatory. */ + if(configyml["property"]) { + if (!gst_parse_props_yaml (cfg_file_path, cfg_params)) { + g_printerr ("Failed to parse group property\n"); + goto done; + } + } + else { + g_printerr ("Could not find group property\n"); + goto done; + } + ret = TRUE; + +done: + if (!ret) { + g_printerr ("** ERROR: <%s:%d>: failed\n", __func__, __LINE__); + } + return ret; +} diff --git a/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h new file mode 100644 index 00000000..f8b188fa --- /dev/null +++ b/src/apps/reference_apps/deepstream-ipc-test-sr/video_template_impl/yaml_parser.h @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __YAML_PARSER_H__ +#define __YAML_PARSER_H__ +#include +#include + + /* +* parameters of calibrator +*/ +struct cfg_params{ + /* model tensor width*/ + int m_tensor_width; + /*sr model tensor height*/ + int m_tensor_height; +}; + +gboolean gst_parse_context_params_yaml (const gchar * cfg_file_path, cfg_params& cal_params); + +#endif \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-masktracker/README.md b/src/apps/reference_apps/deepstream-masktracker/README.md new file mode 100644 index 00000000..11f4bc57 --- /dev/null +++ b/src/apps/reference_apps/deepstream-masktracker/README.md @@ -0,0 +1,105 @@ +# MaskTracker in DeepStream + +## Introduction +This sample application demonstrates using MaskTracker with DeepStream SDK. MaskTracker simultaneously performs multi-object tracking and segmentation using advanced vision foundation models such as Segment Anything Model 2 (SAM2). It uses SAM2 to visually track and segment targets across frames, while automatically adding and removing targets as needed. It stores visual features in previous frames in a memory bank and use them to localize targets in a new frame. For algorithm and setup details, please refer to [DeepStream MaskTracker Documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#masktracker-developer-preview). + +## Prerequisites +Users need to install Ubuntu 24.04 and NVIDIA driver 570.133.20 on x86 with dGPUs supported by DeepStream. Jetson devices may not support running the entire SAM2 network due to resource limitation. Check [here](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. +1. Download the latest DeepStream container image from NGC (e.g., DS 9.0 in the example below) + ```bash + export DS_IMG_NAME="nvcr.io/nvidia/deepstream:9.0-triton-multiarch" + docker pull $DS_IMG_NAME + ``` + +2. Git clone `deepstream_tools` and the current `deepstream_reference_apps` repository to the host machine, and enter MaskTracker directory inside the repository. `deepstream_tools` contains a directory `sam2-onnx-tensorrt`, which will be used to convert SAM2 models for TensorRT inference later. + ```bash + $ git clone https://github.com/NVIDIA/deepstream.git + $ cd deepstream/src/apps/reference_apps/deepstream-masktracker + ``` + +3. Download NVIDIA pretrained `PeopleNet` for detection from [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files?version=deployable_quantized_onnx_v2.6.3)(e.g., PeopleNet v2.6.3 in the example below). + + ```bash + # current directory: deepstream_reference_apps/deepstream-masktracker + mkdir -p models/PeopleNet + cd models/PeopleNet + wget --no-check-certificate --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_onnx_v2.6.3/zip -O peoplenet_deployable_quantized_onnx_v2.6.3.zip + unzip peoplenet_deployable_quantized_onnx_v2.6.3.zip + ``` + + The model files are now stored in `PeopleNet` directory as + + ```bash + deepstream-masktracker + ├── configs + ├── streams + └── models + └── PeopleNet + ├── labels.txt + ├── resnet34_peoplenet.onnx + └── resnet34_peoplenet_int8.txt + ``` +## Running the Application + +Launch the container from current directory, and execute the MaskTracker pipeline inside the container. The current [config](configs/deepstream_app_source1.txt) requires users to run with a display because it uses EGL sink to visualize the overlay results. To run through ssh without display, please change `type=2` to `1` in group `[sink0]` in that file. Users can check [DeepStream sink group](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#sink-group) for the usage of each sink. + + +```bash +# current directory: deepstream_reference_apps/deepstream-masktracker +sudo xhost + # give container access to display +docker run --gpus all -it --rm --net=host --privileged \ + -v /tmp/.X11-unix:/tmp/.X11-unix \ + -v $(pwd):/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker \ + -v $(pwd)/../../../../tools/sam2-onnx-tensorrt:/opt/nvidia/deepstream/deepstream/sources/tracker_ReID/sam2-onnx-tensorrt \ + -v $(pwd)/../deepstream-tracker-3d/streams/Retail02_short.mp4:/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/streams/Retail02_short.mp4 \ + -v $(pwd)/../deepstream-bodypose-3d/streams/bodypose.mp4:/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/streams/bodypose.mp4 \ + -e DISPLAY=$DISPLAY \ + $DS_IMG_NAME +``` + +Inside container, run the following commands. Please note that when `deepstream-app` is launched for the first time, it tries to create model engine files, which may take a couple minutes, depending on HW platforms. + +```bash +# Install prerequisites +cd /opt/nvidia/deepstream/deepstream/ +bash user_additional_install.sh + +# Download and convert SAM2 model +export TRACKER_MODEL_DIR="/opt/nvidia/deepstream/deepstream/samples/models/Tracker" +mkdir -p $TRACKER_MODEL_DIR +cd /opt/nvidia/deepstream/deepstream/sources/tracker_ReID/sam2-onnx-tensorrt +bash run.sh + +# Run MaskTracker pipeline +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-masktracker/configs +mkdir -p track_results +deepstream-app -c deepstream_app_source1.txt +``` + +## Customizing the Video +To run MaskTracker on other videos, in `deepstream_app_source1.txt`, change `uri=file://../streams/bodypose.mp4` to the new video name. For example, to use the retail video, set `uri=file://../streams/Retail02_short.mp4`. + +## Output Retrieval and Visualization + +### DeepStream Direct Visualization +When the pipeline is launced, DeepStream shows the output video like below while processing the input video. Segmentation masks, bounding boxes and IDs are overlaid for each target. The result video is saved as `out.mp4`. Below shows sample video visualization. + +![sample MaskTracker results](figures/.bodypose_osd.gif) +![sample MaskTracker results](figures/.retail_osd.gif) + +### Metadata Processing +The KITTI results for boxes and IDs can be found in `track_results` folder. A file will be created for each frame in each stream, and the data format is defined below. + +| object Label | object Unique Id | blank | blank | blank | bbox left | bbox top | bbox right | bbox bottom | blank | blank | blank | blank | blank | blank | blank | confidence | visibility (N/A) | Foot Image Position X (N/A) | Foot Image Position Y (N/A) | +|--------------|------------------|-------|-------|-------|-----------|----------|------------|-------------|-------|-------|-------|-------|-------|-------|-------|-----------|-----------------------|-----------------------|-----------------------| +| string | long unsigned | float | int | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | + +Each frame is saved as `track_results/00_000_xxxxxx.txt`. Sample output of a frame is like below. Note visibility and foot position are not available for MaskTracker. +```txt +person 1 0.0 0 0.0 964.094116 263.143738 1177.927734 851.131775 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.980957 +person 2 0.0 0 0.0 1298.530762 234.894257 1637.269897 844.630981 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.966797 +person 0 0.0 0 0.0 614.902649 227.807709 911.121948 852.950439 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.981934 +... +``` + +The segmentation mask for each target is generated in the `mask_params` field of `NvDsObjectMeta` in DeepStream meta data. As defined in `/opt/nvidia/deepstream/deepstream/sources/includes/nvll_osd_struct.h`, this data structure stores the segmentation mask as a float array with dimensions matching the target’s bounding box (rounded to integer values). In plugins or probes downstream to tracker, users can implement customized functions to access and store this data. diff --git a/src/apps/reference_apps/deepstream-masktracker/configs/config_infer_primary.txt b/src/apps/reference_apps/deepstream-masktracker/configs/config_infer_primary.txt new file mode 100644 index 00000000..826cf4f0 --- /dev/null +++ b/src/apps/reference_apps/deepstream-masktracker/configs/config_infer_primary.txt @@ -0,0 +1,47 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 + +infer-dims=3;544;960 +int8-calib-file=../models/PeopleNet/resnet34_peoplenet_int8.txt +model-engine-file=../models/PeopleNet/resnet34_peoplenet.onnx_b1_gpu0_int8.engine +labelfile-path=../models/PeopleNet/labels.txt +onnx-file=../models/PeopleNet/resnet34_peoplenet.onnx + +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=3 +interval=0 +gie-unique-id=1 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=3 +#enable-dla=1 +#use-dla-core=0 +#scaling-filter=4 + +filter-out-class-ids=1;2 + +[class-attrs-all] +pre-cluster-threshold=0.1429 +nms-iou-threshold=0.4688 +minBoxes=3 +dbscan-min-score=0.7726 +eps=0.2538 +detected-min-w=10 +detected-min-h=10 diff --git a/src/apps/reference_apps/deepstream-masktracker/configs/deepstream_app_source1.txt b/src/apps/reference_apps/deepstream-masktracker/configs/deepstream_app_source1.txt new file mode 100644 index 00000000..8b2f037a --- /dev/null +++ b/src/apps/reference_apps/deepstream-masktracker/configs/deepstream_app_source1.txt @@ -0,0 +1,108 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=3 +kitti-track-output-dir=track_results + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +nvbuf-memory-type=0 + +[source0] +enable=1 +type=3 +uri=file://../streams/bodypose.mp4 +# uri=file://../streams/Retail02_short.mp4 +num-sources=1 +gpu-id=0 +cudadec-memtype=0 + +[sink0] +enable=1 +type=2 +sync=0 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 +qos=0 + +[sink1] +enable=1 +type=3 +container=1 +codec=1 +enc-type=1 +sync=0 +bitrate=2000000 +profile=0 +output-file=out.mp4 +source-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=2 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Serif +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 +display-mask=1 + +[streammux] +gpu-id=0 +live-source=0 +batch-size=1 +batched-push-timeout=-1 +width=1920 +height=1080 +enable-padding=0 +nvbuf-memory-type=0 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +nvbuf-memory-type=0 +interval=0 +config-file=config_infer_primary.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1080 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_MaskTracker.yml +gpu-id=0 + +[tests] +file-loop=0 + diff --git a/src/apps/reference_apps/deepstream-masktracker/figures/.bodypose_osd.gif b/src/apps/reference_apps/deepstream-masktracker/figures/.bodypose_osd.gif new file mode 100644 index 00000000..1ffe0409 Binary files /dev/null and b/src/apps/reference_apps/deepstream-masktracker/figures/.bodypose_osd.gif differ diff --git a/src/apps/reference_apps/deepstream-masktracker/figures/.retail_osd.gif b/src/apps/reference_apps/deepstream-masktracker/figures/.retail_osd.gif new file mode 100644 index 00000000..d27528cd Binary files /dev/null and b/src/apps/reference_apps/deepstream-masktracker/figures/.retail_osd.gif differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md new file mode 100644 index 00000000..f851cd06 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md @@ -0,0 +1,471 @@ +# Multi-View 3D Tracking in DeepStream + +MV3DT 12-camera live demo + +## Introduction + +This repository provides sample applications for Multi-View 3D Tracking (MV3DT) with DeepStream 9.0 SDK. MV3DT is a distributed, real-time multi-view multi-target 3D tracking framework built for large-scale, calibrated camera networks. It is designed to deliver robust object tracking and identity consistency across complex environments, leveraging camera calibration data as a prerequisite for accurate geometric reasoning. The sample applications support three detector models: [PeopleNet Transformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2?version=deployable_v1.0), a general-purpose people detection transformer model, [PeopleNet v2.6.3](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet), a high-performance people detection model based on DetectNet_v2, and [RT-DETR 2D Warehouse](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/rtdetr_2d_warehouse?version=deployable_efficientvit_l2_v1.0), a real-time DETR model optimized for warehouse environments with multiple object classes. + +This repository aims to demonstrate MV3DT through live visualization of 3D tracking results, and is structured as follows: +- **[Prerequisites](#prerequisites)** - System requirements and setup instructions +- **[Option 1: Sample applications using DeepStream Container](#option-1-running-mv3dt-using-deepstream-container)** - The DeepStream Container has the DeepStream SDK pre-installed. The samples automate MV3DT config generation and launch the DeepStream app inside the container. +- **[Option 2: Sample applications using Inference Builder](#option-2-running-mv3dt-using-inference-builder)** - Inference Builder is an open-source tool that automates inference pipeline generation across AI frameworks and packages them as deployable containers. The samples in this repo are solely intended to demonstrate building and running MV3DT using Inference Builder. For additional capabilities, see the [Inference Builder README](https://github.com/NVIDIA/deepstream/-/tree/master/tools/inference_builder). +- **[Output Visualization Explanations](#output-visualization)** - Expected visualization from DeepStream On-Screen Display (OSD) and real-time Bird's Eye View (BEV) app +- **[Receiving 3D Tracking Metadata from Kafka](#receiving-3d-tracking-metadata-from-kafka)** - How to consume MV3DT tracking metadata from Kafka broker for downstream applications +- **[Customization](#customization)** - How to use MV3DT on custom datasets, and how to convert existing 2D DeepStream tracking pipelines to MV3DT pipeline + +As shown in the repo structure, MV3DT can be run using either DeepStream Container or Inference Builder. You can choose either approach to run the sample applications. We recommend starting with DeepStream Container for quick start and optionally trying out Inference Builder for advanced use cases, for example, integrating it with other AI frameworks or microservices. + +## Prerequisites +The sample applications in this repository require Ubuntu 24.04 and NVIDIA driver version 580.xx or higher; both x86 and Jetson platforms are supported. A graphical display server (e.g., X11) is required to view visualization results. If no physical display is available, a remote desktop solution such as VNC Viewer can be used as an alternative. + +#### Known Issues + +> On **Jetson Thor**, Option 2 Sample 2 (with Inference Builder, on 12-camera dataset) may hang due to file descriptor limitation in the third-party library libmosquitto. This issue is planned to be fixed in the next release. + +> On **DGX Spark**, the RT-DETR model requires TensorRT strongly-typed mode to produce valid inference outputs. Without it, detections may be missing and no bounding boxes will be shown. If you are running on DGX Spark with the RT-DETR detector, please add `strongly-typed=1` to the `[property]` section of `config_templates/config_pgie_rt_detr.txt` before launching the pipeline. + +> On **B200**, the sample apps with PeopleNetTransformer model may occasionally crash with a segmentation fault during inference. If this occurs, add `-e MALLOC_CHECK_=3` to the `docker run` command in launch scripts to mitigate the issue. + +#### Setup + +1. Please check [DeepStream Container Prerequisites](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. + +2. Git clone the current `deepstream_reference_apps` repository to the host machine and enter `deepstream-tracker-3d-multi-view` directory + ```bash + # Install Git LFS + sudo apt install git-lfs + git lfs install + + git clone https://github.com/NVIDIA/deepstream.git + cd deepstream/src/apps/reference_apps/deepstream-tracker-3d-multi-view + git lfs pull # In case repo is already cloned before installing git-lfs + ``` + +3. Run the automated prerequisites setup script: + + The setup script takes about 10-20 minutes to complete. And it may prompt you to enter your password for sudo access and github credentials. After the initial setup, if you close the session or restart your machine, you can still use this script to set up and check prerequisites, then the completion time will be much shorter. + + ```bash + # For DeepStream Container only (Option 1) + ./scripts/setup_prerequisites.sh + + # Additionally, if you want to try out Inference Builder (Option 2) + USE_INFERENCE_BUILDER=true ./scripts/setup_prerequisites.sh + + # [Expected output]: For both options, you should see all items checked with "✓" under "PREREQUISITES CHECK SUMMARY", and in the last line you should see: + # [SUCCESS] Prerequisites check passed! You're ready to use MV3DT. + ``` + + **Environment Variables:** + - `USE_INFERENCE_BUILDER` - Enable Inference Builder setup (default: false, DeepStream Container only) + - `BASE_DIR` - Base directory for Kafka and Inference Builder installations (default: `$HOME`) + - `DEEPSTREAM_IMAGE` - DeepStream Docker image (default: `nvcr.io/nvidia/deepstream:9.0-triton-multiarch` for x86 and Jetson platforms) + + * **Use case 1: If you want to use a different base directory** for Kafka and Inference Builder installations other than `$HOME`, you can set the `BASE_DIR` environment variable before running the script. + ```bash + export BASE_DIR=/path/to/your/preferred/base/directory + # If you want to use Inference Builder (Option 2), uncomment the lines below + # export USE_INFERENCE_BUILDER=true + # export INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + ./scripts/setup_prerequisites.sh + ``` + * **Use case 2: If you are on ARM SBSA platforms**, the DeepStream docker image will be different from the default one. Please set the `DEEPSTREAM_IMAGE` environment variable before running the script. + ```bash + export DEEPSTREAM_IMAGE=nvcr.io/nvidia/deepstream:9.0-triton-arm-sbsa + # If you want to use Inference Builder (Option 2), uncomment the line below + # export USE_INFERENCE_BUILDER=true + ./scripts/setup_prerequisites.sh + ``` + + + For manual setup, troubleshooting, or shutdown instructions for Kafka and Mqtt brokers, see: [Manual Setup Instructions](docs/manual-setup.md) + + +## Option 1: Running MV3DT using DeepStream Container + +The following examples demonstrate running MV3DT using DeepStream container. Note that the configurations are auto-generated from `config_templates` using the [auto-configurator](utils/README.md#deepstream_auto_configuratorpy). + +### Sample 1: 4-camera dataset +--- + +#### Quick Start +Run the provided script to quickly launch the 4-camera DeepStream pipeline. + +```bash +cd + +sudo xhost + # give container access to display, only need to run once per session +# [expected output]: access control disabled, clients can connect from any host + +# chmod +x scripts/test_4cam_ds.sh +./scripts/test_4cam_ds.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_4cam_ds.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_4cam_ds.sh +``` + +Two separate windows will be launched. One named **Bird-Eye View of Multi-View 3D Tracking**, and the other named **DeepStreamTest5App**. You may need to toggle, arrange, or resize the windows to see both views. If anything goes wrong or the windows are not showing, please follow the step-by-step instructions below; otherwise, the quick-start script covers the same processes. + +**Note 1 (Important): When the script is launched for the first time, it tries to create model engine files use by MV3DT, which may take about 15 minutes, depending on HW platforms.** This process is only needed once for each dataset, and subsequent runs will use the generated engine files and launch immediately. + +**Note 2:** It's expected to see the following warnings. Those warnings will not affect the accuracy or performance of the pipeline. +- Load engine failed. Create engine again. +- INT8 calibration file not specified. Trying FP16 mode. +- GStreamer-WARNING + +**Window 1: Bird-Eye View of Multi-View 3D Tracking** +- This window shows the bird's-eye view of the multi-view 3D tracking results. It will show as blank map at the beginning. +- **The BEV visualization window will not exit automatically. To close the window, select the window and press 'q'.** + + +**Window 2: DeepStreamTest5App** +- This window shows the DeepStream on-screen display (OSD) of 4 camera views in a grid. This window is directly from Deepstream pipeline, and will show as black window at the beginning. +- To view a specific camera view in the DeepStream OSD window, left-click on the desired view. To return to the multi-camera grid view, simply right-click anywhere in the window. +- **The DeepStreamTest5App will exit automatically. To quit the DeepStreamTest5App window early, select the window and press 'q'.** After you press 'q', the app will terminate within a few seconds, and finally you will see "App run successful" printed. +- Note about window name: MV3DT is built upon DeepStreamTest5App with specific config files, which is why the window displays "DeepStreamTest5App" as its title. This base application provides essential IoT protocol support (Kafka and MQTT) required by MV3DT. Currently, the window name cannot be changed. For more details, see the [DeepStreamTest5App documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_test5.html). + + +**Expected output during engine generation period from window overview/spread:** + +Screenshot + +**Expected output after pipeline starts:** + +Screenshot + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-1-4-camera-dataset). + +### Sample 2: 12-camera dataset +--- + +#### Quick Start +Run the provided script to quickly launch the 12-camera DeepStream pipeline. For detailed window explanations and important notes, see the [4-camera Quick Start section](#quick-start) above. + +```bash +# sudo xhost + # give container access to display + +# chmod +x scripts/test_12cam_ds.sh +./scripts/test_12cam_ds.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_12cam_ds.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_12cam_ds.sh +``` +#### Step-by-step Instructions +For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-2-12-camera-dataset). + +## Option 2: Running MV3DT using Inference Builder + +Inference Builder is a tool that automatically generates inference pipelines and integrates them into either a microservice or a standalone application. In the samples in this repo, it is used to generate a Python package containing the MV3DT inference flow. + +Compared to traditional DeepStream configuration files, the Inference Builder configuration (e.g., `config_templates/ds_mv3dt.yaml`) is significantly simpler and more concise, making it easier to define and modify. + +### Sample 1: 4-camera dataset +--- + +#### Quick Start + +Run the provided script to quickly start the 4-camera DeepStream Inference Builder pipeline. +```bash +# sudo xhost + # give container access to display + +# If you changed BASE_DIR in the prerequisites setup, you need to export INFERENCE_BUILDER_DIR to the correct path +# export INFERENCE_BUILDER_DIR= + +# chmod +x scripts/test_4cam_ib.sh +./scripts/test_4cam_ib.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_4cam_ib.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_4cam_ib.sh +``` + +For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. Additional notes: + + +**Note 1:** By default, the application waits **1000 seconds** if there is no data being streamed before exiting gracefully. It is expected to see 0 FPS printed during the 1000 seconds wait time. This waiting time is controlled by the `inference_timeout` parameter in `config_templates/ds_mv3dt.yaml`. To avoid this delay: +* Option 1: Pre-generate the engine files and place them in the correct directories before starting the pipeline. Then, set `inference_timeout` to a lower value (e.g., 30 seconds). +* Option 2: Use the default 1000-second timeout for the initial run to allow engine file creation. For later runs, reduce `inference_timeout` (e.g., to 30 seconds) so the container exits promptly after inference completes. + +**Note 2: To quit the app early**, run this command in a separate terminal to stop the container: +`docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. + +If you are not able to see camera view or BEV view, please follow the step-by-step instructions; otherwise, the quick-start script covers the same processes. + + +**Expected output from window overview/spread:** + +Screenshot + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-1-4-camera-dataset). + + +### Sample 2: 12-camera dataset +--- + +#### Quick Start +Run the provided script to quickly start the 12-camera DeepStream Inference Builder pipeline. + +For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. + +```bash +# sudo xhost + # give container access to display + +# If you changed BASE_DIR in the prerequisites setup, you need to export INFERENCE_BUILDER_DIR to the correct path +# export INFERENCE_BUILDER_DIR= + +# chmod +x scripts/test_12cam_ib.sh +./scripts/test_12cam_ib.sh + +# To use RT-DETR detector instead of the default PeopleNetTransformer: +# DETECTOR_MODEL=RTDETR ./scripts/test_12cam_ib.sh + +# To use PeopleNet v2.6.3 detector: +# DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_12cam_ib.sh +``` + +#### Step-by-step Instructions +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-2-12-camera-dataset). + + +## Output Visualization +Whether you use Option 1 (DeepStream Container) or Option 2 (Inference Builder), both approaches launch two windows with similar visualizations. This section explains and demonstrates the content of these visualization windows. + +### DeepStream Direct Visualization +--- +When the pipeline is launched, DeepStream shows the output video like below while processing the input video. In the example frames below, you can see that objects detected across different cameras are assigned globally consistent IDs. And both 2D and 3D bounding boxes are visualized for each tracked object. + +In the Inference Builder OSD window, object IDs are visible directly in the grid view. In the DeepStream Container OSD window, object IDs are only visible when viewing a single camera. To enter single camera view with object IDs, left-click on the desired camera view. And to return to the multi-camera grid view, simply right-click anywhere in the window. + +

+ Example 1: 4-cam dataset, with PeopleNetTransformer, Inference Builder OSD
+ Sample 4-camera cam-view tracking results +

+ +

+ Example 2: 12-cam dataset, with RT-DETR, Deepstream OSD
+ Sample 12-camera cam-view tracking results with RT-DETR
+ Note: a forklift is also detected in the 2nd camera view (row 1, column 2). +

+ +#### Disabling DeepStream Direct Visualization + +If you don't need on-screen display, you can disable it: + +* **For DeepStream Container:** + * Remove the `--enable-osd` option from `deepstream_auto_configurator.py` command in the quick-start script and run again. + +* **For Inference Builder:** + * Comment out the `render_config` section in `config_templates/ds_mv3dt.yaml` and run the quick-start script again. + + + +### Real-time BEV visualization of 3D metadata from Kafka +--- + +The `kafka_bev_visualizer.py` script provides real-time bird's-eye view (BEV) visualization of 3D tracking data streamed via Kafka. + +Note that the BEV visualization script should be launched before launching the MV3DT app. + +* Command: + ```bash + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --show-ids \ + --average-multi-cam + ``` +* Expected outputs (left: 4-camera, right: 12-camera) +
+ Sample 4-camera BEV tracking results + Sample 12-camera BEV tracking results +
+ + +* Note that the BEV visualization shows fused tracks, where trajectories of the same object from multiple cameras are averaged into one. If you want to see the individual trajectories from every camera, simply remove the `--average-multi-cam` option from the command. The output will then look like this: +
+ Sample 4-camera BEV tracking results + Sample 12-camera BEV tracking results +
+ +# Receiving 3D Tracking Metadata from Kafka + +MV3DT streams tracking metadata (frame ID, sensor ID, object IDs, 3D bounding boxes, etc.) to a Kafka topic as protobuf messages. The `kafka_client.py` script demonstrates how to connect to the Kafka broker, deserialize the protobuf messages, and print them as JSON. For building downstream applications using MV3DT tracking metadata, this can be a reference implementation. + +```bash +source mv3dt_venv/bin/activate + +# Default: connects to localhost:9092, topic 'mv3dt' +python utils/kafka_client.py + +# Custom broker and topic +python utils/kafka_client.py --broker localhost:9092 --topic mv3dt +``` + +# Customization + +This section provides customization options for the MV3DT pipeline. If you are new to DeepStream and want to try MV3DT on your own dataset, see [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets). If you already have a working 2D DeepStream tracking pipeline, see [Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT](#converting-your-existing-2d-deepstream-tracking-pipeline-to-mv3dt) for simple transformation to multi-view 3D pipeline. + + + +## Running MV3DT on Custom Datasets + +**Requirements:** +- Multi-view video streams must be synchronized. +- All video streams must have the same resolution. +- Camera calibration parameters (projection matrices) must be available. + +### Steps +--- + +1. **Organize your dataset** with the following structure: + ``` + your_dataset/ + ├── videos/ + │ ├── camera1.mp4 + │ ├── camera2.mp4 + │ └── ... + ├── camInfo/ + │ ├── camera1.yml + │ ├── camera2.yml + │ └── ... + ├── map.png (optional, for BEV visualization) + └── transforms.yml (optional, for BEV visualization) + ``` + +2. **Create camera calibration files** following the format of `datasets/mtmc_4cam/camInfo/Warehouse_Synthetic_Cam001.yml`. Replace the `projectionMatrix_3x4_w2p` values with your camera's projection matrix. For more details about these files, please refer to the [Single-View 3D Tracking](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#single-view-3d-tracking) and [The 3x4 Camera Projection Matrix](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#the-3x4-camera-projection-matrix) sections of the DeepStream documentation. + +3. **Optional: BEV visualization setup** - Prepare a BEV map image and create a `transforms.yml` file specifying the projection matrix that maps world coordinates (in meters) to BEV image coordinates, following the sample format in `datasets/mtmc_4cam/transforms.yml`. + +4. **Generate configurations** using the auto-configurator. Refer to `scripts/test_4cam_ds.sh` for the exact python command and environment variables needed. + +5. **Launch the MV3DT pipeline** using your generated configs. Refer to `scripts/test_4cam_ds.sh` for the exact Docker command and environment variables needed. + + +### How It Works +--- +The auto-configurator generates a complete set of MV3DT config files tailored to your dataset: + +1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files +2. **Generates pipeline configuration** - Creates `config_deepstream.txt` with appropriate source URIs, batch sizes, and display grid layout based on detected cameras +3. **Creates inter-camera communication configs** - Generates `pub_sub_info_config_0.yml` defining camera neighbor relationships for multi-view data sharing +4. **(Optional) Applies overrides** - Uses override files to customize tracker settings for specific datasets or requirements + +For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. + + +## Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT +If you have an existing 2D detection and tracking pipeline using DeepStream, the auto-configurator can automatically enhance it to support multi-view 3D tracking, provided you have camera calibration files available. + +### Steps +--- + +1. **Organize your dataset** as in the previous [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets) section. + +2. **Create camera calibration files** as in the previous section. + +3. **Optional: BEV visualization setup** (required if you want to enable BEV visualization) + - Follow the instructions in the previous section. + +4. **Optional: Set up your model configs** (required if using a different detector than PeopleNetTransformer) + - For example, if you are using PeopleNet as your detector, create a `PeopleNet` folder under `models` and modify `config_templates/config_pgie.txt` to point to your model files. + +5. **Optional: Set up your DeepStream pipeline config** (required if using a custom DeepStream pipeline config) + - Modify `config_templates/config_deepstream.txt` based on your use case. For example, if you have a custom `[pre-process]` section, copy that section to `config_templates/config_deepstream.txt`. Note that `[source%d]` and `[sink%d]` sections will be handled by the auto-configurator. + +6. **Place your tracker config file in the `config_templates` folder.** + + +7. **Generate MV3DT configs** + - Use the auto-configurator with `--tracker-config` argument, i.e. suppose your tracker config file is named `config_tracker_custom_2d.yml`, pass `--tracker-config=config_tracker_custom_2d.yml` to the auto-configurator. The auto-configurator assumes all template configs are located in the `config_templates` folder, so only the file name is needed, not the full path. + +8. **Launch the MV3DT pipeline** using your generated configs. + + +### A Step-by-step Example +--- + +1. To simulate a custom dataset, let's create a 6-camera subset (randomly selected) from the 12-camera dataset. Run the following command to generate a new dataset in `datasets/mtmc_6cam`. + ```bash + ./scripts/create_6cam_subset.sh + ``` + +2. Assume you have an existing 2D tracking config file. In this example, we will use the `config_tracker_2d.yml` file in the `config_templates` folder. + +3. Set up output directories and run the auto-configurator to generate the MV3DT config files. + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_6cam/ + export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + + python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --tracker-config=config_tracker_2d.yml \ + --enable-msg-broker \ + --enable-osd \ + --output-dir=$EXPERIMENT_DIR + ``` + +4. Launch the MV3DT pipeline: + ```bash + export MODEL_REPO=$PWD/models + + docker run -t --privileged --rm --net=host --runtime=nvidia \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + deepstream-test5-app -c config_deepstream.txt + ``` + +* For convenience, the same process is automated in the following scripts: + + ```bash + # Create the 6-camera dataset if not already done + # ./scripts/create_6cam_subset.sh + + # Option 1: using DeepStream Container + ./scripts/test_custom_2d_tracker_ds.sh + + # Option 2: using Inference Builder + ./scripts/test_custom_2d_tracker_ib.sh + ``` + +### How It Works +--- +The auto-configurator generates a complete set of MV3DT config files based on your custom 2D tracker, pipeline, and model configs, as well as your dataset. + +1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files +2. **Generates pipeline configuration** - Uses your updated `config_templates/config_deepstream.txt` as template and generates `$EXPERIMENT_DIR/config_deepstream.txt` with appropriate source URIs, batch sizes, and display grid layout +3. **Extends 2D tracker configs with MV3DT additional sections** - automatically injects MV3DT sections (`ObjectModelProjection` for 3D model projection, `MultiViewAssociator` for multi-view association, and `Communicator` for inter-camera communication) to your 2D tracker config +4. **Creates inter-camera communication configs** - Generates `$EXPERIMENT_DIR/pub_sub_info_config_0.yml` defining camera publish/subscribe relationships for multi-view data sharing +5. **(Optional) Applies overrides** - Uses override files to customize tracker settings for specific datasets or requirements + +For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. + + + + +## Python Util Scripts + +For more details on python utility scripts including auto-configuration generators and visualization tools, see +📁 **[Python Util Scripts Documentation](utils/README.md)** + + + + diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/assets/LICENSE b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/assets/LICENSE new file mode 100644 index 00000000..758d5be4 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/assets/LICENSE @@ -0,0 +1,8 @@ +SPDX-FileCopyrightText: Copyright (c) 2018‑2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: CC-BY-ND-4.0 + +Licensed under the Creative Commons Attribution 4.0 International (CC BY 4.0) License. + +You may copy, distribute, modify, and build upon the material for any purpose, +including commercial use, as long as proper credit is given to the creator +and a link to the license is provided. \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt new file mode 100644 index 00000000..7b3d5651 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_deepstream.txt @@ -0,0 +1,140 @@ +[application] +kitti-track-output-dir=tracker-kitti-dump +gie-kitti-output-dir=infer-kitti-dump +perf-measurement-interval-sec=5 +enable-perf-measurement=1 + +[source0] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam001.mp4 + +[source1] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam002.mp4 + +[source2] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam003.mp4 + +[source3] +type=3 +enable=1 +cudadec-memtype=0 +gpu-id=0 +num-sources=1 +uri=file://videos/Warehouse_Synthetic_Cam004.mp4 + +[streammux] +enable-padding=0 +nvbuf-memory-type=0 +width=1920 +height=1080 +batched-push-timeout=-1 +batch-size=4 +live-source=0 +gpu-id=0 + +[primary-gie] +enable=1 +nvbuf-memory-type=0 +bbox-border-color3=0;1;0;1 +bbox-border-color2=0;0;1;1 +bbox-border-color1=0;1;1;1 +bbox-border-color0=1;0;0;1 +batch-size=4 +gpu-id=0 +gie-unique-id=1 +interval=0 +model-engine-file=/workspace/models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx_b4_gpu0_fp16.engine +config-file=config_pgie.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1088 +display-tracking-id=1 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=config_tracker.yml + +[osd] +enable=1 +nvbuf-memory-type=0 +clock-color=1;0;0;0 +clock-text-size=12 +show-clock=0 +font=Serif +text-size=10 +text-bg-color=0.3;0.3;0.3;1 +text-color=1;1;1;1 +border-width=1 +clock-x-offset=800 +clock-y-offset=820 +gpu-id=0 +display-text=1 + +[tiled-display] +enable=1 +gpu-id=0 +width=1920 +height=1080 +nvbuf-memory-type=0 +rows=2 +columns=2 + +[sink0] +enable=1 +type=1 +qos=0 +gpu-id=0 +nvbuf-memory-type=0 +source-id=0 +sync=0 + +[sink1] +enable=0 +type=2 +qos=0 +gpu-id=0 +nvbuf-memory-type=0 +source-id=0 +sync=0 + +[sink2] +enable=0 +type=3 # file +container=1 # mp4 +codec=1 #h264 +enc-type=0 # hardware +sync=0 +# bitrate=2000000 +profile=0 # baseline +output-file=outVideos/tiled_display_raw.mp4 + +[sink3] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker +type=6 +msg-conv-config=config_msgconv.txt +msg-conv-payload-type=2 +msg-conv-msg2p-new-api=0 +msg-conv-frame-interval=1 +msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv_mega.so +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +msg-broker-conn-str=localhost;9092;mv3dt +topic=mv3dt + +[tests] +file-loop=0 diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt new file mode 100644 index 00000000..1cadee83 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_mqtt.txt @@ -0,0 +1,13 @@ +[message-broker] +username = user +password = password +#client-id = uniqueID +#enable-tls = 1 +#tls-cafile = +#tls-capath = +#tls-certfile = +#tls-keyfile = +#share-connection = 1 +#loop-timeout = 2000 +#keep-alive = 60 +set-threaded = 0 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt new file mode 100644 index 00000000..8a7a2d0f --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_msgconv.txt @@ -0,0 +1,21 @@ +[sensor1] +enable=1 +type=Camera +id=Camera1 + +[sensor2] +enable=1 +type=Camera +id=Camera2 + +[sensor3] +enable=1 +type=Camera +id=Camera3 + +[sensor4] +enable=1 +type=Camera +id=Camera4 + + diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt new file mode 100644 index 00000000..f0a6f0c0 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie.txt @@ -0,0 +1,23 @@ +[property] +gpu-id=0 +offsets=123.675;116.28;103.53 +net-scale-factor=0.0173520735728 +labelfile-path=/workspace/models/PeopleNetTransformer/detector_labels.txt +onnx-file=/workspace/models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx +tlt-model-key=nvidia_tao +batch-size=12 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +filter-out-class-ids=0;2;3 +interval=0 +gie-unique-id=1 +output-blob-names=pred_boxes;pred_logits +infer-dims=3;544;960 +cluster-mode=4 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +custom-lib-path=/workspace/models/PeopleNetTransformer/custom_parser/libnvds_infercustomparser_tao.so + +[class-attrs-all] +pre-cluster-threshold=0.3 +topk=20 diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt new file mode 100644 index 00000000..38bfd991 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_peoplenet.txt @@ -0,0 +1,14 @@ +[property] +net-scale-factor=0.0039215697906911373 +offsets=0.0;0.0;0.0 +labelfile-path=/workspace/models/PeopleNet2.6.3/detector_labels.txt +onnx-file=/workspace/models/PeopleNet2.6.3/resnet34_peoplenet.onnx +gie-unique-id=1 +network-type=0 +network-mode=2 +num-detected-classes=3 +filter-out-class-ids=1;2 +infer-dims=3;544;960 +model-color-format=0 +maintain-aspect-ratio=0 +output-tensor-meta=0 diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt new file mode 100644 index 00000000..542c5e86 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_pgie_rt_detr.txt @@ -0,0 +1,23 @@ +[property] +cluster-mode=4 +custom-lib-path=/workspace/models/RTDETR/custom_parser/libnvds_infercustomparser_tao.so +gie-unique-id=1 +infer-dims=3;544;960 +labelfile-path=/workspace/models/RTDETR/detector_labels.txt +maintain-aspect-ratio=1 +model-color-format=0 +net-scale-factor=0.00392156862745098 +network-mode=2 +network-type=0 +num-detected-classes=7 +filter-out-class-ids=6 +offsets=0;0;0 +onnx-file=/workspace/models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx +output-blob-names=pred_logits;pred_boxes +output-tensor-meta=1 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +workspace-size=1048576 + +[class-attrs-all] +pre-cluster-threshold=0.4652309073592239 +topk=20 diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml new file mode 100644 index 00000000..3c8fc11e --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker.yml @@ -0,0 +1,107 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.027087304322979212 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 0 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.22656630527418112 + minTrackerConfidence: 0.6957540479571296 + probationAge: 5 + maxShadowTrackingAge: 162 + earlyTerminationAge: 1 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.9349462651721144 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 0 + minMatchingScore4Overall: 0.4 + minMatchingScore4SizeSimilarity: 0.4 + minMatchingScore4Iou: 0.1393522182207021 + minMatchingScore4VisualSimilarity: 0.0520394823204932 + matchingScoreWeight4SizeSimilarity: 0.104589699500018 + matchingScoreWeight4Iou: 0.7844652139368062 + matchingScoreWeight4VisualSimilarity: 0.9294872869302965 + tentativeDetectorConfidence: 0.70167245554449 + minMatchingScore4TentativeIou: 0.1768733030811293 + minMatchingScore4PeerAssocIou: 0.2 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 6497.75224242603 + processNoiseVar4Vel: 8035.858212054732 + measurementNoiseVar4Detector: 100.0000 + measurementNoiseVar4Tracker: 883.5847350922555 +ObjectModelProjection: + minPoseConfidence: 0.925 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml + objectModelType: 0 +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 50 + recentlyActiveAge: 178 + minCommonFrames4MatchScore: 2 + maxPeerToPredDistance4Fusion: 1.35 + minPeerVisibility4Fusion: 0.15 + minPeerTrackletMatchScore: 0.48 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 30 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml new file mode 100644 index 00000000..09c3c1ff --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_2d.yml @@ -0,0 +1,63 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.027087304322979212 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 0 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.22656630527418112 + minTrackerConfidence: 0.6957540479571296 + probationAge: 1 + maxShadowTrackingAge: 162 + earlyTerminationAge: 1 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.9349462651721144 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 0 + minMatchingScore4Overall: 0.6671945991661751 + minMatchingScore4SizeSimilarity: 0.6718623956657859 + minMatchingScore4Iou: 0.1393522182207021 + minMatchingScore4VisualSimilarity: 0.0520394823204932 + matchingScoreWeight4SizeSimilarity: 0.104589699500018 + matchingScoreWeight4Iou: 0.7844652139368062 + matchingScoreWeight4VisualSimilarity: 0.9294872869302965 + tentativeDetectorConfidence: 0.70167245554449 + minMatchingScore4TentativeIou: 0.1768733030811293 + minMatchingScore4PeerAssocIou: 0.2 +StateEstimator: + stateEstimatorType: 1 + processNoiseVar4Loc: 6497.75224242603 + processNoiseVar4Vel: 8035.858212054732 + measurementNoiseVar4Detector: 100.0000 + measurementNoiseVar4Tracker: 883.5847350922555 +VisualTracker: + visualTrackerType: 1 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml new file mode 100644 index 00000000..89290a81 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam.yml @@ -0,0 +1,112 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.294 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 1 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.255 + minTrackerConfidence: 0.756 + probationAge: 3 + maxShadowTrackingAge: 150 + earlyTerminationAge: 2 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.935 + minTrackletMatchingScore: 0.2940 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.7981 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.0574 + minBboxSizeSimilarity4TrackletMatching: 0.1013 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.0100 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.0100 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 1 + minMatchingScore4Overall: 0.586 + minMatchingScore4Iou: 0.253 + minMatchingScore4SizeSimilarity: 0.6 + minMatchingScore4VisualSimilarity: 0.6 + minMatchingScore4ReidSimilarity: 0.5 + matchingScoreWeight4Iou: 0.557 + matchingScoreWeight4SizeSimilarity: 0.326 + matchingScoreWeight4VisualSimilarity: 0.654 + matchingScoreWeight4ReidSimilarity: 0.1 + tentativeDetectorConfidence: 0.842 + minMatchingScore4TentativeIou: 0.364 + minMatchingScore4PeerAssocIou: 0.25 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 3561.3 + processNoiseVar4Size: 447.7 + processNoiseVar4Vel: 3663.2 + measurementNoiseVar4Detector: 2078.3 + measurementNoiseVar4Tracker: 2108.0 + +ObjectModelProjection: + objectModelType: 0 + minPoseConfidence: 0.925 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 + +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 30 + recentlyActiveAge: 600 + minCommonFrames4MatchScore: 15 + minPeerTrackletMatchScore: 0.35 + minPeerVisibility4Fusion: 0.0213 + maxPeerToPredDistance4Fusion: 1.78 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 29 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml new file mode 100644 index 00000000..745c956c --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/config_tracker_tuned_12cam_rt_detr.yml @@ -0,0 +1,112 @@ +%YAML:1.0 + +BaseConfig: + minDetectorConfidence: 0.4249794671414514 +TargetManagement: + enableBboxUnClipping: 1 + preserveStreamUpdateOrder: 1 + maxTargetsPerStream: 200 + minIouDiff4NewTarget: 0.38243687471903276 + minTrackerConfidence: 0.741735665044422 + probationAge: 9 + maxShadowTrackingAge: 25 + earlyTerminationAge: 8 + maxTrajectoryBufferLength: -1 + outputTerminatedTracks: 0 +TrajectoryManagement: + useUniqueID: 0 + enableReAssoc: 1 + minMatchingScore4Overall: 0.935 + minTrackletMatchingScore: 0.294 + minMatchingScore4ReidSimilarity: 0 + matchingScoreWeight4TrackletSimilarity: 0.798 + matchingScoreWeight4ReidSimilarity: 0 + minTrajectoryLength4Projection: 34 + prepLength4TrajectoryProjection: 58 + trajectoryProjectionLength: 33 + maxAngle4TrackletMatching: 67 + minSpeedSimilarity4TrackletMatching: 0.057 + minBboxSizeSimilarity4TrackletMatching: 0.101 + maxTrackletMatchingTimeSearchRange: 27 + trajectoryProjectionProcessNoiseScale: 0.010 + trajectoryProjectionMeasurementNoiseScale: 100 + trackletSpacialSearchRegionScale: 0.010 + reidExtractionInterval: 0 +DataAssociator: + dataAssociatorType: 0 + associationMatcherType: 1 + checkClassMatch: 1 + minMatchingScore4Overall: 0.3302893671614782 + minMatchingScore4Iou: 0.24954038423337854 + minMatchingScore4SizeSimilarity: 0.7113775241492641 + minMatchingScore4VisualSimilarity: 0.3 + minMatchingScore4ReidSimilarity: 0.5 + matchingScoreWeight4Iou: 0.7917115752543623 + matchingScoreWeight4SizeSimilarity: 0.153194768090538 + matchingScoreWeight4VisualSimilarity: 0.9123026990374454 + matchingScoreWeight4ReidSimilarity: 0.1 + tentativeDetectorConfidence: 0.8847552837432789 + minMatchingScore4TentativeIou: 0.4499128093756385 + minMatchingScore4PeerAssocIou: 0.8239182971090612 +StateEstimator: + stateEstimatorType: 3 + processNoiseVar4Loc: 580.1268780450512 + processNoiseVar4Size: 4804.018474713682 + processNoiseVar4Vel: 435.8294162749178 + measurementNoiseVar4Detector: 9370.665057327513 + measurementNoiseVar4Tracker: 7510.009244259698 + +ObjectModelProjection: + objectModelType: 0 + minPoseConfidence: 0.9840686678221622 + outputFootLocation: 1 + outputVisibility: 1 + outputConvexHull: 0 + cameraModelFilepath: + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam001.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam002.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam003.yml + - /workspace/inputs/camInfo/Warehouse_Synthetic_Cam004.yml +VisualTracker: + visualTrackerType: 2 + useColorNames: 1 + useHog: 1 + featureImgSizeLevel: 5 + featureFocusOffsetFactor_y: -0.10525549278780495 + filterLr: 0.025008995723548734 + filterChannelWeightsLr: 0.09460260509799209 + gaussianSigma: 0.44967391866177786 + +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 0 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 30 + recentlyActiveAge: 554 + minCommonFrames4MatchScore: 5 + minPeerTrackletMatchScore: 0.4979019568226545 + minPeerVisibility4Fusion: 0.39809962133314714 + maxPeerToPredDistance4Fusion: 1.5012866693280356 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 +Communicator: + communicatorType: 2 + pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml + mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt +PoseEstimator: + poseEstimatorType: 1 + useVPICropScaler: 1 + batchSize: 1 + workspaceSize: 1000 + inferDims: [3, 256, 192] + networkMode: 1 + inputOrder: 0 + colorFormat: 0 + offsets: [123.6750, 116.2800, 103.5300] + netScaleFactor: 0.00392156 + onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx + modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine + poseInferenceInterval: 30 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml new file mode 100644 index 00000000..8017268a --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/ds_mv3dt.yaml @@ -0,0 +1,53 @@ +name: "deepstream-app" +model_repo: "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/workspace/models" +models: +- name: PeopleNetTransformer + backend: deepstream/nvinfer + max_batch_size: 4 + input: + - name: media_url + data_type: TYPE_CUSTOM_BINARY_URLS + dims: [ -1 ] + optional: true + - name: mime + data_type: TYPE_CUSTOM_DS_MIME + dims: [ -1 ] + optional: true + - name: source_config + data_type: TYPE_CUSTOM_DS_SOURCE_CONFIG + dims: [ 1 ] + optional: true + output: + - name: output + data_type: TYPE_CUSTOM_DS_METADATA + dims: [ -1 ] + parameters: + infer_config_path: + - nvdsinfer_config.yaml + resize_video: [1080, 1920] + tracker_config: + ll_lib_file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll_config_file: /workspace/experiments/config_tracker.yml + width: 1920 + height: 1088 + display_tracking_id: true + msgbroker_config: + msgbroker_proto_lib_path: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + msgconv_config_path: /workspace/experiments/config_msgconv.txt + msgbroker_conn_str: localhost;9092;mv3dt + msgbroker_topic: mv3dt + msgconv_payload_type: 2 + msgconv_msg2p_new_api: 0 + msgconv_frame_interval: 1 + msgconv_msg2p_lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv_mega.so + perf_config: + enable_fps_logs: true + enable_latency_logs: true + kitti_output_path: + infer: /workspace/experiments/infer-kitti-dump/ + tracker: /workspace/experiments/tracker-kitti-dump/ + inference_timeout: 1000 + batch_timeout: -1 + render_config: + enable_display: true + enable_osd: true diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml new file mode 100644 index 00000000..2ad8db6f --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_12cam.yml @@ -0,0 +1,15 @@ +MultiViewAssociator: + multiViewAssociatorType: 1 + enableLatePeerReAssoc: 1 + enableIDCorrection: 1 + enableSeeThrough: 1 + enableMsgSync: 1 + maxPeerTrackletSize: 50 + recentlyActiveAge: 178 + minCommonFrames4MatchScore: 2 + minPeerToPredDistance4Fusion: 1.35 + minPeerVisibility4Fusion: 0.15 + minPeerTrackletMatchScore: 0.48 + maxTrackletMatchingTimeSearchRange: 1 + maxPeerFrameDiff4NoDet: 2 + communicatorInitSleepTime: 0 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml new file mode 100644 index 00000000..554fbc1a --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/config_templates/override_tracker_4cam.yml @@ -0,0 +1,8 @@ +VisualTracker: + visualTrackerType: 0 +MultiViewAssociator: + multiViewAssociatorType: 1 + maxPeerToPredDistance4Fusion: 4.0 # mtmc 4.0 # 1.0 # sdg 1.3475168402461755 + minPeerTrackletMatchScore: 0.3 # mtmc 0.3 # sdg 0.48 # 0.48177346044415703 + minPeerVisibility4Fusion: 0.15 # sdg 0.1575362440818593 + recentlyActiveAge: 178 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md new file mode 100644 index 00000000..4f40982f --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md @@ -0,0 +1,214 @@ +## Manual Setup Instructions + + +1. Please check [Deepstream Container Prerequisites](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for Deepstream container setup, and download the latest DeepStream container image. + ```bash + docker pull nvcr.io/nvidia/deepstream:9.0-triton-multiarch + ``` +2. Git clone the current `deepstream_reference_apps` repository to the host machine and enter `deepstream-tracker-3d-multi-view` directory + ```bash + # Install Git LFS + sudo apt install git-lfs + git lfs install + + git clone https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git + cd deepstream_reference_apps/deepstream-tracker-3d-multi-view + git lfs pull # In case repo is already cloned before installing git-lfs + ``` + +3. Unzip the datasets.zip managed by Git LFS + ```bash + unzip assets/datasets.zip + ``` + + +4. Download the `PeopleNetTransformer`, `RTDETR`, `PeopleNet v2.6.3`, and `BodyPose3DNet` models from NGC, and build custom parsers + * Download the models ([PeopleNetTransformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2), [RT-DETR 2D Warehouse](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/rtdetr_2d_warehouse), [PeopleNet v2.6.3](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet), and [BodyPose3DNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/bodypose3dnet)) + ```bash + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O 'models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/rtdetr_2d_warehouse/deployable_efficientvit_l2_v1.0/files?redirect=true&path=rtdetr_warehouse_v1.0.fp16.onnx' -O 'models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' -O 'models/PeopleNet2.6.3/resnet34_peoplenet.onnx' + wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O 'models/BodyPose3DNet/bodypose3dnet_accuracy.onnx' + ``` + + * Build the custom parsers for both PeopleNetTransformer and RTDETR + ```bash + # Build PeopleNetTransformer custom parser + docker run --privileged --rm --net=host --runtime=nvidia \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/PeopleNetTransformer \ + --entrypoint /bin/bash \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + -c "cd custom_parser && make clean && make" + # [Expected output] You should see "libnvds_infercustomparser_tao.so" built under models/PeopleNetTransformer/custom_parser/. Warnings during build are expected. + + # Build RTDETR custom parser + docker run --privileged --rm --net=host --runtime=nvidia \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/RTDETR \ + --entrypoint /bin/bash \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + -c "cd custom_parser && make clean && make" + # [Expected output] You should see "libnvds_infercustomparser_tao.so" built under models/RTDETR/custom_parser/. Warnings during build are expected. + ``` + +5. Install and run the Mosquitto MQTT broker + + * Install Mosquitto and its client tools: + ```bash + sudo apt-add-repository ppa:mosquitto-dev/mosquitto-ppa + sudo apt update + sudo apt install mosquitto mosquitto-clients + ``` + + * Configure Mosquitto for optimal performance by enabling TCP_NODELAY: + ```bash + echo "set_tcp_nodelay true" | sudo tee /etc/mosquitto/conf.d/mv3dt.conf + ``` + + * After the installation, the Mosquitto broker service will be automatically started on port 1883. Restart it to apply the new config, then verify by running the provided test script. If the broker is active, you should see `Hello from Mosquitto test!` in the output: + ```bash + sudo systemctl restart mosquitto + + chmod +x ./scripts/mosquitto_test.sh + ./scripts/mosquitto_test.sh + ``` + + * If the previous step fails (e.g. seeing `Error: Connection refused`), use the following command to start Mosquitto on port 1883, and then run the test script again: + ```bash + mosquitto -p 1883 + # [Expected output] You should see "mosquitto version running" printed. + # You need to keep it running in a separate terminal window. To avoid this, you can use the following command to start it in the background: + # mosquitto -p 1883 -d + # [Expected output] Nothing will be printed. Use the mosquitto_test.sh script to verify the broker is running. + # And to kill it, you can use the following command: + # kill -9 $(lsof -t -i:1883) + ``` + + * Please refer to [Mosquitto documentation](https://mosquitto.org/download/) if you still encounter issues. + +6. Install and start a Kafka broker, and create a `mv3dt` topic: + * Follow the [Kafka quickstart](https://kafka.apache.org/quickstart) to download and start Kafka. The commands are provided below. **Note that please start a separate terminal window to keep the Kafka broker running.** + ```bash + # Kafka requires Java 17+. Check your Java version. + # If you see "Command 'java' not found" or it is older than 17, please install openjdk-17-jdk. + java -version + sudo apt install openjdk-17-jdk + + # Get Kafka + wget https://dlcdn.apache.org/kafka/4.2.0/kafka_2.13-4.2.0.tgz + tar -xzf kafka_2.13-4.2.0.tgz + cd kafka_2.13-4.2.0 + + # Start the Kafka environment + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + + bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/server.properties + # [Expected output] You should see `Formatting dynamic metadata voter directory /tmp/kraft-combined-logs with metadata.version 4.0-IV3.` + + bin/kafka-server-start.sh config/server.properties + # [Expected output] You should see `Kafka Server started.` and it will keep logging `INFO` messages. + ``` + + * Create a `mv3dt` topic under broker server `localhost:9092`, and set the message retention to 30 seconds. + + ```bash + cd + + ./bin/kafka-topics.sh --bootstrap-server localhost:9092 \ + --create \ + --topic mv3dt \ + --partitions 1 \ + --replication-factor 1 \ + --config retention.ms=30000 \ + --if-not-exists + # [Expected output] Seeing `Created topic mv3dt.` or nothing if the topic already exists. + ``` + * After you have followed the above Kafka setup steps, in the future, you only need to run the following command to start Kafka: + ```bash + bin/kafka-server-start.sh config/server.properties + # [Expected output] It is expected to see DUPLICATE_BROKER_REGISTRATION in the logs. As long as the broker keeps running and logging INFO messages, you can proceed. + ``` + + * To stop a Kafka broker running in the background, you can use the following command: + ```bash + cd + bin/kafka-server-stop.sh + ``` + + +7. Install the required Python dependencies. Note that the scripts in this repo expect a virtual environment named `mv3dt_venv` located under the root of the repo. Please make sure to follow the following instructions exactly for quick start. + + ```bash + cd + + # Install required deb packages + sudo apt update + sudo apt install python3-tk python3.12-venv python3.12-dev + + # Create a python virtual enviornment named `mv3dt_venv` and install required python packages + python3 -m venv mv3dt_venv + source mv3dt_venv/bin/activate + + pip install -r requirements.txt + ``` + * Check the virtual environment. If any specific package fails, please install it manually with `pip install `. + ```bash + ls -d mv3dt_venv + # [Expected output] You should see "mv3dt_venv" printed. If you see "No such file or directory", please check the previous step "python3 -m venv mv3dt_venv". + + pip list + # [Expected output] You should see kafka-python, protobuf in the list + ``` + + +8. (Optional) This step is only needed if you choose to use Option 2: Inference Builder. + + Set up [Deepstream Inference Builder](https://github.com/NVIDIA/deepstream/-/tree/master/tools/inference_builder). It is recommended to clone the `inference_builder` repo outside of the current repo. + * Clone the inference builder repo + + ```bash + git clone https://github.com/NVIDIA/deepstream.git + cd tools/inference_builder + git submodule update --init --recursive + ``` + * Create a new virtual environment for inference builder and install prerequisites. Please follow the following instructions exactly for quick start. **Note that there are 2 virtual environments used in this repo, `mv3dt_venv` and `ib_venv`. The scripts provided in the repo assumes that a `mv3dt_venv` folder is under the current repo, and a `ib_venv` folder is under the inference_builder repo.** + + ```bash + # Install required deb packages + sudo apt install protobuf-compiler + + # Deactivate the mv3dt_venv, and create a new virtual environment named ib_venv for inference builder + deactivate + python -m venv ib_venv + source ib_venv/bin/activate + pip3 install -r requirements.txt + ``` + * Check the virtual environment. If any specific package fails, please install it manually with `pip install `. + ```bash + ls -d ib_venv + # [Expected output] You should see "ib_venv" printed. If you see "No such file or directory", please check the previous step "python -m venv ib_venv". + + pip list + # [Expected output] You should see omegaconf 2.3.0 in the list + ``` + * Build a Docker image named `inference-builder-mv3dt:latest` with Inference Builder python dependencies. + ```bash + # Create a temporary Dockerfile + cat > ./Dockerfile.ib_mv3dt << 'EOF' + FROM nvcr.io/nvidia/deepstream:9.0-triton-multiarch + RUN pip3 install torch==2.7.0 omegaconf==2.3.0 + ENV GST_PLUGIN_PATH=/opt/nvidia/deepstream/deepstream/lib/gst-plugins + ENV LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH + ENV NVSTREAMMUX_ADAPTIVE_BATCHING=yes + WORKDIR /mv3dt_app + EOF + + # [Expected output] You should see a Dockerfile.ib_mv3dt file created under the current directory. + + # Build the Docker image + docker build -f ./Dockerfile.ib_mv3dt -t inference-builder-mv3dt:latest . + + # [Expected output] You should see "naming to docker.io/library/inference-builder-mv3dt:latest" printed as the last line. + ``` + diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md new file mode 100644 index 00000000..63ed6cae --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-deepstream.md @@ -0,0 +1,96 @@ +# DeepStream Container: Step-by-step Instructions + +This page provides detailed step-by-step instructions for running MV3DT using the DeepStream Container. For quick start scripts, see the [main README](../README.md#option-1-running-mv3dt-using-deepstream-container). + +## Sample 1: 4-camera dataset + +1. Set up environment variables and prepare experiment directories + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_4cam/ + export EXPERIMENT_DIR=$PWD/experiments/deepstream/4cam + export MODEL_REPO=$PWD/models + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + mkdir -p $EXPERIMENT_DIR/outVideos + ``` +2. Generate DeepStream configuration files using the auto-configurator + + The auto-configurator automatically generates all necessary configuration files based on your dataset. It supports various output options (OSD display, video file output, Kafka streaming) and can work with both 2D and 3D tracker configurations. + + **About Override Files:** The `--config-overrides` parameter allows you to apply dataset-specific settings. For example, `override_tracker_4cam.yml` is optimized for the sample 4-camera dataset (which uses feet as world coordinate units). You can create custom override files for your own datasets. + + For more info on the auto-configurator, see [`utils/README.md`](../utils/README.md#deepstream_auto_configuratorpy). + + ```bash + # Activate the Python environment + source mv3dt_venv/bin/activate + + # Generate configs with 4-camera overrides + python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + + # [Expected output] You should see + # Generated files: + # - config_deepstream.txt (main pipeline config) + # - config_tracker.yml (3D tracker config) + # - config_msgconv.txt (message converter config) + # - pub_sub_info_config_0.yml (communication config) + + ``` +3. (Optional) Launch real-time BEV visualization + + Before launching the main MV3DT pipeline, optionally start the bird's-eye view visualizer to see real-time 3D tracking results. Please keep it running in a separate terminal window or add `&` to the end of the command to run it in the background. + + ```bash + # Start BEV visualization + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids + + # [Expected output] You should see a window named "Bird-Eye View of Multi-View 3D Tracking" pop up and will display the live tracking results. + # Select the window and press 'q' to quit. + ``` + +4. Launch MV3DT + + The following command mounts the necessary folders into the DeepStream container and starts the `deepstream-test5-app` with MV3DT configs. + + ```bash + sudo xhost + # give container access to display + + docker run -t --privileged --rm --net=host --runtime=nvidia \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch \ + deepstream-test5-app -c config_deepstream.txt + + # [Expected output] You should see a window named "DeepStreamTest5App" pop up and will display 4 camera views in a grid. + # Select the window and press 'q' to quit early. + # The pipeline will quit automatically with "App run succesful" as the last line from the logs. + ``` + +## Sample 2: 12-camera dataset + +The steps are the same as for the 4-camera dataset, except setting `DATASET_DIR` and `EXPERIMENT_DIR` to the 12-camera directories. The auto-configurator automatically detects the number of cameras in your dataset and generates required config files for 12-camera dataset. + +```bash +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/12cam + +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --output-dir=$EXPERIMENT_DIR +``` diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md new file mode 100644 index 00000000..cbbfef83 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/step-by-step-inference-builder.md @@ -0,0 +1,104 @@ +# Inference Builder: Step-by-step Instructions + +This page provides detailed step-by-step instructions for running MV3DT using Inference Builder. For quick start scripts, see the [main README](../README.md#option-2-running-mv3dt-using-inference-builder). + +## Sample 1: 4-camera dataset + +1. Set up environment variables and prepare experiment directories + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_4cam/ + export EXPERIMENT_DIR=$PWD/experiments/inference_builder/4cam + export MODEL_REPO=$PWD/models + + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + ``` + +2. Generate Inference Builder configuration files using the auto-configurator + + ```bash + # Activate the Python environment + source mv3dt_venv/bin/activate + + # Generate configs with 4-camera overrides + python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + # [Expected output] You should see + # Generated files: + # - ds_mv3dt.yaml (inference config with max_batch_size: 4) + # - config_tracker.yml (3D tracker config) + # - source_list_static.yaml (source configuration) + # - nvdsinfer_config.yaml (inference engine config with batch_size: 4) + # - config_msgconv.txt (message converter config) + # - pub_sub_info_config_0.yml (communication config) + + # Copy the generated nvdsinfer config to the model directory + cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/PeopleNetTransformer/ + ``` + +3. Generate a Python package at `$INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app` containing the MV3DT inference flow. + ```bash + export INFERENCE_BUILDER_DIR= + cd $INFERENCE_BUILDER_DIR + source ib_venv/bin/activate + python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + ``` + +4. (Optional) Launch real-time BEV visualization. + Please keep it running in a separate terminal window or add `&` to the end of the command to run it in the background. + + ```bash + # Return to the repo directory and activate the Python environment + cd + source mv3dt_venv/bin/activate + + # Start BEV visualization + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids + + # [Expected output] You should see a window named "Bird-Eye View of Multi-View 3D Tracking" pop up and will display the live tracking results. + # Select the window and press 'q' to quit. + ``` + +5. Launch the `inference-builder-mv3dt:latest` container with volume mounts, including the Python package generated in the previous step. + + Note that this container is built during prerequisites setup. Please refer to the Inference Builder setup step in [Manual Setup Instructions](manual-setup.md) for more details. + ```bash + sudo xhost + # give container access to display + + docker run --privileged --rm -it --net=host --runtime=nvidia \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null + + # [Expected output] You should see a window named "python3" pop up and will display 4 camera views in a grid. + # Run this command to quit early: `docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. + # By default, the application waits up to **1000 seconds** if there is no data being streamed before exiting gracefully. You should see "Inference completed" as the last line from the logs. + ``` + + +## Sample 2: 12-camera dataset + +The steps are the same as for the 4-camera dataset, except setting `DATASET_DIR` and `EXPERIMENT_DIR` to the 12-camera directories. + +```bash +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/12cam + +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --output-dir=$EXPERIMENT_DIR +``` diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev.png new file mode 100644 index 00000000..3b5c2711 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png new file mode 100644 index 00000000..006cf189 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_bev_fused.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd.png new file mode 100644 index 00000000..05b13454 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png new file mode 100644 index 00000000..14874463 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/12cam_osd_rtdetr.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev.png new file mode 100644 index 00000000..f3d19c0c Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png new file mode 100644 index 00000000..ec084745 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_bev_fused.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_osd.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_osd.png new file mode 100644 index 00000000..2320b2c5 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/4cam_osd.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif new file mode 100644 index 00000000..614a7d15 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/MV3DT_12cam_live.gif differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_12cam.gif b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_12cam.gif new file mode 100644 index 00000000..34230cc0 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_12cam.gif differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_4cam.gif b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_4cam.gif new file mode 100644 index 00000000..884f5a5f Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/output_4cam.gif differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png new file mode 100644 index 00000000..e7d76c94 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png new file mode 100644 index 00000000..c70350be Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ds_engine_generation.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png new file mode 100644 index 00000000..0a6f306f Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/figures/screenshot_quickstart_4cam_ib.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt new file mode 100644 index 00000000..ef12c0ae --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNet2.6.3/detector_labels.txt @@ -0,0 +1,3 @@ +Person +Bag +Face diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile new file mode 100644 index 00000000..18d8e1f7 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/Makefile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) +CC:= g++ + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes \ + -I/usr/local/cuda/include + +CFLAGS+= -Wall -std=c++11 -shared -fPIC + +LIBS+= -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas + +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinfer_custombboxparser_tao.cpp +TARGET_LIB:= libnvds_infercustomparser_tao.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp new file mode 100644 index 00000000..bb07342e --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/custom_parser/nvdsinfer_custombboxparser_tao.cpp @@ -0,0 +1,422 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "nvdsinfer_custom_impl.h" +#include +#include +#include +#include + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CLIP(a,min,max) (MAX(MIN(a, max), min)) +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +struct MrcnnRawDetection { + float y1, x1, y2, x2, class_id, score; +}; +/* This is a sample bounding box parsing function for the sample FasterRCNN + * + * detector model provided with the SDK. */ + +/* C-linkage to prevent name-mangling */ +extern "C" +bool NvDsInferParseCustomNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + + +extern "C" +bool NvDsInferParseCustomNMSTLT (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 2) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 2 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + // Host memory for "nms" which has 2 output bindings: + // the order is bboxes and keep_count + float* out_nms = (float *) outputLayersInfo[0].buffer; + int * p_keep_count = (int *) outputLayersInfo[1].buffer; + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float* det; + + for (int i = 0; i < p_keep_count[0]; i++) { + det = out_nms + i * 7; + + // Output format for each detection is stored in the below order + // [image_id, label, confidence, xmin, ymin, xmax, ymax] + if ( det[2] < threshold) continue; + assert((int) det[1] < out_class_size); + +#if 0 + std::cout << "id/label/conf/ x/y x/y -- " + << det[0] << " " << det[1] << " " << det[2] << " " + << det[3] << " " << det[4] << " " << det[5] << " " << det[6] << std::endl; +#endif + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) det[1]; + object.detectionConfidence = det[2]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(det[3] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(det[4] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP((det[5] - det[3]) * networkInfo.width, 0, networkInfo.width - 1); + object.height = CLIP((det[6] - det[4]) * networkInfo.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep cout" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(p_bboxes[4*i] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2] * networkInfo.width, 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3] * networkInfo.height, 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomMrcnnTLTV2 (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *detectionLayer = layerFinder("generate_detections"); + const NvDsInferLayerInfo *maskLayer = layerFinder("mask_fcn_logits/BiasAdd"); + + if (!detectionLayer || !maskLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + if(maskLayer->inferDims.numDims != 4U) { + std::cerr << "Network output number of dims is : " << + maskLayer->inferDims.numDims << " expect is 4"<< std::endl; + return false; + } + + const unsigned int det_max_instances = maskLayer->inferDims.d[0]; + const unsigned int num_classes = maskLayer->inferDims.d[1]; + if(num_classes != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" << + detectionParams.numClassesConfigured << ", detected by network: " << + num_classes << std::endl; + } + const unsigned int mask_instance_height= maskLayer->inferDims.d[2]; + const unsigned int mask_instance_width = maskLayer->inferDims.d[3]; + + auto out_det = reinterpret_cast( detectionLayer->buffer); + auto out_mask = reinterpret_cast(maskLayer->buffer); + + for(auto i = 0U; i < det_max_instances; i++) { + MrcnnRawDetection &rawDec = out_det[i]; + + if(rawDec.score < detectionParams.perClassPreclusterThreshold[0]) + continue; + + NvDsInferInstanceMaskInfo obj; + obj.left = CLIP(rawDec.x1, 0, networkInfo.width - 1); + obj.top = CLIP(rawDec.y1, 0, networkInfo.height - 1); + obj.width = CLIP(rawDec.x2, 0, networkInfo.width - 1) - rawDec.x1; + obj.height = CLIP(rawDec.y2, 0, networkInfo.height - 1) - rawDec.y1; + if(obj.width <= 0 || obj.height <= 0) + continue; + obj.classId = static_cast(rawDec.class_id); + obj.detectionConfidence = rawDec.score; + + obj.mask_size = sizeof(float)*mask_instance_width*mask_instance_height; + obj.mask = new float[mask_instance_width*mask_instance_height]; + obj.mask_width = mask_instance_width; + obj.mask_height = mask_instance_height; + + float *rawMask = reinterpret_cast(out_mask + i + * detectionParams.numClassesConfigured + obj.classId); + memcpy (obj.mask, rawMask, sizeof(float)*mask_instance_width*mask_instance_height); + + objectList.push_back(obj); + } + + return true; + +} + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + // Code from NvDsInferParseCustomTfSSD for layer finding + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *boxLayer = layerFinder("pred_boxes"); // 1 x num_queries x 4 + const NvDsInferLayerInfo *classLayer = layerFinder("pred_logits"); // 1 x num_queries x num_classes + + if (!boxLayer || !classLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + const int keep_top_k = 200; + unsigned int numDetections = classLayer->inferDims.d[0]; + unsigned int numClasses = classLayer->inferDims.d[1]; + std::map ordered_objects; + + for (unsigned int idx = 0; idx < numDetections; idx += 1) { + NvDsInferObjectDetectionInfo res = {}; + + unsigned int class_layer_idx = idx * numClasses; + + res.classId = std::max_element(((float*)classLayer->buffer+class_layer_idx), ((float*)classLayer->buffer+class_layer_idx+numClasses)) - ((float*)classLayer->buffer+class_layer_idx); + res.detectionConfidence = ((float*)classLayer->buffer)[class_layer_idx+res.classId]; + + // If model does not have sigmoid layer, perform sigmoid calculation here + res.detectionConfidence = 1.0/(1.0 + exp(-res.detectionConfidence)); + + if(res.classId == 0 || res.detectionConfidence < detectionParams.perClassPreclusterThreshold[res.classId]) { + continue; + } + enum {cx, cy, w, h}; + float rectX1f, rectY1f, rectX2f, rectY2f; + + unsigned int box_layer_idx = idx * 4; + + rectX1f = (((float*)boxLayer->buffer)[box_layer_idx + cx] - (((float*)boxLayer->buffer)[box_layer_idx + w]/2)) * networkInfo.width; + rectY1f = (((float*)boxLayer->buffer)[box_layer_idx + cy] - (((float*)boxLayer->buffer)[box_layer_idx + h]/2)) * networkInfo.height; + rectX2f = rectX1f + ((float*)boxLayer->buffer)[box_layer_idx + w] * networkInfo.width; + rectY2f = rectY1f + ((float*)boxLayer->buffer)[box_layer_idx + h] * networkInfo.height; + + rectX1f = CLIP(rectX1f, 0.0f, networkInfo.width - 1); + rectX2f = CLIP(rectX2f, 0.0f, networkInfo.width - 1); + rectY1f = CLIP(rectY1f, 0.0f, networkInfo.height - 1); + rectY2f = CLIP(rectY2f, 0.0f, networkInfo.height - 1); + + res.left = rectX1f; + res.top = rectY1f; + res.width = rectX2f - rectX1f; + res.height = rectY2f - rectY1f; + + ordered_objects[res.detectionConfidence] = res; + } + + int jdx = 0; + for (auto iter=ordered_objects.rbegin(); iter!=ordered_objects.rend() && jdxsecond.classId != 0){ + objectList.emplace_back(iter->second);} + } + return true; +} + + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + NvDsInferDims inferDims_p_bboxes = outputLayersInfo[1].inferDims; + int numElements_p_bboxes=inferDims_p_bboxes.numElements; + + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float max_bbox=0; + for (int i=0; i < numElements_p_bboxes; i++) + { + // std::cout <<"p_bboxes: " + // < 0) + { + assert (normalized == 0); + for (int i = 0; i < p_keep_count[0]; i++) { + + + if ( p_scores[i] < threshold) continue; + assert((int) p_classes[i] < out_class_size); + + + // std::cout << "label/conf/ x/y x/y -- " + // << (int)p_classes[i] << " " << p_scores[i] << " " + // << p_bboxes[4*i] << " " << p_bboxes[4*i+1] << " " << p_bboxes[4*i+2] << " "<< p_bboxes[4*i+3] << " " << std::endl; + + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) + continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + + object.left=p_bboxes[4*i+1]; + object.top=p_bboxes[4*i]; + object.width=( p_bboxes[4*i+3] - object.left); + object.height= ( p_bboxes[4*i+2] - object.top); + + object.left=CLIP(object.left, 0, networkInfo.width - 1); + object.top=CLIP(object.top, 0, networkInfo.height - 1); + object.width=CLIP(object.width, 0, networkInfo.width - 1); + object.height=CLIP(object.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + } + return true; +} + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomDDETRTAO); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomEfficientDetTAO); diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt new file mode 100644 index 00000000..4b02dbbc --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/detector_labels.txt @@ -0,0 +1,4 @@ +BG +Person +Face +Bag diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile new file mode 100644 index 00000000..18d8e1f7 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/Makefile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) +CC:= g++ + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes \ + -I/usr/local/cuda/include + +CFLAGS+= -Wall -std=c++11 -shared -fPIC + +LIBS+= -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas + +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinfer_custombboxparser_tao.cpp +TARGET_LIB:= libnvds_infercustomparser_tao.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp new file mode 100644 index 00000000..c89a71c3 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/custom_parser/nvdsinfer_custombboxparser_tao.cpp @@ -0,0 +1,425 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "nvdsinfer_custom_impl.h" +#include +#include +#include +#include + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CLIP(a,min,max) (MAX(MIN(a, max), min)) +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +struct MrcnnRawDetection { + float y1, x1, y2, x2, class_id, score; +}; +/* This is a sample bounding box parsing function for the sample FasterRCNN + * + * detector model provided with the SDK. */ + +/* C-linkage to prevent name-mangling */ +extern "C" +bool NvDsInferParseCustomNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + + +extern "C" +bool NvDsInferParseCustomNMSTLT (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 2) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 2 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + // Host memory for "nms" which has 2 output bindings: + // the order is bboxes and keep_count + float* out_nms = (float *) outputLayersInfo[0].buffer; + int * p_keep_count = (int *) outputLayersInfo[1].buffer; + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float* det; + + for (int i = 0; i < p_keep_count[0]; i++) { + det = out_nms + i * 7; + + // Output format for each detection is stored in the below order + // [image_id, label, confidence, xmin, ymin, xmax, ymax] + if ( det[2] < threshold) continue; + assert((int) det[1] < out_class_size); + +#if 0 + std::cout << "id/label/conf/ x/y x/y -- " + << det[0] << " " << det[1] << " " << det[2] << " " + << det[3] << " " << det[4] << " " << det[5] << " " << det[6] << std::endl; +#endif + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) det[1]; + object.detectionConfidence = det[2]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(det[3] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(det[4] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP((det[5] - det[3]) * networkInfo.width, 0, networkInfo.width - 1); + object.height = CLIP((det[6] - det[4]) * networkInfo.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep cout" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(p_bboxes[4*i] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2] * networkInfo.width, 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3] * networkInfo.height, 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomMrcnnTLTV2 (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *detectionLayer = layerFinder("generate_detections"); + const NvDsInferLayerInfo *maskLayer = layerFinder("mask_fcn_logits/BiasAdd"); + + if (!detectionLayer || !maskLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + if(maskLayer->inferDims.numDims != 4U) { + std::cerr << "Network output number of dims is : " << + maskLayer->inferDims.numDims << " expect is 4"<< std::endl; + return false; + } + + const unsigned int det_max_instances = maskLayer->inferDims.d[0]; + const unsigned int num_classes = maskLayer->inferDims.d[1]; + if(num_classes != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" << + detectionParams.numClassesConfigured << ", detected by network: " << + num_classes << std::endl; + } + const unsigned int mask_instance_height= maskLayer->inferDims.d[2]; + const unsigned int mask_instance_width = maskLayer->inferDims.d[3]; + + auto out_det = reinterpret_cast( detectionLayer->buffer); + auto out_mask = reinterpret_cast(maskLayer->buffer); + + for(auto i = 0U; i < det_max_instances; i++) { + MrcnnRawDetection &rawDec = out_det[i]; + + if(rawDec.score < detectionParams.perClassPreclusterThreshold[0]) + continue; + + NvDsInferInstanceMaskInfo obj; + obj.left = CLIP(rawDec.x1, 0, networkInfo.width - 1); + obj.top = CLIP(rawDec.y1, 0, networkInfo.height - 1); + obj.width = CLIP(rawDec.x2, 0, networkInfo.width - 1) - rawDec.x1; + obj.height = CLIP(rawDec.y2, 0, networkInfo.height - 1) - rawDec.y1; + if(obj.width <= 0 || obj.height <= 0) + continue; + obj.classId = static_cast(rawDec.class_id); + obj.detectionConfidence = rawDec.score; + + obj.mask_size = sizeof(float)*mask_instance_width*mask_instance_height; + obj.mask = new float[mask_instance_width*mask_instance_height]; + obj.mask_width = mask_instance_width; + obj.mask_height = mask_instance_height; + + float *rawMask = reinterpret_cast(out_mask + i + * detectionParams.numClassesConfigured + obj.classId); + memcpy (obj.mask, rawMask, sizeof(float)*mask_instance_width*mask_instance_height); + + objectList.push_back(obj); + } + + return true; + +} + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + // Code from NvDsInferParseCustomTfSSD for layer finding + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *boxLayer = layerFinder("pred_boxes"); // 1 x num_queries x 4 + const NvDsInferLayerInfo *classLayer = layerFinder("pred_logits"); // 1 x num_queries x num_classes + + if (!boxLayer || !classLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + const int keep_top_k = 200; + unsigned int numDetections = classLayer->inferDims.d[0]; + unsigned int numClasses = classLayer->inferDims.d[1]; + std::map ordered_objects; + + for (unsigned int idx = 0; idx < numDetections; idx += 1) { + NvDsInferObjectDetectionInfo res = {}; + + unsigned int class_layer_idx = idx * numClasses; + + res.classId = std::max_element(((float*)classLayer->buffer+class_layer_idx), ((float*)classLayer->buffer+class_layer_idx+numClasses)) - ((float*)classLayer->buffer+class_layer_idx); + res.detectionConfidence = ((float*)classLayer->buffer)[class_layer_idx+res.classId]; + + // If model does not have sigmoid layer, perform sigmoid calculation here + res.detectionConfidence = 1.0/(1.0 + exp(-res.detectionConfidence)); + + // Skip forklift and pallet, refer detector_labels.txt for the class ids + // if (res.classId != 0 && res.classId != 1 && res.classId != 2 && res.classId != 3 && res.classId != 4 && res.classId != 5) { + // continue; + // } + if(res.detectionConfidence < detectionParams.perClassPreclusterThreshold[res.classId]) { + continue; + } + enum {cx, cy, w, h}; + float rectX1f, rectY1f, rectX2f, rectY2f; + + unsigned int box_layer_idx = idx * 4; + + rectX1f = (((float*)boxLayer->buffer)[box_layer_idx + cx] - (((float*)boxLayer->buffer)[box_layer_idx + w]/2)) * networkInfo.width; + rectY1f = (((float*)boxLayer->buffer)[box_layer_idx + cy] - (((float*)boxLayer->buffer)[box_layer_idx + h]/2)) * networkInfo.height; + rectX2f = rectX1f + ((float*)boxLayer->buffer)[box_layer_idx + w] * networkInfo.width; + rectY2f = rectY1f + ((float*)boxLayer->buffer)[box_layer_idx + h] * networkInfo.height; + + rectX1f = CLIP(rectX1f, 0.0f, networkInfo.width - 1); + rectX2f = CLIP(rectX2f, 0.0f, networkInfo.width - 1); + rectY1f = CLIP(rectY1f, 0.0f, networkInfo.height - 1); + rectY2f = CLIP(rectY2f, 0.0f, networkInfo.height - 1); + + res.left = rectX1f; + res.top = rectY1f; + res.width = rectX2f - rectX1f; + res.height = rectY2f - rectY1f; + + ordered_objects[res.detectionConfidence] = res; + } + + int jdx = 0; + for (auto iter=ordered_objects.rbegin(); iter!=ordered_objects.rend() && jdxsecond); + } + return true; +} + + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + NvDsInferDims inferDims_p_bboxes = outputLayersInfo[1].inferDims; + int numElements_p_bboxes=inferDims_p_bboxes.numElements; + + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float max_bbox=0; + for (int i=0; i < numElements_p_bboxes; i++) + { + // std::cout <<"p_bboxes: " + // < 0) + { + assert (normalized == 0); + for (int i = 0; i < p_keep_count[0]; i++) { + + + if ( p_scores[i] < threshold) continue; + assert((int) p_classes[i] < out_class_size); + + + // std::cout << "label/conf/ x/y x/y -- " + // << (int)p_classes[i] << " " << p_scores[i] << " " + // << p_bboxes[4*i] << " " << p_bboxes[4*i+1] << " " << p_bboxes[4*i+2] << " "<< p_bboxes[4*i+3] << " " << std::endl; + + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) + continue; + + NvDsInferObjectDetectionInfo object = {}; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + + object.left=p_bboxes[4*i+1]; + object.top=p_bboxes[4*i]; + object.width=( p_bboxes[4*i+3] - object.left); + object.height= ( p_bboxes[4*i+2] - object.top); + + object.left=CLIP(object.left, 0, networkInfo.width - 1); + object.top=CLIP(object.top, 0, networkInfo.height - 1); + object.width=CLIP(object.width, 0, networkInfo.width - 1); + object.height=CLIP(object.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + } + return true; +} + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomDDETRTAO); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomEfficientDetTAO); diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt new file mode 100644 index 00000000..b10e95bb --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/RTDETR/detector_labels.txt @@ -0,0 +1,7 @@ +person +agility_digit +gr1_t2 +nova_carter +transporter +forklift +pallet diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/requirements.txt b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/requirements.txt new file mode 100644 index 00000000..a4aeccc8 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/requirements.txt @@ -0,0 +1,6 @@ +kafka-python>=2.0.2 +protobuf>=3.20.0 +numpy>=1.21.0 +PyYAML>=6.0 +opencv-python>=4.5.0 +tqdm>=4.64.0 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh new file mode 100755 index 00000000..725f56a5 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Global variables - same as setup script +BASE_DIR=${BASE_DIR:-$HOME} +USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} +KAFKA_VERSION="4.2.0" +SCALA_VERSION="2.13" + +# Standardized paths +KAFKA_DIR="$BASE_DIR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}" +INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + +# Initialize status variables +GPU_STATUS="✓" +MQTT_STATUS="✓" +KAFKA_STATUS="✓" +MV3DT_VENV_STATUS="✓" +INFERENCE_BUILDER_STATUS="✓" +DATASETS_MODELS_STATUS="✓" + +# Check 1: Check if NVIDIA GPU is available +if ! nvidia-smi > /dev/null 2>&1; then + echo "NVIDIA GPU is not available" + GPU_STATUS="✗" +fi + +# Check 2: Check if MQTT broker is running on port 1883 +if ! ./scripts/mosquitto_test.sh > /dev/null 2>&1; then + MQTT_STATUS="✗" +fi + +# Check 3: Check if Kafka is running +if [ ! -d "$KAFKA_DIR" ]; then + echo "Kafka directory not found: $KAFKA_DIR" + KAFKA_STATUS="✗" +else + KAFKA_TOPICS_SCRIPT="$KAFKA_DIR/bin/kafka-topics.sh" + if [ -x "$KAFKA_TOPICS_SCRIPT" ]; then + TOPIC_LIST=$(timeout 5s $KAFKA_TOPICS_SCRIPT --bootstrap-server localhost:9092 --list 2>/dev/null) + if ! echo "$TOPIC_LIST" | grep -q "mv3dt"; then + echo "Kafka topics: $TOPIC_LIST" + echo "mv3dt topic not found" + KAFKA_STATUS="✗" + fi + else + echo "Kafka topics script not found or not executable: $KAFKA_TOPICS_SCRIPT" + KAFKA_STATUS="✗" + fi +fi + +# Check 4: Check whether virtual env "mv3dt_venv" setup correctly +if [ ! -d "$PWD/mv3dt_venv" ]; then + echo "Virtual environment 'mv3dt_venv' not found" + MV3DT_VENV_STATUS="✗" +fi + +# Check 5: Check whether inference builder is setup correctly +if [[ "$USE_INFERENCE_BUILDER" != "true" ]]; then + INFERENCE_BUILDER_STATUS="(skipped)" +else + if [ ! -d "$INFERENCE_BUILDER_DIR" ]; then + echo "Inference builder directory not found: $INFERENCE_BUILDER_DIR" + INFERENCE_BUILDER_STATUS="✗" + elif [ ! -d "$INFERENCE_BUILDER_DIR/ib_venv" ]; then + echo "Inference builder virtual environment 'ib_venv' not found: $INFERENCE_BUILDER_DIR/ib_venv" + INFERENCE_BUILDER_STATUS="✗" + fi +fi + +# Check 6: check wether datasets and models are setup correctly +if [ ! -d "$PWD/datasets" ]; then + echo "Datasets directory not found" + DATASETS_MODELS_STATUS="✗" +else + if [ ! -d "$PWD/datasets/mtmc_4cam/camInfo" ]; then + echo "datasets/mtmc_4cam/camInfo directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_4cam/videos" ]; then + echo "datasets/mtmc_4cam/videos directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_12cam/camInfo" ]; then + echo "datasets/mtmc_12cam/camInfo directory not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/datasets/mtmc_12cam/videos" ]; then + echo "datasets/mtmc_12cam/videos directory not found" + DATASETS_MODELS_STATUS="✗" + fi +fi + +if [ ! -d "$PWD/models" ]; then + echo "Models directory not found" + DATASETS_MODELS_STATUS="✗" +else + if [ ! -d "$PWD/models/BodyPose3DNet" ]; then + echo "BodyPose3DNet model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/PeopleNetTransformer" ]; then + echo "PeopleNetTransformer model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -f "$PWD/models/PeopleNetTransformer/custom_parser/libnvds_infercustomparser_tao.so" ]; then + echo "PeopleNetTransformer custom parser libnvds_infercustomparser_tao.so not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/RTDETR" ]; then + echo "RTDETR model not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -f "$PWD/models/RTDETR/custom_parser/libnvds_infercustomparser_tao.so" ]; then + echo "RTDETR custom parser libnvds_infercustomparser_tao.so not found" + DATASETS_MODELS_STATUS="✗" + fi + if [ ! -d "$PWD/models/PeopleNet2.6.3" ]; then + echo "PeopleNet2.6.3 model not found" + DATASETS_MODELS_STATUS="✗" + fi +fi + +# Summary of all checks +echo "" +echo "---- PREREQUISITES CHECK SUMMARY ----" +echo "1. NVIDIA GPU: $GPU_STATUS" +echo "2. MQTT Broker: $MQTT_STATUS" +echo "3. Kafka Broker: $KAFKA_STATUS" +echo "4. Python Environment: $MV3DT_VENV_STATUS" +echo "5. Datasets & Models: $DATASETS_MODELS_STATUS" +if [[ "$USE_INFERENCE_BUILDER" == "true" ]]; then + echo "6. Inference Builder: $INFERENCE_BUILDER_STATUS" +fi +echo "-------------------------------------" + +# Check overall status and exit with appropriate code +inference_check_passed=true +if [[ "$USE_INFERENCE_BUILDER" == "true" && "$INFERENCE_BUILDER_STATUS" != "✓" ]]; then + inference_check_passed=false +fi + +if [[ "$GPU_STATUS" = "✓" && "$MQTT_STATUS" = "✓" && "$KAFKA_STATUS" = "✓" && + "$MV3DT_VENV_STATUS" = "✓" && "$DATASETS_MODELS_STATUS" = "✓" && + "$inference_check_passed" = true ]]; then + echo "" + echo "✅ All prerequisites are properly set up!" + exit 0 +else + echo "" + echo "❌ Some prerequisites are missing or not properly configured." + echo "Please run the setup script or follow manual setup instructions." + exit 1 +fi \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh new file mode 100755 index 00000000..701b9e31 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/create_6cam_subset.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Simple script to create a random 6-camera subset from 12-camera dataset + +set -e + +DATASET_DIR="datasets" +SOURCE_DIR="$DATASET_DIR/mtmc_12cam" +TARGET_DIR="$DATASET_DIR/mtmc_6cam" + +# Check if source dataset exists +if [[ ! -d "$SOURCE_DIR" ]]; then + echo "Error: Source dataset not found: $SOURCE_DIR" + exit 1 +fi + +# Create target directory +if [[ -d "$TARGET_DIR" ]]; then + echo "Target directory already exists: $TARGET_DIR" + read -p "Remove existing directory and continue? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$TARGET_DIR" + else + exit 1 + fi +fi + +echo "Creating 6-camera subset from 12-camera dataset..." + +# Create directory structure +mkdir -p "$TARGET_DIR/camInfo" +mkdir -p "$TARGET_DIR/videos" + +# Randomly select 6 cameras from 1-12 +selected_cams=($(seq -f "%03g" 1 12 | shuf -n 6 | sort)) + +echo "Selected cameras: ${selected_cams[@]}" + +# Copy selected camera files +for cam in "${selected_cams[@]}"; do + cam_file="Warehouse_Synthetic_Cam${cam}" + + # Copy camera info file + if [[ -f "$SOURCE_DIR/camInfo/${cam_file}.yml" ]]; then + cp "$SOURCE_DIR/camInfo/${cam_file}.yml" "$TARGET_DIR/camInfo/" + echo "Copied: camInfo/${cam_file}.yml" + fi + + # Copy video file + if [[ -f "$SOURCE_DIR/videos/${cam_file}.mp4" ]]; then + cp "$SOURCE_DIR/videos/${cam_file}.mp4" "$TARGET_DIR/videos/" + echo "Copied: videos/${cam_file}.mp4" + fi +done + +# Copy common files +cp "$SOURCE_DIR/map.png" "$TARGET_DIR/" +cp "$SOURCE_DIR/transforms.yml" "$TARGET_DIR/" + +echo "" +echo "✅ Successfully created 6-camera dataset at: $TARGET_DIR" +echo "📁 Selected cameras: ${selected_cams[@]}" +echo "" +echo "To use this subset, update your experiment config to point to: $TARGET_DIR" diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh new file mode 100755 index 00000000..0a496c40 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/mosquitto_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Test parameters +BROKER="localhost" # Change if the broker is on a different machine or IP address +PORT=1883 # Default MQTT port for Mosquitto +TOPIC="test/topic" # The topic to test +MESSAGE="Hello from Mosquitto test!" # The test message + + +# Subscribe to the topic in the background +echo "Subscribing to topic '$TOPIC'..." +mosquitto_sub -h $BROKER -p $PORT -t $TOPIC & +SUB_PID=$! + +# Give the subscriber some time to start up +sleep 1 + +# Publish a message to the topic +echo "Publishing message to topic '$TOPIC'..." +mosquitto_pub -h $BROKER -p $PORT -t $TOPIC -m "$MESSAGE" + +# Wait a few seconds to allow the subscriber to receive the message +sleep 2 + +# Stop the mosquitto_sub process +kill $SUB_PID +wait $SUB_PID + +# The script will automatically exit, and the message should be received by the subscriber. +echo "Test completed." diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh new file mode 100755 index 00000000..1027e2eb --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh @@ -0,0 +1,724 @@ +#!/bin/bash + +# setup_prerequisites.sh - Automated prerequisites setup for MV3DT +# This script automates all the manual setup steps described in README.md + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Global variables +export DEEPSTREAM_IMAGE="${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BASE_DIR=${BASE_DIR:-$HOME} +USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} +KAFKA_VERSION="4.2.0" +SCALA_VERSION="2.13" + +# Standardized paths +KAFKA_DIR="$BASE_DIR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}" +INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + +# Utility functions +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +check_os() { + if [[ "$OSTYPE" != "linux-gnu"* ]]; then + log_error "This script only supports Linux. Detected OS: $OSTYPE" + exit 1 + fi + + if ! command_exists lsb_release; then + log_warning "lsb_release not found, checking if lsb-release package is installed" + if ! dpkg-query -W -f='${Status}' lsb-release 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing lsb-release package..." + sudo apt update && sudo apt install -y lsb-release || true + else + log_info "lsb-release package is installed but command not found, may need PATH update" + fi + fi + + if command_exists lsb_release; then + local distro=$(lsb_release -si) + local version=$(lsb_release -sr) + log_info "Detected OS: $distro $version" + + if [[ "$distro" != "Ubuntu" ]]; then + log_warning "This script is optimized for Ubuntu 24.04, but will attempt to continue on $distro" + fi + fi +} + +check_gpu() { + log_info "Checking NVIDIA GPU availability..." + if command_exists nvidia-smi; then + if nvidia-smi >/dev/null 2>&1; then + local driver_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader,nounits | head -1) + local major_version=$(echo "$driver_version" | cut -d'.' -f1) + + if [[ $major_version -ge 580 ]]; then + log_success "NVIDIA GPU detected with driver version: $driver_version" + return 0 + else + log_error "NVIDIA driver version $driver_version is too old (need 580+ series)" + return 1 + fi + else + log_error "nvidia-smi command failed to execute" + fi + else + log_error "nvidia-smi not found" + fi + + log_error "NVIDIA GPU or drivers not properly installed" + log_info "Please install NVIDIA drivers (version 580.xx or 590.xx) and try again" + log_info "Visit: https://docs.nvidia.com/cuda/cuda-installation-guide-linux/" + return 1 +} + + +setup_deepstream_container() { + # Check if DeepStream container is already available + if docker images --format "table {{.Repository}}:{{.Tag}}" | grep -q "^$DEEPSTREAM_IMAGE$"; then + log_success "DeepStream container already available" + else + log_info "Pulling DeepStream container image..." + if docker pull $DEEPSTREAM_IMAGE; then + log_success "DeepStream container pulled successfully" + else + log_error "Failed to pull DeepStream container" + log_info "Please check your network connection and Docker credentials" + return 1 + fi + fi +} + +setup_git_lfs() { + log_info "Setting up Git LFS and extracting datasets and models..." + + cd "$REPO_ROOT" + + # Check if datasets are already extracted + if [[ -d "datasets/mtmc_4cam" && -d "datasets/mtmc_12cam" ]]; then + log_success "Datasets already extracted" + else + # Extract assets if they exist + if [[ -f "assets/datasets.zip" ]]; then + log_info "Extracting datasets..." + unzip -q assets/datasets.zip + log_success "Datasets extracted" + else + log_error "assets/datasets.zip not found" + return 1 + fi + fi + + if [[ -f "models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx" && -f "models/BodyPose3DNet/bodypose3dnet_accuracy.onnx" && -f "models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx" && -f "models/PeopleNet2.6.3/resnet34_peoplenet.onnx" ]]; then + log_success "Models already extracted" + else + log_info "Downloading PeopleNet Transformer model..." + mkdir -p models/PeopleNetTransformer + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O 'models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx'; then + log_error "Failed to download PeopleNet Transformer model" + return 1 + fi + + log_info "Downloading RT-DETR model..." + mkdir -p models/RTDETR + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/rtdetr_2d_warehouse/deployable_efficientvit_l2_v1.0/files?redirect=true&path=rtdetr_warehouse_v1.0.fp16.onnx' -O 'models/RTDETR/rtdetr_warehouse_v1.0.fp16.onnx'; then + log_error "Failed to download RT-DETR model" + return 1 + fi + + log_info "Downloading PeopleNet v2.6.3 model..." + mkdir -p models/PeopleNet2.6.3 + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' -O 'models/PeopleNet2.6.3/resnet34_peoplenet.onnx'; then + log_error "Failed to download PeopleNet v2.6.3 model" + return 1 + fi + + log_info "Downloading BodyPose3DNet model..." + mkdir -p models/BodyPose3DNet + if ! wget -q --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O 'models/BodyPose3DNet/bodypose3dnet_accuracy.onnx'; then + log_error "Failed to download BodyPose3DNet model" + return 1 + fi + fi +} + +setup_mosquitto() { + log_info "Setting up Mosquitto MQTT broker..." + + if ! command_exists mosquitto; then + log_info "Checking Mosquitto packages..." + local mosquitto_packages=("mosquitto" "mosquitto-clients") + local packages_to_install=() + + for package in "${mosquitto_packages[@]}"; do + if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then + packages_to_install+=("$package") + else + log_info "$package is already installed" + fi + done + + if [[ ${#packages_to_install[@]} -gt 0 ]]; then + log_info "Installing missing Mosquitto packages: ${packages_to_install[*]}" + sudo apt-add-repository -y ppa:mosquitto-dev/mosquitto-ppa + sudo apt update + sudo apt install -y "${packages_to_install[@]}" + else + log_success "All Mosquitto packages are already installed" + fi + fi + + # Configure mosquitto + local mosquitto_conf="/etc/mosquitto/conf.d/mv3dt.conf" + if [[ ! -f "$mosquitto_conf" ]] || ! grep -q "set_tcp_nodelay true" "$mosquitto_conf"; then + log_info "Adding set_tcp_nodelay config to Mosquitto..." + echo "set_tcp_nodelay true" | sudo tee "$mosquitto_conf" > /dev/null + fi + + # Start mosquitto service + if ! systemctl is-active --quiet mosquitto; then + log_info "Starting Mosquitto service..." + sudo systemctl enable mosquitto + sudo systemctl start mosquitto + fi + + # Test MQTT broker + sleep 2 + cd "$REPO_ROOT" + if [[ -x "scripts/mosquitto_test.sh" ]]; then + if ./scripts/mosquitto_test.sh >/dev/null 2>&1; then + log_success "Mosquitto broker is running and accessible" + else + log_warning "Mosquitto service test failed, trying manual start..." + # Try starting manually + mosquitto -p 1883 -d + sleep 2 + if ./scripts/mosquitto_test.sh >/dev/null 2>&1; then + log_success "Mosquitto broker started manually" + else + log_error "Failed to start Mosquitto broker" + return 1 + fi + fi + else + log_warning "Mosquitto test script not found, assuming broker is working" + fi +} + +setup_java() { + log_info "Setting up Java for Kafka..." + + if command_exists java; then + local java_version=$(java -version 2>&1 | head -n 1 | cut -d'"' -f2 | cut -d'.' -f1) + if [[ "$java_version" -ge 17 ]]; then + log_success "Java $java_version already installed" + return 0 + fi + fi + + log_info "Checking OpenJDK 17 package..." + if ! dpkg-query -W -f='${Status}' openjdk-17-jdk 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing OpenJDK 17..." + sudo apt update && sudo apt install -y openjdk-17-jdk + else + log_success "OpenJDK 17 package is already installed" + fi + + # Verify installation + if java -version >/dev/null 2>&1; then + log_success "Java installed successfully" + else + log_error "Java installation failed" + return 1 + fi +} + +setup_kafka() { + log_info "Setting up Kafka..." + + # Check if Kafka is already running + if netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_info "Kafka appears to be running on port 9092, checking topic..." + if check_kafka_topic; then + log_success "Kafka is already running with mv3dt topic" + return 0 + else + log_warning "Kafka is running but mv3dt topic not found, stopping old Kafka broker..." + for old_kafka in "$BASE_DIR"/kafka_*/bin/kafka-server-stop.sh; do + [[ -x "$old_kafka" ]] && "$old_kafka" 2>/dev/null || true + done + sleep 3 + fi + fi + + setup_java + + # Download and extract Kafka if not exists + if [[ ! -d "$KAFKA_DIR" ]]; then + log_info "Downloading Kafka ${KAFKA_VERSION}..." + cd "$BASE_DIR" + + local kafka_url="https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + + if ! wget -q "$kafka_url" -O "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"; then + log_error "Failed to download Kafka from $kafka_url" + return 1 + fi + + log_info "Extracting Kafka..." + tar -xzf "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + rm "kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" + log_success "Kafka extracted to $KAFKA_DIR" + fi + + cd "$KAFKA_DIR" + + # Check for existing cluster ID or create new one + local meta_properties="/tmp/kraft-combined-logs/meta.properties" + if [[ -f "$meta_properties" ]]; then + # Extract existing cluster ID from meta.properties + export KAFKA_CLUSTER_ID=$(grep "cluster.id=" "$meta_properties" | cut -d'=' -f2) + if [[ -n "$KAFKA_CLUSTER_ID" ]]; then + log_info "Reusing existing Kafka cluster ID: $KAFKA_CLUSTER_ID" + else + log_warning "Found meta.properties but no cluster.id, generating new one" + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + fi + else + log_info "No existing Kafka cluster found, generating new cluster ID" + export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" + fi + + # Format storage only if meta.properties doesn't exist or is invalid + if [[ ! -f "$meta_properties" || -z "$KAFKA_CLUSTER_ID" ]]; then + log_info "Formatting Kafka storage with cluster ID: $KAFKA_CLUSTER_ID" + if ! bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/server.properties >/dev/null 2>&1; then + log_error "Failed to format Kafka storage" + return 1 + fi + else + log_info "Using existing Kafka storage (skipping format)" + fi + + # Start Kafka + log_info "Starting Kafka..." + + # Start Kafka in background + nohup bin/kafka-server-start.sh config/server.properties >/dev/null 2>&1 & + local kafka_pid=$! + + # Wait for Kafka to start + log_info "Waiting for Kafka to start..." + for i in {1..30}; do + if netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_success "Kafka started successfully" + break + fi + sleep 2 + if ! kill -0 $kafka_pid 2>/dev/null; then + log_error "Kafka startup failed" + return 1 + fi + done + + if ! netstat -ln 2>/dev/null | grep -q ":9092 "; then + log_error "Kafka failed to start on port 9092" + return 1 + fi + + # Create mv3dt topic + create_kafka_topic "$KAFKA_DIR" +} + +check_kafka_topic() { + if [[ ! -d "$KAFKA_DIR" || ! -x "$KAFKA_DIR/bin/kafka-topics.sh" ]]; then + return 1 + fi + + local topics=$("$KAFKA_DIR/bin/kafka-topics.sh" --bootstrap-server localhost:9092 --list 2>/dev/null) + echo "$topics" | grep -q "mv3dt" +} + +create_kafka_topic() { + local kafka_dir="$1" + + log_info "Creating mv3dt topic..." + if "$kafka_dir/bin/kafka-topics.sh" --bootstrap-server localhost:9092 \ + --create --topic mv3dt --partitions 1 --replication-factor 1 \ + --config retention.ms=30000 --if-not-exists >/dev/null 2>&1; then + log_success "mv3dt topic created successfully" + else + log_error "Failed to create mv3dt topic" + return 1 + fi +} + +setup_python_env() { + log_info "Setting up Python environment..." + + cd "$REPO_ROOT" + + # Check and install system packages + log_info "Checking system Python packages..." + local packages_to_install=() + local required_packages=("python3-tk" "python3.12-venv" "python3.12-dev" "python3-pip") + + for package in "${required_packages[@]}"; do + if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then + packages_to_install+=("$package") + else + log_info "$package is already installed" + fi + done + + if [[ ${#packages_to_install[@]} -gt 0 ]]; then + log_info "Installing missing packages: ${packages_to_install[*]}" + sudo apt update + sudo apt install -y "${packages_to_install[@]}" + else + log_success "All required Python packages are already installed" + fi + + # Create virtual environment if it doesn't exist + if [[ ! -d "mv3dt_venv" ]]; then + log_info "Creating mv3dt_venv virtual environment..." + python3 -m venv mv3dt_venv + fi + + # Activate and install requirements + log_info "Installing Python dependencies..." + source mv3dt_venv/bin/activate + + if [[ -f "requirements.txt" ]]; then + pip install --quiet --upgrade pip + pip install --quiet -r requirements.txt + log_success "Python dependencies installed" + else + log_error "requirements.txt not found" + return 1 + fi + + # Verify key packages + python -c "import kafka, google.protobuf" 2>/dev/null + if [[ $? -eq 0 ]]; then + log_success "Python environment setup completed successfully" + else + log_error "Failed to import required Python packages" + return 1 + fi +} + +build_custom_parser() { + log_info "Building custom parsers..." + + # Set correct GPU flag considering diffent platforms + if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" + elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" + else + echo "No GPU support found in Docker." + exit 1 + fi + + local models=("PeopleNetTransformer" "RTDETR") + for model in "${models[@]}"; do + if [[ -f "$PWD/models/$model/custom_parser/libnvds_infercustomparser_tao.so" ]]; then + log_success "$model custom parser already built" + continue + fi + + log_info "Building custom parser for $model..." + build_output=$(docker run --privileged --rm --net=host $GPU_FLAG \ + -v $PWD/models:/workspace/models \ + -w /workspace/models/$model \ + --entrypoint /bin/bash \ + $DEEPSTREAM_IMAGE \ + -c "cd custom_parser && make clean 2>&1 && make 2>&1" 2>&1) + build_exit_code=$? + + if [[ $build_exit_code -ne 0 ]]; then + log_error "Build failed for $model with errors:" + echo "$build_output" + return 1 + fi + + if [[ -f "$PWD/models/$model/custom_parser/libnvds_infercustomparser_tao.so" ]]; then + log_success "$model custom parser built successfully" + else + log_error "Failed to build $model custom parser: libnvds_infercustomparser_tao.so not found" + echo "$build_output" + return 1 + fi + done +} + + +setup_inference_builder() { + if [[ "$USE_INFERENCE_BUILDER" != "true" ]]; then + log_info "Skipping Inference Builder setup (USE_INFERENCE_BUILDER not set to true)" + return 0 + fi + + log_info "Setting up DeepStream Inference Builder..." + + if [[ ! -d "$INFERENCE_BUILDER_DIR" ]]; then + log_info "Cloning Inference Builder repository..." + cd "$BASE_DIR" + if ! git clone https://github.com/NVIDIA-AI-IOT/inference_builder.git; then + log_warning "Failed to clone Inference Builder repository" + log_info "This is optional - Inference Builder is skipped by default. Set USE_INFERENCE_BUILDER=true to enable it." + return 0 + fi + + cd inference_builder + git submodule update --init --recursive + else + log_success "Inference Builder directory already exists" + cd "$INFERENCE_BUILDER_DIR" + fi + + # Install system dependencies + log_info "Checking protobuf-compiler package..." + if ! dpkg-query -W -f='${Status}' protobuf-compiler 2>/dev/null | grep -q "install ok installed"; then + log_info "Installing protobuf-compiler..." + sudo apt update && sudo apt install -y protobuf-compiler + else + log_success "protobuf-compiler is already installed" + fi + + # Create virtual environment + if [[ ! -d "ib_venv" ]]; then + log_info "Creating Inference Builder virtual environment..." + python -m venv ib_venv + fi + + # Install requirements + log_info "Installing Inference Builder dependencies..." + source ib_venv/bin/activate + + if [[ -f "requirements.txt" ]]; then + pip install --quiet --upgrade pip + pip install --quiet -r requirements.txt + else + log_warning "Inference Builder requirements.txt not found, skipping pip install" + fi + + # Create custom DeepStream Docker image with MV3DT packages (always) + log_info "Installing inference builder dependencies into DeepStream container..." + + # Create Dockerfile in temporary location + cat > ./Dockerfile.mv3dt << EOF +FROM ${DEEPSTREAM_IMAGE} + +# Install IB python packages +RUN pip3 install torch==2.7.0 omegaconf==2.3.0 + +# Set environment variables +ENV GST_PLUGIN_PATH=/opt/nvidia/deepstream/deepstream/lib/gst-plugins +ENV LD_LIBRARY_PATH=/opt/nvidia/deepstream/deepstream/lib:$LD_LIBRARY_PATH +ENV NVSTREAMMUX_ADAPTIVE_BATCHING=yes + +# Set working directory +WORKDIR /mv3dt_app +EOF + + # Build the custom image + # Before building, check whether the image already exists + if docker inspect inference-builder-mv3dt:latest >/dev/null 2>&1; then + log_info "Target docker image inference-builder-mv3dt:latest already exists" + log_success "Inference Builder setup completed" + return 0 + fi + + # Build the Docker image + if docker build -f ./Dockerfile.mv3dt -t inference-builder-mv3dt:latest .; then + log_info "DeepStream container with Inference Builder dependencies saved as docker image inference-builder-mv3dt:latest" + rm -f ./Dockerfile.mv3dt + else + log_error "Failed to build custom DeepStream Docker image" + rm -f ./Dockerfile.mv3dt + return 1 + fi + + log_success "Inference Builder setup completed" +} + +run_prerequisites_check() { + log_info "Running prerequisites check..." + cd "$REPO_ROOT" + + if [[ -x "scripts/check_prerequisites.sh" ]]; then + # Make mosquitto test script executable if it exists + [[ -f "scripts/mosquitto_test.sh" ]] && chmod +x scripts/mosquitto_test.sh + + if bash scripts/check_prerequisites.sh; then + return 0 + else + log_error "Prerequisites check failed" + return 1 + fi + else + log_error "Prerequisites check script not found" + return 1 + fi +} + +print_usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Automated setup script for MV3DT prerequisites. + +OPTIONS: + -h, --help Show this help message + --check-only Only run prerequisites check without setup + +ENVIRONMENT VARIABLES: + BASE_DIR Base directory for installations (default: $HOME) + USE_INFERENCE_BUILDER Set to 'true' to enable Inference Builder setup (default: false) + +EOF +} + +main() { + local check_only=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_usage + exit 0 + ;; + --check-only) + check_only=true + shift + ;; + *) + log_error "Unknown option: $1" + print_usage + exit 1 + ;; + esac + done + + echo "==============================================" + echo " MV3DT Prerequisites Setup Script" + echo "==============================================" + echo + + if [[ "$check_only" == "true" ]]; then + run_prerequisites_check + exit $? + fi + + check_os + + # Check if we're in the right directory + if [[ ! -f "$REPO_ROOT/README.md" ]] || ! grep -q "Multi-View 3D Tracking" "$REPO_ROOT/README.md"; then + log_error "Please run this script from the deepstream-tracker-3d-multi-view repository root" + exit 1 + fi + + log_info "Starting automated prerequisites setup..." + + # Run setup steps + local failed_steps=() + + if ! check_gpu; then + failed_steps+=("GPU/NVIDIA drivers") + fi + + if ! setup_deepstream_container; then + failed_steps+=("DeepStream container") + fi + + if ! setup_git_lfs; then + failed_steps+=("Git LFS/Datasets/Models") + fi + + if ! build_custom_parser; then + failed_steps+=("Build Custom parser") + fi + + if ! setup_mosquitto; then + failed_steps+=("Mosquitto MQTT") + fi + + if ! setup_kafka; then + failed_steps+=("Kafka") + fi + + if ! setup_python_env; then + failed_steps+=("Python environment") + fi + + if ! setup_inference_builder; then + failed_steps+=("Inference Builder") + fi + + echo + echo "==============================================" + echo " Setup Summary" + echo "==============================================" + + if [[ ${#failed_steps[@]} -eq 0 ]]; then + log_success "All setup steps completed successfully!" + echo + log_info "Configured Paths:" + echo -e " ${BLUE}Base Directory:${NC} $BASE_DIR" + echo -e " ${BLUE}Kafka Installation:${NC} $KAFKA_DIR" + if [[ "$USE_INFERENCE_BUILDER" == "true" ]]; then + echo -e " ${BLUE}Inference Builder:${NC} $INFERENCE_BUILDER_DIR" + fi + echo -e " ${BLUE}MV3DT Repo:${NC} $REPO_ROOT" + echo + + # Run final check + if run_prerequisites_check; then + log_success "Prerequisites check passed! You're ready to use MV3DT." + echo + else + log_error "Setup completed but prerequisites check still failed" + echo "Please review the error messages and run the script again" + exit 1 + fi + else + log_error "The following setup steps failed:" + for step in "${failed_steps[@]}"; do + echo " - $step" + done + echo + echo "Please review the error messages above and:" + echo " 1. Fix the issues manually" + echo " 2. Run this script again" + echo " 3. Or run specific setup steps as needed" + exit 1 + fi + + echo "==============================================" +} + +# Handle script interruption +trap 'log_error "Setup interrupted by user"; exit 130' INT TERM + +# Run main function +main "$@" \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh new file mode 100755 index 00000000..f6afadd6 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ds.sh @@ -0,0 +1,63 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/12cam +export MODEL_REPO=$PWD/models + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam_rt_detr.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-osd \ + --tracker-config=$TRACKER_CONFIG \ + --enable-msg-broker \ + --detector-config=$DETECTOR_CONFIG \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh new file mode 100755 index 00000000..da4e97bf --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_12cam_ib.sh @@ -0,0 +1,75 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_12cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/12cam +export MODEL_REPO=$PWD/models + +REPO_DIR=$PWD +INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker_tuned_12cam.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Clean and create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/$DETECTOR_MODEL/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh new file mode 100755 index 00000000..0e9da4ec --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ds.sh @@ -0,0 +1,65 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_4cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/4cam +export MODEL_REPO=$PWD/models + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + + +# Launch MV3DT pipeline +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh new file mode 100755 index 00000000..0cfefd54 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_4cam_ib.sh @@ -0,0 +1,76 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_4cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/4cam +export MODEL_REPO=$PWD/models + +REPO_DIR=$PWD +INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Select detector model: PeopleNetTransformer (default), RTDETR, or PeopleNet2.6.3 +export DETECTOR_MODEL=${DETECTOR_MODEL:-PeopleNetTransformer} + +if [ "$DETECTOR_MODEL" = "RTDETR" ]; then + DETECTOR_CONFIG="config_pgie_rt_detr.txt" + TRACKER_CONFIG="config_tracker.yml" +elif [ "$DETECTOR_MODEL" = "PeopleNet2.6.3" ]; then + DETECTOR_CONFIG="config_pgie_peoplenet.txt" + TRACKER_CONFIG="config_tracker.yml" +else + DETECTOR_CONFIG="config_pgie.txt" + TRACKER_CONFIG="config_tracker.yml" +fi + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --detector-config=$DETECTOR_CONFIG \ + --tracker-config=$TRACKER_CONFIG \ + --config-overrides=override_tracker_4cam.yml \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/$DETECTOR_MODEL/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# # Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh new file mode 100755 index 00000000..0324b073 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ds.sh @@ -0,0 +1,46 @@ +# Set dataset, model, and experiment directories +export DATASET_DIR=$PWD/datasets/mtmc_6cam/ +export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam +export MODEL_REPO=$PWD/models + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump +mkdir -p $EXPERIMENT_DIR/outVideos + +# Auto-generate DeepStream configuration files with custom 2D tracker config +source mv3dt_venv/bin/activate +python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --enable-msg-broker \ + --enable-osd \ + --tracker-config=config_tracker_2d.yml \ + --output-dir=$EXPERIMENT_DIR + +# Launch real-time BEV visualization +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with custom 2D tracker config +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + ${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.0-triton-multiarch} \ + deepstream-test5-app -c config_deepstream.txt \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh new file mode 100755 index 00000000..94087009 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/test_custom_2d_tracker_ib.sh @@ -0,0 +1,58 @@ +# Set dataset, model, experiment, and repo directories +export DATASET_DIR=$PWD/datasets/mtmc_6cam/ +export EXPERIMENT_DIR=$PWD/experiments/inference_builder/6cam +export MODEL_REPO=$PWD/models + +export REPO_DIR=$PWD +export INFERENCE_BUILDER_DIR=${INFERENCE_BUILDER_DIR:-$HOME/inference_builder} + +# Set correct GPU flag considering diffent platforms +if docker info | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." + exit 1 +fi + +# Create directories for output +mkdir -p $EXPERIMENT_DIR/infer-kitti-dump +mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + +# Generate Inference Builder configuration files with custom 2D tracker config +source mv3dt_venv/bin/activate +python utils/inference_builder_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --tracker-config=config_tracker_2d.yml \ + --output-dir=$EXPERIMENT_DIR + +cp $EXPERIMENT_DIR/nvdsinfer_config.yaml $MODEL_REPO/PeopleNetTransformer/ + +# Generate Python package for MV3DT inference flow using Inference Builder +cd $INFERENCE_BUILDER_DIR +source ib_venv/bin/activate +python builder/main.py $EXPERIMENT_DIR/ds_mv3dt.yaml \ + -o builder/samples/mv3dt_app \ + --server-type serverless + +# Launch real-time BEV visualization +cd $REPO_DIR +source mv3dt_venv/bin/activate +python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --average-multi-cam \ + --show-ids & + +# Launch MV3DT pipeline with Inference Builder and custom 2D tracker config +docker run --privileged --rm -it --net=host $GPU_FLAG \ + -v $INFERENCE_BUILDER_DIR/builder/samples/mv3dt_app/deepstream-app:/mv3dt_app \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /mv3dt_app \ + inference-builder-mv3dt:latest \ + python3 __main__.py --source-config /workspace/experiments/source_list_static.yaml -s /dev/null diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/README.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/README.md new file mode 100644 index 00000000..6be7107e --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/README.md @@ -0,0 +1,103 @@ +## Python Utility Scripts + +This directory contains Python utility scripts to help with configuration generation and visualization for the DeepStream Multi-View 3D Tracking project. + + +### `deepstream_auto_configurator.py` + +The Deepstream auto-configurator automatically generates DeepStream configuration files based on your dataset. + +**Usage:** +```bash +python deepstream_auto_configurator.py --dataset-dir DATASET_DIR [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-dir`: Dataset directory containing `videos/` and `camInfo/` subdirectories +- `--output-dir`: Output directory for generated configs (default: `temp_outputs`) +- `--enable-osd`: Enable OSD display sink +- `--enable-file-output`: Enable video file output +- `--enable-msg-broker`: Enable Kafka message broker output +- `--num_vision_neighbor`: Number of vision neighbors per camera +- `--tracker-config`: Base tracker configuration file (default: `config_tracker.yml`) +- `--config-overrides`: YAML file containing section overrides for tracker configuration +``` + + +### `inference_builder_auto_configurator.py` + +The Inference Builder auto-configurator automatically generates Inference Builder config files based on your dataset. + +**Usage:** +```bash +python inference_builder_auto_configurator.py --dataset-dir DATASET_DIR [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-dir`: Dataset directory containing `videos/` and `camInfo/` subdirectories +- `--output-dir`: Output directory for generated configs (default: `temp_outputs`) +- `--tracker-config`: Base tracker configuration file (default: `config_tracker.yml`) +- `--config-overrides`: YAML file containing section overrides for tracker configuration +``` + + + +### `kafka_bev_visualizer.py` + +Real-time bird's-eye view visualization of 3D tracking data from Kafka streams. + +**Usage:** +```bash +python kafka_bev_visualizer.py [OPTIONS] +``` + +**Arguments:** +``` +- `--dataset-path`: Path to dataset containing map.png and transforms.yml +- `--msgconv-config`: Path to message converter config file +- `--output-path`: Output directory for videos and screenshots +- `--offline`: Run in offline mode to save video from all messages instead of real-time visualization +- `--show-ids`: Show object IDs near trajectory heads +- `--average-multi-cam`: Average trajectory points from multiple cameras for the same object (shows 1 point per object instead of multiple points from different cameras) +``` + +**Interactive Controls (Real-time mode):** +- `q`: Quit application +- `s`: Save current frame as screenshot +- `c`: Clear all trajectories +- `r`: Start/stop recording video + + + +## Additional Utilities + +### `generate_pub_sub_configs.py` + +Generates communication configurations for multi-camera tracking systems, including peer-to-peer relationships and camera neighbor mappings. + +**Usage:** +```bash +python generate_pub_sub_configs.py --deployment_config_path CONFIG_PATH [OPTIONS] +``` + +**Arguments:** +``` +- `--deployment_config_path`: Path to YAML file containing deployment configurations +- `--cam_info_path`: Directory containing camera calibration information +- `--output_path`: Directory to store output configuration files +- `--neighbor_criteria`: Criteria for selecting neighboring cameras + - Format: `"top_N:{N}"` or `"overlap_threshold:{threshold}"` +- `--minimum_object_size`: Minimum object size in pixels for visibility +- `--range_of_interest`: Range of interest in world coordinates + - Format: `"x1,y1,x2,y2"` (min and max corners) +``` + + +**Note:** This script is typically called automatically by the `deepstream_auto_configurator.py` script. + + +### `schema_pb2.py` + +Contains Protocol Buffer schema definitions for message serialization used in Kafka communication. diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py new file mode 100644 index 00000000..07fe2409 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/deepstream_auto_configurator.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +import os +import argparse +import math +import glob +import re +import shutil +import subprocess +import yaml +import cv2 +from pathlib import Path +from typing import List, Tuple, Dict + +class DeepStreamAutoConfigurator: + def __init__(self, dataset_dir: str, output_dir: str): + self.dataset_dir = Path(dataset_dir) + self.output_dir = Path(output_dir) + + def compute_grid_layout(self, num_videos: int) -> Tuple[int, int]: + if num_videos <= 1: return 1, 1 + elif num_videos == 2: return 1, 2 + elif num_videos <= 4: return 2, 2 + elif num_videos <= 6: return 2, 3 + elif num_videos <= 9: return 3, 3 + elif num_videos <= 12: return 3, 4 + elif num_videos <= 16: return 4, 4 + else: + side = math.ceil(math.sqrt(num_videos)) + return side, side + + def get_video_resolution(self, video_path: str) -> Tuple[int, int]: + cap = cv2.VideoCapture(str(video_path)) + if not cap.isOpened(): + raise RuntimeError(f"Could not open video file: {video_path}") + + width, height = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + cap.release() + + if width <= 0 or height <= 0: + raise RuntimeError(f"Invalid resolution detected: {width}x{height}") + return width, height + + def get_engine_file_from_detector_config(self, detector_config: str, num_videos: int) -> str: + """Extract onnx-file from detector config and generate engine file path.""" + config_path = Path("config_templates") / detector_config + if not config_path.exists(): + return None + + with open(config_path, 'r') as f: + for line in f: + if line.strip().startswith('onnx-file='): + onnx_file = line.strip().split('=', 1)[1] + engine_file = f"{onnx_file}_b{num_videos}_gpu0_fp16.engine" + print(f"Generated engine file: {engine_file}") + return engine_file + return None + + def generate_deepstream_config(self, video_files: List[str], enabled_sinks: List[str], detector_config: str = None) -> str: + template_path = Path("config_templates/config_deepstream.txt") + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + num_videos = len(video_files) + rows, columns = self.compute_grid_layout(num_videos) + + # Detect video resolution from the first video file + first_video_path = self.dataset_dir / "videos" / video_files[0] + video_width, video_height = self.get_video_resolution(first_video_path) + + print(f"Detected video resolution: {video_width}x{video_height}") + + # Get engine file path from detector config + engine_file_path = self.get_engine_file_from_detector_config(detector_config, num_videos) if detector_config else None + print (f"Detector config: {detector_config}") + print (f"Engine file path: {engine_file_path}") + + # Update source sections and URIs + content = self._update_sources(content, video_files) + + # Simple line-by-line replacements + lines = content.split('\n') + current_section = None + + for i, line in enumerate(lines): + stripped = line.strip() + + if stripped.startswith('[') and stripped.endswith(']'): + current_section = stripped[1:-1] + elif stripped.startswith('batch-size='): + lines[i] = f"batch-size={num_videos}" + elif stripped.startswith('model-engine-file='): + lines[i] = f"model-engine-file={engine_file_path}" if engine_file_path else re.sub(r'_b\d+_', f'_b{num_videos}_', line) + elif stripped.startswith('rows='): + lines[i] = f"rows={rows}" + elif stripped.startswith('columns='): + lines[i] = f"columns={columns}" + elif stripped.startswith('enable=') and current_section in ['sink0', 'sink1', 'sink2', 'sink3']: + lines[i] = f"enable={'1' if current_section in enabled_sinks else '0'}" + # Update streammux section resolution + elif stripped.startswith('width=') and current_section == 'streammux': + lines[i] = f"width={video_width}" + elif stripped.startswith('height=') and current_section == 'streammux': + lines[i] = f"height={video_height}" + # Update tracker section resolution + elif stripped.startswith('tracker-width=') and current_section == 'tracker': + lines[i] = f"tracker-width={video_width}" + elif stripped.startswith('tracker-height=') and current_section == 'tracker': + lines[i] = f"tracker-height={video_height}" + + return '\n'.join(lines) + + def _update_sources(self, content: str, video_files: List[str]) -> str: + lines = content.split('\n') + result = [] + i = 0 + sources_added = 0 + + while i < len(lines): + line = lines[i] + + if line.strip().startswith('[source') and sources_added < len(video_files): + # Keep/add source section + result.append(line) + i += 1 + while i < len(lines) and not (lines[i].strip().startswith('[') and lines[i].strip().endswith(']')): + if lines[i].strip().startswith('uri='): + result.append(f"uri=file:///workspace/inputs/videos/{video_files[sources_added]}") + else: + result.append(lines[i]) + i += 1 + sources_added += 1 + continue + elif line.strip().startswith('[source') and sources_added >= len(video_files): + # Skip extra source sections + i += 1 + while i < len(lines) and not (lines[i].strip().startswith('[') and lines[i].strip().endswith(']')): + i += 1 + continue + elif line.strip().startswith('[streammux]'): + # Add missing sources before streammux + while sources_added < len(video_files): + result.extend([ + "", f"[source{sources_added}]", "type=3", "enable=1", + "cudadec-memtype=0", "gpu-id=0", "num-sources=1", + f"uri=file:///workspace/inputs/videos/{video_files[sources_added]}" + ]) + sources_added += 1 + + # Add blank line before streammux section + result.append("") + result.append(line) + else: + result.append(line) + i += 1 + + return '\n'.join(result) + + def generate_msgconv_config(self, video_files: List[str]) -> str: + """Generate message converter configuration based on video files.""" + lines = [] + + # Sort video files to ensure consistent ordering + sorted_video_files = sorted(video_files) + + for i, video_file in enumerate(sorted_video_files, 1): + # Use index-based camera IDs (1, 2, 3, 4...) + camera_id = i + + lines.extend([ + f"[sensor{i-1}]", + "enable=1", + "type=Camera", + f"id=Camera{camera_id}", + "" # Empty line between sections + ]) + + # Add extra empty line at the end to match template format + lines.append("") + + return '\n'.join(lines) + + def generate_tracker_config(self, video_files: List[str], calib_files: List[str], config_overrides: str = None, tracker_config: str = None) -> str: + # Use specified tracker config or default to config_tracker.yml + tracker_template = tracker_config or "config_tracker.yml" + template_path = Path("config_templates") / tracker_template + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + # check if this is a 2d config + is_2d_config = self._is_2d_tracker_config(content) + + # Load config overrides if provided + overrides = self._load_section_overrides(config_overrides) + if overrides and not is_2d_config: + print(f"Applying section overrides from: {config_overrides}") + content = self._apply_section_overrides(content, overrides) + + # Augment 2D config if needed + if is_2d_config: + print("Detected 2D tracker config - adding ObjectModelProjection, MultiViewAssociator, and Communicator sections") + # Always add default multiview sections first + content = self._add_multiview_sections(content, calib_files) + # Then apply any user overrides on top + if overrides: + content = self._apply_section_overrides(content, overrides) + + lines = content.split('\n') + num_videos = len(video_files) + i = 0 + + while i < len(lines): + stripped = lines[i].strip() + + if stripped.startswith('cameraModelFilepath:'): + i += 1 + # Remove existing entries + start_idx = i + while i < len(lines) and lines[i].startswith(' - '): + i += 1 + # Delete the old entries + del lines[start_idx:i] + i = start_idx # Reset index to where we deleted + + # Add new camera paths - one for each video file + print(f"Mapping calibration files to {num_videos} videos:") + for j, video_file in enumerate(video_files): + # Try to find corresponding calibration file + calib_file = self._find_matching_calib_file(video_file, calib_files, j) + print(f" Video {j+1}: {video_file} -> {calib_file}") + lines.insert(i, f" - /workspace/inputs/camInfo/{calib_file}") + i += 1 + continue + if stripped.startswith('stateEstimatorType:') and is_2d_config: + old_value = lines[i].split(':')[1].strip() + if old_value != '3': + lines[i] = " stateEstimatorType: 3" + print(f"stateEstimatorType updated to 3") + elif stripped.startswith('visualTrackerType:') and is_2d_config: + old_value = lines[i].split(':')[1].strip() + if old_value != '2': + lines[i] = " visualTrackerType: 2" + print(f"visualTrackerType updated to 2") + + i += 1 + + return '\n'.join(lines) + + def extract_camera_ids(self, video_files: List[str]) -> List[int]: + """Extract camera IDs by using sorted index + 1.""" + sorted_files = sorted(video_files) + return [i + 1 for i in range(len(sorted_files))] + + def generate_deployment_config(self, cam_ids: List[int]) -> str: + """Generate deployment config file for production mode.""" + deployment_config = { + 'mqtt_broker_per_instance': ["127.0.0.1:1883"], + 'topic_template': "/trck/cam%d", + 'ds_instance_cam_assignment': [cam_ids], + 'ds_instance_gpu_assignment': [0] + } + + # Save to temporary file + deployment_config_path = self.output_dir / "deployment_config.yml" + with open(deployment_config_path, 'w') as f: + yaml.dump(deployment_config, f, default_flow_style=False) + + print(f"Generated deployment config: {deployment_config_path}") + return str(deployment_config_path) + + def _is_2d_tracker_config(self, content: str) -> bool: + """Check if the tracker config is missing 3D multi-view sections.""" + return not any(section in content for section in [ + 'ObjectModelProjection:', 'MultiViewAssociator:', 'Communicator:' + ]) + + def _load_section_overrides(self, override_file: str) -> Dict: + """Load section overrides from a YAML file.""" + if not override_file: + return {} + + override_path = Path(override_file) + if not override_path.exists(): + # Try relative to config_templates directory + override_path = Path("config_templates") / override_file + if not override_path.exists(): + print(f"Warning: Override file not found: {override_file}") + return {} + + try: + with open(override_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + print(f"Error loading override file {override_path}: {e}") + return {} + + def _apply_section_overrides(self, content: str, overrides: Dict) -> str: + """Apply section overrides to the config content.""" + if not overrides: + return content + + lines = content.split('\n') + result_lines = [] + i = 0 + + while i < len(lines): + line = lines[i] + section_name = None + + # Check if this line starts a section that has an override + stripped = line.strip() + if stripped.endswith(':') and not stripped.startswith('-') and not stripped.startswith('#'): + potential_section = stripped[:-1] # Remove the colon + if potential_section in overrides: + section_name = potential_section + + if section_name: + # Skip the original section + result_lines.append(f"# Original {section_name} section replaced by override") + i += 1 + # Skip all lines until next section + while i < len(lines): + next_line = lines[i].strip() + # Stop if we hit another top-level section (no leading spaces and ends with :) + if (next_line.endswith(':') and not next_line.startswith('-') and + not next_line.startswith('#') and not lines[i].startswith(' ')): + break + i += 1 + + # Add the override section + result_lines.append(f"{section_name}:") + section_data = overrides[section_name] + if isinstance(section_data, dict): + for key, value in section_data.items(): + if isinstance(value, list): + result_lines.append(f" {key}:") + for item in value: + result_lines.append(f" - {item}") + else: + result_lines.append(f" {key}: {value}") + else: + result_lines.append(line) + i += 1 + + return '\n'.join(result_lines) + + def _add_multiview_sections(self, content: str, calib_files: List[str]) -> str: + """Add ObjectModelProjection, MultiViewAssociator, Communicator, and PoseEstimator sections to 2D config.""" + lines = content.split('\n') + + # Check which sections already exist + has_object_model_projection = any('ObjectModelProjection:' in line for line in lines) + has_multi_view_associator = any('MultiViewAssociator:' in line for line in lines) + has_communicator = any('Communicator:' in line for line in lines) + has_pose_estimator = any('PoseEstimator:' in line for line in lines) + + # Append to the bottom of the config + insertion_point = len(lines) + + # Add ObjectModelProjection section if it doesn't exist + if not has_object_model_projection: + print("Adding ObjectModelProjection section") + object_model_sections = [ + 'ObjectModelProjection:', + ' outputFootLocation: 1', + ' outputVisibility: 1', + ' outputConvexHull: 0', + ' objectModelType: 0', + ' cameraModelFilepath:' + ] + + # Add camera model file paths + for calib_file in calib_files: + object_model_sections.append(f' - /workspace/inputs/camInfo/{calib_file}') + + # Insert ObjectModelProjection section + for j, section_line in enumerate(object_model_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(object_model_sections) + + # Add MultiViewAssociator section if it doesn't exist + if not has_multi_view_associator: + print("Adding MultiViewAssociator section") + multiview_sections = [ + 'MultiViewAssociator:', + ' multiViewAssociatorType: 1', + ' enableLatePeerReAssoc: 1', + ' enableIDCorrection: 1', + ' enableSeeThrough: 1', + ' enableMsgSync: 1', + ' maxPeerTrackletSize: 50', + ' recentlyActiveAge: 178', + ' minCommonFrames4MatchScore: 2', + ' maxPeerToPredDistance4Fusion: 1.35', + ' minPeerVisibility4Fusion: 0.15', + ' minPeerTrackletMatchScore: 0.48', + ' maxTrackletMatchingTimeSearchRange: 1', + ' maxPeerFrameDiff4NoDet: 2', + ' communicatorInitSleepTime: 0', + ] + + # Insert MultiViewAssociator section + for j, section_line in enumerate(multiview_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(multiview_sections) + + # Add Communicator section if it doesn't exist + if not has_communicator: + print("Adding Communicator section") + communicator_sections = [ + 'Communicator:', + ' communicatorType: 2', + ' pubSubInfoConfigPath: /workspace/experiments/pub_sub_info_config_0.yml', + ' mqttProtoAdaptorConfigPath: /workspace/experiments/config_mqtt.txt', + ] + + # Insert Communicator section + for j, section_line in enumerate(communicator_sections): + lines.insert(insertion_point + j, section_line) + insertion_point += len(communicator_sections) + + # Add PoseEstimator section if it doesn't exist + if not has_pose_estimator: + print("Adding PoseEstimator section") + pose_estimator_sections = [ + 'PoseEstimator:', + ' poseEstimatorType: 1', + ' useVPICropScaler: 1', + ' batchSize: 1', + ' workspaceSize: 1000', + ' inferDims: [3, 256, 192]', + ' networkMode: 1', + ' inputOrder: 0', + ' colorFormat: 0', + ' offsets: [123.6750, 116.2800, 103.5300]', + ' netScaleFactor: 0.00392156', + ' onnxFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx', + ' modelEngineFile: /workspace/models/BodyPose3DNet/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine', + ' poseInferenceInterval: -1', + '' # Empty line + ] + + # Insert PoseEstimator section + for j, section_line in enumerate(pose_estimator_sections): + lines.insert(insertion_point + j, section_line) + + return '\n'.join(lines) + + def _find_matching_calib_file(self, video_file: str, calib_files: List[str], index: int) -> str: + """Find the corresponding calibration file for a video file.""" + # Remove extension from video file to get base name + video_base = os.path.splitext(video_file)[0] + + # Try to find exact match by replacing video extension with .yml + yml_candidate = video_base + ".yml" + if yml_candidate in calib_files: + return yml_candidate + + # Try to find match by extracting camera number/ID from video filename + video_match = re.search(r'[Cc]am(\d+)', video_file) + if video_match: + cam_id = video_match.group(1) + # Look for calibration files with the same camera ID + for calib_file in calib_files: + calib_match = re.search(r'[Cc]am(\d+)', calib_file) + if calib_match and calib_match.group(1) == cam_id: + return calib_file + + # Fallback: use index-based matching (sorted order) + if index < len(calib_files): + return calib_files[index] + + # Last resort: generate expected filename based on pattern + # If we have calib files, try to follow their naming pattern + if calib_files: + # Get the pattern from the first calibration file + first_calib = calib_files[0] + calib_match = re.search(r'(.+[Cc]am)(\d+)(\.yml)$', first_calib) + if calib_match: + prefix, _, suffix = calib_match.groups() + cam_id = str(index + 1).zfill(len(calib_match.group(2))) + expected_file = f"{prefix}{cam_id}{suffix}" + print(f" Warning: Expected calibration file {expected_file} not found!") + return expected_file + + # Final fallback: generate a generic name + expected_file = f"camera_{index + 1:03d}.yml" + print(f" Warning: Generated fallback calibration filename {expected_file}") + return expected_file + + def generate_pub_sub_config(self, dataset_dir: str, video_files: List[str], num_vision_neighbor: int = 3, use_debug_communicator: bool = False): + """Generate pub_sub_info_config_0.yml using the existing script.""" + cam_ids = self.extract_camera_ids(video_files) + cam_subset = ','.join([str(cam_id) for cam_id in cam_ids]) + + # Check if the script exists + script_path = Path("utils/generate_pub_sub_configs.py") + if not script_path.exists(): + print(f"Error: Script not found at {script_path}") + print(f"Current working directory: {os.getcwd()}") + return + + command = [ + "python", str(script_path), + "--cam_info_path", os.path.join(dataset_dir, "camInfo"), + "--neighbor_criteria", f"top_N:{num_vision_neighbor}", + "--output_path", str(self.output_dir) + ] + + if use_debug_communicator: + command.extend(["--use_debug_communicator", "--cam_subset", cam_subset]) + else: + # Generate deployment config for production mode + deployment_config_path = self.generate_deployment_config(cam_ids) + command.extend(["--deployment_config_path", deployment_config_path]) + + try: + print(f"Generating pub_sub config for cameras: {cam_subset} (vision neighbors: {num_vision_neighbor})") + print(f"Command: {' '.join(command)}") + result = subprocess.run(command, check=True, capture_output=True, text=True) + + # Check if file was actually created + pub_sub_file = self.output_dir / "pub_sub_info_config_0.yml" + if pub_sub_file.exists(): + print(f"Generated: {pub_sub_file}") + else: + print(f"Warning: pub_sub_info_config_0.yml not found at {pub_sub_file}") + print(f"Command stdout: {result.stdout}") + print(f"Command stderr: {result.stderr}") + except subprocess.CalledProcessError as e: + print(f"Error: Failed to generate pub_sub config: {e}") + print(f"Command output: {e.stdout}") + print(f"Command error: {e.stderr}") + except Exception as e: + print(f"Unexpected error during pub_sub generation: {e}") + + def generate_configs(self, enabled_sinks: List[str] = None, config_overrides: str = None, tracker_config: str = None, detector_config: str = None) -> Dict[str, str]: + if enabled_sinks is None: + enabled_sinks = ['sink0'] + + # Detect MP4 video files + video_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "videos" / "*.mp4"))]) + if not video_files: + raise ValueError(f"No MP4 files found in {self.dataset_dir}/videos") + + # Detect YML calibration files + calib_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "camInfo" / "*.yml"))]) + + print(f"Found {len(video_files)} videos, {len(calib_files)} calibration files") + + # Generate pub_sub config (will be called from main with num_peers) + # Note: num_peers will be passed from main function + + return { + 'config_deepstream.txt': self.generate_deepstream_config(video_files, enabled_sinks, detector_config), + 'config_tracker.yml': self.generate_tracker_config(video_files, calib_files, config_overrides, tracker_config), + 'config_msgconv.txt': self.generate_msgconv_config(video_files) + } + + def save_configs(self, configs: Dict[str, str]): + self.output_dir.mkdir(parents=True, exist_ok=True) + for filename, content in configs.items(): + path = self.output_dir / filename + with open(path, 'w') as f: + f.write(content) + print(f"Generated: {path}") + + def copy_static_configs(self, detector_config: str = None): + """Copy static config files from templates to output directory.""" + self.output_dir.mkdir(parents=True, exist_ok=True) + + # copy mqtt config + src_path = Path('config_templates') / 'config_mqtt.txt' + dst_path = self.output_dir / 'config_mqtt.txt' + if src_path.exists(): + shutil.copy2(src_path, dst_path) + print(f"Copied: {dst_path}") + else: + print(f"Warning: Template file not found: {src_path}") + + # copy detector config + src_path = Path('config_templates') / detector_config + dst_path = self.output_dir / 'config_pgie.txt' + if src_path.exists(): + shutil.copy2(src_path, dst_path) + print(f"Copied: {dst_path}") + else: + print(f"Warning: Template file not found: {src_path}") + + +def main(): + parser = argparse.ArgumentParser(description='DeepStream Auto-Configurator') + parser.add_argument('--dataset-dir', default='datasets/mtmc_4cam', help='Dataset directory with videos/ and camInfo/') + parser.add_argument('--output-dir', default='temp_outputs', help='Output directory') + parser.add_argument('--enable-osd', action='store_true', help='Enable OSD sink (sink1 - EglSink)') + parser.add_argument('--enable-file-output', action='store_true', help='Enable video file output (sink2 - MP4)') + parser.add_argument('--enable-msg-broker', action='store_true', help='Enable message broker output (sink3 - Kafka)') + parser.add_argument('--num_vision_neighbor', type=int, default=None, help='Number of vision neighbors per camera') + parser.add_argument('--use_debug_communicator', action='store_true', help='Use debug communicator for pub_sub config') + parser.add_argument('--config-overrides', type=str, help='YAML file with section overrides (e.g., override_tracker_4cam.yml, override_tracker_12cam.yml)') + parser.add_argument('--tracker-config', type=str, default='config_tracker.yml', help='Tracker configuration template (e.g., config_tracker_2d.yml)') + parser.add_argument('--detector-config', type=str, default='config_pgie.txt', help='Detector configuration template (e.g., config_pgie.txt, config_pgie_rt_deter.txt)') + + args = parser.parse_args() + + enabled_sinks = ['sink0'] # sink0 always enabled (fake sink, type=1) + if args.enable_osd: enabled_sinks.append('sink1') # sink1 (OSD/EglSink, type=2) + if args.enable_file_output: enabled_sinks.append('sink2') # sink2 (MP4 file output, type=3) + if args.enable_msg_broker: enabled_sinks.append('sink3') # sink3 (Kafka message broker, type=6) + + try: + configurator = DeepStreamAutoConfigurator(args.dataset_dir, args.output_dir) + + # Get video files for pub_sub generation + video_files = sorted([os.path.basename(f) for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + if args.num_vision_neighbor is None: + args.num_vision_neighbor = len(video_files) - 1 + configurator.generate_pub_sub_config(args.dataset_dir, video_files, args.num_vision_neighbor, args.use_debug_communicator) + + configs = configurator.generate_configs(enabled_sinks, args.config_overrides, args.tracker_config, args.detector_config) + configurator.save_configs(configs) + configurator.copy_static_configs(args.detector_config) + + # Get video count for summary + num_videos = len([f for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + rows, columns = configurator.compute_grid_layout(num_videos) + + print(f"✅ Generated configs for {num_videos} videos ({rows}x{columns} grid)") + print(f"📁 Output: {args.output_dir}") + print(f"🔧 Sinks: {', '.join(enabled_sinks)}") + print(f"🤝 Vision neighbors: {args.num_vision_neighbor}") + print(f"📋 Generated files:") + print(f" - config_deepstream.txt (main pipeline config)") + print(f" - config_tracker.yml (3D tracker config)") + print(f" - config_msgconv.txt (message converter config)") + print(f" - pub_sub_info_config_0.yml (communication config)") + print(f"\n🚀 Run with DeepStream container using the generated configs") + + except Exception as e: + print(f"❌ Error: {e}") + return 1 + + return 0 + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py new file mode 100644 index 00000000..dddd9272 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/generate_pub_sub_configs.py @@ -0,0 +1,267 @@ +import argparse +import numpy as np +import tqdm +import yaml +import os +import cv2 +import glob +import re +from os.path import join +METERS_TO_FEET = 3.28084 + +def get_camera_fov_mask(cam_calib, num_pix, range_of_interest=None): + P, Q, K, R, _, pos, end, height = cam_calib + cx, cy = K[0, 2], K[1, 2] + cam_h, cam_w = int(cy * 2), int(cx * 2) + limits_ov = range_of_interest + # print ("range_of_interests", limits_ov[0, 0, 0], limits_ov[1, 0, 0], limits_ov[0, 0, 1], limits_ov[1, 0, 1]) + x_ov = np.arange(np.round(limits_ov[0]), np.round(limits_ov[2])) + y_ov = np.arange(np.round(limits_ov[1]), np.round(limits_ov[3])) + xy_ov = np.array(np.meshgrid(x_ov, y_ov), dtype=float).reshape(2, -1) + # Create 3D points in z=0, z=h/2, z=h + xyz_0 = np.vstack([xy_ov, np.zeros(xy_ov.shape[1])]).T.reshape(len(x_ov), len(y_ov), 3) + xyz_h = np.vstack([xy_ov, height * np.ones(xy_ov.shape[1])]).T.reshape(len(x_ov), len(y_ov), 3) + # Project to the image plane + xy0_cam = cv2.perspectiveTransform(xyz_0, P).reshape(-1, 2).T + xyh_cam = cv2.perspectiveTransform(xyz_h, P).reshape(-1, 2).T + # Create the mask + mask1 = (xy0_cam[0] > 0) & (xy0_cam[0] < cam_w) + mask2 = (xy0_cam[1] > 0) & (xy0_cam[1] < cam_h) + mask3 = np.linalg.norm(xyh_cam - xy0_cam, ord=np.inf, axis=0) > num_pix # object size: OK + mask4 = ((xy_ov.T - pos.T) @ (end - pos)).flatten() > 0 # direction: OK + # Update the mask + mask = mask1 & mask2 & mask3 & mask4 + return mask + +def load_and_process_camera_matrices(cam_info_path): + # Load all .yml and .yaml files + cam_files = glob.glob(os.path.join(cam_info_path, "*.yml")) + glob.glob(os.path.join(cam_info_path, "*.yaml")) + + cam_matrices = {} + for idx, cam_file in enumerate(sorted(cam_files)): + cam = idx + 1 + + with open(cam_file, 'r') as file: + yaml_data = yaml.safe_load(file) + if isinstance(yaml_data['modelInfo'], list): + heights = [yaml_data['modelInfo'][i]['height'] for i in range(len(yaml_data['modelInfo']))] + height = max(heights) * METERS_TO_FEET # meters to feet (OV) + else: + height = yaml_data['modelInfo']["height"] * METERS_TO_FEET # meters to feet (OV) + P = np.array(yaml_data["projectionMatrix_3x4_w2p"]).reshape(3, 4) + Q = np.linalg.pinv(P) + K, R, t, _, _, _, _ = cv2.decomposeProjectionMatrix(P) + K = K / K[2, 2] + # Translation vector in OV (-R.t @ t) + pos = t[:2] / t[-1] + # point at 3 OV units (feet) in front of the camera + end = pos + R[-1:, :2].T * (3 / np.linalg.norm(R[-1, :2])) + # pos_px = cv2.perspectiveTransform(pos.reshape(1, 1, 2), T_ov2px).reshape(2) + # end_px = cv2.perspectiveTransform(end.reshape(1, 1, 2), T_ov2px).reshape(2) + cam_matrices[cam] = P, Q, K, R, t, pos, end, height + return cam_matrices + + +def get_overlap_of_2_masks(mask1, mask2): + overlap = np.logical_and(mask1, mask2) + overlap_count = np.sum(overlap) + mask1_ratio = overlap_count / np.sum(mask1) + mask2_ratio = overlap_count / np.sum(mask2) + return mask1_ratio, mask2_ratio, overlap_count + + +def get_overlap_matrix(cam_matrices, minimum_object_size, range_of_interest): + overlap_matrix = {} + masks = {} + + # Generate masks for all cameras + for cam in tqdm.tqdm(cam_matrices, desc="Generating masks"): + mask = get_camera_fov_mask(cam_matrices[cam], num_pix=minimum_object_size, range_of_interest=range_of_interest) + masks[cam] = mask + + # Calculate overlap ratios for all camera pairs + for cam1 in tqdm.tqdm(cam_matrices, desc="Calculating overlaps"): + overlap_matrix[cam1] = {} + mask1 = masks[cam1] + for cam2 in cam_matrices: + if cam1 == cam2: continue + mask2 = masks[cam2] + mask1_ratio, mask2_ratio, _ = get_overlap_of_2_masks(mask1, mask2) + overlap_matrix[cam1][cam2] = mask1_ratio + + return overlap_matrix + + +def get_subscription_map(overlap_matrix, criteria): + criteria_type, value = criteria.split(':') + subscription_map = {} + if criteria_type == 'top_N': + N = int(value) + for cam in overlap_matrix: + subscription_map[cam] = [] + neighbors = list(overlap_matrix[cam].keys()) + top_cam_idxs = np.argpartition([overlap_matrix[cam][nei] for nei in neighbors], -N)[-N:].tolist() + top_cams = [neighbors[i] for i in top_cam_idxs] + subscription_map[cam] = top_cams + + + elif criteria_type == 'overlap_threshold': + threshold = float(value) + for cam in overlap_matrix: + subscription_map[cam] = [] + for neighbor in overlap_matrix[cam]: + if overlap_matrix[cam][neighbor] >= threshold: + subscription_map[cam].append(neighbor) + + return subscription_map + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--deployment_config_path', + type=str, + default=join(os.getenv("HOME"), "Documents", "deepstream", "workspace", "mqtt", "config_templates", "deployment_configs", "repro_1ds_100cam_orig_mtmc.yml"), + help='Path to yaml file containing deployment related configs' + ) + + parser.add_argument( + '--use_debug_communicator', + action='store_true', + help='Use debug communicator' + ) + + parser.add_argument( + '--cam_info_path', + type=str, + default=join(os.getenv("PWD"), "camInfo"), + help='Directory containing camera calibration info (intrinsic & extrinsic params)' + ) + + parser.add_argument( + '--minimum_object_size', + type=int, + default=150, + help='Number of pixels (in height) to consider an object visible when rendering FOV' + ) + + parser.add_argument( + '--neighbor_criteria', + type=str, + default='overlap_threshold:%f' % (2 / (1920 * 1080)), # 'top_N:3', # 'overlap_threshold:%f' % (2 / (1920 * 1080)), + help='Format: "top_N:{N}" or "overlap_threshold:{thres}". Determines neighbor selection method' + ) + + parser.add_argument( + '--output_path', + type=str, + default='./peer_configs', + help='Directory to store output peer cam config files' + ) + + parser.add_argument( + '--range_of_interest', + type=str, + default=None, + help='Range of interest of world plane in format "x1,y1,x2,y2" where (x1,y1) is min corner and (x2,y2) is max corner' + ) + + return parser.parse_args() + +if __name__ == '__main__': + args = parse_args() + if args.use_debug_communicator: + # For debug mode, use all cameras found in directory + cam_matrices = load_and_process_camera_matrices(args.cam_info_path) + cam_subset = list(cam_matrices.keys()) + else: + # Load deployment config and camera matrices + with open(args.deployment_config_path, 'r') as f: + deployment_config = yaml.safe_load(f) + cam_subset = [cam for cam_list in deployment_config['ds_instance_cam_assignment'] for cam in cam_list] + cam2instance = {} + for instance_id, cam_list in enumerate(deployment_config['ds_instance_cam_assignment']): + for cam in cam_list: + cam2instance[cam] = instance_id + + # Load all cameras, then filter to cam_subset + cam_matrices = load_and_process_camera_matrices(args.cam_info_path) + + # Filter cam_matrices to only include cameras in cam_subset + filtered_cam_matrices = {cam: cam_matrices[cam] for cam in cam_subset if cam in cam_matrices} + + # Parse range of interest + if args.range_of_interest: + x1, y1, x2, y2 = map(float, args.range_of_interest.split(',')) + range_of_interest_ov = np.array([x1, y1, x2, y2], dtype=float) + else: + range_padding = 100 + cam_poses = [filtered_cam_matrices[cam][5] for cam in filtered_cam_matrices] + min_x = min([pose[0][0] for pose in cam_poses]) + max_x = max([pose[0][0] for pose in cam_poses]) + min_y = min([pose[1][0] for pose in cam_poses]) + max_y = max([pose[1][0] for pose in cam_poses]) + range_of_interest_ov = np.array([min_x - range_padding, min_y - range_padding, max_x + range_padding, max_y + range_padding], dtype=float) + + # print (filtered_cam_matrices) + # Calculate overlap matrix for all cameras + overlap_matrix = get_overlap_matrix(filtered_cam_matrices, args.minimum_object_size, range_of_interest_ov) + # for cam in overlap_matrix: + # print (cam, overlap_matrix[cam]) + + # Parse neighbor criteria + subscription_map = get_subscription_map(overlap_matrix, args.neighbor_criteria) + subscription_map = "" + print ('subscription_map:', subscription_map) + + subscription_map = get_subscription_map(overlap_matrix, args.neighbor_criteria) + for cam in subscription_map: + print (" %d:" % (cam), subscription_map[cam]) + + # # sudo apt install graphviz + # # pip install graphviz + # import graphviz + # dot = graphviz.Digraph(comment='Camera Subscriptions') + # nodes_expanded = set() + # nodes_to_expand = set([27]) + # while nodes_to_expand: + # node = nodes_to_expand.pop() + # nodes_expanded.add(node) + # for nei in subscription_map[node]: + # if nei not in nodes_expanded: + # nodes_to_expand.add(nei) + # dot.edge(str(node), str(nei)) + # dot.render('camera_subscriptions', format='png') + + # Get average number of neighbors + num_neighbors = np.mean([len(subscription_map[cam]) for cam in subscription_map]) + print ('Average number of neighbors:', num_neighbors) + + + if args.use_debug_communicator: + peer_config = {'camIds': cam_subset, 'subToCamIds': []} + for cam in cam_subset: + peer_config['subToCamIds'].append(subscription_map[cam]) + if not os.path.exists(args.output_path): + os.makedirs(args.output_path) + with open(os.path.join(args.output_path, 'pub_sub_info_config_0.yml'), 'w') as f: + yaml.dump(peer_config, f, default_flow_style=False) + else: + # Generate pub_sub_info_config.yml files + for instance_id, cam_list in enumerate(deployment_config['ds_instance_cam_assignment']): + instance_config = {"pubBrokerTopicStr": [], "subPeerBrokerTopicStrs": []} + for cam_idx, cam in enumerate(cam_list): + # cam_url = deployment_config['cam_url_map'][cam] + cam_url = deployment_config['mqtt_broker_per_instance'][instance_id] + cam_topic = deployment_config['topic_template'] % cam + instance_config["pubBrokerTopicStr"].append(cam_url + ';' + cam_topic) + instance_config["subPeerBrokerTopicStrs"].append([]) + for nei in subscription_map[cam]: + nei_instance_id = cam2instance[nei] + nei_url = deployment_config['mqtt_broker_per_instance'][nei_instance_id] + nei_topic = deployment_config['topic_template'] % nei + instance_config["subPeerBrokerTopicStrs"][cam_idx].append(nei_url + ';' + nei_topic) + if not os.path.exists(args.output_path): + os.makedirs(args.output_path) + with open(os.path.join(args.output_path, f"pub_sub_info_config_{instance_id}.yml"), 'w') as f: + yaml.dump(instance_config, f, default_flow_style=False) diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py new file mode 100644 index 00000000..9ea81808 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/inference_builder_auto_configurator.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +import argparse +import os +import glob +from pathlib import Path +from typing import List, Dict +import yaml + +# Import the base configurator to reuse functionality +from deepstream_auto_configurator import DeepStreamAutoConfigurator + + +class InferenceBuilderAutoConfigurator(DeepStreamAutoConfigurator): + """ + Auto-configurator for inference-builder based on DeepStream auto-configurator. + Generates ds_mv3dt.yaml and source_list_static.yaml instead of config_deepstream.txt. + """ + + @staticmethod + def parse_model_name(detector_config: str) -> str: + """Extract model name from the onnx-file path in the detector config.""" + config_path = Path("config_templates") / detector_config + if not config_path.exists(): + raise FileNotFoundError(f"Detector config not found: {config_path}") + with open(config_path, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('onnx-file='): + onnx_path = Path(line.split('=', 1)[1]) + return onnx_path.parent.name + raise ValueError(f"No onnx-file entry found in {detector_config}") + + def generate_ds_mv3dt_config(self, video_files: List[str], detector_config: str = None) -> str: + """Generate ds_mv3dt.yaml with correct max_batch_size and model name.""" + template_path = Path("config_templates/ds_mv3dt.yaml") + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + config = yaml.safe_load(f) + + # Detect video resolution from the first video file + first_video_path = self.dataset_dir / "videos" / video_files[0] + video_width, video_height = self.get_video_resolution(first_video_path) + print(f"Detected video resolution: {video_width}x{video_height}") + + # Update configuration + num_videos = len(video_files) + model_config = config['models'][0] + model_config['max_batch_size'] = num_videos + model_config['parameters']['resize_video'] = [video_height, video_width] + + detector_config = detector_config or "config_pgie.txt" + model_name = self.parse_model_name(detector_config) + model_config['name'] = model_name + + # Update tracker resolution + if 'tracker_config' in model_config['parameters']: + model_config['parameters']['tracker_config']['width'] = video_width + model_config['parameters']['tracker_config']['height'] = video_height + + return yaml.dump(config, default_flow_style=False, sort_keys=False) + + def generate_source_list_static(self, video_files: List[str]) -> str: + """Generate source_list_static.yaml based on video files.""" + header = """source-list:""" + + # Generate source entries + source_entries = [] + for i, video_file in enumerate(sorted(video_files), 1): + source_entries.extend([ + f'- uri: "file:///workspace/inputs/videos/{video_file}"', + f' sensor-id: Camera{i}', + f' sensor-name: UniqueSensorName{i}' + ]) + + footer = """source-config: + source-bin: "nvurisrcbin" + properties: + file-loop: false""" + + return '\n'.join([header] + source_entries + [footer]) + + def generate_nvdsinfer_config(self, video_files: List[str], detector_config: str = None) -> str: + """Generate nvdsinfer_config.yaml based on detector config template.""" + detector_config = detector_config or "config_pgie.txt" + template_path = Path("config_templates") / detector_config + if not template_path.exists(): + raise FileNotFoundError(f"Template not found: {template_path}") + + with open(template_path, 'r') as f: + content = f.read() + + num_videos = len(video_files) + + # Simple format conversion: INI to YAML with batch-size adjustment + yaml_lines = [] + current_section = None + + for line in content.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + elif line.startswith('[') and line.endswith(']'): + current_section = line[1:-1] # Remove brackets + yaml_lines.append(f'{current_section}:') + elif '=' in line and current_section: + key, value = line.split('=', 1) + if key == 'batch-size': + yaml_lines.append(f' {key}: {num_videos}') + else: + yaml_lines.append(f' {key}: {value}') + + return '\n'.join(yaml_lines) + + def generate_configs(self, enabled_sinks: List[str] = None, config_overrides: str = None, tracker_config: str = None, detector_config: str = None) -> Dict[str, str]: + """Generate inference-builder configs instead of DeepStream configs.""" + # Detect MP4 video files + video_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "videos" / "*.mp4"))]) + if not video_files: + raise ValueError(f"No MP4 files found in {self.dataset_dir}/videos") + + # Detect YML calibration files + calib_files = sorted([os.path.basename(f) for f in glob.glob(str(self.dataset_dir / "camInfo" / "*.yml"))]) + + print(f"Found {len(video_files)} videos, {len(calib_files)} calibration files") + + return { + 'ds_mv3dt.yaml': self.generate_ds_mv3dt_config(video_files, detector_config), + 'config_tracker.yml': self.generate_tracker_config(video_files, calib_files, config_overrides, tracker_config), + 'source_list_static.yaml': self.generate_source_list_static(video_files), + 'nvdsinfer_config.yaml': self.generate_nvdsinfer_config(video_files, detector_config), + 'config_msgconv.txt': self.generate_msgconv_config(video_files) + } + + +def main(): + parser = argparse.ArgumentParser(description='Inference Builder Auto-Configurator') + parser.add_argument('--dataset-dir', default='datasets/mtmc_4cam', help='Dataset directory with videos/ and camInfo/') + parser.add_argument('--output-dir', default='temp_outputs', help='Output directory') + parser.add_argument('--config-overrides', type=str, help='YAML file with section overrides (e.g., override_tracker_4cam.yml, override_tracker_12cam.yml)') + parser.add_argument('--tracker-config', type=str, default='config_tracker.yml', help='Tracker configuration template (e.g., config_tracker_2d.yml)') + parser.add_argument('--detector-config', type=str, default='config_pgie.txt', help='Detector configuration template (e.g., config_pgie.txt, config_pgie_rt_detr.txt)') + parser.add_argument('--num_vision_neighbor', type=int, default=None, help='Number of vision neighbors per camera') + parser.add_argument('--use_debug_communicator', action='store_true', help='Use debug communicator for pub_sub config') + + args = parser.parse_args() + + try: + configurator = InferenceBuilderAutoConfigurator(args.dataset_dir, args.output_dir) + + # Generate pub_sub config (reuse from parent class) + video_files = sorted([os.path.basename(f) for f in glob.glob(str(Path(args.dataset_dir) / "videos" / "*.mp4"))]) + if args.num_vision_neighbor is None: + args.num_vision_neighbor = len(video_files) - 1 + configurator.generate_pub_sub_config(args.dataset_dir, video_files, args.num_vision_neighbor, args.use_debug_communicator) + + # Generate inference-builder configs + configs = configurator.generate_configs(config_overrides=args.config_overrides, tracker_config=args.tracker_config, detector_config=args.detector_config) + configurator.save_configs(configs) + configurator.copy_static_configs(args.detector_config) + + # Get video count for summary + num_videos = len(video_files) + + print(f"✅ Generated inference-builder configs for {num_videos} videos") + print(f"📁 Output: {args.output_dir}") + print(f"🤝 Vision neighbors: {args.num_vision_neighbor}") + print(f"🔍 Detector config: {args.detector_config}") + print(f"🐛 Debug communicator: {'enabled' if args.use_debug_communicator else 'disabled'}") + if args.config_overrides: + print(f"🔧 Config overrides: {args.config_overrides}") + print(f"📋 Generated files:") + print(f" - ds_mv3dt.yaml (inference config with max_batch_size: {num_videos})") + print(f" - config_tracker.yml (3D tracker config)") + print(f" - source_list_static.yaml (source configuration)") + print(f" - nvdsinfer_config.yaml (inference engine config with batch_size: {num_videos})") + print(f" - config_msgconv.txt (message converter config)") + print(f" - pub_sub_info_config_0.yml (communication config)") + print(f"\n🚀 Run with inference builder using the generated configs") + + except Exception as e: + print(f"❌ Error: {e}") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py new file mode 100644 index 00000000..2a9edc85 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_bev_visualizer.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 + +import os, tkinter as tk, argparse +from datetime import datetime +from kafka import KafkaConsumer +from google.protobuf.json_format import MessageToDict +from schema_pb2 import Frame +from collections import defaultdict +import numpy as np, yaml, cv2, time +from tqdm import tqdm + +def extract_expected_sensors(msgconv_config): + """Extract expected sensor IDs from config_msgconv.txt file.""" + try: + expected_sensors = [] + + with open(msgconv_config, 'r') as f: + lines = f.readlines() + + current_section = None + for line in lines: + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + # Check for section header [sensorX] + if line.startswith('[sensor') and line.endswith(']'): + current_section = line[1:-1] # Remove brackets + continue + + # Look for id= lines within sensor sections + if current_section and line.startswith('id='): + sensor_id = line.split('=', 1)[1].strip() + if sensor_id: + expected_sensors.append(sensor_id) + + if not expected_sensors: + raise ValueError("No sensor IDs found in msgconv config file") + + return expected_sensors + + except (FileNotFoundError, ValueError) as e: + raise Exception(f"Could not parse msgconv config file {msgconv_config}: {e}") + + +class FrameBuffer: + def __init__(self, expected_sensors=None, timeout=0.5, lookahead_frames=1): + """ + For a frame t to be considered complete, 1 of the 3 conditions must be met: + 1. Messages from all expected sensors are received at frame t + 2. Current time - message sent time of any sensor at frame t > timeout + 3. At least 1 of the future frames (t+lookahead_frames) are received + + The current settings is designed for single-container MV3DT. + For multi-container MV3DT, consider increasing the timeout and lookahead_frames. + """ + + self.frame_data = defaultdict(dict) + self.expected_sensors = expected_sensors or set() + self.timeout = timeout + self.timestamps = {} + self.lookahead_frames = lookahead_frames + + def add_frame(self, frame_id, sensor_id, frame_dict): + try: + frame_id = int(frame_id) + except ValueError: + pass + if frame_id not in self.timestamps: + self.timestamps[frame_id] = time.time() + self.frame_data[frame_id][sensor_id] = frame_dict + if not self.expected_sensors: + self.expected_sensors.add(sensor_id) + + def get_complete_frame(self): + current_time = time.time() + num_expected = len(self.expected_sensors) + # Process frames in ascending order when possible + def _key_fn(x): + try: + return int(x) + except Exception: + return float('inf') + for frame_id in sorted(list(self.frame_data.keys()), key=_key_fn): + sensors = set(self.frame_data[frame_id].keys()) + frame_time = self.timestamps.get(frame_id, current_time) + # Condition 1: full set received for this frame + full_received = (num_expected > 0 and len(sensors) >= num_expected and sensors.issubset(self.expected_sensors)) + # Condition 2: timeout + timed_out = (current_time - frame_time) > self.timeout + # Condition 3: at least 1 of future frame (t+lookahead_frames) received + future_received = False + if num_expected > 0: + try: + base_id = int(frame_id) + future_id = base_id + self.lookahead_frames + future_data = self.frame_data.get(future_id) + if future_data is not None: + future_received = True + except Exception: + pass + if full_received or timed_out or future_received: + # print ('frame_id', frame_id, 'full_received', full_received, 'timed_out', timed_out, 'future_received', future_received) + frame_data = self.frame_data.pop(frame_id) + self.timestamps.pop(frame_id, None) + return frame_id, frame_data + return None, None + + def get_all_complete_frames(self): + complete_frames = [] + for frame_id, sensors_data in self.frame_data.items(): + if len(sensors_data) > 0: + try: + complete_frames.append((int(frame_id), frame_id, sensors_data)) + except (ValueError, TypeError): + complete_frames.append((0, frame_id, sensors_data)) + complete_frames.sort(key=lambda x: x[0]) + return [(frame_id, data) for _, frame_id, data in complete_frames] + + def get_frame(self, frame_id): + """Get frame data for specific frame_id without waiting for conditions, and remove it from buffer""" + if frame_id in self.frame_data: + frame_data = self.frame_data.pop(frame_id) + self.timestamps.pop(frame_id, None) + return frame_id, frame_data + return None, None + +def display_frame(vis_img, frame_id, video_writer): + """Add frame info, display frame, and write to video""" + recording = "REC" if video_writer else "" + info = f"Frame: {frame_id} {recording}" + cv2.putText(vis_img, info, (10, vis_img.shape[0] - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) + + cv2.imshow('Bird-Eye View of Multi-View 3D Tracking', vis_img) + + if video_writer: + video_writer.write(vis_img) + +def load_map_and_transforms(dataset_path): + """Load map image and transformation matrix from dataset""" + map_path = os.path.join(dataset_path, 'map.png') + transforms_path = os.path.join(dataset_path, 'transforms.yml') + + with open(transforms_path, 'r') as f: + transforms = yaml.safe_load(f) + T_ov2px = np.array(transforms['T_ov2px']).reshape(3, 3) + + map_img = cv2.imread(map_path) + if map_img is None: + raise FileNotFoundError(f"Map image not found at {map_path}") + + return map_img, T_ov2px + +def setup_map_scaling(map_img, target_width_ratio=0.8, target_height_ratio=0.8): + """Setup map scaling based on screen size""" + root = tk.Tk() + screen_width, screen_height = root.winfo_screenwidth(), root.winfo_screenheight() + root.destroy() + + target_width = int(screen_width * target_width_ratio) + target_height = int(screen_height * target_height_ratio) + + map_height, map_width = map_img.shape[:2] + scale = min(target_width / map_width, target_height / map_height) + new_width, new_height = int(map_width * scale), int(map_height * scale) + + map_img_resized = cv2.resize(map_img, (new_width, new_height)) + scale_matrix = np.array([[scale, 0, 0], [0, scale, 0], [0, 0, 1]]) + + return map_img_resized, scale_matrix, new_width, new_height + +def create_kafka_consumer(group_id, consumer_timeout_ms=None): + """Create and configure Kafka consumer""" + config = { + 'bootstrap_servers': 'localhost:9092', + 'auto_offset_reset': 'earliest', + 'value_deserializer': lambda x: x, + 'group_id': group_id, + 'enable_auto_commit': True + } + + if consumer_timeout_ms is not None: + config['consumer_timeout_ms'] = consumer_timeout_ms + + consumer = KafkaConsumer(**config) + consumer.subscribe(['mv3dt']) + print("Connected to Kafka and subscribed to 'mv3dt' topic") + return consumer + +def draw_objects_on_map(frame_data, T_ov2px, map_img, trajectories, object_colors, frame_id, frame_history, show_ids=False, average_multi_cam=False): + vis_img = map_img.copy() + colors = [ + (255, 0, 0), (0, 255, 255), (139, 69, 19), (0, 255, 0), (255, 0, 255), + (50, 205, 50), (255, 140, 0), (0, 0, 255), (255, 165, 0), (255, 105, 180), + (75, 0, 130), (255, 255, 0), (0, 128, 128), (0, 191, 255), (154, 205, 50), + (255, 20, 147), (30, 144, 255), (128, 0, 128), (220, 20, 60), (0, 206, 209) + ] + + all_objects = [obj for frame_dict in frame_data.values() for obj in frame_dict.get('objects', [])] + + try: + current_frame_num = int(frame_id) + frame_history.append(current_frame_num) + if len(frame_history) > 240: + frame_history = frame_history[-240:] + except: + current_frame_num = 0 + + current_objects = set() + + if average_multi_cam: + # Group objects by ID across all cameras for averaging + objects_by_id = defaultdict(list) + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + object_id = obj.get('id', 0) + objects_by_id[object_id].append((world_x, world_y)) + except: + continue + + # Calculate average positions and add to trajectories + for object_id, positions in objects_by_id.items(): + if positions: + # Calculate average world position + avg_world_x = sum(pos[0] for pos in positions) / len(positions) + avg_world_y = sum(pos[1] for pos in positions) / len(positions) + + try: + # Convert to pixel coordinates + pt_ov_h = np.array([avg_world_x, avg_world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + current_objects.add(object_id) + + if object_id not in object_colors: + object_colors[object_id] = colors[len(object_colors) % len(colors)] + + trajectories[object_id].append((px_x, px_y, current_frame_num)) + except: + continue + else: + # Original behavior: show all trajectory points from all cameras + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if not bbox_3d: + continue + try: + world_x, world_y = bbox_3d[:2] + pt_ov_h = np.array([world_x, world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + object_id = obj.get('id', 0) + current_objects.add(object_id) + + if object_id not in object_colors: + object_colors[object_id] = colors[len(object_colors) % len(colors)] + + trajectories[object_id].append((px_x, px_y, current_frame_num)) + except: + continue + + # Cleanup old trajectory points + frame_threshold = current_frame_num - 240 + for object_id in list(trajectories.keys()): + trajectories[object_id] = [(x, y, f) for x, y, f in trajectories[object_id] if f >= frame_threshold] + if not trajectories[object_id]: + del trajectories[object_id] + object_colors.pop(object_id, None) + + # Draw trajectories + for object_id, traj_points in trajectories.items(): + if not traj_points: + continue + color = object_colors.get(object_id, (128, 128, 128)) + base_alpha = 0.9 if object_id in current_objects else 0.6 + min_alpha = 0.3 # Minimum brightness to prevent complete black + + for i, (x, y, _) in enumerate(traj_points): + # Slower fade: use square root for gentler curve + fade_ratio = (i / max(1, len(traj_points) - 1)) ** 0.5 + fade = min_alpha + (base_alpha - min_alpha) * fade_ratio + fade_color = tuple(int(c * fade) for c in color) + cv2.circle(vis_img, (x, y), 1, fade_color, -1) + + # Draw current positions with ID labels + if average_multi_cam: + # Draw averaged positions for each object + objects_by_id = defaultdict(list) + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + object_id = obj.get('id', 0) + objects_by_id[object_id].append((world_x, world_y)) + except: + continue + + for object_id, positions in objects_by_id.items(): + if positions and object_id in object_colors: + # Calculate average world position + avg_world_x = sum(pos[0] for pos in positions) / len(positions) + avg_world_y = sum(pos[1] for pos in positions) / len(positions) + + try: + # Convert to pixel coordinates + pt_ov_h = np.array([avg_world_x, avg_world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + # Draw object circle + cv2.circle(vis_img, (px_x, px_y), 3, object_colors[object_id], -1) + + # Draw ID label near the object (if enabled) + if show_ids: + label_x = px_x + 8 + label_y = px_y - 8 + cv2.putText(vis_img, str(object_id), (label_x, label_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + except: + continue + else: + # Original behavior: draw all object positions from all cameras + for obj in all_objects: + bbox_3d = obj.get('bbox3d', {}).get('coordinates', {}) + if bbox_3d: + try: + world_x, world_y = bbox_3d[:2] + pt_ov_h = np.array([world_x, world_y, 1.0]) + pt_px_h = np.dot(T_ov2px, pt_ov_h) + pt_px_h /= pt_px_h[2] + px_x, px_y = int(pt_px_h[0]), int(pt_px_h[1]) + + object_id = obj.get('id', 0) + if object_id in object_colors: + # Draw object circle + cv2.circle(vis_img, (px_x, px_y), 3, object_colors[object_id], -1) + + # Draw ID label near the object (if enabled) + if show_ids: + label_x = px_x + 8 + label_y = px_y - 8 + cv2.putText(vis_img, str(object_id), (label_x, label_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + except: + continue + + return vis_img, frame_history + +def collect_all_messages(consumer, expected_sensors, max_timeout=300, verbose=False): + frame_buffer = FrameBuffer(expected_sensors=expected_sensors) + start_time = time.time() + last_msg_time = start_time + count = 0 + first_frame = True + + consumer._consumer_timeout_ms = 1000 + + while True: + current_time = time.time() + if current_time - start_time > max_timeout or current_time - last_msg_time > 10: + break + + batch = consumer.poll(timeout_ms=1000) + if not batch: + continue + + for _, messages in batch.items(): + for msg in messages: + try: + frame = Frame() + frame.ParseFromString(msg.value) + frame_dict = MessageToDict(frame) + frame_id = frame_dict.get('id', 'unknown') + + if first_frame and frame_id != 'unknown': + try: + if int(frame_id) > 100: + if verbose: + print(f"Discarding frame {frame_id} (from previous run)") + first_frame = False + continue + except (ValueError, TypeError): + pass + first_frame = False + + frame_buffer.add_frame(frame_id, frame_dict.get('sensorId', 'unknown'), frame_dict) + count += 1 + last_msg_time = current_time + except: + continue + + print(f"Collected {count} messages") + return frame_buffer + +def generate_video(dataset_path, output_path, show_ids, expected_sensors, average_multi_cam, verbose=False): + os.makedirs(output_path, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + video_output_path = os.path.join(output_path, f"trajectory_video_{timestamp}.mp4") + + # Setup common components + map_img, T_ov2px = load_map_and_transforms(dataset_path) + map_img_resized, scale_matrix, new_width, new_height = setup_map_scaling(map_img) + T_ov2px_scaled = np.dot(scale_matrix, T_ov2px) + + try: + consumer = create_kafka_consumer('mv3dt_bev_video') + + frame_buffer = collect_all_messages(consumer, expected_sensors, verbose=verbose) + complete_frames = frame_buffer.get_all_complete_frames() + print(frame_buffer.frame_data) + + if not complete_frames: + return + + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + video_writer = cv2.VideoWriter(video_output_path, fourcc, 30, (new_width, new_height)) + + trajectories, object_colors, frame_history = defaultdict(list), {}, [] + + for frame_id, frame_data in tqdm(complete_frames): + vis_img, frame_history = draw_objects_on_map( + frame_data, T_ov2px_scaled, map_img_resized, trajectories, object_colors, frame_id, frame_history, show_ids, average_multi_cam) + + cv2.putText(vis_img, f"Frame: {frame_id}", + (10, vis_img.shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) + video_writer.write(vis_img) + + video_writer.release() + print(f"Video saved: {video_output_path}") + + except Exception as e: + print(f"Error: {e}") + finally: + if 'consumer' in locals(): + consumer.close() + +def real_time_visualization(dataset_path, output_path, show_ids, expected_sensors, average_multi_cam, verbose=False): + # Setup common components + map_img, T_ov2px = load_map_and_transforms(dataset_path) + map_img_resized, scale_matrix, new_width, new_height = setup_map_scaling(map_img) + T_ov2px_scaled = np.dot(scale_matrix, T_ov2px) + + frame_buffer = FrameBuffer(expected_sensors=expected_sensors) + trajectories, object_colors, frame_history = defaultdict(list), {}, [] + + cv2.namedWindow('Bird-Eye View of Multi-View 3D Tracking', cv2.WINDOW_NORMAL) + cv2.resizeWindow('Bird-Eye View of Multi-View 3D Tracking', new_width, new_height) + + # Display blank map initially + initial_img = map_img_resized.copy() + cv2.imshow('Bird-Eye View of Multi-View 3D Tracking', initial_img) + cv2.waitKey(1) # Process window events + + consumer = None + video_writer = None + + try: + consumer = create_kafka_consumer('mv3dt_visualizer', consumer_timeout_ms=50) + + # Wait for partition assignment (up to 5 seconds) + assignment_timeout = 5.0 + start_time = time.time() + while not consumer.assignment() and (time.time() - start_time) < assignment_timeout: + consumer.poll(timeout_ms=100) # This triggers partition assignment + time.sleep(0.1) + + print(f"Consumer assignment: {consumer.assignment()}") + if not consumer.assignment(): + print("Warning: No partitions assigned. Topic might not exist or have no partitions.") + + # Try to get topic metadata for debugging + try: + metadata = consumer.list_consumer_group_offsets() + topics = consumer.topics() + print(f"Available topics: {topics}") + if 'mv3dt' in topics: + partitions = consumer.partitions_for_topic('mv3dt') + print(f"Partitions for 'mv3dt' topic: {partitions}") + else: + print("Topic 'mv3dt' not found!") + except Exception as debug_e: + print(f"Could not get topic metadata: {debug_e}") + + except Exception as e: + print(f"Kafka connection failed: {e}") + + print("Controls: 'q'-quit, 'c'-clear, 'r'-record") + + last_update = time.time() + last_frame_id = None + try: + while True: + current_time = time.time() + + if consumer: + try: + batch = consumer.poll(timeout_ms=10) + for _, messages in batch.items(): + for msg in messages: + try: + frame = Frame() + frame.ParseFromString(msg.value) + frame_dict = MessageToDict(frame) + # print ('Received frame', frame_dict.get('id', 'unknown'), 'from sensor', frame_dict.get('sensorId', 'unknown')) + frame_buffer.add_frame(frame_dict.get('id', 'unknown'), + frame_dict.get('sensorId', 'unknown'), frame_dict) + except: + continue + except: + pass + + while True: + frame_id, frame_data = frame_buffer.get_complete_frame() + if not frame_data: + break + + # If the first frame is > 100, it's likely from previous run. + if last_frame_id is None and frame_id > 100: + if verbose: + print(f"Discarding frame {frame_id} (from previous run)") + continue + + if last_frame_id is not None and frame_id - last_frame_id > 1: + # Process missing frames in between + for missing_frame_id in range(last_frame_id + 1, frame_id): + missing_id, missing_data = frame_buffer.get_frame(missing_frame_id) + if missing_data: + # Render frame with tracking data + vis_img, frame_history = draw_objects_on_map(missing_data, T_ov2px_scaled, + map_img_resized, trajectories, object_colors, missing_frame_id, frame_history, show_ids, average_multi_cam) + display_frame(vis_img, missing_frame_id, video_writer) + else: + # Render empty frame with correct frame_id + empty_vis_img = map_img_resized.copy() + display_frame(empty_vis_img, missing_frame_id, video_writer) + if last_frame_id is not None and 0 < last_frame_id - frame_id < 30: + if verbose: + print(f"Received late message from frame {frame_id}, last_frame_id was {last_frame_id}") + print(f"Discarding frame {frame_id} (late message)") + continue + last_frame_id = frame_id + vis_img, frame_history = draw_objects_on_map(frame_data, T_ov2px_scaled, + map_img_resized, trajectories, object_colors, frame_id, frame_history, show_ids, average_multi_cam) + + display_frame(vis_img, frame_id, video_writer) + + last_update = current_time + + key = cv2.waitKey(1) & 0xFF + try: + window_closed = cv2.getWindowProperty('Bird-Eye View of Multi-View 3D Tracking', cv2.WND_PROP_VISIBLE) < 1 + except cv2.error: + window_closed = True + if key == ord('q') or window_closed: + break + elif key == ord('c'): + trajectories.clear() + object_colors.clear() + frame_history.clear() + print("Cleared trajectories") + elif key == ord('r'): + if video_writer is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + live_video_path = os.path.join(output_path, f"live_trajectory_{timestamp}.mp4") + os.makedirs(output_path, exist_ok=True) + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + video_writer = cv2.VideoWriter(live_video_path, fourcc, 30, (new_width, new_height)) + print(f"Started recording: {live_video_path}" if video_writer.isOpened() else "Recording failed") + else: + video_writer.release() + video_writer = None + print("Stopped recording") + + finally: + if video_writer: + video_writer.release() + if consumer: + consumer.close() + cv2.destroyAllWindows() + +def parse_args(): + parser = argparse.ArgumentParser(description='Kafka BEV Online Visualizer') + parser.add_argument('--dataset-path', type=str, + default="datasets/mtmc_4cam", + help='Path to dataset)') + parser.add_argument('--msgconv-config', type=str, + default='config_msgconv.txt', + help='Path to message converter config file (config_msgconv.txt)') + parser.add_argument('--output-path', type=str, + default='output_videos', + help='Output directory for videos') + parser.add_argument('--offline', action='store_true', + help='Run in offline mode (save a video from all messages instead of real-time visualization). \ + Please run this script after launching the MV3DT app.') + parser.add_argument('--show-ids', action='store_true', + help='Show object IDs near trajectory heads') + parser.add_argument('--average-multi-cam', action='store_true', + help='Average trajectory points from multiple cameras for the same object') + parser.add_argument('--verbose', action='store_true', + help='Print warnings and diagnostic messages') + + return parser.parse_args() + +def main(): + args = parse_args() + expected_sensors = extract_expected_sensors(args.msgconv_config) + print(f"Expected sensors: {expected_sensors}") + if args.offline: + generate_video(args.dataset_path, args.output_path, args.show_ids, expected_sensors, args.average_multi_cam, args.verbose) + else: + real_time_visualization(args.dataset_path, args.output_path, args.show_ids, expected_sensors, args.average_multi_cam, args.verbose) + + +if __name__ == "__main__": + main() diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_client.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_client.py new file mode 100644 index 00000000..d22dbf32 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/kafka_client.py @@ -0,0 +1,70 @@ +"""Simple Kafka client for receiving and decoding MV3DT protobuf messages.""" + +import json +from kafka import KafkaConsumer +from kafka.errors import KafkaError +from google.protobuf.json_format import MessageToDict +from schema_pb2 import Frame +import argparse + +def main(): + """Main function to consume and print Kafka messages.""" + parser = argparse.ArgumentParser() + parser.add_argument('--topic', type=str, default='mv3dt', help='Kafka topic to subscribe to') + parser.add_argument('--broker', type=str, default='localhost:9092', help='Kafka broker server (e.g., localhost:9092)') + args = parser.parse_args() + + topic = args.topic + broker = args.broker + + print(f"Starting Kafka client reading from broker '{broker}' topic '{topic}'...") + + try: + # Create Kafka consumer + consumer = KafkaConsumer( + topic, + bootstrap_servers=broker, + auto_offset_reset='earliest', + value_deserializer=lambda x: x # Keep as bytes for protobuf + ) + + print("Connected to Kafka. Waiting for messages...") + + message_count = 0 + + # Consume messages + for msg in consumer: + try: + # Parse protobuf message + frame = Frame() + frame.ParseFromString(msg.value) + # print ('frame', frame) + # Convert to dictionary and then to JSON + frame_dict = MessageToDict(frame) + json_str = json.dumps(frame_dict, indent=2) + + message_count += 1 + print(f"\n--- Message {message_count} ---") + # print(f"Frame ID: {frame_id}") + # print(f"Sensor ID: {sensor_id}") + # print(f"Objects: {object_count}") + print(f"JSON Data:") + print(json_str) + print("-" * 50) + + except Exception as e: + print(f"Error parsing message: {e}") + continue + + except KafkaError as e: + print(f"Kafka error: {e}") + except KeyboardInterrupt: + print("\nStopped by user") + except Exception as e: + print(f"Unexpected error: {e}") + finally: + print("Consumer stopped") + + +if __name__ == "__main__": + main() diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/schema_pb2.py b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/schema_pb2.py new file mode 100644 index 00000000..f7d91d5d --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/utils/schema_pb2.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: schema.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cschema.proto\x12\x02nv\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa5\x03\n\x05\x46rame\x12\x0f\n\x07version\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\t\x12-\n\ttimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x10\n\x08sensorId\x18\x04 \x01(\t\x12\x1b\n\x07objects\x18\x05 \x03(\x0b\x32\n.nv.Object\x12\x1c\n\x03\x66ov\x18\x06 \x03(\x0b\x32\x0f.nv.TypeMetrics\x12\x1d\n\x04rois\x18\x07 \x03(\x0b\x32\x0f.nv.TypeMetrics\x12 \n\x10socialDistancing\x18\x08 \x01(\x0b\x32\x06.nv.SD\x12&\n\x0csegmentation\x18\t \x01(\x0b\x32\x10.nv.Segmentation\x12%\n\x0cinteractions\x18\n \x03(\x0b\x32\x0f.nv.Interaction\x12#\n\x0b\x63ongestions\x18\x0b \x03(\x0b\x32\x0e.nv.Congestion\x12!\n\x04info\x18\x0c \x03(\x0b\x32\x13.nv.Frame.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x93\x03\n\x06Object\x12\n\n\x02id\x18\x01 \x01(\t\x12\x16\n\x04\x62\x62ox\x18\x02 \x01(\x0b\x32\x08.nv.Bbox\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x12\n\nconfidence\x18\x04 \x01(\x02\x12\"\n\x04info\x18\x05 \x03(\x0b\x32\x14.nv.Object.InfoEntry\x12 \n\tembedding\x18\x06 \x01(\x0b\x32\r.nv.Embedding\x12\x16\n\x04pose\x18\x07 \x01(\x0b\x32\x08.nv.Pose\x12\x16\n\x04gaze\x18\x08 \x01(\x0b\x32\x08.nv.Gaze\x12$\n\x0blipActivity\x18\t \x01(\x0b\x32\x0f.nv.LipActivity\x12\r\n\x05speed\x18\n \x01(\x02\x12\x0b\n\x03\x64ir\x18\x0b \x03(\x02\x12\"\n\ncoordinate\x18\x0c \x01(\x0b\x32\x0e.nv.Coordinate\x12\x1e\n\x08location\x18\r \x01(\x0b\x32\x0c.nv.Location\x12\x1a\n\x06\x62\x62ox3d\x18\x0e \x01(\x0b\x32\n.nv.Bbox3d\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"-\n\nCoordinate\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\x12\t\n\x01z\x18\x03 \x01(\x01\"1\n\x08Location\x12\x0b\n\x03lat\x18\x01 \x01(\x01\x12\x0b\n\x03lon\x18\x02 \x01(\x01\x12\x0b\n\x03\x61lt\x18\x03 \x01(\x01\"\xca\x01\n\x04\x42\x62ox\x12\r\n\x05leftX\x18\x01 \x01(\x02\x12\x0c\n\x04topY\x18\x02 \x01(\x02\x12\x0e\n\x06rightX\x18\x03 \x01(\x02\x12\x0f\n\x07\x62ottomY\x18\x04 \x01(\x02\x12!\n\nembeddings\x18\x05 \x03(\x0b\x32\r.nv.Embedding\x12\x12\n\nconfidence\x18\x06 \x01(\x02\x12 \n\x04info\x18\x07 \x03(\x0b\x32\x12.nv.Bbox.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa5\x01\n\x06\x42\x62ox3d\x12\x13\n\x0b\x63oordinates\x18\x01 \x03(\x01\x12!\n\nembeddings\x18\x02 \x03(\x0b\x32\r.nv.Embedding\x12\x12\n\nconfidence\x18\x03 \x01(\x02\x12\"\n\x04info\x18\x04 \x03(\x0b\x32\x14.nv.Bbox3d.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"s\n\x0cSegmentation\x12\x0c\n\x04mask\x18\x01 \x03(\x05\x12(\n\x04info\x18\x02 \x03(\x0b\x32\x1a.nv.Segmentation.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc4\x01\n\x0bTypeMetrics\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05\x63ount\x18\x03 \x01(\x05\x12#\n\x0b\x63oordinates\x18\x04 \x03(\x0b\x32\x0e.nv.Coordinate\x12\x11\n\tobjectIds\x18\x05 \x03(\t\x12\'\n\x04info\x18\x06 \x03(\x0b\x32\x19.nv.TypeMetrics.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"&\n\x07\x43luster\x12\x1b\n\x06points\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\"\x1f\n\x07Point2D\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\"\xa0\x01\n\x02SD\x12\x11\n\tthreshold\x18\x01 \x01(\x01\x12\x1b\n\x13proximityDetections\x18\x02 \x01(\x05\x12\x1d\n\x08\x63lusters\x18\x03 \x03(\x0b\x32\x0b.nv.Cluster\x12\x1e\n\x04info\x18\x04 \x03(\x0b\x32\x10.nv.SD.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"K\n\x07Polygon\x12 \n\x0b\x63oordinates\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\x12\x1e\n\x05holes\x18\x02 \x03(\x0b\x32\x0f.nv.PolygonHole\"/\n\x0bPolygonHole\x12 \n\x0b\x63oordinates\x18\x01 \x03(\x0b\x32\x0b.nv.Point2D\"\xbc\x01\n\x0bInteraction\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tobjectIds\x18\x02 \x03(\t\x12#\n\x0b\x63oordinates\x18\x03 \x03(\x0b\x32\x0e.nv.Coordinate\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\'\n\x04info\x18\x05 \x03(\x0b\x32\x19.nv.Interaction.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x90\x01\n\nCongestion\x12\n\n\x02id\x18\x01 \x01(\t\x12\x11\n\tobjectIds\x18\x02 \x03(\t\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x02\x12&\n\x04info\x18\x04 \x03(\x0b\x32\x18.nv.Congestion.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x02\n\x04Pose\x12\x0c\n\x04type\x18\x01 \x01(\t\x12$\n\tkeypoints\x18\x02 \x03(\x0b\x32\x11.nv.Pose.Keypoint\x12 \n\x07\x61\x63tions\x18\x03 \x03(\x0b\x32\x0f.nv.Pose.Action\x12 \n\x04info\x18\x04 \x03(\x0b\x32\x12.nv.Pose.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x41\n\x08Keypoint\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x63oordinates\x18\x02 \x03(\x02\x12\x12\n\nquaternion\x18\x03 \x03(\x02\x1a*\n\x06\x41\x63tion\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x12\n\nconfidence\x18\x02 \x01(\x02\"C\n\x04Gaze\x12\t\n\x01x\x18\x01 \x01(\x02\x12\t\n\x01y\x18\x02 \x01(\x02\x12\t\n\x01z\x18\x03 \x01(\x02\x12\r\n\x05theta\x18\x04 \x01(\x02\x12\x0b\n\x03phi\x18\x05 \x01(\x02\"!\n\x0bLipActivity\x12\x12\n\nclassLabel\x18\x01 \x01(\t\"q\n\x05\x45vent\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12!\n\x04info\x18\x05 \x03(\x0b\x32\x13.nv.Event.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xad\x01\n\x0f\x41nalyticsModule\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0e\n\x06source\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\x12+\n\x04info\x18\x05 \x03(\x0b\x32\x1d.nv.AnalyticsModule.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x06Sensor\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x1e\n\x08location\x18\x04 \x01(\x0b\x32\x0c.nv.Location\x12\"\n\ncoordinate\x18\x05 \x01(\x0b\x32\x0e.nv.Coordinate\x12\"\n\x04info\x18\x06 \x03(\x0b\x32\x14.nv.Sensor.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc3\x01\n\x05Place\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1e\n\x08location\x18\x04 \x01(\x0b\x32\x0c.nv.Location\x12\"\n\ncoordinate\x18\x05 \x01(\x0b\x32\x0e.nv.Coordinate\x12!\n\x04info\x18\x06 \x03(\x0b\x32\x13.nv.Place.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x02\n\x07Message\x12\x11\n\tmessageid\x18\x01 \x01(\t\x12\x12\n\nmdsversion\x18\x02 \x01(\t\x12-\n\ttimestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x18\n\x05place\x18\x04 \x01(\x0b\x32\t.nv.Place\x12\x1a\n\x06sensor\x18\x05 \x01(\x0b\x32\n.nv.Sensor\x12,\n\x0f\x61nalyticsModule\x18\x06 \x01(\x0b\x32\x13.nv.AnalyticsModule\x12\x1a\n\x06object\x18\x07 \x01(\x0b\x32\n.nv.Object\x12\x18\n\x05\x65vent\x18\x08 \x01(\x0b\x32\t.nv.Event\x12\x11\n\tvideoPath\x18\t \x01(\t\"s\n\tEmbedding\x12\x12\n\x06vector\x18\x01 \x03(\x02\x42\x02\x10\x01\x12%\n\x04info\x18\x02 \x03(\x0b\x32\x17.nv.Embedding.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xae\x01\n\tImageData\x12\x1f\n\x06\x66ormat\x18\x01 \x01(\x0e\x32\x0f.nv.ImageFormat\x12\x10\n\x08\x65ncoding\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12%\n\x04info\x18\x05 \x03(\x0b\x32\x17.nv.ImageData.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa4\x02\n\tVisionLLM\x12\x0f\n\x07version\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cstartFrameId\x18\x04 \x01(\t\x12\x12\n\nendFrameId\x18\x05 \x01(\t\x12\x1a\n\x06sensor\x18\x06 \x01(\x0b\x32\n.nv.Sensor\x12\x14\n\x03llm\x18\x07 \x01(\x0b\x32\x07.nv.LLM\x12%\n\x04info\x18\x08 \x03(\x0b\x32\x17.nv.VisionLLM.InfoEntry\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9f\x01\n\x03LLM\x12\x1f\n\x04info\x18\x01 \x03(\x0b\x32\x11.nv.LLM.InfoEntry\x12\'\n\rconversations\x18\x02 \x03(\x0b\x32\x10.nv.Conversation\x12!\n\nembeddings\x18\x03 \x03(\x0b\x32\r.nv.Embedding\x1a+\n\tInfoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\";\n\x0c\x43onversation\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08response\x18\x03 \x01(\t*2\n\x0bImageFormat\x12\x07\n\x03RAW\x10\x00\x12\x07\n\x03JPG\x10\x01\x12\x08\n\x04JPEG\x10\x02\x12\x07\n\x03PNG\x10\x03\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'schema_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _FRAME_INFOENTRY._options = None + _FRAME_INFOENTRY._serialized_options = b'8\001' + _OBJECT_INFOENTRY._options = None + _OBJECT_INFOENTRY._serialized_options = b'8\001' + _BBOX_INFOENTRY._options = None + _BBOX_INFOENTRY._serialized_options = b'8\001' + _BBOX3D_INFOENTRY._options = None + _BBOX3D_INFOENTRY._serialized_options = b'8\001' + _SEGMENTATION_INFOENTRY._options = None + _SEGMENTATION_INFOENTRY._serialized_options = b'8\001' + _TYPEMETRICS_INFOENTRY._options = None + _TYPEMETRICS_INFOENTRY._serialized_options = b'8\001' + _SD_INFOENTRY._options = None + _SD_INFOENTRY._serialized_options = b'8\001' + _INTERACTION_INFOENTRY._options = None + _INTERACTION_INFOENTRY._serialized_options = b'8\001' + _CONGESTION_INFOENTRY._options = None + _CONGESTION_INFOENTRY._serialized_options = b'8\001' + _POSE_INFOENTRY._options = None + _POSE_INFOENTRY._serialized_options = b'8\001' + _EVENT_INFOENTRY._options = None + _EVENT_INFOENTRY._serialized_options = b'8\001' + _ANALYTICSMODULE_INFOENTRY._options = None + _ANALYTICSMODULE_INFOENTRY._serialized_options = b'8\001' + _SENSOR_INFOENTRY._options = None + _SENSOR_INFOENTRY._serialized_options = b'8\001' + _PLACE_INFOENTRY._options = None + _PLACE_INFOENTRY._serialized_options = b'8\001' + _EMBEDDING_INFOENTRY._options = None + _EMBEDDING_INFOENTRY._serialized_options = b'8\001' + _EMBEDDING.fields_by_name['vector']._options = None + _EMBEDDING.fields_by_name['vector']._serialized_options = b'\020\001' + _IMAGEDATA_INFOENTRY._options = None + _IMAGEDATA_INFOENTRY._serialized_options = b'8\001' + _VISIONLLM_INFOENTRY._options = None + _VISIONLLM_INFOENTRY._serialized_options = b'8\001' + _LLM_INFOENTRY._options = None + _LLM_INFOENTRY._serialized_options = b'8\001' + _IMAGEFORMAT._serialized_start=4538 + _IMAGEFORMAT._serialized_end=4588 + _FRAME._serialized_start=54 + _FRAME._serialized_end=475 + _FRAME_INFOENTRY._serialized_start=432 + _FRAME_INFOENTRY._serialized_end=475 + _OBJECT._serialized_start=478 + _OBJECT._serialized_end=881 + _OBJECT_INFOENTRY._serialized_start=432 + _OBJECT_INFOENTRY._serialized_end=475 + _COORDINATE._serialized_start=883 + _COORDINATE._serialized_end=928 + _LOCATION._serialized_start=930 + _LOCATION._serialized_end=979 + _BBOX._serialized_start=982 + _BBOX._serialized_end=1184 + _BBOX_INFOENTRY._serialized_start=432 + _BBOX_INFOENTRY._serialized_end=475 + _BBOX3D._serialized_start=1187 + _BBOX3D._serialized_end=1352 + _BBOX3D_INFOENTRY._serialized_start=432 + _BBOX3D_INFOENTRY._serialized_end=475 + _SEGMENTATION._serialized_start=1354 + _SEGMENTATION._serialized_end=1469 + _SEGMENTATION_INFOENTRY._serialized_start=432 + _SEGMENTATION_INFOENTRY._serialized_end=475 + _TYPEMETRICS._serialized_start=1472 + _TYPEMETRICS._serialized_end=1668 + _TYPEMETRICS_INFOENTRY._serialized_start=432 + _TYPEMETRICS_INFOENTRY._serialized_end=475 + _CLUSTER._serialized_start=1670 + _CLUSTER._serialized_end=1708 + _POINT2D._serialized_start=1710 + _POINT2D._serialized_end=1741 + _SD._serialized_start=1744 + _SD._serialized_end=1904 + _SD_INFOENTRY._serialized_start=432 + _SD_INFOENTRY._serialized_end=475 + _POLYGON._serialized_start=1906 + _POLYGON._serialized_end=1981 + _POLYGONHOLE._serialized_start=1983 + _POLYGONHOLE._serialized_end=2030 + _INTERACTION._serialized_start=2033 + _INTERACTION._serialized_end=2221 + _INTERACTION_INFOENTRY._serialized_start=432 + _INTERACTION_INFOENTRY._serialized_end=475 + _CONGESTION._serialized_start=2224 + _CONGESTION._serialized_end=2368 + _CONGESTION_INFOENTRY._serialized_start=432 + _CONGESTION_INFOENTRY._serialized_end=475 + _POSE._serialized_start=2371 + _POSE._serialized_end=2653 + _POSE_INFOENTRY._serialized_start=432 + _POSE_INFOENTRY._serialized_end=475 + _POSE_KEYPOINT._serialized_start=2544 + _POSE_KEYPOINT._serialized_end=2609 + _POSE_ACTION._serialized_start=2611 + _POSE_ACTION._serialized_end=2653 + _GAZE._serialized_start=2655 + _GAZE._serialized_end=2722 + _LIPACTIVITY._serialized_start=2724 + _LIPACTIVITY._serialized_end=2757 + _EVENT._serialized_start=2759 + _EVENT._serialized_end=2872 + _EVENT_INFOENTRY._serialized_start=432 + _EVENT_INFOENTRY._serialized_end=475 + _ANALYTICSMODULE._serialized_start=2875 + _ANALYTICSMODULE._serialized_end=3048 + _ANALYTICSMODULE_INFOENTRY._serialized_start=432 + _ANALYTICSMODULE_INFOENTRY._serialized_end=475 + _SENSOR._serialized_start=3051 + _SENSOR._serialized_end=3255 + _SENSOR_INFOENTRY._serialized_start=432 + _SENSOR_INFOENTRY._serialized_end=475 + _PLACE._serialized_start=3258 + _PLACE._serialized_end=3453 + _PLACE_INFOENTRY._serialized_start=432 + _PLACE_INFOENTRY._serialized_end=475 + _MESSAGE._serialized_start=3456 + _MESSAGE._serialized_end=3724 + _EMBEDDING._serialized_start=3726 + _EMBEDDING._serialized_end=3841 + _EMBEDDING_INFOENTRY._serialized_start=432 + _EMBEDDING_INFOENTRY._serialized_end=475 + _IMAGEDATA._serialized_start=3844 + _IMAGEDATA._serialized_end=4018 + _IMAGEDATA_INFOENTRY._serialized_start=432 + _IMAGEDATA_INFOENTRY._serialized_end=475 + _VISIONLLM._serialized_start=4021 + _VISIONLLM._serialized_end=4313 + _VISIONLLM_INFOENTRY._serialized_start=432 + _VISIONLLM_INFOENTRY._serialized_end=475 + _LLM._serialized_start=4316 + _LLM._serialized_end=4475 + _LLM_INFOENTRY._serialized_start=432 + _LLM_INFOENTRY._serialized_end=475 + _CONVERSATION._serialized_start=4477 + _CONVERSATION._serialized_end=4536 +# @@protoc_insertion_point(module_scope) diff --git a/src/apps/reference_apps/deepstream-tracker-3d/README.md b/src/apps/reference_apps/deepstream-tracker-3d/README.md new file mode 100644 index 00000000..7481878c --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d/README.md @@ -0,0 +1,121 @@ +# Single-View 3D Tracking in DeepStream + +## Introduction +This sample application demonstrates the single-view 3D tracking with DeepStream SDK. Given the [camera matrix and human model](configs/camInfo.yml) of a static camera, this application estimates and keeps tracking of object states in the 3D physical world. It can recover the complete bounding box, foot location and body convex hulls precisely from partial occlusions. For algorithm and setup details, please refer to [DeepStream Single View 3D Tracking Documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#single-view-3d-tracking-alpha). + +The currnet SV3DT configuration in this sample uses an 2D pose estimator, which estimates human key-points on the 2D image plane. The SV3DT algorithm uses height and waist key-points as anchor to precisely estimate each person's 3D human height. If setting `poseEstimatorType: 0` in `configs/config_tracker_NvDCF_accuracy_3D.yml`, pose estimator will be disabled, and the algorithm uses 2D detection bounding boxes and a human model with fixed height. It estimates the 3D location by matching the head with 2D bounding box's top edge. + +## Prerequisites +This sample application can be run on both x86 and Jetson platforms inside DeepStream container. Check [here](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. +1. Download the latest DeepStream container image from NGC (e.g., DS 9.0 in the example below) + ```bash + export DS_IMG_NAME="nvcr.io/nvidia/deepstream:9.0-triton-multiarch" + docker pull $DS_IMG_NAME + ``` + +2. Git clone the current `deepstream_reference_apps` repository to the host machine, and enter single-view 3D tracking directory inside the repository. + ```bash + git clone https://github.com/NVIDIA/deepstream.git + cd deepstream/src/apps/reference_apps/deepstream-tracker-3d + ``` + +3. Download NVIDIA pretrained [`PeopleNet`](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files?version=deployable_quantized_onnx_v2.6.3) for 2D detection. + + ```bash + # current directory: deepstream/src/apps/reference_apps/deepstream-tracker-3d + mkdir -p models/PeopleNet + cd models/PeopleNet; wget --no-check-certificate --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/deployable_quantized_onnx_v2.6.3/zip -O peoplenet_deployable_quantized_onnx_v2.6.3.zip; unzip peoplenet_deployable_quantized_onnx_v2.6.3.zip + ``` + + The model files are now stored in `PeopleNet` directory as + + ```bash + deepstream-tracker-3d + ├── configs + ├── streams + └── models + └── PeopleNet + ├── labels.txt + ├── resnet34_peoplenet.onnx + └── resnet34_peoplenet_int8.txt + ``` +## Running the Application +Launch the container from current directory, and execute the 3D tracking pipeline inside the container. The current [config](configs/deepstream_app_source1_3d_tracking.txt) requires users to run with a display because it uses EGL sink to visualize the overlay results. To run through ssh without display, please change `type=2` to `1` in group `[sink0]` in that file. Users can check [DeepStream sink group](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#sink-group) for the usage of each sink. + +```bash +cd ../.. +sudo xhost + # give container access to display +# current directory: deepstream_reference_apps/deepstream-tracker-3d +docker run --runtime=nvidia -it --rm --net=host --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -v $(pwd):/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d -e DISPLAY=$DISPLAY $DS_IMG_NAME +``` + +Inside container, run the following commands. Please note that when `deepstream-app` is launched for the first time, it tries to create model engine files, which may take a couple minutes, depending on HW platforms. + +```bash +# Install prerequisites +cd /opt/nvidia/deepstream/deepstream/ +bash user_additional_install.sh + +# Download ReID and BodyPose3DNet model +export MODEL_DIR="/opt/nvidia/deepstream/deepstream/samples/models/Tracker" +mkdir -p $MODEL_DIR +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/reidentificationnet/versions/deployable_v1.0/files/resnet50_market1501.etlt' -P $MODEL_DIR +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/bodypose3dnet/versions/deployable_accuracy_onnx_1.0/files/bodypose3dnet_accuracy.onnx' -P $MODEL_DIR + +# Run 3D tracking pipeline +cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d/configs +mkdir -p track_results +deepstream-app -c deepstream_app_source1_3d_tracking.txt +``` + +## Output Retrival and Visualization + +### DeepStream Direct Visualization +When the pipeline is launced, DeepStream shows the output video like below while processing the input video. The 3D bounding boxes of the people are reconstructed and plotted even though there are partial occlusions. The result video is saved as `out.mp4`. + +![sample 3d tracking results](figures/.retail_osd.png) + +### 3D Metadata Processing and Visualization +The extracted metadata (e.g., bounding box, frame num, target ID, etc.) is saved in both extended MOT and KITTI format. For detailed explanation, please refer to [DeepStream Tracker Miscellaneous Data Output](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#miscellaneous-data-output). + +The MOT results can be found in `track_dump_0.txt` file, which contains all the object metadata for one stream, and the data format is defined below. The foot image position and the convex hull of the projected cylindrical human model are defined in video frame coordinates, and can be used to draw the visualization figures below. Users can create such overlay video or image using their favorite tools like [OpenCV](https://github.com/opencv/opencv). The foot world position is defined in the 3D world ground plane corresponding to the 3x4 camera projection matrix. + +| frame number(starting from 1) | object unique id | bbox left | bbox top | bbox width | bbox height | confidence | Foot World Position X | Foot World Position Y | blank | class id | tracker state | visibility | Foot Image Position X | Foot Image Position Y | ConvexHull Points Relative to bbox center | +|-------------------------------|------------------|-----------|----------|------------|-------------|------------|-----------------------|-----------------------|-------|--------------|---------------|------------|-----------------------|-----------------------|--------------------------------| +| unsigned int |long unsigned int | int | int | int | int | float | float | float | int | unsigned int | int | float | float | float | int separated by vertical bar | + +Sample output is like below. The green dot in the visualization figure is plotted as `(Foot Image Position X, Foot Image Position Y)`, and the cylinder is plotted by connecting convex hull points. For example, the bbox in the first line of the output file has center `(1433, -50)`. Then its convex hull points are `(1433 - 94, -50 - 170), (1433 - 87, -50 - 176), ..., (1433 - 23, -50 + 190)`. Users can plot the foot location and convex hull on their own as shown in the figure below. +```txt +1,1,1366,-195,134,290,1.000,-171.080,1058.295,-1,0,2,0.220,1458,80,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +2,1,1365,-194,135,290,0.989,-170.646,1045.254,-1,0,2,0.230,1458,84,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +3,1,1366,-196,134,290,0.860,-170.679,1054.089,-1,0,2,0.229,1458,82,-94|-170|-87|-176|-71|-183|-49|-191|-23|-196|0|-200|18|-201|29|-198|95|165|95|173|85|183|66|191|42|198|16|202|-4|201|-18|197|-23|190 +... +``` +![sample 3d tracking results](figures/.retail_viz.png) + +The KITTI results can be found in `track_results` folder. A file will be created for each frame in each stream, and the data format is defined below. + +| object Label | object Unique Id | blank | blank | blank | bbox left | bbox top | bbox right | bbox bottom | blank | blank | blank | blank | blank | blank | blank |confidence | visibility (optional) | Foot Image Position X (optional) | Foot Image Position Y (optional) | +|--------------|------------------|-------|-------|-------|-----------|----------|------------|-------------|-------|-------|-------|-------|-------|-------|-------|-----------|-----------------------|-----------------------|-----------------------| +| string | long unsigned | float | int | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | float | + +Each frame is saved as `track_results/00_000_xxxxxx.txt`. Sample output of a frame is like below. Note that if a object is found in past frame data, it wouldn't have visibility and foot position in KITTI dump. +```txt +person 1 0.0 0 0.0 1365.907227 -196.875290 1500.249146 93.554581 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.890186 0.229870 1458.166992 81.585060 +person 6 0.0 0 0.0 1419.655151 72.774818 1647.446167 561.540894 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.841420 0.408513 1575.264160 531.851746 +person 0 0.0 0 0.0 1008.387817 36.228714 1202.886353 421.609314 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.632065 0.504738 1148.117188 399.992645 +... +``` + +## Customizing the Video +To run the single view 3D tracking on other videos, the following changes are required. +1. In `deepstream_app_source1_3d_tracking.txt`, change `uri=file://../streams/Retail02_short.mp4` to the new video name, `width=1920, height=1080, tracker-width=1920, tracker-height=1080` to the new video's resolution. +2. Generate camera projection matrix for the new video. Change `projectionMatrix_3x4` in `camInfo.yml` into the new matrix. + +Note: Multiple streams can run at the same time. Add all the sources to `deepstream_app_source1_3d_tracking.txt`, and all the camera information files to `config_tracker_NvDCF_accuracy_3D.yml`. +```yaml + cameraModelFilepath: # In order of the source streams + - 'camInfo-01.yml' + - 'camInfo-02.yml' + - ... +``` diff --git a/src/apps/reference_apps/deepstream-tracker-3d/configs/camInfo.yml b/src/apps/reference_apps/deepstream-tracker-3d/configs/camInfo.yml new file mode 100644 index 00000000..1e4c41f0 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d/configs/camInfo.yml @@ -0,0 +1,38 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# row-major, so the actual matrix is like +# 2582.5691623002185 -485.10283397043617 650.27745033162591 89466.605755471101 +# -423.46809686390498 1044.6870098337931 2461.1283636622838 -214284.36100320917 +# -0.25563255317172684 -0.90495941862094287 0.34014768617197644 -1181.960782357068 + + +projectionMatrix_3x4: + - 2582.5691623002185 + - -485.10283397043617 + - 650.27745033162591 + - -89466.605755471101 + - -423.46809686390498 + - 1044.6870098337931 + - 2461.1283636622838 + - -214284.36100320917 + - -0.25563255317172684 + - -0.90495941862094287 + - 0.34014768617197644 + - -1181.960782357068 + +# the height and radius of the cylinder model +modelInfo: + height: 205 + radius: 33 diff --git a/src/apps/reference_apps/deepstream-tracker-3d/configs/config_infer_primary.txt b/src/apps/reference_apps/deepstream-tracker-3d/configs/config_infer_primary.txt new file mode 100644 index 00000000..0ed48eba --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d/configs/config_infer_primary.txt @@ -0,0 +1,47 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 + +infer-dims=3;544;960 +int8-calib-file=../models/PeopleNet/resnet34_peoplenet_int8.txt +model-engine-file=../models/PeopleNet/resnet34_peoplenet.onnx_b1_gpu0_fp16.engine +labelfile-path=../models/PeopleNet/labels.txt +onnx-file=../models/PeopleNet/resnet34_peoplenet.onnx + +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +interval=0 +gie-unique-id=1 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=3 +#enable-dla=1 +#use-dla-core=0 +#scaling-filter=4 + +filter-out-class-ids=1;2 + +[class-attrs-all] +pre-cluster-threshold=0.1429 +nms-iou-threshold=0.4688 +minBoxes=3 +dbscan-min-score=0.7726 +eps=0.2538 +detected-min-w=20 +detected-min-h=20 diff --git a/src/apps/reference_apps/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml b/src/apps/reference_apps/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml new file mode 100644 index 00000000..461a14c4 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d/configs/config_tracker_NvDCF_accuracy_3D.yml @@ -0,0 +1,158 @@ +%YAML:1.0 +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +BaseConfig: + minDetectorConfidence: 0.1894 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking + +TargetManagement: + enableBboxUnClipping: 1 # In case the bbox is likely to be clipped by image border, unclip bbox + preserveStreamUpdateOrder: 0 # When assigning new target ids, preserve input streams' order to keep target ids in a deterministic order over multuple runs + maxTargetsPerStream: 150 # Max number of targets to track per stream. Recommended to set >10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity + + # [Creation & Termination Policy] + minIouDiff4NewTarget: 0.3686 # If the IOU between the newly detected object and any of the existing targets is higher than this threshold, this newly detected object will be discarded. + minTrackerConfidence: 0.1513 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] + probationAge: 2 # If the target's age exceeds this, the target will be considered to be valid. + maxShadowTrackingAge: 42 # Max length of shadow tracking. If the shadowTrackingAge exceeds this limit, the tracker will be terminated. + earlyTerminationAge: 1 # If the shadowTrackingAge reaches this threshold while in TENTATIVE period, the target will be terminated prematurely. + + # dump tracklets in txt file + outputTerminatedTracks: 1 # save terminated tracklets + terminatedTrackFilename: track_dump_ # file name: "terminatedTrackFilename"0.txt, "terminatedTrackFilename"1_2.txt, ... + +TrajectoryManagement: + useUniqueID: 0 # Use 64-bit long Unique ID when assignining tracker ID. Default is [true] + enableReAssoc: 1 # Enable Re-Assoc + + # [Re-Assoc Metric: Thresholds for valid candidates] + minMatchingScore4Overall: 0.6622 # min matching score for overall + minTrackletMatchingScore: 0.2940 # min tracklet similarity score for re-assoc + minMatchingScore4ReidSimilarity: 0.0771 # min reid similarity score for re-assoc + + # [Re-Assoc Metric: Weights] + matchingScoreWeight4TrackletSimilarity: 0.7981 # weight for tracklet similarity score + matchingScoreWeight4ReidSimilarity: 0.3848 # weight for reid similarity score + + # [Re-Assoc: Motion-based] + minTrajectoryLength4Projection: 34 # min trajectory length required to make projected trajectory + prepLength4TrajectoryProjection: 58 # the length of the trajectory during which the state estimator is updated to make projections + trajectoryProjectionLength: 33 # the length of the projected trajectory + maxAngle4TrackletMatching: 67 # max angle difference for tracklet matching [degree] + minSpeedSimilarity4TrackletMatching: 0.0574 # min speed similarity for tracklet matching + minBboxSizeSimilarity4TrackletMatching: 0.1013 # min bbox size similarity for tracklet matching + maxTrackletMatchingTimeSearchRange: 27 # the search space in time for max tracklet similarity + trajectoryProjectionProcessNoiseScale: 0.0100 # trajectory projector's process noise scale w.r.t. state estimator + trajectoryProjectionMeasurementNoiseScale: 100 # trajectory projector's measurement noise scale w.r.t. state estimator + trackletSpacialSearchRegionScale: 0.0100 # the search region scale for peer tracklet + + # [Re-Assoc: Reid based. Reid model params are set in ReID section] + reidExtractionInterval: 8 # frame interval to extract reid features per target + +DataAssociator: + dataAssociatorType: 0 # the type of data associator among { DEFAULT= 0 } + associationMatcherType: 1 # the type of matching algorithm among { GREEDY=0, CASCADED=1 } + checkClassMatch: 1 # If checked, only the same-class objects are associated with each other. Default: true + + # [Association Metric: Thresholds for valid candidates] + minMatchingScore4Overall: 0.0222 # Min total score + minMatchingScore4SizeSimilarity: 0.3552 # Min bbox size similarity score + minMatchingScore4Iou: 0.0548 # Min IOU score + minMatchingScore4VisualSimilarity: 0.5043 # Min visual similarity score + + # [Association Metric: Weights] + matchingScoreWeight4VisualSimilarity: 0.3951 # Weight for the visual similarity (in terms of correlation response ratio) + matchingScoreWeight4SizeSimilarity: 0.6003 # Weight for the Size-similarity score + matchingScoreWeight4Iou: 0.4033 # Weight for the IOU score + + # [Association Metric: Tentative detections] only uses iou similarity for tentative detections + tentativeDetectorConfidence: 0.1024 # If a detection's confidence is lower than this but higher than minDetectorConfidence, then it's considered as a tentative detection + minMatchingScore4TentativeIou: 0.2852 # Min iou threshold to match targets and tentative detection + +StateEstimator: + stateEstimatorType: 3 # the type of state estimator among { DUMMY=0, SIMPLE=1, REGULAR=2, 3D=3 } + + # [Dynamics Modeling] + processNoiseVar4Loc: 6810.8668 # Process noise variance for bbox center + processNoiseVar4Vel: 1348.4874 # Process noise variance for velocity + measurementNoiseVar4Detector: 100.0000 # Measurement noise variance for detector's detection + measurementNoiseVar4Tracker: 293.3238 # Measurement noise variance for tracker's localization + +ObjectModelProjection: + minPoseConfidence: 0.925 + cameraModelFilepath: # camera calibration file for each stream + - /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-tracker-3d/configs/camInfo.yml + outputVisibility: 1 # output visibility by occlusion + outputFootLocation: 1 # output foot location estimated from 3D model + outputConvexHull: 0 # output convex hull for each object estimated from 3D cylinder model + +VisualTracker: + visualTrackerType: 2 # the type of visual tracker among { DUMMY=0, NvDCF=1, NvDCF_VPI=2 } + vpiBackend4DcfTracker: 1 # the type of compute backend among {CUDA=1, PVA=2} + + # [NvDCF: Feature Extraction] + useColorNames: 1 # Use ColorNames feature + useHog: 1 # Use Histogram-of-Oriented-Gradient (HOG) feature + featureImgSizeLevel: 3 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest + featureFocusOffsetFactor_y: -0.1054 # The offset for the center of hanning window relative to the feature height. The center of hanning window would move by (featureFocusOffsetFactor_y*featureMatSize.height) in vertical direction + + # [NvDCF: Correlation Filter] + filterLr: 0.0767 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] + filterChannelWeightsLr: 0.0339 # learning rate for the channel weights among feature channels. Valid Range: [0.0, 1.0] + gaussianSigma: 0.5687 # Standard deviation for Gaussian for desired response when creating DCF filter [pixels] + +ReID: + reidType: 2 # The type of reid among { DUMMY=0, NvDEEPSORT=1, Reid based reassoc=2, both NvDEEPSORT and reid based reassoc=3} + + # [Reid Network Info] + batchSize: 100 # Batch size of reid network + workspaceSize: 1000 # Workspace size to be used by reid engine, in MB + reidFeatureSize: 256 # Size of reid feature + reidHistorySize: 100 # Max number of reid features kept for one object + inferDims: [3, 256, 128] # Reid network input dimension CHW or HWC based on inputOrder + networkMode: 1 # Reid network inference precision mode among {fp32=0, fp16=1, int8=2 } + + # [Input Preprocessing] + inputOrder: 0 # Reid network input order among { NCHW=0, NHWC=1 }. Batch will be converted to the specified order before reid input. + colorFormat: 0 # Reid network input color format among {RGB=0, BGR=1 }. Batch will be converted to the specified color before reid input. + offsets: [123.6750, 116.2800, 103.5300] # Array of values to be subtracted from each input channel, with length equal to number of channels + netScaleFactor: 0.01735207 # Scaling factor for reid network input after substracting offsets + keepAspc: 1 # Whether to keep aspc ratio when resizing input objects for reid + useVPICropScaler: 1 # Use VPI backend crop and scaler + + # [Output Postprocessing] + addFeatureNormalization: 1 # If reid feature is not normalized in network, adding normalization on output so each reid feature has l2 norm equal to 1 + minVisibility4GalleryUpdate: 0.6 # Add ReID embedding to the gallery only if the visibility is not lower than this + + # [Paths and Names] + tltEncodedModel: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/resnet50_market1501.etlt" # NVIDIA TAO model path + tltModelKey: "nvidia_tao" # NVIDIA TAO model key + modelEngineFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/resnet50_market1501.etlt_b100_gpu0_fp16.engine" # Engine file path + +PoseEstimator: + poseEstimatorType: 1 # Type of pose estimator used + useVPICropScaler: 1 # Use VPI backend for cropping and scaling + batchSize: 1 # Batch size for pose estimation + workspaceSize: 1000 # Workspace size in MB for the pose estimator engine + inferDims: [3, 256, 192] # Input dimensions for the pose estimator network (C, H, W) + networkMode: 1 # Inference precision mode (fp32=0, fp16=1, int8=2) + inputOrder: 0 # Input order for the network (NCHW=0, NHWC=1) + colorFormat: 0 # Input color format (RGB=0, BGR=1) + offsets: [123.6750, 116.2800, 103.5300] # Channel-wise mean subtraction values + netScaleFactor: 0.00392156 # Scaling factor for input normalization + onnxFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/bodypose3dnet_accuracy.onnx" # Path to the ONNX model file + modelEngineFile: "/opt/nvidia/deepstream/deepstream/samples/models/Tracker/bodypose3dnet_accuracy.onnx_b1_gpu0_fp16.engine" # Path to the engine file + poseInferenceInterval: -1 # Pose inference frame interval. -1 means only for the first frame of each target and use it to determine the target height. diff --git a/src/apps/reference_apps/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt b/src/apps/reference_apps/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt new file mode 100644 index 00000000..84ef117e --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d/configs/deepstream_app_source1_3d_tracking.txt @@ -0,0 +1,107 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=3 +kitti-track-output-dir=track_results + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +nvbuf-memory-type=0 + +[source0] +enable=1 +type=3 +uri=file://../streams/Retail02_short.mp4 +num-sources=1 +gpu-id=0 +cudadec-memtype=0 + +[sink0] +enable=1 +type=2 +sync=1 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 +qos=0 + +[sink1] +enable=1 +type=3 +container=1 +codec=1 +enc-type=0 +sync=0 +bitrate=2000000 +profile=0 +output-file=out.mp4 +source-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=2 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Serif +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +live-source=0 +batch-size=1 +batched-push-timeout=-1 +width=1920 +height=1080 +enable-padding=0 +nvbuf-memory-type=0 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +nvbuf-memory-type=0 +interval=0 +config-file=config_infer_primary.txt + +[tracker] +enable=1 +tracker-width=1920 +tracker-height=1080 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=config_tracker_NvDCF_accuracy_3D.yml +#ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +gpu-id=0 + +[tests] +file-loop=0 + diff --git a/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_osd.png b/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_osd.png new file mode 100644 index 00000000..313b3737 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_osd.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_viz.png b/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_viz.png new file mode 100644 index 00000000..2dbebcc5 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d/figures/.retail_viz.png differ diff --git a/src/apps/reference_apps/deepstream-tracker-3d/streams/Retail02_short.mp4 b/src/apps/reference_apps/deepstream-tracker-3d/streams/Retail02_short.mp4 new file mode 100644 index 00000000..a88268b8 Binary files /dev/null and b/src/apps/reference_apps/deepstream-tracker-3d/streams/Retail02_short.mp4 differ diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/README.md b/src/apps/reference_apps/deepstream-vllm-plugin/README.md new file mode 100644 index 00000000..4888abb0 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/README.md @@ -0,0 +1,284 @@ +# VLLM DeepStream Plugin + +A GStreamer plugin for NVIDIA DeepStream that integrates Vision-Language Models (VLM) using VLLM for real-time video understanding and analysis. + +## Table of Contents + +- [Overview](#overview) +- [Key Features](#key-features) +- [Requirements](#requirements) +- [Quick Start](#quick-start) +- [Configuration](#configuration) +- [Model Support](#model-support) + +## Overview + +The `nvvllmvlm` plugin enables integration of vision-language models (Cosmos-Reason2, etc.) into DeepStream pipelines. It processes video frames in configurable time segments and performs batch VLM inference asynchronously. + +### Key Features + +- **Segment-Based Processing**: Collects frames into time windows for batch inference +- **Flexible Frame Sampling**: FPS-based or interval-based subsampling +- **Async Inference**: Background worker thread for non-blocking processing +- **Multi-Stream Support**: One plugin instance handles multiple streams efficiently +- **Per-Stream Prompts**: Different prompts and settings for each stream +- **Configurable Models**: Supports video-native and image-only VLM models +- **GPU-Optimized**: Zero-copy GPU operations, shared model across streams +- **Flexible Input Formats**: PyTorch tensors, PIL Images, or numpy arrays + +## Requirements + +- NVIDIA GPU with CUDA support +- NVIDIA DeepStream SDK 9.0.0+ +- Docker with NVIDIA Container Runtime +- Python 3.12+ +- 40GB+ GPU memory required +- Currently supported on x86 based GPU platforms + +## Quick Start + +### Installation + +```bash +# Launch DeepStream container +sudo docker run -it --rm --runtime=nvidia --gpus all --network=host \ + -v $(pwd):/home/vllm_ds_plugin \ + nvcr.io/nvidia/deepstream:9.0-triton-multiarch + +# Inside DeepStream container install dependencies +cd /home/vllm_ds_plugin/deepstream-vllm-plugin +./install.sh +``` + +### Steps to get Hugging face Token + +1. Log in at huggingface.co +2. Go to Profile → Access Tokens +3. Create and save the generated token +4. To use cosmos-reason2 model, go to https://huggingface.co/nvidia/Cosmos-Reason2-8B, review and agree Nvidia Open Model License Agreement + +### Single Stream and Multi-Stream Processing +```bash +# Run the following inside DeepStream container: + +# Select GPU (optional) +export CUDA_VISIBLE_DEVICES=0 + +# Export Huggingface token from previous step to download models from HF +export HF_TOKEN= + +# Copy streams to the container or use streams that are already part of the container + +# Single stream (dry-run: results printed to console) +python3 vllm_ds_app_kafka_publish.py --dry-run + +For example, +python3 vllm_ds_app_kafka_publish.py /opt/nvidia/deepstream/deepstream-9.0/samples/streams/sample_1080p_h264.mp4 --dry-run + +# Multi-stream with shared model (dry-run) +python3 vllm_ds_app_kafka_publish.py --dry-run +``` + +### Kafka Integration + +Stream results to Kafka in real-time: +```bash +# Bring up kafka containers by running the following on host outside the DeepStream container: + +# Start Kafka +docker compose -f docker-compose-kafka.yml up -d + +# Run kafka publishing application and consumer script inside DeepStream container: + +# Run with Kafka publishing (single or multi-stream) +python3 vllm_ds_app_kafka_publish.py \ + --kafka-bootstrap localhost:9092 --topic vlm-results + +# On another terminal start the consumer test script +python3 test_consumer.py --topic vlm-results +``` + + +## Configuration + +### Configuration File (config.yaml) + +Place `config.yaml` in the plugin directory or current working directory. + +#### Complete Example + +```yaml +# Model Configuration +model: + path: "nvidia/Cosmos-Reason2-8B" + max_model_len: 20480 # Max context length + gpu_memory_utilization: 0.7 # GPU memory fraction to use. Update depending on platform and availabe gpu memory. + trust_remote_code: true + gpu_id: 0 # GPU device ID (-1 for auto) + + # Video processing mode + video_mode: 1 # 1=video metadata (native video support), 0=multi-image mode + + # Tensor format for image inputs + tensor_format: "pytorch" # pytorch/pil/numpy + +# Segment Processing +segment: + length_sec: 30 # Segment length in seconds + overlap_sec: 0 # Segment overlap in seconds + subsample_interval: 1 # Keep every Nth frame + selection_fps: 30 # Target FPS (0 to disable FPS-based sampling) + +# Inference Configuration +inference: + # User prompt with optional placeholders: {num_frames}, {stream_id}, {timestamps} + # Control timestamp inclusion by using or omitting {timestamps} placeholder + user_prompt: "These are {num_frames} images from stream {stream_id} sampled at timestamps {timestamps}. Describe the scene in detail." + + # System prompt (optional - omit for no system prompt) + system_prompt: | + Provide captions with timestamps using format: + caption of event. + + # Sampling parameters (all optional) + max_tokens: 2048 # Max tokens to generate + temperature: 0.7 # Sampling temperature (0-1) + + # Advanced sampling parameters (optional) + # top_p: 0.9 # Top-p nucleus sampling + # top_k: 100 # Top-k sampling + # repetition_penalty: 1.1 # Repetition penalty + + # Per-stream prompt overrides (multi-stream mode) + stream_prompts: + 0: # Stream 0 + user_prompt: "Stream {stream_id} at {timestamps}: Monitor for security threats." + system_prompt: "You are a security analyst." + 1: # Stream 1 + user_prompt: "Analyze traffic flow." # No {timestamps} = no timestamps + +# Pipeline Configuration +pipeline: + queue_maxsize: 20 # Max inference queue size + max_wait_timeout: 300 # Max wait time for segment completion (seconds) + +# Video Configuration +video: + default_fps_numerator: 30 # Default FPS numerator (30/1 = 30 fps) + default_fps_denominator: 1 # Default FPS denominator (used if stream lacks FPS) +``` + +#### Key Settings + +**Video Mode**: +- `video_mode: 1` - Video metadata (For models that has native video support) +- `video_mode: 0` - Multi-image mode (image-only models) + +**Tensor Format** (image modes only): +- `tensor_format: "pytorch"` - PyTorch tensors (default) +- `tensor_format: "pil"` - PIL Images +- `tensor_format: "numpy"` - numpy arrays + +**System Prompt**: +- Specified: Uses that value +- Omitted: No system prompt (None) +- Empty string `""`: Sends empty system prompt + +**Prompt Placeholders**: +- `{num_frames}` - Number of frames in segment +- `{stream_id}` - Stream identifier +- `{timestamps}` - Timestamp string (e.g., "0.00s 1.00s 2.00s") +- Include `{timestamps}` to show timestamps, omit to exclude them + +**Per-Stream Prompts**: +- Override any inference setting for specific streams +- Streams without overrides use global defaults +- Supports all inference settings per-stream + +#### Plugin Properties + +Properties can override config values at runtime: + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `model` | string | from config | HuggingFace model ID | +| `user-prompt` | string | from config | User prompt with placeholders | +| `system-prompt` | string | None | System prompt (optional) | +| `segment-length-sec` | int | 10 | Segment length in seconds | +| `overlap-sec` | int | 0 | Segment overlap | +| `selection-fps` | int | 1 | Target FPS (0=disabled) | +| `subsample-interval` | int | 1 | Keep every Nth frame | +| `max-tokens` | int | 2048 | Max tokens to generate | +| `temperature` | float | 0.7 | Sampling temperature | +| `top-p` | float | 0.9 | Top-p nucleus sampling | +| `top-k` | int | 100 | Top-k sampling | +| `repetition-penalty` | float | 1.1 | Repetition penalty | +| `max-model-len` | int | 20480 | Max model context length | +| `trust-remote-code` | bool | true | Trust remote code | +| `gpu-memory-utilization` | float | 0.7 | GPU memory fraction (0.0-1.0) | +| `gpu-id` | int | 0 | GPU device ID (-1=auto) | +| `video-mode` | int | 1 | 1=video, 0=multi-image | +| `tensor-format` | string | pytorch | pytorch/pil/numpy | +| `queue-maxsize` | int | 20 | Inference queue size | +| `max-wait-timeout` | int | 300 | Shutdown timeout (seconds) | +| `default-fps-numerator` | int | 30 | Default FPS numerator | +| `default-fps-denominator` | int | 1 | Default FPS denominator | + + +## Model Support + +### Supported Models + +**Video-Native Models** (`video_mode: 1`): +- `Cosmos-Reason2-8B` (default) +- Models with native video metadata support + +**Image-Only Models** (`video_mode: 0`): +- Models that only support image inputs + +### Model Configuration Examples + +**Cosmos-Reason2-8B**: +```yaml +model: + path: "nvidia/Cosmos-Reason2-8B" + video_mode: 1 + tensor_format: "pytorch" +``` + +### Custom Prompts with Placeholders + +Control prompt content and timestamps using placeholders: + +**With timestamps**: +```yaml +inference: + user_prompt: "These are {num_frames} images from stream {stream_id} sampled at timestamps {timestamps}. Describe the scene." +``` + +**Without timestamps**: +```yaml +inference: + user_prompt: "Describe what you see in stream {stream_id}." +``` + +**Per-stream custom prompts**: +```yaml +inference: + stream_prompts: + 0: + user_prompt: "Stream {stream_id} at {timestamps}: Security analysis." + 1: + user_prompt: "Analyse vehicles." # Minimal, no placeholders +``` + + +### Signal-Based Results + +Access results via GObject signals: +```python +def on_vlm_result(element, stream_id, start_time, end_time, text, user_data): + print(f"Stream {stream_id} [{start_time:.2f}s-{end_time:.2f}s]: {text}") + +vlm.connect("vlm-result", on_vlm_result, None) +``` diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/config.yaml b/src/apps/reference_apps/deepstream-vllm-plugin/config.yaml new file mode 100644 index 00000000..e0efbdc2 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/config.yaml @@ -0,0 +1,107 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### +# +# Configuration for gstnvvllmvlm.py +# VLM Plugin Configuration + +# Model Configuration +model: + # HuggingFace model path/ID + path: "nvidia/Cosmos-Reason2-8B" + + # Maximum model context length + max_model_len: 20480 + + # Trust remote code when loading model + trust_remote_code: true + + # GPU memory utilization (0.0 to 1.0) + gpu_memory_utilization: 0.7 + + # GPU device ID to use (-1 for auto from CUDA_VISIBLE_DEVICES, 0-15 for specific GPU) + gpu_id: 0 + + # Video mode: 1 = use video metadata (multi-frame), 0 = image-only mode (to pass frames as images to VLLM) + # Set to 0 for models that don't support video + # Set to 1 for models that support video + video_mode: 1 + + # Tensor format for image inputs: "pytorch", "pil", or "numpy" + # Note: Video mode (video_mode=1, multi-frame) always uses PyTorch tensors + # Image modes can use any format based on model requirements + tensor_format: "pytorch" + +# Segment Processing Configuration +segment: + # Length of each segment in seconds + length_sec: 10 + + # Overlap between consecutive segments in seconds + overlap_sec: 0 + + # Keep every Nth frame (subsample interval) + subsample_interval: 1 + + # Target frames per second per segment (0 = disabled, use subsample_interval instead) + selection_fps: 1 + +# Inference Configuration +inference: + # User prompt with optional placeholders: {num_frames}, {stream_id}, {timestamps} + # Control timestamp inclusion by using or omitting {timestamps} placeholder + user_prompt: "Describe what you see in detail" + + # System prompt (optional - omit for no system prompt) + # system_prompt: + + # Sampling parameters (all optional) + # Max tokens + max_tokens: 2048 + + # Temperature + temperature: 0.7 + + # Top-p (nucleus) sampling + top_p: 0.9 + + # Top-k sampling + top_k: 100 + + # Repetition penalty + repetition_penalty: 1.1 + + # Per-stream prompt overrides (optional) + # stream_prompts: + # 0: # Stream 0 + # user_prompt: "Frames are sampled at {timestamps}: Monitor for warehouse safety hazards." + # system_prompt: "You are a warehouse analyst." + # 1: # Stream 1 + # user_prompt: "Analyse traffic flow" # No {timestamps} = no timestamps + +# Pipeline Configuration +pipeline: + # Maximum size of inference queue + queue_maxsize: 20 + + # Maximum wait time for segment completion during shutdown (seconds) + max_wait_timeout: 300 + +# Video Configuration +video: + # Default FPS if not detected from stream + default_fps_numerator: 30 + default_fps_denominator: 1 diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/config_loader.py b/src/apps/reference_apps/deepstream-vllm-plugin/config_loader.py new file mode 100644 index 00000000..73c4fb67 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/config_loader.py @@ -0,0 +1,200 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +"""Configuration loader for VLM plugin""" + +import os +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml + + +class Config: + """Configuration manager for VLM plugin""" + + def __init__(self, config_path: Optional[str] = None): + """ + Load configuration from YAML file + + Args: + config_path: Path to config file. If None, searches for + config.yaml in: + 1. Current directory + 2. Script directory + 3. Uses default values + """ + self._config = self._load_config(config_path) + + def _load_config(self, config_path: Optional[str]) -> Dict[str, Any]: + """Load config from file or use defaults""" + + # Try to find config file + if config_path is None: + # Search in common locations + search_paths = [ + Path.cwd() / "config.yaml", + Path(__file__).parent / "config.yaml", + ] + + for path in search_paths: + if path.exists(): + config_path = str(path) + break + + # Load from file if found + if config_path and os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) + + # Return empty dict if no config found (will use property defaults) + return {} + + # Model properties + @property + def model_path(self) -> str: + return self._config.get("model", {}).get( # noqa: BLK100 + "path", "nvidia/Cosmos-Reason2-8B" + ) + + @property + def max_model_len(self) -> int: + return self._config.get("model", {}).get("max_model_len", 20480) + + @property + def trust_remote_code(self) -> bool: + return self._config.get("model", {}).get("trust_remote_code", True) + + @property + def gpu_memory_utilization(self) -> float: + return self._config.get("model", {}).get("gpu_memory_utilization", 0.4) + + @property + def gpu_id(self) -> int: + return self._config.get("model", {}).get("gpu_id", 0) + + @property + def video_mode(self) -> int: + return self._config.get("model", {}).get("video_mode", 1) + + @property + def tensor_format(self) -> str: + return self._config.get("model", {}).get("tensor_format", "pytorch") + + # Segment properties + @property + def segment_length_sec(self) -> int: + return self._config.get("segment", {}).get("length_sec", 10) + + @property + def overlap_sec(self) -> int: + return self._config.get("segment", {}).get("overlap_sec", 0) + + @property + def subsample_interval(self) -> int: + return self._config.get("segment", {}).get("subsample_interval", 1) + + @property + def selection_fps(self) -> int: + return self._config.get("segment", {}).get("selection_fps", 1) + + # Inference properties + @property + def user_prompt(self) -> str: + return self._config.get("inference", {}).get( + "user_prompt", "Describe the scene in detail." + ) + + @property + def system_prompt(self) -> str: + # Return None if not specified in config (no default system prompt) + return self._config.get("inference", {}).get("system_prompt", None) + + @property + def max_tokens(self) -> int: + """Max tokens default: 2048""" + return self._config.get("inference", {}).get("max_tokens", 2048) + + @property + def temperature(self) -> float: + """Temperature default: 0.7""" + return self._config.get("inference", {}).get("temperature", 0.7) + + @property + def top_p(self) -> Optional[float]: + """Top-p (nucleus) sampling parameter""" + return self._config.get("inference", {}).get("top_p", None) + + @property + def top_k(self) -> Optional[int]: + """Top-k sampling parameter""" + value = self._config.get("inference", {}).get("top_k", None) + return int(value) if value is not None else None + + @property + def repetition_penalty(self) -> Optional[float]: + """Repetition penalty parameter""" + return self._config.get("inference", {}).get( # noqa: BLK100 + "repetition_penalty", None + ) + + @property + def stream_prompts(self) -> dict: + """ + Get per-stream prompt overrides + Returns dict: {stream_id: {setting: value, ...}} + """ + return self._config.get("inference", {}).get("stream_prompts", {}) + + # Pipeline properties + @property + def queue_maxsize(self) -> int: + return self._config.get("pipeline", {}).get("queue_maxsize", 20) + + @property + def max_wait_timeout(self) -> int: + return self._config.get("pipeline", {}).get("max_wait_timeout", 300) + + # Video properties + @property + def default_fps(self) -> tuple: + numerator = self._config.get("video", {}).get( # noqa: BLK100 + "default_fps_numerator", 30 + ) + denominator = self._config.get("video", {}).get( # noqa: BLK100 + "default_fps_denominator", 1 + ) + return (numerator, denominator) + + +# Global config instance +_config_instance: Optional[Config] = None + + +def get_config(config_path: Optional[str] = None) -> Config: + """Get or create global config instance""" + global _config_instance + if _config_instance is None: + _config_instance = Config(config_path) + return _config_instance + + +def reload_config(config_path: Optional[str] = None) -> Config: + """Reload configuration from file""" + global _config_instance + _config_instance = Config(config_path) + return _config_instance diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/docker-compose-kafka.yml b/src/apps/reference_apps/deepstream-vllm-plugin/docker-compose-kafka.yml new file mode 100644 index 00000000..2db2f1cc --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/docker-compose-kafka.yml @@ -0,0 +1,43 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### +version: '3.8' + +# RECOMMENDED: Use host networking for compatibility with DeepStream containers +# This allows the DeepStream container to access Kafka on localhost:9092 + +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.5.0 + container_name: zookeeper + network_mode: host + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:7.5.0 + container_name: kafka + network_mode: host + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: localhost:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/gstnvvllmvlm.py b/src/apps/reference_apps/deepstream-vllm-plugin/gstnvvllmvlm.py new file mode 100644 index 00000000..8b336c32 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/gstnvvllmvlm.py @@ -0,0 +1,1485 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +import gi + +gi.require_version("Gst", "1.0") +gi.require_version("GstBase", "1.0") +import multiprocessing as mp # noqa: E402 +import threading # noqa: E402 +from queue import Empty, Queue # noqa: E402 +from typing import Any, Dict, List, Optional # noqa: E402 + +import torch # noqa: E402 +from gi.repository import GObject, Gst, GstBase # noqa: E402 +from PIL import Image # noqa: E402 +from pyservicemaker import Buffer # noqa: E402 +from transformers import AutoTokenizer # noqa: E402 +from vllm import LLM, SamplingParams # noqa: E402 + +# Set multiprocessing start method for CUDA compatibility +try: + mp.set_start_method("spawn", force=True) +except RuntimeError: + pass # Already set + +# Load configuration +from config_loader import get_config # noqa: E402 + +Gst.init(None) + +GST_PLUGIN_NAME = "nvvllmvlm" + +# Load config instance +config = get_config() + +# Configuration values (can be overridden via config.yaml) +MODEL_PATH = config.model_path +DEFAULT_SEGMENT_LEN_SEC = config.segment_length_sec +DEFAULT_OVERLAP_SEC = config.overlap_sec +DEFAULT_SUBSAMPLE_INTERVAL = config.subsample_interval +DEFAULT_SELECTION_FPS = config.selection_fps + + +class BufferData: + """Frame data for VLM processing""" + + def __init__( + self, + frame_number: int, + pts: int, + dts: int, + duration: int, + tensor_gpu: torch.Tensor, + ) -> None: + self.frame_number = frame_number + self.pts = pts + self.tensor_gpu = tensor_gpu + + +class Segment: + """Temporal segment containing multiple frames""" + + def __init__( + self, stream_id: int, start_pts_ns: int, end_pts_ns: int, batch_id: int + ) -> None: + self.stream_id = stream_id # Which stream this segment belongs to + self.start_pts_ns = start_pts_ns + self.end_pts_ns = end_pts_ns + self.batch_id = batch_id + self.frames: List[BufferData] = [] + self.last_saved_pts_ns: Optional[int] = None + + +class StreamContext: + """Per-stream context for multi-stream processing""" + + def __init__(self, stream_id: int): + self.stream_id = stream_id + + # Segment management + self.open_segments: List[Segment] = [] + self.next_segment_start_pts: Optional[int] = None + self.base_pts: Optional[int] = None + self.frame_counter: int = 0 + + # Statistics + self.segments_submitted: int = 0 + self.segments_completed: int = 0 + self.segments_dropped: int = 0 + self.total_frames_in_segments: int = 0 + + # Latest result + self.latest_text: Optional[str] = None + self.latest_lock: threading.Lock = threading.Lock() + + def update_result(self, text: str, start_sec: float, end_sec: float): + """Update the latest result for this stream""" + with self.latest_lock: + self.latest_text = ( + f"[Stream {self.stream_id}] " + f"[{start_sec:.2f}s-{end_sec:.2f}s] {text}" + ) + self.segments_completed += 1 + + +class SegmentRequest: + """Request for VLM inference on a segment""" + + def __init__(self, stream_id: int, segment: Segment, prompt_config: Dict): + self.stream_id = stream_id + self.segment = segment + self.prompt_config = prompt_config + + +class NvVllmVLM(GstBase.BaseTransform): + __gstmetadata__ = ( + "NvVllmVLM", + "Generic/Analyzer", + "vLLM inference with multi-stream support", + "VSS", + ) + + src_format = Gst.Caps.from_string( + "video/x-raw(memory:NVMM), format=RGB, " + "width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], " + "framerate=(fraction)[ 0/1, 2147483647/1 ]" + ) + sink_format = Gst.Caps.from_string( + "video/x-raw(memory:NVMM), format=RGB, " + "width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], " + "framerate=(fraction)[ 0/1, 2147483647/1 ]" + ) + + src_pad_template = Gst.PadTemplate.new( + "src", Gst.PadDirection.SRC, Gst.PadPresence.ALWAYS, src_format + ) + sink_pad_template = Gst.PadTemplate.new( + "sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, sink_format + ) + __gsttemplates__ = (src_pad_template, sink_pad_template) + + __gsignals__ = { + "vlm-result": ( + GObject.SignalFlags.RUN_LAST, + None, + ( + int, + float, + float, + str, + ), + ) + # Signal emitted when VLM inference completes for a segment + # Args: stream_id (int), start_time (float), end_time (float), + # result_text (str) + } + + __gproperties__ = { + "segment-length-sec": ( + int, + "Segment Length (sec)", + "Length of each segment in seconds", + 1, + 3600, + DEFAULT_SEGMENT_LEN_SEC, + GObject.ParamFlags.READWRITE, + ), + "overlap-sec": ( + int, + "Overlap (sec)", + "Overlap between consecutive segments", + -600, + 600, + DEFAULT_OVERLAP_SEC, + GObject.ParamFlags.READWRITE, + ), + "subsample-interval": ( + int, + "Subsample Interval", + "Keep every Nth frame", + 1, + 100, + DEFAULT_SUBSAMPLE_INTERVAL, + GObject.ParamFlags.READWRITE, + ), + "selection-fps": ( + int, + "Selection FPS", + "Target frames per second per segment (0 = disabled)", + 0, + 240, + DEFAULT_SELECTION_FPS, + GObject.ParamFlags.READWRITE, + ), + "model": ( + str, + "Model", + "HuggingFace model id", + MODEL_PATH, + GObject.ParamFlags.READWRITE, + ), + "user-prompt": ( + str, + "User Prompt", + "User prompt with optional placeholders: " + "{num_frames}, {stream_id}, {timestamps}", + "Describe what you see in detail", + GObject.ParamFlags.READWRITE, + ), + "max-tokens": ( + int, + "Max Tokens", + "Maximum tokens to generate", + 1, + 8192, + 2048, + GObject.ParamFlags.READWRITE, + ), + "temperature": ( + float, + "Temperature", + "Sampling temperature", + 0.0, + 2.0, + 0.7, + GObject.ParamFlags.READWRITE, + ), + "gpu-id": ( + int, + "GPU ID", + ( + "GPU device ID to use " + "(default: 0, -1 = auto from CUDA_VISIBLE_DEVICES)" + ), + -1, + 15, + 0, + GObject.ParamFlags.READWRITE, + ), + "video-mode": ( + int, + "Video Mode", + "Video mode (1=video metadata, 0=image-only)", + 0, + 1, + 1, + GObject.ParamFlags.READWRITE, + ), + "tensor-format": ( + str, + "Tensor Format", + "Image tensor format: pytorch, pil, or numpy", + "pytorch", + GObject.ParamFlags.READWRITE, + ), + "top-p": ( + float, + "Top P", + "Top-p nucleus sampling", + 0.0, + 1.0, + 0.9, + GObject.ParamFlags.READWRITE, + ), + "top-k": ( + int, + "Top K", + "Top-k sampling", + -1, + 1000, + 100, + GObject.ParamFlags.READWRITE, + ), + "repetition-penalty": ( + float, + "Repetition Penalty", + "Repetition penalty", + 1.0, + 2.0, + 1.1, + GObject.ParamFlags.READWRITE, + ), + "max-model-len": ( + int, + "Max Model Length", + "Maximum model context length", + 512, + 65536, + 20480, + GObject.ParamFlags.READWRITE, + ), + "trust-remote-code": ( + bool, + "Trust Remote Code", + "Trust remote code when loading model", + True, + GObject.ParamFlags.READWRITE, + ), + "gpu-memory-utilization": ( + float, + "GPU Memory Utilization", + "GPU memory fraction (0.0 to 1.0)", + 0.1, + 1.0, + 0.7, + GObject.ParamFlags.READWRITE, + ), + "system-prompt": ( + str, + "System Prompt", + "System prompt for inference (optional)", + None, + GObject.ParamFlags.READWRITE, + ), + "queue-maxsize": ( + int, + "Queue Maxsize", + "Maximum size of inference queue", + 1, + 1000, + 20, + GObject.ParamFlags.READWRITE, + ), + "max-wait-timeout": ( + int, + "Max Wait Timeout", + "Maximum wait time for segment completion (seconds)", + 1, + 3600, + 300, + GObject.ParamFlags.READWRITE, + ), + "default-fps-numerator": ( + int, + "Default FPS Numerator", + "Default FPS numerator if not detected from stream", + 1, + 240, + 30, + GObject.ParamFlags.READWRITE, + ), + "default-fps-denominator": ( + int, + "Default FPS Denominator", + "Default FPS denominator if not detected from stream", + 1, + 1000, + 1, + GObject.ParamFlags.READWRITE, + ), + } + + def __init__(self) -> None: + GstBase.BaseTransform.__init__(self) + + # Segment configuration + self.segment_length_sec: int = DEFAULT_SEGMENT_LEN_SEC + self.overlap_sec: int = DEFAULT_OVERLAP_SEC + self.subsample_interval: int = DEFAULT_SUBSAMPLE_INTERVAL + self.selection_fps: int = DEFAULT_SELECTION_FPS + + # Video format info + self.width: Optional[int] = None + self.height: Optional[int] = None + self.format: Optional[str] = None + self.fps: Optional[tuple] = None # (numerator, denominator) + + # Computed values + self._step_ns: int = self._compute_step_ns() + self._seg_len_ns: int = self.segment_length_sec * 1_000_000_000 + self._sample_interval_ns: Optional[int] = None + + # Model configuration (load from config) + self.model: str = MODEL_PATH + self.user_prompt: str = config.user_prompt + self.max_tokens: int = config.max_tokens + self.temperature: float = config.temperature + self.gpu_id: int = config.gpu_id + self.video_mode: int = config.video_mode + self.tensor_format: str = config.tensor_format + # Can be None if not specified + self._system_prompt = config.system_prompt + + # Additional sampling parameters (optional) + self.top_p: Optional[float] = config.top_p + self.top_k: Optional[int] = config.top_k + self.repetition_penalty: Optional[float] = config.repetition_penalty + + # Model initialization parameters + self.max_model_len: int = config.max_model_len + self.trust_remote_code: bool = config.trust_remote_code + self.gpu_memory_utilization: float = config.gpu_memory_utilization + + # Pipeline parameters + self.queue_maxsize: int = config.queue_maxsize + self.max_wait_timeout: int = config.max_wait_timeout + + # Video parameters + self.default_fps_numerator: int = config.default_fps[0] + self.default_fps_denominator: int = config.default_fps[1] + + # Per-stream prompt overrides + self._stream_prompts: Dict[int, Dict[str, Any]] = config.stream_prompts + + # Single VLM model instance (shared across all streams naturally) + self.llm: Optional[LLM] = None + self.tokenizer: Optional[AutoTokenizer] = None + + # Per-stream contexts (keyed by pad_index/source_id) + self.stream_contexts: Dict[int, StreamContext] = {} + self.stream_contexts_lock: threading.Lock = threading.Lock() + + # Shared inference queue for all streams + self._infer_queue: Queue = Queue(maxsize=self.queue_maxsize) + self._infer_thread: Optional[threading.Thread] = None + self._stop_event: threading.Event = threading.Event() + + # Load model once + try: + Gst.info(f"{GST_PLUGIN_NAME}: Loading VLM model '{self.model}'") + + # Initialize CUDA + if torch.cuda.is_available(): + torch.cuda.init() + + # Determine which GPU to use + if self.gpu_id >= 0: + # Explicit GPU ID specified + if self.gpu_id >= torch.cuda.device_count(): + Gst.warning( + f"{GST_PLUGIN_NAME}: Requested GPU " + f"{self.gpu_id} not available. Using GPU 0 " + f"(available: {torch.cuda.device_count()} GPUs)" + ) + self.gpu_id = 0 + torch.cuda.set_device(self.gpu_id) + device_name = torch.cuda.get_device_name(self.gpu_id) + Gst.info( + f"{GST_PLUGIN_NAME}: CUDA initialized on GPU " + f"{self.gpu_id} ({device_name})" + ) + else: + # Auto-select from CUDA_VISIBLE_DEVICES + # (use device 0 of visible devices) + torch.cuda.set_device(0) + device_name = torch.cuda.get_device_name(0) + Gst.info( + f"{GST_PLUGIN_NAME}: CUDA initialized on GPU 0 " + f"(auto from CUDA_VISIBLE_DEVICES, {device_name})" + ) + else: + Gst.error(f"{GST_PLUGIN_NAME}: CUDA not available!") + raise RuntimeError("CUDA not available") + + self.llm = LLM( + model=self.model, + max_model_len=self.max_model_len, + trust_remote_code=self.trust_remote_code, + gpu_memory_utilization=self.gpu_memory_utilization, + ) + try: + self.tokenizer = AutoTokenizer.from_pretrained( + self.model, trust_remote_code=self.trust_remote_code + ) + except Exception: + self.tokenizer = None + + Gst.info(f"{GST_PLUGIN_NAME}: VLM model loaded successfully") + except Exception as e: + Gst.error(f"{GST_PLUGIN_NAME}: Failed to initialize vLLM - {e}") + import traceback + + traceback.print_exc() + + def _compute_step_ns(self) -> int: + step = max(1, (self.segment_length_sec - self.overlap_sec)) + return step * 1_000_000_000 + + def _update_sample_interval(self) -> None: + if self.selection_fps and self.selection_fps > 0: + self._sample_interval_ns = int(1_000_000_000 / self.selection_fps) + Gst.info( + f"{GST_PLUGIN_NAME}: selection_fps={self.selection_fps}, " + f"sample_interval_ns={self._sample_interval_ns}" + ) + else: + self._sample_interval_ns = None + Gst.info( + f"{GST_PLUGIN_NAME}: selection_fps disabled, using " + f"subsample-interval={self.subsample_interval}" + ) + + def do_get_property(self, prop: GObject.ParamSpec) -> Any: + if prop.name == "model": + return self.model + if prop.name == "user-prompt": + return self.user_prompt + if prop.name == "max-tokens": + return self.max_tokens + if prop.name == "temperature": + return self.temperature + if prop.name == "gpu-id": + return self.gpu_id + if prop.name == "video-mode": + return self.video_mode + if prop.name == "tensor-format": + return self.tensor_format + if prop.name == "segment-length-sec": + return self.segment_length_sec + if prop.name == "overlap-sec": + return self.overlap_sec + if prop.name == "subsample-interval": + return self.subsample_interval + if prop.name == "selection-fps": + return self.selection_fps + if prop.name == "top-p": + return self.top_p + if prop.name == "top-k": + return self.top_k + if prop.name == "repetition-penalty": + return self.repetition_penalty + if prop.name == "max-model-len": + return self.max_model_len + if prop.name == "trust-remote-code": + return self.trust_remote_code + if prop.name == "gpu-memory-utilization": + return self.gpu_memory_utilization + if prop.name == "system-prompt": + return self._system_prompt + if prop.name == "queue-maxsize": + return self.queue_maxsize + if prop.name == "max-wait-timeout": + return self.max_wait_timeout + if prop.name == "default-fps-numerator": + return self.default_fps_numerator + if prop.name == "default-fps-denominator": + return self.default_fps_denominator + msg = f"{GST_PLUGIN_NAME}: Unknown property '{prop.name}'" + raise AttributeError(msg) + + def do_set_property(self, prop: GObject.ParamSpec, value: Any) -> None: + if prop.name == "model": + self.model = value + elif prop.name == "user-prompt": + self.user_prompt = value + elif prop.name == "max-tokens": + self.max_tokens = value + elif prop.name == "temperature": + self.temperature = value + elif prop.name == "gpu-id": + self.gpu_id = int(value) + elif prop.name == "video-mode": + self.video_mode = int(value) + elif prop.name == "tensor-format": + self.tensor_format = str(value).lower() + if self.tensor_format not in ["pytorch", "pil", "numpy"]: + Gst.warning( + f"{GST_PLUGIN_NAME}: Invalid tensor-format '{value}', " + f"using 'pytorch'" + ) + self.tensor_format = "pytorch" + elif prop.name == "segment-length-sec": + self.segment_length_sec = int(value) + self._seg_len_ns = self.segment_length_sec * 1_000_000_000 + self._step_ns = self._compute_step_ns() + elif prop.name == "overlap-sec": + self.overlap_sec = int(value) + self._step_ns = self._compute_step_ns() + elif prop.name == "subsample-interval": + self.subsample_interval = max(1, int(value)) + elif prop.name == "selection-fps": + self.selection_fps = max(0, int(value)) + self._update_sample_interval() + elif prop.name == "top-p": + self.top_p = float(value) if value is not None else None + elif prop.name == "top-k": + self.top_k = int(value) if value is not None else None + elif prop.name == "repetition-penalty": + self.repetition_penalty = ( # noqa: BLK100 + float(value) if value is not None else None + ) + elif prop.name == "max-model-len": + self.max_model_len = int(value) + elif prop.name == "trust-remote-code": + self.trust_remote_code = bool(value) + elif prop.name == "gpu-memory-utilization": + self.gpu_memory_utilization = float(value) + elif prop.name == "system-prompt": + self._system_prompt = str(value) if value is not None else None + elif prop.name == "queue-maxsize": + self.queue_maxsize = int(value) + elif prop.name == "max-wait-timeout": + self.max_wait_timeout = int(value) + elif prop.name == "default-fps-numerator": + self.default_fps_numerator = int(value) + elif prop.name == "default-fps-denominator": + self.default_fps_denominator = int(value) + else: + msg = f"{GST_PLUGIN_NAME}: Unknown property '{prop.name}'" + raise AttributeError(msg) + + def do_start(self) -> bool: + if self.llm is None: + Gst.error(f"{GST_PLUGIN_NAME}: vLLM not initialized") + return False + + # Start inference worker thread + self._stop_event.clear() + self._infer_thread = threading.Thread( + target=self._inference_worker, name="vlm-worker", daemon=True + ) + self._infer_thread.start() + + self._update_sample_interval() + msg = f"{GST_PLUGIN_NAME}: Plugin started - ready for multi-stream" + Gst.info(msg) + return True + + def do_set_caps(self, incaps: Gst.Caps, outcaps: Gst.Caps) -> bool: + struct = incaps.get_structure(0) + self.width = struct.get_int("width").value + self.height = struct.get_int("height").value + self.format = struct.get_string("format") + + # get_fraction returns (success, numerator, denominator) + fps_result = struct.get_fraction("framerate") + if fps_result[0]: # success + self.fps = ( + fps_result[1], + fps_result[2], + ) # Store as tuple (numerator, denominator) + Gst.info( + f"{GST_PLUGIN_NAME}: caps set - {self.width}x{self.height} " + f"format={self.format} fps={self.fps[0]}/{self.fps[1]}" + ) + else: + self.fps = ( + self.default_fps_numerator, + self.default_fps_denominator, + ) + Gst.info( + f"{GST_PLUGIN_NAME}: caps set - {self.width}x{self.height} " + f"format={self.format} fps={self.fps[0]}/{self.fps[1]} " + f"(default)" + ) + + return True + + def _get_or_create_stream_context(self, stream_id: int) -> StreamContext: + """Get or create context for a stream""" + with self.stream_contexts_lock: + if stream_id not in self.stream_contexts: + ctx = StreamContext(stream_id) + self.stream_contexts[stream_id] = ctx + Gst.info( + f"{GST_PLUGIN_NAME}: Created context for stream " + f"{stream_id} (total streams: {len(self.stream_contexts)})" + ) + return self.stream_contexts[stream_id] + + def _ensure_segments_until( + self, ctx: StreamContext, pts_ns: int, batch_id: int + ) -> None: + """Create segments for a stream until covering pts_ns""" + if ctx.next_segment_start_pts is None: + ctx.next_segment_start_pts = pts_ns + + while ( + ctx.next_segment_start_pts is not None + and ctx.next_segment_start_pts <= pts_ns + ): + start = ctx.next_segment_start_pts + end = start + self._seg_len_ns + seg = Segment(ctx.stream_id, start, end, batch_id) + ctx.open_segments.append(seg) + Gst.debug( + f"{GST_PLUGIN_NAME}[Stream {ctx.stream_id}]: " + f"opened segment [{start/1e9:.2f}s - {end/1e9:.2f}s]" + ) + ctx.next_segment_start_pts = start + self._step_ns + + def _finalize_segments_up_to( + self, ctx: StreamContext, pts_ns: int, batch_id: int + ) -> None: + """Finalize completed segments for a stream""" + to_finalize = [] + for s in ctx.open_segments: + batch_match = s.batch_id == batch_id or batch_id is None + if s.end_pts_ns <= pts_ns and batch_match: + to_finalize.append(s) + + # Determine if we're in multi-stream mode for cleaner logging + num_streams = len(self.stream_contexts) + stream_label = f"[Stream {ctx.stream_id}]" if num_streams > 1 else "" + + for seg in to_finalize: + if seg.frames: + start_sec = seg.start_pts_ns / 1_000_000_000 + end_sec = seg.end_pts_ns / 1_000_000_000 + print( + f"{GST_PLUGIN_NAME}{stream_label}: Finalizing segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(seg.frames)} frames" + ) + + # Build stream-specific prompt config with fallback to global + prompt_config = { + "user_prompt": self._get_stream_config( + ctx.stream_id, "user_prompt", self.user_prompt + ), + "system_prompt": self._get_stream_config( + ctx.stream_id, "system_prompt", self._system_prompt + ), + "max_tokens": self._get_stream_config( + ctx.stream_id, "max_tokens", self.max_tokens + ), + "temperature": self._get_stream_config( + ctx.stream_id, "temperature", self.temperature + ), + } + + # Add optional sampling parameters if specified + top_p = self._get_stream_config( # noqa: BLK100 + ctx.stream_id, "top_p", self.top_p + ) + if top_p is not None: + prompt_config["top_p"] = top_p + + top_k = self._get_stream_config( # noqa: BLK100 + ctx.stream_id, "top_k", self.top_k + ) + if top_k is not None: + prompt_config["top_k"] = top_k + + repetition_penalty = self._get_stream_config( + ctx.stream_id, + "repetition_penalty", + self.repetition_penalty, + ) + if repetition_penalty is not None: + prompt_config["repetition_penalty"] = repetition_penalty + + request = SegmentRequest(ctx.stream_id, seg, prompt_config) + try: + self._infer_queue.put_nowait(request) + ctx.segments_submitted += 1 + ctx.total_frames_in_segments += len(seg.frames) + print( + f"{GST_PLUGIN_NAME}{stream_label}: Submitted " + f"(total: {ctx.segments_submitted})" + ) + except Exception as e: + ctx.segments_dropped += 1 + print(f"{GST_PLUGIN_NAME}{stream_label}: Dropped - {e}") + + try: + ctx.open_segments.remove(seg) + except ValueError: + pass + + def do_transform_ip(self, gst_buffer: Gst.Buffer) -> Gst.FlowReturn: + """Process batched frames from multiple streams""" + + buffer = Buffer(gst_buffer) + batch_meta = buffer.batch_meta + + if batch_meta.n_frames == 0: + return Gst.FlowReturn.OK + + # Process each frame in the batch + for frame_meta in batch_meta.frame_items: + try: + # Identify which stream this frame belongs to + stream_id = frame_meta.pad_index # or frame_meta.source_id + + # Get or create context for this stream + ctx = self._get_or_create_stream_context(stream_id) + + # Process frame for this stream's context + ctx.frame_counter += 1 + subsample_mod = ctx.frame_counter % self.subsample_interval + keep_by_subsample = subsample_mod == 0 + + current_pts = frame_meta.buffer_pts + + # Ensure segments and finalize completed ones + batch_id = frame_meta.batch_id + self._ensure_segments_until(ctx, current_pts, batch_id) + prev_pts = current_pts - 1 + self._finalize_segments_up_to(ctx, prev_pts, batch_id) + + # Extract frame data + tensor = buffer.extract(batch_id) + torch_frame_data = torch.utils.dlpack.from_dlpack(tensor) + + if torch_frame_data is None or torch_frame_data.numel() == 0: + continue + + bd_created = False + bd = None + + # Add frame to appropriate segments for this stream + if ( + self._sample_interval_ns is not None + and self._sample_interval_ns > 0 + ): + # FPS-based sampling + for seg in ctx.open_segments: + if seg.batch_id != frame_meta.batch_id: + continue + if seg.start_pts_ns <= current_pts <= seg.end_pts_ns: + should_keep = ( + seg.last_saved_pts_ns is None + or (current_pts - seg.last_saved_pts_ns) + >= self._sample_interval_ns + ) + if should_keep: + if not bd_created: + bd = BufferData( + -1, + current_pts, + -1, + -1, + torch_frame_data.clone(), + ) + bd_created = True + seg.frames.append(bd) + seg.last_saved_pts_ns = current_pts + else: + # Interval-based sampling + if keep_by_subsample: + bd = BufferData( + -1, current_pts, -1, -1, torch_frame_data.clone() + ) + for seg in ctx.open_segments: + start = seg.start_pts_ns + end = seg.end_pts_ns + if start <= current_pts <= end: + seg.frames.append(bd) + seg.last_saved_pts_ns = current_pts + + except Exception as e: + msg = f"{GST_PLUGIN_NAME}: Frame processing failed - {e}" + Gst.warning(msg) + + return Gst.FlowReturn.OK + + def _get_stream_config( # noqa: BLK100 + self, stream_id: int, setting: str, default: Any + ) -> Any: + """ + Get config value for a specific stream with fallback to global + + Priority: + 1. Stream-specific setting (if exists in stream_prompts) + 2. Global setting (self.setting) + 3. Default value + + Args: + stream_id: Stream identifier + setting: Setting name (e.g., 'user_prompt', 'system_prompt') + default: Default value if not found + + Returns: + Config value for this stream + """ + # Check if stream has specific override + if ( + stream_id in self._stream_prompts + and setting in self._stream_prompts[stream_id] + ): + value = self._stream_prompts[stream_id][setting] + Gst.debug( + f"{GST_PLUGIN_NAME}[Stream {stream_id}]: " + f"Using stream-specific {setting}: {value}" + ) + return value + + # Fall back to global setting or default + return default + + def _format_user_prompt( + self, + user_prompt: str, + stream_id: int, + num_frames: int, + timestamps: str, + ) -> str: + """ + Format user prompt by replacing placeholders. + + Available placeholders: + - {num_frames}: Number of frames in segment + - {stream_id}: Stream identifier + - {timestamps}: Timestamp string (e.g., "0.00s 1.00s 2.00s") + + Args: + user_prompt: User's prompt string with placeholders + stream_id: Stream ID + num_frames: Number of frames + timestamps: Timestamp string + + Returns: + Formatted prompt string + """ + try: + return user_prompt.format( + num_frames=num_frames, + stream_id=stream_id, + timestamps=timestamps, + ) + except KeyError as e: + msg = f"{GST_PLUGIN_NAME}: Invalid placeholder in user_prompt: {e}" + Gst.warning(msg) + return user_prompt + + def _convert_tensor_to_format( + self, tensor: torch.Tensor, target_format: str + ): # noqa: BLK100 + """ + Convert PyTorch tensor to specified format (pytorch, pil, or numpy) + + Args: + tensor: PyTorch tensor with shape [C, H, W], RGB format + target_format: "pytorch", "pil", or "numpy" + + Returns: + Converted tensor in requested format + """ + if target_format == "pytorch": + return tensor.cpu() + + elif target_format == "pil": + # Convert [C, H, W] to [H, W, C] for PIL + tensor = tensor.cpu() + + # Handle different dtypes + if tensor.dtype in (torch.float32, torch.float16): + # Assume normalized [0, 1], convert to [0, 255] uint8 + tensor = (tensor * 255).clamp(0, 255).byte() + elif tensor.dtype == torch.uint8: + # Already uint8 + pass + else: + # Convert to uint8 + tensor = tensor.byte() + + # Convert to numpy and create PIL Image + # [C, H, W] -> [H, W, C] + np_array = tensor.permute(1, 2, 0).numpy() + return Image.fromarray(np_array, mode="RGB") + + elif target_format == "numpy": + # Convert to numpy, keep shape [C, H, W] + return tensor.cpu().numpy() + + else: + Gst.warning( + f"{GST_PLUGIN_NAME}: Unknown tensor format " + f"'{target_format}', using pytorch" + ) + return tensor.cpu() + + def _inference_worker(self) -> None: + """Worker thread processes segments from all streams""" + print(f"{GST_PLUGIN_NAME}: Inference worker started (multi-stream)") + + while not self._stop_event.is_set(): + try: + request = self._infer_queue.get(timeout=0.1) + except Empty: + continue + + stream_id = request.stream_id + segment = request.segment + start_sec = segment.start_pts_ns / 1_000_000_000 + end_sec = segment.end_pts_ns / 1_000_000_000 + + # Determine stream label based on number of active streams + num_streams = len(self.stream_contexts) + stream_label = f"[Stream {stream_id}]" if num_streams > 1 else "" + + msg = ( + f"{GST_PLUGIN_NAME}{stream_label}: Processing segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(segment.frames)} frames" + ) + print(msg) + + try: + result_text = self._run_vlm_batch( + segment, request.prompt_config + ) # noqa: BLK100 + if result_text: + ctx = self.stream_contexts.get(stream_id) + if ctx: + ctx.update_result(result_text, start_sec, end_sec) + print( + f"{GST_PLUGIN_NAME}{stream_label}: Completed " + f"(total: {ctx.segments_completed})" + ) + # Print full result with timestamp prefix + print( + f"{GST_PLUGIN_NAME}{stream_label}: Result: " + f"{start_sec:.2f}s-{end_sec:.2f}s {result_text}" + ) + + # Emit signal with result + self.emit( + "vlm-result", + stream_id, + start_sec, + end_sec, + result_text, + ) + else: + print( + f"{GST_PLUGIN_NAME}{stream_label}: " + f"VLM returned empty result" + ) + except Exception as e: + msg = f"{GST_PLUGIN_NAME}{stream_label}: Worker error - {e}" + print(msg) + import traceback + + traceback.print_exc() + finally: + self._infer_queue.task_done() + + print(f"{GST_PLUGIN_NAME}: Inference worker stopped") + + def _run_vlm_batch( + self, segment: Segment, prompt_config: Dict + ) -> Optional[str]: # noqa: BLK100 + """Run VLM inference on a segment""" + if self.llm is None: + return None + + try: + # Collect frame tensors and timestamps + frame_tensors = [] + frame_times = [] + + for i, frame_data in enumerate(segment.frames): + tensor = frame_data.tensor_gpu + if tensor.dim() == 3: + tensor = tensor.permute(2, 0, 1) + frame_tensors.append(tensor) + frame_time_sec = frame_data.pts / 1_000_000_000.0 + frame_times.append(frame_time_sec) + + if not frame_tensors: + return None + + # Stack into batch + batch_tensor = torch.stack(frame_tensors) + + # Calculate FPS + if len(frame_times) > 1: + time_diff = frame_times[-1] - frame_times[0] + num_intervals = len(frame_times) - 1 + fps = num_intervals / time_diff if time_diff > 0 else 1.0 + else: + fps = 1.0 + + # Video metadata + if len(frame_times) > 1: + duration = frame_times[-1] - frame_times[0] + else: + duration = 0.0 + video_metadata = { + "total_num_frames": len(frame_tensors), + "frames_indices": [int(t * fps) for t in frame_times], + "fps": fps, + "duration": duration, + } + + # Build timestamp string + string_of_times = " ".join([f"{t:.2f}s" for t in frame_times]) + num_frames = len(frame_tensors) + + # Build SamplingParams with required parameters + sampling_params_dict = { + "temperature": prompt_config.get("temperature", 0.2), + "max_tokens": prompt_config.get("max_tokens", 64), + } + + # Add optional parameters if specified + if "top_p" in prompt_config and prompt_config["top_p"] is not None: + sampling_params_dict["top_p"] = prompt_config["top_p"] + + if "top_k" in prompt_config and prompt_config["top_k"] is not None: + sampling_params_dict["top_k"] = prompt_config["top_k"] + + if ( + "repetition_penalty" in prompt_config + and prompt_config["repetition_penalty"] is not None + ): + sampling_params_dict["repetition_penalty"] = prompt_config[ + "repetition_penalty" + ] + + sampling_params = SamplingParams(**sampling_params_dict) + + # Use chat template + has_chat_template = hasattr(self.tokenizer, "apply_chat_template") + if self.tokenizer and has_chat_template: + # Determine mode: video_mode=1 uses video input (all frames), + # video_mode=0 uses multi-image input + if self.video_mode == 0: + # Image mode: pass all frames as separate images + # Get user prompt with default + user_prompt = prompt_config.get( + "user_prompt", + "These are {num_frames} images from stream " + "{stream_id} sampled at timestamps {timestamps}. " + "Describe the scene in detail.", + ) + + # Format prompt with placeholders + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + # Build content with text + multiple images + content = [{"type": "text", "text": prompt_text}] + for i in range(num_frames): + img_entry = {"type": "image", "image": f"frame{i}.jpg"} + content.append(img_entry) + + # Build messages - only include system prompt if not None + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": content}, + ] + else: + messages = [ + {"role": "user", "content": content}, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + # Convert frames to target format (pytorch/pil/numpy) + image_tensors = [ + self._convert_tensor_to_format( + batch_tensor[i], self.tensor_format + ) + for i in range(num_frames) + ] + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"image": image_tensors}, + } + + elif num_frames == 1: + # Single frame in video_mode=1: use image input. + # Video processors (e.g. Qwen3VL) require >=2 frames; + # passing as image avoids that constraint. + user_prompt = prompt_config.get( + "user_prompt", + "This is an image from stream {stream_id} at " + "timestamp {timestamps}. Describe the scene.", + ) + + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "image", "image": "frame.jpg"}, + ], + }, + ] + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "image", "image": "frame.jpg"}, + ], + }, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + # Convert [C, H, W] tensor to [H, W, C] numpy array. + # vLLM interprets a torch.Tensor in image mm_data as + # pre-computed image_embeds; numpy avoids that error. + image_np = ( + batch_tensor[0].cpu().permute(1, 2, 0).numpy() + ) + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"image": image_np}, + } + + else: + # Video mode - multiple frames with video metadata + # Get user prompt with default + user_prompt = prompt_config.get( + "user_prompt", + "This is a video from stream {stream_id} with " + "{num_frames} frames sampled at timestamps " + "{timestamps}. Describe the video content.", + ) + + # Format prompt with placeholders + prompt_text = self._format_user_prompt( + user_prompt, + segment.stream_id, + num_frames, + string_of_times, + ) + + # Build messages - only include system prompt if not None + system_prompt = prompt_config.get( + "system_prompt", self._system_prompt + ) + if system_prompt is not None: + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "video", "video": "segment.mp4"}, + ], + }, + ] + else: + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + {"type": "video", "video": "segment.mp4"}, + ], + }, + ] + + prompt_text = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + video_input = (batch_tensor.cpu(), video_metadata) + inputs = { + "prompt": prompt_text, + "multi_modal_data": {"video": video_input}, + } + + outputs = self.llm.generate( + inputs, sampling_params=sampling_params + ) # noqa: BLK100 + if outputs: + return outputs[0].outputs[0].text + + return None + + except Exception as e: + msg = f"{GST_PLUGIN_NAME}: VLM inference failed - {e}" + print(msg) + import traceback + + traceback.print_exc() + return None + + def do_stop(self) -> bool: + """Stop processing and finalize all streams""" + n = len(self.stream_contexts) + lbl = "stream" if n == 1 else "stream(s)" + print(f"{GST_PLUGIN_NAME}: Stopping - finalizing {n} {lbl}") + + # Finalize remaining segments for each stream + with self.stream_contexts_lock: + for stream_id, ctx in self.stream_contexts.items(): + ctx_label = f"[Stream {stream_id}]" if n > 1 else "" + + if ctx.open_segments: + print( + f"{GST_PLUGIN_NAME}{ctx_label}: Finalizing " + f"{len(ctx.open_segments)} remaining segments" + ) + for seg in ctx.open_segments[:]: + if seg.frames: + start_sec = seg.start_pts_ns / 1_000_000_000 + end_sec = seg.end_pts_ns / 1_000_000_000 + pts_diff = seg.frames[-1].pts - seg.frames[0].pts + if len(seg.frames) > 1: + duration_sec = pts_diff / 1_000_000_000 + else: + duration_sec = 0 + + msg = ( + f"{GST_PLUGIN_NAME}{ctx_label}: Submitting " + f"incomplete segment " + f"[{start_sec:.2f}s - {end_sec:.2f}s] " + f"with {len(seg.frames)} frames " + f"(duration: {duration_sec:.2f}s)" + ) + print(msg) + + prompt_config = { + "user_prompt": self.user_prompt, + "system_prompt": self._system_prompt, + "max_tokens": self.max_tokens, + "temperature": self.temperature, + } + + # Add optional sampling parameters if specified + if self.top_p is not None: + prompt_config["top_p"] = self.top_p + if self.top_k is not None: + prompt_config["top_k"] = self.top_k + if self.repetition_penalty is not None: + prompt_config["repetition_penalty"] = ( + self.repetition_penalty + ) + + request = SegmentRequest( + stream_id, seg, prompt_config + ) # noqa: BLK100 + try: + self._infer_queue.put_nowait(request) + ctx.segments_submitted += 1 + ctx.total_frames_in_segments += len(seg.frames) + except Exception: + ctx.segments_dropped += 1 + + # Wait for queue to drain AND all processing to complete + import time + + max_wait = self.max_wait_timeout + elapsed = 0.0 + + print(f"{GST_PLUGIN_NAME}: Waiting for all segments to complete...") + + while elapsed < max_wait: + # Check both queue and per-stream completion status + queue_empty = self._infer_queue.empty() + + all_complete = True + pending_count = 0 + with self.stream_contexts_lock: + for ctx in self.stream_contexts.values(): + pending = ( + ctx.segments_submitted + - ctx.segments_completed + - ctx.segments_dropped + ) + if pending > 0: + all_complete = False + pending_count += pending + + if queue_empty and all_complete: + print( + f"{GST_PLUGIN_NAME}: All segments completed " + f"successfully (waited {elapsed:.1f}s)" + ) + break + + time.sleep(0.5) + elapsed += 0.5 + + if int(elapsed) % 10 == 0 and pending_count > 0: + print( + f"{GST_PLUGIN_NAME}: Still processing... " + f"{pending_count} segment(s) remaining " + f"({elapsed:.0f}s elapsed)" + ) + + if not all_complete: + print( + f"{GST_PLUGIN_NAME}: WARNING: Timeout waiting for " + f"segments. Some segments may not be fully processed." + ) + + # Give a small grace period to ensure any final result updates + # are complete + time.sleep(0.5) + + # Stop worker thread + print(f"\n{GST_PLUGIN_NAME}: Stopping inference worker...") + self._stop_event.set() + if self._infer_thread: + self._infer_thread.join(timeout=5.0) + self._infer_thread = None + + # Shut down vLLM engine to release GPU memory + if self.llm is not None: + print(f"{GST_PLUGIN_NAME}: Shutting down vLLM engine...") + try: + if hasattr(self.llm, "shutdown"): + self.llm.shutdown() + elif hasattr(self.llm, "llm_engine") and hasattr( + self.llm.llm_engine, "shutdown" + ): + self.llm.llm_engine.shutdown() + except Exception as e: + print( + f"{GST_PLUGIN_NAME}: Warning: vLLM shutdown error: {e}" + ) + finally: + self.llm = None + + # Print statistics for each stream + # (AFTER worker stops to ensure final counts) + num_streams = len(self.stream_contexts) + if num_streams == 1: + stats_label = "Statistics" + else: + stats_label = "Multi-Stream Statistics" + print(f"\n{GST_PLUGIN_NAME}: Final {stats_label}:") + + with self.stream_contexts_lock: + for stream_id, ctx in self.stream_contexts.items(): + # For single stream, omit "Stream 0:" for cleaner output + if num_streams == 1: + print(f" Frames processed: {ctx.frame_counter}") + print(f" Segments submitted: {ctx.segments_submitted}") + print(f" Segments completed: {ctx.segments_completed}") + print(f" Segments dropped: {ctx.segments_dropped}") + total_frames = ctx.total_frames_in_segments + print(f" Total frames in segments: {total_frames}") + if ctx.segments_submitted > 0: + n_frames = ctx.total_frames_in_segments + n_segs = ctx.segments_submitted + avg_frames = n_frames / n_segs + print(f" Avg frames per segment: {avg_frames:.1f}") + else: + print(f"\n Stream {stream_id}:") + print(f" Frames processed: {ctx.frame_counter}") + print(f" Segments submitted: {ctx.segments_submitted}") + print(f" Segments completed: {ctx.segments_completed}") + print(f" Segments dropped: {ctx.segments_dropped}") + total_frames = ctx.total_frames_in_segments + print(f" Total frames in segments: {total_frames}") + if ctx.segments_submitted > 0: + n_frames = ctx.total_frames_in_segments + n_segs = ctx.segments_submitted + avg_frames = n_frames / n_segs + print(f" Avg frames per segment: {avg_frames:.1f}") + + print(f"{GST_PLUGIN_NAME}: Shutdown complete") + return True + + +GObject.type_register(NvVllmVLM) +__gstelementfactory__ = (GST_PLUGIN_NAME, Gst.Rank.NONE, NvVllmVLM) diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/install.sh b/src/apps/reference_apps/deepstream-vllm-plugin/install.sh new file mode 100644 index 00000000..7ae7754e --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/install.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Script to set up environment for VLLM DS plugin + +set -e # Exit immediately if a command exits with a non-zero status + +# 1) Install Python dependencies from requirements.txt +# --ignore-installed forces reinstall, useful if system packages conflict +pip install -r requirements.txt --ignore-installed + +# 2) Update apt package index to get latest package metadata +apt update + +# 3) Install Python GObject Introspection bindings (used with GStreamer) +apt install -y python3-gi + +# 4) Install GStreamer Python 3 bindings +apt install -y python3-gst-1.0 + +# 5) Install the GStreamer Python plugin loader +apt install -y gstreamer1.0-python3-plugin-loader + diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/requirements.txt b/src/apps/reference_apps/deepstream-vllm-plugin/requirements.txt new file mode 100644 index 00000000..4adf55c8 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/requirements.txt @@ -0,0 +1,5 @@ +vllm==0.15.1 +torch==2.9.1 +kafka-python +PyYAML +Pillow diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/test_consumer.py b/src/apps/reference_apps/deepstream-vllm-plugin/test_consumer.py new file mode 100644 index 00000000..2355c004 --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/test_consumer.py @@ -0,0 +1,229 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +""" +Kafka consumer for VLM results. +Continuously listens for VLM inference results and prints them in real-time. +""" + +import argparse +import json +import sys +import uuid +from datetime import datetime + +try: + from kafka import KafkaConsumer +except ImportError: + print("Error: kafka-python not installed") + print("Install with: pip install kafka-python") + sys.exit(1) + + +def main(): + # Parse arguments + parser = argparse.ArgumentParser( + description="Kafka consumer for VLM results", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Listen for messages with 5-minute timeout (default behavior) + python3 test_consumer.py + + # Specify custom Kafka broker and topic + python3 test_consumer.py --broker localhost:9092 --topic vlm-result + + # Read all existing messages from beginning with a fresh consumer group + python3 test_consumer.py --reset --topic vlm-result + + # Keep running forever until Ctrl+C (no timeout) + python3 test_consumer.py --timeout 0 + + # Exit after 30 seconds of no new messages + python3 test_consumer.py --timeout 30000 + + # Start from latest (only read new messages going forward) + python3 test_consumer.py --from-latest + """, + ) + parser.add_argument( + "--broker", + default="localhost:9092", + help="Kafka bootstrap servers (default: localhost:9092)", + ) + parser.add_argument( + "--topic", + default="vlm-results", + help="Kafka topic name (default: vlm-results)", + ) + parser.add_argument( + "--reset", + action="store_true", + help="Use unique consumer group to read all messages from beginning", + ) + parser.add_argument( + "--timeout", + type=int, + default=300000, + help=( + "Exit after N milliseconds of no messages " + "(default: 300000 = 5 minutes, use 0 for no timeout)" + ), + ) + parser.add_argument( + "--from-latest", + action="store_true", + help="Start from latest messages instead of earliest", + ) + + args = parser.parse_args() + + bootstrap_servers = args.broker + topic = args.topic + + # Consumer group + if args.reset: + group_id = f"vlm-consumer-{uuid.uuid4().hex[:8]}" + print( + "🔄 RESET MODE: Using unique consumer group " # noqa: BLK100 + "(will read from beginning)" + ) + else: + group_id = "vlm-consumer-group" + + # Offset reset + offset_reset = "latest" if args.from_latest else "earliest" + + # Handle timeout (0 means no timeout) + timeout_ms = None if args.timeout == 0 else args.timeout + + print("Starting Kafka consumer...") + print(f" Bootstrap servers: {bootstrap_servers}") + print(f" Topic: {topic}") + print(f" Consumer group: {group_id}") + print(f" Reading from: {offset_reset}") + if timeout_ms: + timeout_mins = timeout_ms / 60000 + print( + f" Timeout: {timeout_ms}ms ({timeout_mins:.1f} minutes - " + "will exit if no messages)" + ) + else: + print(" Mode: Continuous (will keep running until Ctrl+C)") + print("=" * 80) + + try: + consumer = KafkaConsumer( + topic, + bootstrap_servers=bootstrap_servers, + value_deserializer=lambda m: json.loads(m.decode("utf-8")), + auto_offset_reset=offset_reset, + enable_auto_commit=True, + group_id=group_id, + consumer_timeout_ms=timeout_ms, # None = run forever + ) + except Exception as e: + print(f"\n✗ Failed to connect to Kafka: {e}") + print("\nTroubleshooting:") + print(" 1. Ensure Kafka is running: docker ps | grep kafka") + print(" 2. Check connection: telnet localhost 9092") + print( # noqa: BLK100 + " 3. Start Kafka: " # noqa: BLK100 + "docker-compose -f docker-compose-kafka.yml up -d" + ) + sys.exit(1) + + print("\n✓ Connected successfully!") + print("Listening for messages... (Press Ctrl+C to stop)\n") + print("=" * 80) + + message_count = 0 + start_time = datetime.now() + + try: + for message in consumer: + message_count += 1 + data = message.value + + timestamp = datetime.now().strftime('%H:%M:%S') # noqa: BLK100 + print(f"\n📥 Message #{message_count} [{timestamp}]") + print("─" * 80) + print(" Kafka Metadata:") + print(f" Partition: {message.partition}") + print(f" Offset: {message.offset}") + key_str = ( + message.key.decode('utf-8') if message.key else 'None' + ) + print(f" Key: {key_str}") + msg_time = datetime.fromtimestamp( + message.timestamp / 1000 + ).strftime('%Y-%m-%d %H:%M:%S') + print(f" Timestamp: {msg_time}") + print("\n VLM Result:") + print(f" Stream ID: {data['stream_id']}") + start_t = data['segment']['start_time'] + end_t = data['segment']['end_time'] + print(f" Time Range: {start_t:.1f}s - {end_t:.1f}s") + print(f" Duration: {data['segment']['duration']:.1f}s") + + # Format result text with wrapping + result_text = data["result"] + if len(result_text) > 200: + # Wrap long text + print(" Result:") + for i in range(0, len(result_text), 100): + print(f" {result_text[i:i+100]}") + else: + print(f" Result: {result_text}") + + print("\n Metadata:") + print(f" Source: {data['metadata']['source']}") + print(f" Version: {data['metadata']['version']}") + print(f" Publish Time: {data.get('timestamp', 'N/A')}") + print("─" * 80) + + # If we reach here naturally (not via exception), it's a timeout + if timeout_ms: + timeout_secs = timeout_ms / 1000 + print( + f"\n⏱️ Timeout reached " + f"({timeout_secs:.0f}s of no new messages)" + ) + + except KeyboardInterrupt: + print("\n\n⚠️ Interrupted by user (Ctrl+C)") + except Exception as e: + print(f"\n✗ Error consuming messages: {e}") + import traceback + + traceback.print_exc() + + consumer.close() + + elapsed = (datetime.now() - start_time).total_seconds() + print("\n" + "=" * 80) + print("Session Summary") + print("=" * 80) + print(f" Messages consumed: {message_count}") + print(f" Duration: {elapsed:.1f}s") + if message_count > 0 and elapsed > 0: + print(f" Rate: {message_count / elapsed:.2f} messages/sec") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/src/apps/reference_apps/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py b/src/apps/reference_apps/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py new file mode 100644 index 00000000..e3d1494d --- /dev/null +++ b/src/apps/reference_apps/deepstream-vllm-plugin/vllm_ds_app_kafka_publish.py @@ -0,0 +1,500 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +""" +DeepStream VLM application with Kafka publishing and signal-based result +handling. +Supports both single-stream and multi-stream processing with file and RTSP +sources. + +Features: +- Single-stream and multi-stream VLM processing +- Real-time result delivery via GObject signals +- Kafka topic publishing for downstream processing +- Dry-run mode for testing without Kafka +- Efficient event-driven architecture +- File and RTSP source support via uridecodebin +""" + +import json +import os +import sys +import time +from typing import Optional + +import gi +gi.require_version("Gst", "1.0") # noqa: E402, I003, BLK100 +from gi.repository import GLib, Gst # noqa: E402, I003 + +# Register the custom plugin +import gstnvvllmvlm # noqa: E402 + +Gst.Element.register(None, "nvvllmvlm", Gst.Rank.NONE, gstnvvllmvlm.NvVllmVLM) + +# Kafka imports (with graceful fallback) +try: + from kafka import KafkaProducer + from kafka.errors import KafkaError + + KAFKA_AVAILABLE = True +except ImportError: + KAFKA_AVAILABLE = False + print("Warning: kafka-python not installed. Run: pip install kafka-python") + + +def to_uri(path_or_uri: str) -> str: + """Convert a file path or URI string to a GStreamer-compatible URI.""" + if "://" in path_or_uri: + return path_or_uri + return "file://" + os.path.abspath(path_or_uri) + + +class VLMKafkaSignalPublisher: + """ + Kafka publisher that uses GObject signals to receive VLM results. + More efficient than polling - publishes immediately when results are + available. + """ + + def __init__(self, kafka_config: dict, topic: str, dry_run: bool = False): + """ + Initialize Kafka publisher. + + Args: + kafka_config: Kafka connection configuration + topic: Topic name to publish to + dry_run: If True, print messages instead of sending to Kafka + """ + self.topic = topic + self.dry_run = dry_run + self.producer: Optional[KafkaProducer] = None + self.messages_sent = 0 + self.messages_failed = 0 + + # Initialize Kafka producer + if not dry_run and KAFKA_AVAILABLE: + try: + self.producer = KafkaProducer( + bootstrap_servers=kafka_config.get( + "bootstrap_servers", "localhost:9092" + ), + value_serializer=lambda v: json.dumps(v).encode("utf-8"), + key_serializer=lambda k: k.encode("utf-8") if k else None, + acks="all", + retries=3, + # Required for idempotence + max_in_flight_requests_per_connection=1, + enable_idempotence=True, + compression_type="gzip", + linger_ms=100, + batch_size=16384, + ) + print(f"✓ Kafka producer initialized (topic: {self.topic})") + except Exception as e: + print(f"✗ Failed to initialize Kafka producer: {e}") + print(" Falling back to dry-run mode (console output only)") + self.dry_run = True + self.producer = None + print("✓ Dry-run mode enabled") + else: + if not KAFKA_AVAILABLE: + print("✗ Kafka not available - dry-run mode enabled") + else: + print("✓ Dry-run mode enabled (console output only)") + self.producer = None + + def on_vlm_result( + self, element, stream_id, start_time, end_time, result_text + ): + """ + Signal handler for vlm-result signal. + Called immediately when VLM inference completes. + + Args: + element: The nvvllmvlm element that emitted the signal + stream_id: Stream identifier + start_time: Segment start time in seconds + end_time: Segment end time in seconds + result_text: VLM inference result + """ + # Construct message + message = { + "stream_id": stream_id, + "timestamp": time.time(), + "segment": { + "start_time": start_time, + "end_time": end_time, + "duration": end_time - start_time, + }, + "result": result_text, + "metadata": {"source": "vllm-ds-plugin", "version": "1.0"}, + } + + # Publish to Kafka or print to console + self.publish(message, stream_id) + + def publish(self, message: dict, stream_id: int): + """ + Publish message to Kafka or print to console. + + Args: + message: Message payload + stream_id: Stream ID (used as partition key) + """ + # Use stream_id as partition key for ordering + partition_key = f"stream_{stream_id}" + + if self.dry_run or self.producer is None: + # Dry-run mode: print to console + print(f"\n{'='*80}") + print("📤 KAFKA MESSAGE (Dry-Run)") + print(f"{'='*80}") + print(f"Topic: {self.topic}") + print(f"Key: {partition_key}") + print(f"Value: {json.dumps(message, indent=2)}") + print(f"{'='*80}\n") + self.messages_sent += 1 + else: + # Send to Kafka + try: + future = self.producer.send( + self.topic, key=partition_key, value=message + ) + + # Optional: wait for acknowledgment + record_metadata = future.get(timeout=10) + + self.messages_sent += 1 + print( + f"✓ Published to Kafka: stream={stream_id}, " + f"time={message['segment']['start_time']:.1f}s-" + f"{message['segment']['end_time']:.1f}s, " + f"partition={record_metadata.partition}, " + f"offset={record_metadata.offset}" + ) + + except KafkaError as e: + self.messages_failed += 1 + print(f"✗ Kafka publish failed: {e}") + except Exception as e: + self.messages_failed += 1 + print(f"✗ Unexpected error during publish: {e}") + + def close(self): + """Close Kafka producer and print statistics""" + if self.producer: + print("\nFlushing Kafka producer...") + self.producer.flush(timeout=10) + self.producer.close() + + print(f"\n{'='*80}") + print("KAFKA PUBLISHER STATISTICS") + print(f"{'='*80}") + print(f"Messages sent: {self.messages_sent}") + print(f"Messages failed: {self.messages_failed}") + print(f"{'='*80}\n") + + +class VLMKafkaApp: + """DeepStream VLM app with Kafka publishing via signals + (single or multi-stream, file or RTSP sources)""" + + def __init__(self, input_uris, kafka_config, topic, dry_run=False): + """ + Initialize application. + + Args: + input_uris: List of GStreamer-compatible URIs (file:// or rtsp://) + kafka_config: Kafka connection configuration + topic: Kafka topic name + dry_run: If True, print messages instead of sending to Kafka + """ + self.input_uris = input_uris + self.num_sources = len(input_uris) + self.pipeline = None + self.loop = None + self.streams_eos = set() + + # Initialize Kafka publisher + self.kafka_publisher = VLMKafkaSignalPublisher( + kafka_config, topic, dry_run + ) + + def bus_call(self, bus, message, loop): + """Handle GStreamer bus messages""" + t = message.type + + if t == Gst.MessageType.EOS: + print("End-of-stream") + loop.quit() + elif t == Gst.MessageType.WARNING: + err, debug = message.parse_warning() + print(f"Warning: {err}: {debug}") + elif t == Gst.MessageType.ERROR: + err, debug = message.parse_error() + print(f"Error: {err}: {debug}") + loop.quit() + + return True + + def pad_probe_callback(self, pad, info, stream_id): + """Probe to detect per-stream EOS""" + gst_buffer = info.get_buffer() + if gst_buffer: + if gst_buffer.pts == Gst.CLOCK_TIME_NONE: + print(f"Stream {stream_id}: Received EOS") + self.streams_eos.add(stream_id) + + if len(self.streams_eos) == self.num_sources: + print(f"All {self.num_sources} stream(s) finished") + + return Gst.PadProbeReturn.OK + + def build_pipeline(self): + """Build the GStreamer pipeline. + + Uses uridecodebin per stream so that both file:// and rtsp:// URIs + are supported transparently. uridecodebin selects the appropriate + source plugin, demuxer, parser, and hardware decoder automatically. + """ + print(f"Building pipeline for {self.num_sources} source(s)...") + + has_live = any( + uri.startswith("rtsp://") or uri.startswith("rtsps://") + for uri in self.input_uris + ) + + # Create pipeline + self.pipeline = Gst.Pipeline.new("vlm-kafka-signal-pipeline") + + # Create streammux + streammux = Gst.ElementFactory.make("nvstreammux", "stream-muxer") + streammux.set_property("width", 1920) + streammux.set_property("height", 1080) + streammux.set_property("batch-size", self.num_sources) + streammux.set_property("live-source", has_live) + if not has_live: + streammux.set_property("batched-push-timeout", 4000000) + + # Add to pipeline + self.pipeline.add(streammux) + + # Pre-request mux sink pads so pad-added callbacks can link into them + mux_sink_pads = [] + for i in range(self.num_sources): + sink_pad = streammux.request_pad_simple(f"sink_{i}") + if not sink_pad: + print(f"Error: Could not get sink pad {i} from streammux") + return None + sink_pad.add_probe( + Gst.PadProbeType.BUFFER, self.pad_probe_callback, i + ) + mux_sink_pads.append(sink_pad) + + # Create one uridecodebin per source + for i, uri in enumerate(self.input_uris): + print(f" Source {i}: {uri}") + + uri_decode_bin = Gst.ElementFactory.make( + "uridecodebin", f"uri-decode-bin-{i}" + ) + if not uri_decode_bin: + print( + f"Error: Could not create uridecodebin for stream {i}" + ) + return None + + uri_decode_bin.set_property("uri", uri) + self.pipeline.add(uri_decode_bin) + + # Capture loop variables via default args + def on_pad_added( + element, + pad, + mux_sinkpad=mux_sink_pads[i], + stream_id=i, + ): + caps = pad.get_current_caps() + if not caps: + caps = pad.query_caps() + if not caps: + return + structure = caps.get_structure(0) + if "video" in structure.get_name(): + if not mux_sinkpad.is_linked(): + if pad.link(mux_sinkpad) == Gst.PadLinkReturn.OK: + print( + f" Linked uridecodebin → " + f"streammux.sink_{stream_id}" + ) + + uri_decode_bin.connect("pad-added", on_pad_added) + + # Video converter + nvvidconv = Gst.ElementFactory.make("nvvideoconvert", "convertor") + nvvidconv.set_property("nvbuf-memory-type", 0) + + # Caps filter for RGB + caps_filter = Gst.ElementFactory.make("capsfilter", "caps-filter") + caps_rgb = Gst.Caps.from_string("video/x-raw(memory:NVMM), format=RGB") + caps_filter.set_property("caps", caps_rgb) + + # VLM plugin - uses configuration from config.yaml + nvvllm = Gst.ElementFactory.make("nvvllmvlm", "vlm-infer") + + # Connect signal to Kafka publisher + nvvllm.connect("vlm-result", self.kafka_publisher.on_vlm_result) + print("✓ Connected vlm-result signal to Kafka publisher") + + # Fakesink + sink = Gst.ElementFactory.make("fakesink", "fake-sink") + sink.set_property("sync", False) + + # Add elements to pipeline + self.pipeline.add(nvvidconv) + self.pipeline.add(caps_filter) + self.pipeline.add(nvvllm) + self.pipeline.add(sink) + + # Link pipeline + streammux.link(nvvidconv) + nvvidconv.link(caps_filter) + caps_filter.link(nvvllm) + nvvllm.link(sink) + + print("Pipeline built successfully\n") + + return self.pipeline + + def run(self): + """Run the application""" + # Build pipeline + pipeline = self.build_pipeline() + + # Set up bus + bus = pipeline.get_bus() + bus.add_signal_watch() + + # Create main loop + self.loop = GLib.MainLoop() + bus.connect("message", self.bus_call, self.loop) + + # Start pipeline + print("Starting pipeline...") + pipeline.set_state(Gst.State.PLAYING) + + try: + print("Running... (Press Ctrl+C to stop)\n") + self.loop.run() + except KeyboardInterrupt: + print("\nInterrupted by user") + + # Cleanup + print("\nStopping pipeline...") + pipeline.set_state(Gst.State.NULL) + + # Close Kafka publisher + self.kafka_publisher.close() + + +def main(): + """Main entry point""" + import argparse + + parser = argparse.ArgumentParser( + description="DeepStream VLM app with Kafka publishing " + "(single-stream or multi-stream, file or RTSP sources)", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +URIs can be: + File paths: /path/to/video.mp4 (auto-converted to file:// URI) + File URIs: file:///path/to/video.mp4 + RTSP streams: rtsp://user:pass@host:port/stream + +Examples: + # Single file with dry-run (console output) + python3 vllm_ds_app_kafka_publish.py video1.mp4 --dry-run + + # RTSP stream with dry-run + python3 vllm_ds_app_kafka_publish.py rtsp://192.168.1.100:8554/stream \\ + --dry-run + + # Single file with Kafka publishing + python3 vllm_ds_app_kafka_publish.py video1.mp4 \\ + --kafka-bootstrap localhost:9092 \\ + --topic vlm-results + + # Multi-stream with mixed sources and Kafka + python3 vllm_ds_app_kafka_publish.py \\ + video1.mp4 rtsp://192.168.1.100:8554/stream \\ + --kafka-bootstrap localhost:9092 \\ + --topic vlm-results + """, + ) + + parser.add_argument( + "sources", + nargs="+", + help="Video file paths or URIs to process (file paths, file://, rtsp://)", + ) + parser.add_argument( + "--kafka-bootstrap", + default="localhost:9092", + help="Kafka bootstrap servers (default: localhost:9092)", + ) + parser.add_argument( + "--topic", + default="vlm-results", + help="Kafka topic name (default: vlm-results)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print messages to console instead of sending to Kafka", + ) + + args = parser.parse_args() + + # Initialize GStreamer + Gst.init(None) + + # Convert bare file paths to file:// URIs; validate files exist + input_uris = [] + for src in args.sources: + uri = to_uri(src) + if uri.startswith("file://"): + file_path = uri[len("file://"):] + if not os.path.exists(file_path): + print(f"Error: File not found: {file_path}") + sys.exit(1) + input_uris.append(uri) + + # Kafka configuration + kafka_config = {"bootstrap_servers": args.kafka_bootstrap} + + # Create and run app + app = VLMKafkaApp( + input_uris=input_uris, + kafka_config=kafka_config, + topic=args.topic, + dry_run=args.dry_run, + ) + app.run() + + +if __name__ == "__main__": + main() diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/Makefile b/src/apps/reference_apps/deepstream_parallel_inference_app/Makefile new file mode 100644 index 00000000..fbb5968f --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/Makefile @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +SUBDIR := tritonclient/sample + +all: + @$(MAKE) -C $(SUBDIR) + +install: + @$(MAKE) install -C $(SUBDIR) + +clean: + @$(MAKE) clean -C $(SUBDIR) diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/README.md b/src/apps/reference_apps/deepstream_parallel_inference_app/README.md new file mode 100644 index 00000000..ae6be56c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/README.md @@ -0,0 +1,230 @@ +# Parallel Multiple Models App +## Introduction +The parallel inferencing application constructs the parallel inferencing branches pipeline as the following graph, so that the multiple models can run in parallel in one piepline. + +![Pipeline_Diagram](common.png) + +## Main Features + +* Support multiple models inference with [nvinfer](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinfer.html)(TensorRT) or [nvinferserver](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinferserver.html)(Triton) in parallel +* Support sources selection for different models with [nvstreammux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreammux.html) and [nvstreamdemux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreamdemux.html) +* Support [new nvstreammux](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvstreammux2.html). you can set "export USE_NEW_NVSTREAMMUX=yes" to use new streammux. +* Support to mux output meta from different sources and different models with **gst-nvdsmetamux** plugin newly introduced in DeepStream 6.1.1 or above +* Support latency measurement and frame rate measurement with the enviroment varialble enabling. + + The latency measurement can be enabled by set the following enviroment varialbles: + ``` + export NVDS_ENABLE_COMPONENT_LATENCY_MEASUREMENT=1 + export NVDS_ENABLE_LATENCY_MEASUREMENT=1 + ``` + + +# Prerequisites +- If you are using a deepstream docker above DeepStream 6.1.1 version, please execute /opt/nvidia/deepstream/deepstream/user_additional_install.sh to install tools such as x264enc which is used in this sample. +- DeepStream 6.1.1 or above, especially + - [nvmsgbroker](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvmsgbroker.html) if you want to enable nvmsgbroker sink, e.g. [Kafka](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvmsgbroker.html#nvds-kafka-proto-kafka-protocol-adapter) + - [nvinferserver](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinferserver.html) if running model with Triton +- Cloud server, e.g. Kafka server (version >= kafka_2.12-3.2.0), if you want to enable broker sink + + + +# Steps To Run + +The sample should be downloaded and built with **root** permission. + +1. Download + + ``` + apt install git-lfs + git lfs install --skip-repo + cd deepstream/src/apps/reference_apps/deepstream_parallel_inference_app + git lfs pull + ``` + + If git-lfs download fails for bodypose2d and YoloV4 models, get them from download [link](https://nvidia.box.com/s/rqb5t5koigvpmobkxj1245gv09d9u2sx) + +2. Generate Inference Engines + + Below instructions are only needed on **Jetson** ([Jetpack 5.0.2 or above](https://developer.nvidia.com/embedded/jetpack-sdk-502)) + + ``` + apt-get install -y libjson-glib-dev libgstrtspserver-1.0-dev + /opt/nvidia/deepstream/deepstream/samples/triton_backend_setup.sh + ## Only DeepStream 6.1.1 GA need to copy the metamux plugin library. Skip this copy command if DeepStream version is above 6.1.1 GA + cp tritonclient/sample/gst-plugins/gst-nvdsmetamux/libnvdsgst_metamux.so /opt/nvidia/deepstream/deepstream/lib/gst-plugins/libnvdsgst_metamux.so + ## set power model and boost CPU/GPU/EMC clocks + nvpmodel -m 0 && jetson_clocks + ``` + + Below instructions are needed for both **Jetson** and **dGPU** (DeepStream Triton docker - [6.1.1-triton or above](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream/tags)) + + ``` + cd tritonserver/ + ./build_engine.sh + ``` + +3. Build and Run + + ``` + cd tritonclient/sample/ + source build.sh + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml + ``` + + + +# Directory + +![Files](files.PNG) + + + +# Application Configuration Semantics + +The parallel inferencing app uses the YAML configuration file to config GIEs, sources, and other features of the pipeline. The basic group semantics is the same as [deepstream-app](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#expected-output-for-the-deepstream-reference-application-deepstream-app). + +Please refer to deepstream-app [Configuration Groups](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_deepstream.html#configuration-groups) part for the semantics of corresponding groups. + +There are additional new groups introduced by the parallel inferencing app which enable the app to select sources for different inferencing branches and to select output metadata for different inferencing GIEs: + +### Branch Group +The branch group specifies the sources to be infered by the specific inferencing branch. The selected sources are identified by the source IDs list. The inferencing branch is identified by the first PGIE unique-id in this branch. For example: +``` +branch0: + key1: value1 + key2: value2 +``` + +The branch group properties are: + +| Key | Meaning | Type and Value | Example | Plateforms | +|-------|-------------------------------------------------------|----------------------------------|----------------|------------| +|pgie-id|the first PGIE unique-id in this branch | Integer, >0 |pgie-id: 8 |dGPU, Jetson| +|src-ids|The source-id list of selected sources for this branch | Semicolon separated integer array|src-ids: 0;2;5;6|dGPU, Jetson| + +### Metamux Group + +The metamux group specifies the configuration file of gst-dsmetamux plugin. For example: +``` +meta-mux: + key1: value1 + key2: value2 +``` +The metamux group properties are: + +| Key | Meaning | Type and Value | Example | Plateforms | +|-----------|------------------------------------------------------------|----------------|---------------------------------|------------| +| enable |Indicates whether the MetaMux must be enabled. | Boolean |enable=1 |dGPU, Jetson| +|config-file|Pathname of the configuration file for gst-dsmetamux plugin | String |config-file: ./config_metamux.txt|dGPU, Jetson| + +The gst-dsmetamux configuration details are introduced in gst-dsmetamux plugin README. + +# Sample Models + +The sample application uses the following models as samples. + +|Model Name | Inference Plugin | source | +|-----------|-------------------|-----------------------------------------------------------| +|bodypose2d |nvinfer|https://github.com/NVIDIA-AI-IOT/deepstream_pose_estimation| +|YoloV4 |nvinfer|https://github.com/NVIDIA-AI-IOT/yolov4_deepstream| +|peoplenet|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet| +|Primary Car detection|nvinferserver|DeepStream SDK| +|Secondary Car maker|nvinferserver|DeepStream SDK| +|Secondary Car type|nvinferserver|DeepStream SDK| +|trafficcamnet|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/trafficcamnet| +|LPD|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lpdnet| +|LPR|nvinferserver|https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lprnet| + +# Sample Configuration Files + +The application will create new inferencing branch for the designated primary GIE. The secondary GIEs should identify the primary GIE on which they work by setting "operate-on-gie-id" in nvinfer or nvinfereserver configuration file. + +To make every inferencing branch unique and identifiable, the "unique-id" for every GIE should be different and unique. The gst-dsmetamux module will rely on the "unique-id" to identify the metadata comes from which model. + +There are five sample configurations in current project for reference. + +* The sample configuration for the open source YoloV4, bodypose2d and TAO car license plate identification models with nvinferserver. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo_lpr + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. + + - Pipeline Graph: + + ![Yolov4_BODY_LPR](pipeline_0.png) + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml + `` + +* The sample configuration for the TAO vehicle classifications, carlicense plate identification and peopleNet models with nvinferserver. + + - Configuration folder + + tritonclient/sample/configs/apps/vehicle_lpr_analytic + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](new_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the TAO vehicle classifications, carlicense plate identification and peopleNet models with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/vehicle0_lpr_analytic + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The vehicle branch uses nvinfer, the car plate and the peoplenet branches use nvinferserver. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](new_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the open source YoloV4, bodypose2d with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo/ + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The bodypose branch uses nvinfer, the yolov4 branch use nvinferserver. The output streams is tiled. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](demo_pipe.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml + `` +* The sample configuration for the open source YoloV4, bodypose2d with nvinferserver and nvinfer. + + - Configuration folder + + tritonclient/sample/configs/apps/bodypose_yolo_win1/ + + "source4_1080p_dec_parallel_infer.yml" is the application configuration file. The other configuration files are for different modules in the pipeline, the application configuration file uses these files to configure different modules. The bodypose branch uses nvinfer, the yolov4 branch use nvinferserver. The output streams is source 2. + + - Pipeline Graph: + + ![LPR_Vehicle_peopleNet](demo_pipe_src2.png) + + - App Command: + + `` + ./apps/deepstream-parallel-infer/deepstream-parallel-infer -c configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml + `` diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/common.png b/src/apps/reference_apps/deepstream_parallel_inference_app/common.png new file mode 100644 index 00000000..08a48011 Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/common.png differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe.png b/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe.png new file mode 100644 index 00000000..a52d9e12 Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe.png differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe_src2.png b/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe_src2.png new file mode 100644 index 00000000..e4f3e877 Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/demo_pipe_src2.png differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/files.PNG b/src/apps/reference_apps/deepstream_parallel_inference_app/files.PNG new file mode 100644 index 00000000..89285cf6 Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/files.PNG differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/new_pipe.png b/src/apps/reference_apps/deepstream_parallel_inference_app/new_pipe.png new file mode 100644 index 00000000..970a3430 Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/new_pipe.png differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/pipeline_0.png b/src/apps/reference_apps/deepstream_parallel_inference_app/pipeline_0.png new file mode 100644 index 00000000..a9beac8f Binary files /dev/null and b/src/apps/reference_apps/deepstream_parallel_inference_app/pipeline_0.png differ diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/Makefile b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/Makefile new file mode 100644 index 00000000..6aec16c3 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/Makefile @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: + @make -C apps/deepstream-parallel-infer + @make -C gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo + @make -C gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser + +install: + @make install -C apps/deepstream-parallel-infer + @make install -C gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo + @make install -C gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser + +clean: + @make clean -C apps/deepstream-parallel-infer + @make clean -C gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo + @make clean -C gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md new file mode 100644 index 00000000..2581c660 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/CLA_LICENSE.md @@ -0,0 +1,25 @@ +# Individual Contributor License Agreement (CLA) + +Thank you for submitting your contributions to this project. + +By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions to the project. + +## License. +You hereby represent that all present, past and future contributions are governed by the MIT License copyright statement. + +This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights of the code or documents you contribute to the project itself or its maintainers. Furthermore you also represent that you have the authority to perform the above waiver with respect to the entirety of you contributions. + +## Moral Rights. +To the fullest extent permitted under applicable law, you hereby waive, and agree not to assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. + +## Third Party Content. +If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, specifications, documentation, data, materials, feedback, information or other works of authorship that were not authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary rights associated with your Contribution (“Third Party Rights”), then you agree to include with the submission of your Contribution full details respecting such Third Party Content and Third Party Rights, including, without limitation, identification of which aspects of your Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. + +## Representations. +You represent that, other than the Third Party Content and Third Party Rights identified by you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were created in the course of your employment with your past or present employer(s), you represent that such employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer (s) has waived all of their right, title or interest in or to your Contributions. + +## Disclaimer. +To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" basis, without any warranties or conditions, express or implied, including, without limitation, any implied warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not required to provide support for your Contributions, except to the extent you desire to provide support. + +## No Obligation. +You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions into the project. The decision to use or incorporate your contributions into the project will be made at the sole discretion of the maintainers or their authorized delegates. \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md new file mode 100644 index 00000000..59479b3d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/LICENSE.md @@ -0,0 +1,14 @@ +SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile new file mode 100644 index 00000000..16114e70 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/Makefile @@ -0,0 +1,67 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +APP:= deepstream-parallel-infer + + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +#SRCS:= $(wildcard *.c) $(wildcard *.cpp) +SRCS:= deepstream_parallel_infer_app.cpp \ + deepstream_parallel_infer_config_parser.cpp + +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/*.c) +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/deepstream-yaml/*.cpp) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(SRCS:.c=.o) +OBJS:= $(OBJS:.cpp=.o) + +CFLAGS+= -g -I./ -I/opt/nvidia/deepstream/deepstream/sources/apps/apps-common/includes \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda/include + +LIBS:= -L/usr/local/cuda/lib64/ -lcudart + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper \ + -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm -lyaml-cpp \ + -lcuda -lgstrtspserver-1.0 -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) -lnvbufsurface + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md new file mode 100644 index 00000000..0390105a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/README.md @@ -0,0 +1,31 @@ +# DeepStream Parallel Inference + +DeepStream Parallel Inference is one application which run inference in parallel. tee will send batched buffer to different models. nvdsmetamux will mux batch meta from different models which run in parallel. User can configure the pipeline in application configuration yaml file. User can select source ids which need inference in preprocess configuration file. User can select source ids which need mux in nvdsmetamux configuration file. + +## Prerequisites +DeepStream SDK 6.1.1 GA or above + + +## Getting Started: +To get started, please follow these steps. +1. Install [DeepStream](https://developer.nvidia.com/deepstream-sdk) on your platform, verify it is working by running deepstream-app. +2. Install nvdsmetamux. +3. Install triton client video template libraries. +4. Clone the repository preferably in `$DEEPSTREAM_DIR/sources/apps/sample_apps`. +5. Compile and run the program + + ``` + $ cd deepstream-parallel-infer/ + $ sudo make + $ sudo ./deepstream-parallel-infer -c +``` +Run parallel inference app with default configure file: $ ./deepstream-parallel-infer -c source4_1080p_dec_parallel_infer.yml + +For any issues or questions, please feel free to make a new post on the [DeepStreamSDK forums](https://forums.developer.nvidia.com/c/accelerated-computing/intelligent-video-analytics/deepstream-sdk/). + +#### Known Issues + +On **DGX Spark** (NVIDIA GB10, compute capability 12.1 / sm_121), the ONNX Runtime (1.23.0) bundled in the `nvcr.io/nvidia/deepstream:9.0-triton-sbsa-dgx-spark` container was not compatible with sm_121. +You can apply the following workaround to the affected **bodypose2d** model: +1.Run inference through the nvinfer plugin +2.If the nvinferserver plugin must be used, configure the model to use the TensorRT backend (platform: "tensorrt_plan" in config.pbtxt) instead of the ONNX Runtime backend diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp new file mode 100644 index 00000000..588a3c64 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_metamux_yaml.cpp @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "deepstream_common.h" +#include "deepstream_config_yaml.h" +#include +#include +#include + +using std::cout; +using std::endl; + +gboolean +parse_metamux_yaml (NvDsMetaMuxConfig *config, gchar* cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["meta-mux"].begin(); + itr != configyml["meta-mux"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->config_file_path)) { + g_printerr ("Error: Could not parse config-file-path in metamux.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in metamux: " << paramKey << endl; + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h new file mode 100644 index 00000000..42886871 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer.h @@ -0,0 +1,232 @@ +/* +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __NVGSTDS_APP_H__ +#define __NVGSTDS_APP_H__ + +#include +#ifdef __cplusplus +extern "C" +{ +#endif + +#include + +#include "deepstream_app_version.h" +#include "deepstream_common.h" +#include "deepstream_config.h" +#include "deepstream_osd.h" +#include "deepstream_perf.h" +#include "deepstream_preprocess.h" +#include "deepstream_primary_gie.h" +#include "deepstream_sinks.h" +#include "deepstream_sources.h" +#include "deepstream_streammux.h" +#include "deepstream_tiled_display.h" +#include "deepstream_dsanalytics.h" +#include "deepstream_dsexample.h" +#include "deepstream_tracker.h" +#include "deepstream_secondary_gie.h" +#include "deepstream_c2d_msg.h" +#include "deepstream_image_save.h" + +#define MAX_PRIMARY_GIE_BINS (16) +#define MAX_PRE_PROCESS_BINS (16) +#define MAX_VIDEO_TEMPLATE_PROPS (32) + +typedef struct _AppCtx AppCtx; + +typedef void (*bbox_generated_callback) (AppCtx *appCtx, GstBuffer *buf, + NvDsBatchMeta *batch_meta, guint index); +typedef gboolean (*overlay_graphics_callback) (AppCtx *appCtx, GstBuffer *buf, + NvDsBatchMeta *batch_meta, guint index); + +typedef struct +{ + gboolean enable; + + gchar *config_file_path; +} NvDsMetaMuxConfig; + +typedef struct +{ + gboolean enable; + + guint num_customlib_props; + gchar *customlib_name; + gchar *customlib_props[MAX_VIDEO_TEMPLATE_PROPS]; +} NvDsVideoTemplateConfig; + +typedef struct +{ + GstElement *bin; + GstElement *tee; + GstElement *muxer; + gulong muxer_buffer_probe_id; + GstElement *source_tee[MAX_PRIMARY_GIE_BINS]; + GstElement *demuxer; + GstElement *streammux[MAX_PRIMARY_GIE_BINS]; + NvDsPrimaryGieBin primary_gie_bin[MAX_PRIMARY_GIE_BINS]; + NvDsTrackerBin tracker_bin[MAX_PRIMARY_GIE_BINS]; + NvDsSecondaryGieBin secondary_gie_bin[MAX_PRIMARY_GIE_BINS]; + NvDsDsAnalyticsBin dsanalytics_bin[MAX_PRIMARY_GIE_BINS]; + NvDsPreProcessBin preprocess_bin[MAX_PRIMARY_GIE_BINS]; +} NvDsParallelGieBin; + +typedef struct +{ + guint index; + gulong all_bbox_buffer_probe_id; + gulong primary_bbox_buffer_probe_id; + gulong fps_buffer_probe_id; + GstElement *bin; + GstElement *tee; + GstElement *sink_tee; + GstElement *msg_conv; + NvDsPreProcessBin preprocess_bin; + NvDsPrimaryGieBin primary_gie_bin; + NvDsOSDBin osd_bin; + NvDsSecondaryGieBin secondary_gie_bin; + NvDsTrackerBin tracker_bin; + NvDsSinkBin sink_bin; + NvDsSinkBin demux_sink_bin; + NvDsDsAnalyticsBin dsanalytics_bin; + NvDsDsExampleBin dsexample_bin; + AppCtx *appCtx; +} NvDsInstanceBin; + +typedef struct +{ + gulong primary_bbox_buffer_probe_id; + guint bus_id; + GstElement *pipeline; + NvDsSrcParentBin multi_src_bin; + NvDsParallelGieBin parallel_infer_bin; + NvDsInstanceBin instance_bins[MAX_SOURCE_BINS]; + NvDsInstanceBin demux_instance_bins[MAX_SOURCE_BINS]; + NvDsInstanceBin common_elements; + GstElement *tiler_tee; + NvDsTiledDisplayBin tiled_display_bin; + GstElement *demuxer; + NvDsDsExampleBin dsexample_bin; + AppCtx *appCtx; +} NvDsPipeline; + +typedef struct +{ + gint pgie_id; + gchar *src_ids; +} NvDsStrcIDConfig; + +typedef struct +{ + gboolean enable_perf_measurement; + gint file_loop; + gint pipeline_recreate_sec; + gboolean source_list_enabled; + guint total_num_sources; + guint num_source_sub_bins; + guint num_secondary_gie_num[MAX_PRIMARY_GIE_BINS]; + guint num_secondary_gie_sub_bins; + guint num_pre_process_sub_bins; + guint num_primary_gie_sub_bins; + guint num_src_ids_sub_bins; + guint num_tracker_sub_bins; + guint num_analysis_sub_bins; + guint num_sink_sub_bins; + guint num_message_consumers; + guint perf_measurement_interval_sec; + guint sgie_batch_size; + gchar *bbox_dir_path; + gchar *kitti_track_dir_path; + guint show_source; + + gchar **uri_list; + NvDsSourceConfig multi_source_config[MAX_SOURCE_BINS]; + NvDsStreammuxConfig streammux_config; + NvDsOSDConfig osd_config; + NvDsPreProcessConfig pre_process_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsGieConfig primary_gie_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsVideoTemplateConfig video_template_sub_bin_config[MAX_PRIMARY_GIE_BINS]; + NvDsMetaMuxConfig meta_mux_config; + NvDsTrackerConfig tracker_config[MAX_PRIMARY_GIE_BINS]; + NvDsStrcIDConfig srcids_config[MAX_PRIMARY_GIE_BINS]; + NvDsGieConfig secondary_gie_sub_bin_config[MAX_PRIMARY_GIE_BINS][MAX_SECONDARY_GIE_BINS]; + NvDsSinkSubBinConfig sink_bin_sub_bin_config[MAX_SINK_BINS]; + NvDsMsgConsumerConfig message_consumer_config[MAX_MESSAGE_CONSUMERS]; + NvDsTiledDisplayConfig tiled_display_config; + NvDsDsAnalyticsConfig dsanalytics_config[MAX_PRIMARY_GIE_BINS]; + NvDsDsExampleConfig dsexample_config; + NvDsSinkMsgConvBrokerConfig msg_conv_config; + NvDsImageSave image_save_config; + +} NvDsConfig; + +typedef struct +{ + gulong frame_num; +} NvDsInstanceData; + +struct _AppCtx +{ + gboolean version; + gboolean cintr; + gboolean show_bbox_text; + gboolean seeking; + gboolean quit; + gint person_class_id; + gint car_class_id; + gint return_value; + guint index; + gint active_source_index; + + GMutex app_lock; + GCond app_cond; + + NvDsPipeline pipeline; + NvDsConfig config; + NvDsConfig override_config; + NvDsInstanceData instance_data[MAX_SOURCE_BINS]; + NvDsC2DContext *c2d_ctx[MAX_MESSAGE_CONSUMERS]; + NvDsAppPerfStructInt perf_struct; + bbox_generated_callback bbox_generated_post_analytics_cb; + bbox_generated_callback all_bbox_generated_cb; + overlay_graphics_callback overlay_graphics_cb; + NvDsFrameLatencyInfo *latency_info; + GMutex latency_lock; + GThread *ota_handler_thread; + guint ota_inotify_fd; + guint ota_watch_desc; +}; + +/** + * Function to read properties from YML configuration file. + * + * @param[in] config pointer to @ref NvDsConfig + * @param[in] cfg_file_path path of configuration file. + * + * @return true if parsed successfully. + */ +gboolean +parse_config_file_yaml (NvDsConfig * config, gchar * cfg_file_path); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp new file mode 100644 index 00000000..2e938ba1 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_app.cpp @@ -0,0 +1,1617 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "deepstream_parallel_infer.h" +#include "post_process/body_pose/post_process.cpp" +#include "nvds_version.h" + +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" +#include "nvdsmeta_schema.h" +#include "deepstream_perf.h" + +#include +#include +#include +#include +#include + +#define EPS 1e-6 + +#define MAX_DISPLAY_LEN 64 + +#define MAX_TIME_STAMP_LEN 32 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 +#define MAX_STR_LEN 2048 + +AppCtx *appCtx; +static guint cintr = FALSE; +static GMainLoop *main_loop = NULL; +static gchar **cfg_files = NULL; +static gboolean print_version = FALSE; +static gboolean show_bbox_text = FALSE; +static gboolean print_dependencies_version = FALSE; +static gboolean quit = FALSE; +static gint return_value = 0; +static guint num_input_uris; +static gint frame_interval = 30; + +template +using Vec1D = std::vector; + +template +using Vec2D = std::vector>; + +template +using Vec3D = std::vector>; + +gint frame_number = 0; + +#define MAX_STREAMS 64 + +typedef struct +{ + /** identifies the stream ID */ + guint32 stream_index; + gdouble fps[MAX_STREAMS]; + gdouble fps_avg[MAX_STREAMS]; + guint32 num_instances; + guint header_print_cnt; + GMutex fps_lock; + gpointer context; + + /** Test specific info */ + guint32 set_batch_size; +}DemoPerfCtx; + +typedef struct { + GMutex *lock; + int num_sources; +}LatencyCtx; + +/** + * callback function to print the performance numbers of each stream. + */ +static void +perf_cb (gpointer context, NvDsAppPerfStruct * str) +{ + DemoPerfCtx *thCtx = (DemoPerfCtx *) context; + + g_mutex_lock(&thCtx->fps_lock); + /** str->num_instances is == num_sources */ + guint32 numf = str->num_instances; + guint32 i; + + for (i = 0; i < numf; i++) { + thCtx->fps[i] = str->fps[i]; + thCtx->fps_avg[i] = str->fps_avg[i]; + } + thCtx->context = thCtx; + g_print ("**PERF: "); + for (i = 0; i < numf; i++) { + g_print ("%.2f (%.2f)\t", thCtx->fps[i], thCtx->fps_avg[i]); + } + g_print ("\n"); + g_mutex_unlock(&thCtx->fps_lock); +} + +/** + * callback function to print the latency of each component in the pipeline. + */ + +static GstPadProbeReturn +latency_measurement_buf_prob(GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + LatencyCtx *ctx = (LatencyCtx *) u_data; + static int batch_num = 0; + guint i = 0, num_sources_in_batch = 0; + if(nvds_enable_latency_measurement) + { + GstBuffer *buf = (GstBuffer *) info->data; + NvDsFrameLatencyInfo *latency_info = NULL; + g_mutex_lock (ctx->lock); + latency_info = (NvDsFrameLatencyInfo *) + calloc(1, ctx->num_sources * sizeof(NvDsFrameLatencyInfo));; + g_print("\n************BATCH-NUM = %d soure %d**************\n",batch_num,ctx->num_sources); + num_sources_in_batch = nvds_measure_buffer_latency(buf, latency_info); + + for(i = 0; i < num_sources_in_batch; i++) + { + g_print("Source id = %d Frame_num = %d Frame latency = %lf (ms) \n", + latency_info[i].source_id, + latency_info[i].frame_num, + latency_info[i].latency); + } + g_mutex_unlock (ctx->lock); + batch_num++; + } + + return GST_PAD_PROBE_OK; +} + +GST_DEBUG_CATEGORY (NVDS_APP); + +GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version, + "Print DeepStreamSDK version", NULL} + , + {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text, + "Display Bounding box labels in tiled mode", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version, + "Print DeepStreamSDK and dependencies version", NULL} + , + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files, + "Set the config file", NULL} + , + {NULL} + , +}; + +/*Method to parse information returned from the model*/ +std::tuple, Vec3D> +parse_objects_from_tensor_meta(NvDsInferTensorMeta *tensor_meta) +{ + Vec1D counts; + Vec3D peaks; + + float threshold = 0.1; + int window_size = 5; + int max_num_parts = 20; + int num_integral_samples = 7; + float link_threshold = 0.1; + int max_num_objects = 100; + + void *cmap_data = tensor_meta->out_buf_ptrs_host[0]; + NvDsInferDims &cmap_dims = tensor_meta->output_layers_info[0].inferDims; + void *paf_data = tensor_meta->out_buf_ptrs_host[1]; + NvDsInferDims &paf_dims = tensor_meta->output_layers_info[1].inferDims; + + /* Finding peaks within a given window */ + find_peaks(counts, peaks, cmap_data, cmap_dims, threshold, window_size, max_num_parts); + /* Non-Maximum Suppression */ + Vec3D refined_peaks = refine_peaks(counts, peaks, cmap_data, cmap_dims, window_size); + /* Create a Bipartite graph to assign detected body-parts to a unique person in the frame */ + Vec3D score_graph = paf_score_graph(paf_data, paf_dims, topology, counts, refined_peaks, num_integral_samples); + /* Assign weights to all edges in the bipartite graph generated */ + Vec3D connections = assignment(score_graph, topology, counts, link_threshold, max_num_parts); + /* Connecting all the Body Parts and Forming a Human Skeleton */ + Vec2D objects = connect_parts(connections, topology, counts, max_num_objects); + return {objects, refined_peaks}; +} + +/* MetaData to handle drawing onto the on-screen-display */ +static void +create_display_meta(Vec2D &objects, Vec3D &normalized_peaks, NvDsFrameMeta *frame_meta, int frame_width, int frame_height) +{ + int K = topology.size(); + int count = objects.size(); + NvDsBatchMeta *bmeta = frame_meta->base_meta.batch_meta; + NvDsDisplayMeta *dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + + for (auto &object : objects) + { + int C = object.size(); + for (int j = 0; j < C; j++) + { + int k = object[j]; + if (k >= 0) + { + auto &peak = normalized_peaks[j][k]; + int x = peak[1] * MUXER_OUTPUT_WIDTH; + int y = peak[0] * MUXER_OUTPUT_HEIGHT; + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META) + { + dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + } + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = x; + cparams.yc = y; + cparams.radius = 8; + cparams.circle_color = NvOSD_ColorParams{244, 67, 54, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0, 255, 0, 1}; + dmeta->num_circles++; + } + } + + for (int k = 0; k < K; k++) + { + int c_a = topology[k][2]; + int c_b = topology[k][3]; + if (object[c_a] >= 0 && object[c_b] >= 0) + { + auto &peak0 = normalized_peaks[c_a][object[c_a]]; + auto &peak1 = normalized_peaks[c_b][object[c_b]]; + int x0 = peak0[1] * MUXER_OUTPUT_WIDTH; + int y0 = peak0[0] * MUXER_OUTPUT_HEIGHT; + int x1 = peak1[1] * MUXER_OUTPUT_WIDTH; + int y1 = peak1[0] * MUXER_OUTPUT_HEIGHT; + if (dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) + { + dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + } + NvOSD_LineParams &lparams = dmeta->line_params[dmeta->num_lines]; + lparams.x1 = x0; + lparams.x2 = x1; + lparams.y1 = y0; + lparams.y2 = y1; + lparams.line_width = 3; + lparams.line_color = NvOSD_ColorParams{0, 255, 0, 1}; + dmeta->num_lines++; + } + } + } +} + +/* body_pose_gie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +body_pose_gie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + if (frame_meta->batch_id == 0) + g_print("Processing frame number = %d\t\n", frame_meta->frame_num); + + for (l_user = frame_meta->frame_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + Vec2D objects; + Vec3D normalized_peaks; + tie(objects, normalized_peaks) = parse_objects_from_tensor_meta(tensor_meta); + create_display_meta(objects, normalized_peaks, frame_meta, frame_meta->source_frame_width, frame_meta->source_frame_height); + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + Vec2D objects; + Vec3D normalized_peaks; + tie(objects, normalized_peaks) = parse_objects_from_tensor_meta(tensor_meta); + create_display_meta(objects, normalized_peaks, frame_meta, frame_meta->source_frame_width, frame_meta->source_frame_height); + } + } + } + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +yolov4_gie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + + for (l_user = frame_meta->frame_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + } + } + + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) + { + NvDsInferTensorMeta *tensor_meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + } + } + } + } + return GST_PAD_PROBE_OK; +} + +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = (NvDsEventMsgMeta *) g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = (gdouble *) g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *srcObj = (NvDsVehicleObject *) srcMeta->extMsg; + NvDsVehicleObject *obj = + (NvDsVehicleObject *) g_malloc0 (sizeof (NvDsVehicleObject)); + if (srcObj->type) + obj->type = g_strdup (srcObj->type); + if (srcObj->make) + obj->make = g_strdup (srcObj->make); + if (srcObj->model) + obj->model = g_strdup (srcObj->model); + if (srcObj->color) + obj->color = g_strdup (srcObj->color); + if (srcObj->license) + obj->license = g_strdup (srcObj->license); + if (srcObj->region) + obj->region = g_strdup (srcObj->region); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsVehicleObject); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + obj->age = srcObj->age; + + if (srcObj->gender) + obj->gender = g_strdup (srcObj->gender); + if (srcObj->cap) + obj->cap = g_strdup (srcObj->cap); + if (srcObj->hair) + obj->hair = g_strdup (srcObj->hair); + if (srcObj->apparel) + obj->apparel = g_strdup (srcObj->apparel); + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + +static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *obj = (NvDsVehicleObject *) srcMeta->extMsg; + if (obj->type) + g_free (obj->type); + if (obj->color) + g_free (obj->color); + if (obj->make) + g_free (obj->make); + if (obj->model) + g_free (obj->model); + if (obj->license) + g_free (obj->license); + if (obj->region) + g_free (obj->region); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->gender) + g_free (obj->gender); + if (obj->cap) + g_free (obj->cap); + if (obj->hair) + g_free (obj->hair); + if (obj->apparel) + g_free (obj->apparel); + } + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + } + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +static void +generate_vehicle_meta (gpointer data) +{ + NvDsVehicleObject *obj = (NvDsVehicleObject *) data; + + obj->type = g_strdup ("sedan"); + obj->color = g_strdup ("blue"); + obj->make = g_strdup ("Bugatti"); + obj->model = g_strdup ("M"); + obj->license = g_strdup ("XX1234"); + obj->region = g_strdup ("CA"); +} + +static void +generate_person_meta (gpointer data) +{ + NvDsPersonObject *obj = (NvDsPersonObject *) data; + obj->age = 45; + obj->cap = g_strdup ("none"); + obj->hair = g_strdup ("black"); + obj->gender = g_strdup ("male"); + obj->apparel = g_strdup ("formal"); +} + +static void +generate_event_msg_meta (gpointer data, gint class_id, + NvDsObjectMeta * obj_params) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + meta->sensorId = 0; + meta->placeId = 0; + meta->moduleId = 0; + meta->sensorStr = g_strdup ("sensor-0"); + + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + + /* + * This demonstrates how to attach custom objects. + * Any custom object as per requirement can be generated and attached + * like NvDsVehicleObject / NvDsPersonObject. Then that object should + * be handled in payload generator library (nvmsgconv.cpp) accordingly. + */ + if (class_id == PGIE_CLASS_ID_VEHICLE) { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_VEHICLE; + meta->objClassId = PGIE_CLASS_ID_VEHICLE; + + NvDsVehicleObject *obj = + (NvDsVehicleObject *) g_malloc0 (sizeof (NvDsVehicleObject)); + generate_vehicle_meta (obj); + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsVehicleObject); + } else if (class_id == PGIE_CLASS_ID_PERSON) { + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PGIE_CLASS_ID_PERSON; + + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + generate_person_meta (obj); + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsPersonObject); + } +} +/* osd_sink_pad_buffer_probe will extract metadata received from OSD + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + gboolean is_first_object = TRUE; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + is_first_object = TRUE; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) + { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + /** Generate NvDsEventMsgMeta for every object */ + if (is_first_object && !(frame_number % frame_interval)) { + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->frameId = frame_number; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->confidence = obj_meta->confidence; + generate_event_msg_meta (msg_meta, obj_meta->class_id, obj_meta); + + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + + /* Parameters to draw text onto the On-Screen-Display */ + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Frame Number = %d", frame_number); + offset = snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, ""); + + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + txt_params->font_params.font_name = "Mono"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call(GstBus *bus, GstMessage *msg, gpointer data) +{ + GMainLoop *main_loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_EOS: + g_print("End of Stream\n"); + g_main_loop_quit(main_loop); + break; + + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", + GST_OBJECT_NAME(msg->src), error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(main_loop); + break; + } + + default: + break; + } + return TRUE; +} + +gboolean +link_element_to_metamux_sink_pad (GstElement *metamux, GstElement *elem, + gint index) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *src_pad = NULL; + gchar pad_name[16]; + + if (index >= 0) { + g_snprintf (pad_name, 16, "sink_%u", index); + pad_name[15] = '\0'; + } else { + strcpy (pad_name, "sink_%u"); + } + + mux_sink_pad = gst_element_get_request_pad (metamux, pad_name); + if (!mux_sink_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get sink pad from metamux"); + goto done; + } + + src_pad = gst_element_get_static_pad (elem, "src"); + if (!src_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get src pad from '%s'", + GST_ELEMENT_NAME (elem)); + goto done; + } + + if (gst_pad_link (src_pad, mux_sink_pad) != GST_PAD_LINK_OK) { + NVGSTDS_ERR_MSG_V ("Failed to link '%s' and '%s'", GST_ELEMENT_NAME (metamux), + GST_ELEMENT_NAME (elem)); + goto done; + } + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (src_pad) { + gst_object_unref (src_pad); + } + return ret; +} + +gboolean +unlink_element_from_metamux_sink_pad (GstElement *metamux, GstElement *elem) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *src_pad = NULL; + + src_pad = gst_element_get_static_pad (elem, "src"); + if (!src_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get src pad from '%s'", + GST_ELEMENT_NAME (elem)); + goto done; + } + + mux_sink_pad = gst_pad_get_peer (src_pad); + if (!mux_sink_pad) { + NVGSTDS_ERR_MSG_V ("Failed to get sink pad from metamux"); + goto done; + } + + if (!gst_pad_unlink (src_pad, mux_sink_pad)) { + NVGSTDS_ERR_MSG_V ("Failed to unlink '%s' and '%s'", GST_ELEMENT_NAME (metamux), + GST_ELEMENT_NAME (elem)); + goto done; + } + + gst_element_release_request_pad(metamux, mux_sink_pad); + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (src_pad) { + gst_object_unref (src_pad); + } + return ret; +} + +gboolean +link_streamdemux_to_streammux (NvDsParallelGieBin *bin, GstElement *demux, GstElement *mux, + gint index) +{ + gboolean ret = FALSE; + GstPad *mux_sink_pad = NULL; + GstPad *source_tee_src_pad = NULL; + GstElement *queue = NULL; + gchar pad_name[16]; + + if (!bin->source_tee[index]) { + bin->source_tee[index] = gst_element_factory_make (NVDS_ELEM_TEE, NULL); + if (!bin->source_tee[index]) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_source_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->source_tee[index]); + + link_element_to_demux_src_pad (demux, bin->source_tee[index], index); + } + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + link_element_to_streammux_sink_pad (mux, queue, index); + + link_element_to_tee_src_pad (bin->source_tee[index], queue); + + ret = TRUE; + +done: + if (mux_sink_pad) { + gst_object_unref (mux_sink_pad); + } + if (source_tee_src_pad) { + gst_object_unref (source_tee_src_pad); + } + return ret; +} + +gboolean +create_primary_gie_videotemplate_bin (NvDsVideoTemplateConfig *config, NvDsPrimaryGieBin *bin) +{ + gboolean ret = FALSE; + guint i; + + bin->bin = gst_bin_new ("primary_gie_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_bin'"); + goto done; + } + + bin->nvvidconv = + gst_element_factory_make (NVDS_ELEM_VIDEO_CONV, "primary_gie_conv"); + if (!bin->nvvidconv) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_conv'"); + goto done; + } + + bin->queue = gst_element_factory_make (NVDS_ELEM_QUEUE, "primary_gie_queue"); + if (!bin->queue) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie_queue'"); + goto done; + } + + bin->primary_gie = + gst_element_factory_make ("nvdsvideotemplate", "primary_gie"); + if (!bin->primary_gie) { + NVGSTDS_ERR_MSG_V ("Failed to create 'primary_gie'"); + goto done; + } + + g_object_set (G_OBJECT (bin->primary_gie), + "customlib-name", config->customlib_name, NULL); + + for (i = 0; i < config->num_customlib_props; i ++) { + g_object_set (G_OBJECT (bin->primary_gie), + "customlib-props", config->customlib_props[i], NULL); + } + + /* + g_object_set (G_OBJECT (bin->nvvidconv), "gpu-id", config->gpu_id, NULL); + g_object_set (G_OBJECT (bin->nvvidconv), "nvbuf-memory-type", + config->nvbuf_memory_type, NULL); + */ + + gst_bin_add_many (GST_BIN (bin->bin), bin->queue, + bin->nvvidconv, bin->primary_gie, NULL); + + NVGSTDS_LINK_ELEMENT (bin->queue, bin->nvvidconv); + + NVGSTDS_LINK_ELEMENT (bin->nvvidconv, bin->primary_gie); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->primary_gie, "src"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->queue, "sink"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} + +#if 0 +static gboolean +create_parallel_infer_bin (guint num_sub_bins, NvDsConfig *config, + NvDsParallelGieBin *bin, AppCtx *appCtx) +{ + gboolean ret = FALSE; + GstElement *sink_elem = NULL; + GstElement *src_elem = NULL; + GstElement *nvvidconv = NULL, *caps_filter = NULL; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + gchar name[50]; + guint i = 0; + + bin->bin = gst_bin_new ("parallel_infer_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'parallel_infer_bin'"); + goto done; + } + + bin->tee = gst_element_factory_make (NVDS_ELEM_TEE, "infer_bin_tee"); + if (!bin->tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->tee); + + bin->muxer = gst_element_factory_make ("nvdsmetamux", "infer_bin_muxer"); + if (!bin->muxer) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_muxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->muxer), "config-file", + GET_FILE_PATH (config->meta_mux_config.config_file_path), NULL); + + NVGSTDS_ELEM_ADD_PROBE (bin->muxer_buffer_probe_id, bin->muxer, "src", + body_pose_gie_src_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, + appCtx); + + gst_bin_add (GST_BIN (bin->bin), bin->muxer); + + for (i = 0; i < num_sub_bins; i++) { + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + if (config->video_template_sub_bin_config[i].enable) { + if (!create_primary_gie_videotemplate_bin (&config->video_template_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } else { + if (!create_primary_gie_bin (&config->primary_gie_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } + g_snprintf (name, sizeof (name), "primary_gie_%d_bin", i); + gst_element_set_name (bin->primary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->primary_gie_bin[i].bin); + + sink_elem = bin->primary_gie_bin[i].bin; + src_elem = bin->primary_gie_bin[i].bin; + } + + if (config->pre_process_sub_bin_config[i].enable) { + if (!create_preprocess_bin (&config->pre_process_sub_bin_config[i], + &bin->preprocess_bin[i])) { + g_print ("creating preprocess bin failed\n"); + goto done; + } + g_snprintf (name, sizeof (name), "preprocess_%d_bin", i); + gst_element_set_name (bin->preprocess_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->preprocess_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->preprocess_bin[i].bin, sink_elem); + } + + sink_elem = bin->preprocess_bin[i].bin; + } + + /* Add video convert to avoid parallel infer operate on the same batch meta */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", NULL); + caps_filter = gst_element_factory_make ("capsfilter", NULL); + caps = + gst_caps_new_simple ("video/x-raw", + "width", G_TYPE_INT, 1920, + "height", G_TYPE_INT, 1082, + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (caps_filter), "caps", caps, NULL); + gst_bin_add (GST_BIN (bin->bin), nvvidconv); + gst_bin_add (GST_BIN (bin->bin), caps_filter); + NVGSTDS_LINK_ELEMENT (nvvidconv, caps_filter); + NVGSTDS_LINK_ELEMENT (caps_filter, sink_elem); + sink_elem = nvvidconv; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + link_element_to_metamux_sink_pad (bin->muxer, src_elem, i); + } + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->tee, "sink"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->muxer, "src"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} +#endif + +/* Separate a config file entry with delimiters + * into strings. */ +std::vector split_string (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ';') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +/* select streamdemux output sources, then link sources and streammux */ +static gboolean link_streamdemux_to_streammux(NvDsConfig *config, NvDsParallelGieBin *bin, int i){ + gboolean ret = FALSE; + if (config->primary_gie_sub_bin_config[i].unique_id != config->srcids_config[i].pgie_id) { + NVGSTDS_ERR_MSG_V ("pgieid %d != branch pgieid %d\n", + config->primary_gie_sub_bin_config[i].unique_id, + config->srcids_config[i].pgie_id); + return ret; + } else { + std::string str = config->srcids_config[i].src_ids; + std::vector vec = split_string (str); + for(int j = 0; j < vec.size(); j++) { + int id = std::stoi(vec[j]); + g_print("link_streamdemux_to_streammux, srid:%d, mux:%d\n", id, i); + if (!link_streamdemux_to_streammux (bin, bin->demuxer, bin->streammux[i], id)) { + NVGSTDS_ERR_MSG_V ("source %d cannot be linked to mux's sink pad %p\n", + id, bin->streammux[i]); + return ret; + } + } + } + return true; +} + +static gboolean +create_parallel_infer_bin (guint num_sub_bins, NvDsConfig *config, + NvDsParallelGieBin *bin, AppCtx *appCtx) +{ + gboolean ret = FALSE; + GstElement *sink_elem = NULL; + GstElement *src_elem = NULL; + GstElement *queue = NULL; + GstElement *nvvidconv = NULL, *caps_filter = NULL; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + gchar name[50]; + guint i, j; + std::string str; + std::vector vec; + guint src_id_num; + + bin->bin = gst_bin_new ("parallel_infer_bin"); + if (!bin->bin) { + NVGSTDS_ERR_MSG_V ("Failed to create 'parallel_infer_bin'"); + goto done; + } + + bin->tee = gst_element_factory_make (NVDS_ELEM_TEE, "infer_bin_tee"); + if (!bin->tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_tee'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->tee); + + bin->muxer = gst_element_factory_make ("nvdsmetamux", "infer_bin_muxer"); + if (!bin->muxer) { + NVGSTDS_ERR_MSG_V ("Failed to create 'infer_bin_muxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->muxer), "config-file", + GET_FILE_PATH (config->meta_mux_config.config_file_path), NULL); + + NVGSTDS_ELEM_ADD_PROBE (bin->muxer_buffer_probe_id, bin->muxer, "src", + body_pose_gie_src_pad_buffer_probe, GST_PAD_PROBE_TYPE_BUFFER, + appCtx); + + gst_bin_add (GST_BIN (bin->bin), bin->muxer); + sink_elem = bin->muxer; + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + link_element_to_metamux_sink_pad (bin->muxer, queue, 0); + sink_elem = queue; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + + bin->demuxer = + gst_element_factory_make (NVDS_ELEM_STREAM_DEMUX, NULL); + if (!bin->demuxer) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'demuxer'"); + goto done; + } + g_object_set (G_OBJECT (bin->demuxer), "per-stream-eos", TRUE, NULL); + gst_bin_add (GST_BIN (bin->bin), bin->demuxer); + sink_elem = bin->demuxer; + + queue = gst_element_factory_make (NVDS_ELEM_QUEUE, NULL); + if (!queue) { + NVGSTDS_ERR_MSG_V ("Could not create 'queue'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), queue); + NVGSTDS_LINK_ELEMENT (queue, sink_elem); + sink_elem = queue; + + link_element_to_tee_src_pad (bin->tee, sink_elem); + + + for (i = 0; i < num_sub_bins; i++) { + sink_elem = src_elem = NULL; + + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + + if (config->num_secondary_gie_sub_bins > 0 && config->num_secondary_gie_num[i] > 0) { + if (!create_secondary_gie_bin (config->num_secondary_gie_num[i], + config->primary_gie_sub_bin_config[i].unique_id, + config->secondary_gie_sub_bin_config[i], + &bin->secondary_gie_bin[i])) { + g_print("create_secondary_gie_bin failed"); + goto done; + } + g_snprintf (name, sizeof (name), "sgie_%d_bin", i); + gst_element_set_name (bin->secondary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->secondary_gie_bin[i].bin); + sink_elem = bin->secondary_gie_bin[i].bin; + src_elem = bin->secondary_gie_bin[i].bin; + } + } + //add analysis + if (config->tracker_config[i].enable && config->dsanalytics_config[i].enable) { + if (!create_dsanalytics_bin (&config->dsanalytics_config[i], + &bin->dsanalytics_bin[i])) { + g_print ("creating dsanalytics bin failed\n"); + goto done; + } + + g_snprintf (name, sizeof (name), "analytics_%d_bin", i); + gst_element_set_name (bin->dsanalytics_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->dsanalytics_bin[i].bin); + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->dsanalytics_bin[i].bin, sink_elem); + } + sink_elem = bin->dsanalytics_bin[i].bin; + if (!src_elem) { + src_elem = bin->dsanalytics_bin[i].bin; + } + } + //add tracker + if (config->tracker_config[i].enable) { + if (!create_tracking_bin (&config->tracker_config[i], + &bin->tracker_bin[i])) { + g_print ("creating tracker bin failed\n"); + goto done; + } + + g_snprintf (name, sizeof (name), "tracking_%d_bin", i); + gst_element_set_name (bin->tracker_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), + bin->tracker_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->tracker_bin[i].bin, sink_elem); + } + sink_elem = bin->tracker_bin[i].bin; + if (!src_elem) { + src_elem = bin->tracker_bin[i].bin; + } + } + + if (config->primary_gie_sub_bin_config[i].enable + || config->video_template_sub_bin_config[i].enable) { + if (config->video_template_sub_bin_config[i].enable) { + if (!create_primary_gie_videotemplate_bin (&config->video_template_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } else { + if (!create_primary_gie_bin (&config->primary_gie_sub_bin_config[i], + &bin->primary_gie_bin[i])) { + goto done; + } + } + g_snprintf (name, sizeof (name), "primary_gie_%d_bin", i); + gst_element_set_name (bin->primary_gie_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->primary_gie_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->primary_gie_bin[i].bin, sink_elem); + } + + sink_elem = bin->primary_gie_bin[i].bin; + if (!src_elem) { + src_elem = bin->primary_gie_bin[i].bin; + } + } + + if (config->pre_process_sub_bin_config[i].enable) { + if (!create_preprocess_bin (&config->pre_process_sub_bin_config[i], + &bin->preprocess_bin[i])) { + g_print ("creating preprocess bin failed\n"); + goto done; + } + g_snprintf (name, sizeof (name), "preprocess_%d_bin", i); + gst_element_set_name (bin->preprocess_bin[i].bin, name); + gst_bin_add (GST_BIN (bin->bin), bin->preprocess_bin[i].bin); + + if (sink_elem) { + NVGSTDS_LINK_ELEMENT (bin->preprocess_bin[i].bin, sink_elem); + } + + sink_elem = bin->preprocess_bin[i].bin; + } + + /* streamdemux and streammux to select source to inference */ + bin->streammux[i] = + gst_element_factory_make (NVDS_ELEM_STREAM_MUX, NULL); + if (!bin->streammux[i]) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'streammux'"); + goto done; + } + gst_bin_add (GST_BIN (bin->bin), bin->streammux[i]); + if (config->streammux_config.is_parsed){ + if(!set_streammux_properties (&config->streammux_config, + bin->streammux[i])){ + NVGSTDS_WARN_MSG_V("Failed to set streammux properties"); + } + } + + str = config->srcids_config[i].src_ids; + vec = split_string (str); + src_id_num = vec.size(); + g_print("i:%d, src_id_num:%d\n", i, src_id_num); + g_object_set (G_OBJECT (bin->streammux[i]), "batch-size", src_id_num, NULL); + + if(!link_streamdemux_to_streammux(config, bin, i)){ + goto done; + } + + NVGSTDS_LINK_ELEMENT (bin->streammux[i], sink_elem); + + link_element_to_metamux_sink_pad (bin->muxer, src_elem, i+1); + } + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->tee, "sink"); + + NVGSTDS_BIN_ADD_GHOST_PAD (bin->bin, bin->muxer, "src"); + + ret = TRUE; +done: + if (!ret) { + NVGSTDS_ERR_MSG_V ("%s failed", __func__); + } + return ret; +} + +static gboolean +add_and_link_broker_sink (AppCtx * appCtx) +{ + NvDsConfig *config = &appCtx->config; + /** Only first instance_bin broker sink + * employed as there's only one analytics path for N sources + * NOTE: There shall be only one [sink] group + * with type=6 (NV_DS_SINK_MSG_CONV_BROKER) + * a) Multiple of them does not make sense as we have only + * one analytics pipe generating the data for broker sink + * b) If Multiple broker sinks are configured by the user + * in config file, only the first in the order of + * appearance will be considered + * and others shall be ignored + * c) Ideally it should be documented (or obvious) that: + * multiple [sink] groups with type=6 (NV_DS_SINK_MSG_CONV_BROKER) + * is invalid + */ + NvDsInstanceBin *instance_bin = &appCtx->pipeline.instance_bins[0]; + NvDsPipeline *pipeline = &appCtx->pipeline; + + for (guint i = 0; i < config->num_sink_sub_bins; i++) { + if(config->sink_bin_sub_bin_config[i].type == NV_DS_SINK_MSG_CONV_BROKER) + { + /** add the broker sink bin to pipeline */ + if(!gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_bin.sub_bins[i].bin)) { + return FALSE; + } + g_print("add_and_link_broker_sink\n"); + // link the broker sink bin to the sink tee + if (!link_element_to_tee_src_pad (instance_bin->sink_tee, instance_bin->sink_bin.sub_bins[i].bin)) { + return FALSE; + } + } + } + return TRUE; +} + +int main(int argc, char *argv[]) +{ + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GstElement *last_elem = NULL; + NvDsInstanceBin *instance_bin; + NvDsPipeline *pipeline; + NvDsConfig *config; + GstBus *bus = NULL; + guint bus_watch_id = 0; + GstPad *osd_sink_pad = NULL; + GError *error = NULL; + guint i; + const gchar *new_mux_str = NULL; + gboolean use_new_mux = FALSE; + + ctx = g_option_context_new ("Nvidia DeepStream Parallel Demo"); + group = g_option_group_new ("abc", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + return -1; + } + + if (print_version) { + g_print ("deepstream-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + nvds_version_print (); + return 0; + } + + if (print_dependencies_version) { + g_print ("deepstream-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + nvds_version_print (); + nvds_dependencies_version_print (); + return 0; + } + + if (!cfg_files) { + NVGSTDS_ERR_MSG_V ("Specify config file with -c option"); + return_value = -1; + goto done; + } + + appCtx = (AppCtx *)g_malloc0 (sizeof (AppCtx)); + appCtx->person_class_id = -1; + appCtx->car_class_id = -1; + appCtx->index = i; + appCtx->active_source_index = -1; + if (show_bbox_text) { + appCtx->show_bbox_text = TRUE; + } + + if (!parse_config_file_yaml (&appCtx->config, cfg_files[0])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[0]); + appCtx->return_value = -1; + goto done; + } + + /* Standard GStreamer initialization */ + gst_init(&argc, &argv); + main_loop = g_main_loop_new(NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + instance_bin = &appCtx->pipeline.instance_bins[0]; + pipeline = &appCtx->pipeline; + config = &appCtx->config; + + pipeline->pipeline = gst_pipeline_new("deepstream-tensorrt-openpose-pipeline"); + + /* + * Add muxer and < N > source components to the pipeline based + * on the settings in configuration file. + */ + if (!create_multi_source_bin (config->num_source_sub_bins, + config->multi_source_config, &pipeline->multi_src_bin)) { + g_print ("creating multi source bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->multi_src_bin.bin); + + /* if using new sreammux, nvvideocovnert will scale input resolutions to the same resolution */ + new_mux_str = g_getenv ("USE_NEW_NVSTREAMMUX"); + use_new_mux = !g_strcmp0 (new_mux_str, "yes"); + if (use_new_mux) { + GstCaps* caps = NULL; + gchar * caps_string = NULL; + GstCaps* fiter_caps = NULL; + char strCaps[MAX_STR_LEN] = {0}; + for (int i = 0; i < config->num_source_sub_bins; i++){ + g_object_get (G_OBJECT (pipeline->multi_src_bin.sub_bins[i].cap_filter1), "caps", &caps, NULL); + caps_string = gst_caps_to_string (caps); + snprintf(strCaps, MAX_STR_LEN-1, "%s,width=%d, height=%d", caps_string, + config->streammux_config.pipeline_width, config->streammux_config.pipeline_height); + fiter_caps = gst_caps_from_string (strCaps); + g_object_set (G_OBJECT (pipeline->multi_src_bin.sub_bins[i].cap_filter1), "caps", fiter_caps, NULL); + printf("strCaps:%s\n", strCaps); + + gst_caps_unref (caps); + gst_caps_unref (fiter_caps); + g_free (caps_string); + } + } + + if (config->streammux_config.is_parsed){ + if(!set_streammux_properties (&config->streammux_config, + pipeline->multi_src_bin.streammux)){ + NVGSTDS_WARN_MSG_V("Failed to set streammux properties"); + } + } + + if (!create_parallel_infer_bin (config->num_primary_gie_sub_bins, + config, &pipeline->parallel_infer_bin, appCtx)) { + g_print ("creating parallel infer bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->parallel_infer_bin.bin); + last_elem = pipeline->parallel_infer_bin.bin; + NVGSTDS_LINK_ELEMENT (pipeline->multi_src_bin.bin, last_elem); + + /* Add common message converter */ + if (config->msg_conv_config.enable) { + NvDsSinkMsgConvBrokerConfig *convConfig = &config->msg_conv_config; + instance_bin->msg_conv = gst_element_factory_make (NVDS_ELEM_MSG_CONV, "common_msg_conv"); + if (!instance_bin->msg_conv) { + NVGSTDS_ERR_MSG_V ("Failed to create element 'common_msg_conv'"); + goto done; + } + + g_object_set (G_OBJECT(instance_bin->msg_conv), + "config", convConfig->config_file_path, + "msg2p-lib", (convConfig->conv_msg2p_lib ? convConfig->conv_msg2p_lib : "null"), + "payload-type", convConfig->conv_payload_type, + "comp-id", convConfig->conv_comp_id, + "debug-payload-dir", convConfig->debug_payload_dir, + "multiple-payloads", convConfig->multiple_payloads, + "msg2p-newapi", convConfig->conv_msg2p_new_api, + "frame-interval", convConfig->conv_frame_interval, + NULL); + + gst_bin_add (GST_BIN (pipeline->pipeline), + instance_bin->msg_conv); + + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->msg_conv); + last_elem = instance_bin->msg_conv; + } + + + + if (config->tiled_display_config.enable) { + if (config->tiled_display_config.columns * + config->tiled_display_config.rows < config->num_source_sub_bins) { + if (config->tiled_display_config.columns == 0) { + config->tiled_display_config.columns = + (guint) (sqrt (config->num_source_sub_bins) + 0.5); + } + config->tiled_display_config.rows = + (guint) ceil (1.0 * config->num_source_sub_bins / + config->tiled_display_config.columns); + NVGSTDS_WARN_MSG_V + ("Num of Tiles less than number of sources, readjusting to " + "%u rows, %u columns", config->tiled_display_config.rows, + config->tiled_display_config.columns); + } + + if (!create_tiled_display_bin (&config->tiled_display_config, + &pipeline->tiled_display_bin)) { + g_print ("creating tiled display bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->tiled_display_bin.bin); + + if(config->show_source != -1){ + //default -1 means composite and show all sources + g_object_set(G_OBJECT(pipeline->tiled_display_bin.tiler), "show-source", config->show_source, NULL); + g_print("show-source:%d\n", config->show_source); + } + + + NVGSTDS_LINK_ELEMENT (last_elem, pipeline->tiled_display_bin.bin); + last_elem = pipeline->tiled_display_bin.bin; + osd_sink_pad = gst_element_get_static_pad(pipeline->tiled_display_bin.tiler, "sink"); + NvDsAppPerfStructInt *str = (NvDsAppPerfStructInt *)g_malloc0(sizeof(NvDsAppPerfStructInt)); + DemoPerfCtx *perf_ctx = (DemoPerfCtx *)g_malloc0(sizeof(DemoPerfCtx)); + g_mutex_init(&perf_ctx->fps_lock); + str->context = perf_ctx; + enable_perf_measurement (str, osd_sink_pad, config->num_source_sub_bins, 1, 0, perf_cb); + gst_object_unref(osd_sink_pad); + } + + if (config->osd_config.enable) { + if (!create_osd_bin (&config->osd_config, &instance_bin->osd_bin)) { + g_print ("creating osd bin failed\n"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->osd_bin.bin); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->osd_bin.bin); + last_elem = instance_bin->osd_bin.bin; + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + osd_sink_pad = gst_element_get_static_pad(instance_bin->osd_bin.nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else { + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, NULL, NULL); + LatencyCtx *ctx = (LatencyCtx *)g_malloc0(sizeof(LatencyCtx)); + ctx->lock = (GMutex *)g_malloc0(sizeof(GMutex)); + ctx->num_sources = config->num_source_sub_bins; + gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + latency_measurement_buf_prob, ctx, NULL); + } + } + + //create sink_tee + instance_bin->sink_tee = gst_element_factory_make (NVDS_ELEM_TEE, "sink_tee"); + if (!instance_bin->sink_tee) { + NVGSTDS_ERR_MSG_V ("Failed to create 'sink_tee'"); + goto done; + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_tee); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->sink_tee); + last_elem = instance_bin->sink_tee; + + if (!create_sink_bin (config->num_sink_sub_bins, + config->sink_bin_sub_bin_config, &instance_bin->sink_bin, 0)) { + g_print ("creating sink bin failed\n"); + goto done; + } + //x264enc will output one buffer after input 66 buffers at default, enable zerolatency property. + for(int i = 0; i < config->num_sink_sub_bins; i++){ + if(config->sink_bin_sub_bin_config[i].encoder_config.enc_type == NV_DS_ENCODER_TYPE_SW) + g_object_set (G_OBJECT (instance_bin->sink_bin.sub_bins[i].encoder), "tune", 0x4, NULL); + } + gst_bin_add (GST_BIN (pipeline->pipeline), instance_bin->sink_bin.bin); + NVGSTDS_LINK_ELEMENT (last_elem, instance_bin->sink_bin.bin); + + //link broker to sink-tee + add_and_link_broker_sink(appCtx); + + /* we add a message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline->pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, main_loop); + gst_object_unref(bus); + + /* Set the pipeline to "playing" state */ + gst_element_set_state(pipeline->pipeline, GST_STATE_PLAYING); + + GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline->pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(main_loop); + +done: + + g_print ("Quitting\n"); + if (bus_watch_id) { + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline->pipeline, GST_STATE_NULL); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline->pipeline)); + g_source_remove(bus_watch_id); + } + + if (appCtx) { + if (appCtx->return_value == -1) + return_value = -1; + g_free (appCtx); + } + + if (main_loop) { + g_main_loop_unref (main_loop); + } + + if (ctx) { + g_option_context_free (ctx); + } + + if (return_value == 0) { + g_print ("App run successful\n"); + } else { + g_print ("App run failed\n"); + } + + gst_deinit (); + + return return_value; +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp new file mode 100644 index 00000000..d2c5c734 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/deepstream_parallel_infer_config_parser.cpp @@ -0,0 +1,615 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "deepstream_parallel_infer.h" +#include "deepstream_config_yaml.h" +#include + +#include +#include + +using std::cout; +using std::endl; + +static gboolean +parse_tests_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["tests"].begin(); + itr != configyml["tests"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "file-loop") { + config->file_loop = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group tests" << endl; + } + } + + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static gboolean +parse_app_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["application"].begin(); + itr != configyml["application"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable-perf-measurement") { + config->enable_perf_measurement = + itr->second.as(); + } else if (paramKey == "perf-measurement-interval-sec") { + config->perf_measurement_interval_sec = + itr->second.as(); + } else if (paramKey == "gie-kitti-output-dir") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->bbox_dir_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, config->bbox_dir_path); + g_free(str); + } else if (paramKey == "kitti-track-output-dir") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->kitti_track_dir_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, config->kitti_track_dir_path); + g_free(str); + } + else { + cout << "Unknown key " << paramKey << " for group application" << endl; + } + } + + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_multi_preprocess_yaml (NvDsPreProcessConfig *config, std::string group, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->config_file_path, temp.c_str(), 1024); + } else { + cout << "[WARNING] Unknown param found in pre-process: " << paramKey << endl; + } + } + + ret = TRUE; + done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_video_template_yaml (NvDsVideoTemplateConfig *config, std::string group, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "customlib-name") { + std::string temp = itr->second.as(); + config->customlib_name = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->customlib_name, temp.c_str(), 1024); + } else if (paramKey == "customlib-props") { + std::string temp = itr->second.as(); + config->customlib_props[config->num_customlib_props] = (char*) malloc(sizeof(char) * 1024); + std::strncpy (config->customlib_props[config->num_customlib_props], temp.c_str(), 1024); + config->num_customlib_props ++; + if (config->num_customlib_props == MAX_VIDEO_TEMPLATE_PROPS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_VIDEO_TEMPLATE_PROPS); + ret = FALSE; + goto done; + } + } else { + cout << "[WARNING] Unknown param found in pre-process: " << paramKey << endl; + } + } + + ret = TRUE; + done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_metamux_yaml (NvDsMetaMuxConfig *config, gchar* cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["meta-mux"].begin(); + itr != configyml["meta-mux"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = itr->second.as(); + } else if (paramKey == "config-file") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->config_file_path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->config_file_path)) { + g_printerr ("Error: Could not parse config-file-path in metamux.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in metamux: " << paramKey << endl; + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static gboolean +parse_sgie_yaml (NvDsConfig *config, NvDsGieConfig *gieConfig, std::string group, gchar *cfg_file_path, + gboolean enable) +{ + gboolean ret = FALSE; + gboolean parse_err = false; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + int sgie_num = 0; + gboolean have_one_enable = false; + + for(YAML::const_iterator itr = configyml.begin(); + itr != configyml.end(); ++itr) + { + std::string paramKey = itr->first.as(); + parse_err = + !parse_gie_yaml (gieConfig + sgie_num, paramKey, cfg_file_path); + if(enable && gieConfig[sgie_num].enable){ + have_one_enable = true; + sgie_num++; + } + } + config->num_secondary_gie_num[config->num_secondary_gie_sub_bins] = sgie_num; + config->num_secondary_gie_sub_bins++; + + ret = TRUE; + + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +static std::vector +split_csv_entries (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ',') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +gboolean +parse_tiled_display_yaml (NvDsConfig *appConfig, NvDsTiledDisplayConfig *config, gchar *cfg_file_path) +{ + gboolean ret = FALSE; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + for(YAML::const_iterator itr = configyml["tiled-display"].begin(); + itr != configyml["tiled-display"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + config->enable = + (NvDsTiledDisplayEnable) itr->second.as(); + } else if (paramKey == "rows") { + config->rows = itr->second.as(); + } else if (paramKey == "columns") { + config->columns = itr->second.as(); + } else if (paramKey == "width") { + config->width = itr->second.as(); + } else if (paramKey == "height") { + config->height = itr->second.as(); + } else if (paramKey == "gpu-id") { + config->gpu_id = itr->second.as(); + } else if (paramKey == "nvbuf-memory-type") { + config->nvbuf_memory_type = itr->second.as(); + } else if (paramKey == "show-source") { + appConfig->show_source = itr->second.as(); + } else { + cout << "[WARNING] Unknown param found in tiled-display: " << paramKey << endl; + } + } + ret = TRUE; + + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} + +gboolean +parse_config_file_yaml (NvDsConfig *config, gchar *cfg_file_path) +{ + gboolean parse_err = false; + gboolean ret = FALSE; + gboolean enable = false; + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + std::string source_str = "source"; + std::string sink_str = "sink"; + std::string pre_process_str = "pre-process"; + std::string pgie_str = "primary-gie"; + std::string branch_str = "branch"; + std::string tracker_str = "tracker"; + std::string sgie_str = "secondary-gie"; + std::string video_template_str = "video-template"; + std::string msgcons_str = "message-consumer"; + std::string analytics_str = "nvds-analytics"; + config->source_list_enabled = FALSE; + config->show_source = -1; //default: show all source + + for(YAML::const_iterator itr = configyml.begin(); + itr != configyml.end(); ++itr) { + std::string paramKey = itr->first.as(); + + if (paramKey == "application") { + parse_err = !parse_app_yaml (config, cfg_file_path); + } + else if (paramKey == "source") { + if(configyml["source"]["csv-file-path"]) { + std::string csv_file_path = configyml["source"]["csv-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + + std::ifstream inputFile (abs_csv_path); + if (!inputFile.is_open()) { + cout << "Couldn't open CSV file " << abs_csv_path << endl; + } + std::string line, temp; + /* Separating header field and inserting as strings into the vector. + */ + getline(inputFile, line); + std::vector headers = split_csv_entries(line); + /*Parsing each csv entry as an input source */ + while(getline(inputFile, line)) { + std::vector source_values = split_csv_entries(line); + if (config->num_source_sub_bins == MAX_SOURCE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS); + ret = FALSE; + goto done; + } + guint source_id = 0; + source_id = config->num_source_sub_bins; + parse_err = !parse_source_yaml (&config->multi_source_config[source_id], headers, source_values, cfg_file_path); + if (config->multi_source_config[source_id].enable) + config->num_source_sub_bins++; + } + } else { + NVGSTDS_ERR_MSG_V ("CSV file not specified\n"); + ret = FALSE; + goto done; + } + } + else if (paramKey == "streammux") { + parse_err = !parse_streammux_yaml(&config->streammux_config, cfg_file_path); + } + else if (paramKey == "osd") { + parse_err = !parse_osd_yaml(&config->osd_config, cfg_file_path); + } + else if (paramKey.compare(0, pre_process_str.size(), pre_process_str) == 0) { + if (config->num_pre_process_sub_bins == MAX_PRE_PROCESS_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d pre-process", MAX_PRE_PROCESS_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_multi_preprocess_yaml (&config->pre_process_sub_bin_config[config-> + num_pre_process_sub_bins], + paramKey, cfg_file_path); + if (config->pre_process_sub_bin_config[config->num_pre_process_sub_bins].enable){ + config->num_pre_process_sub_bins++; + } + } + else if (paramKey.compare(0, pgie_str.size(), pgie_str) == 0) { + if (config->num_primary_gie_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_gie_yaml (&config->primary_gie_sub_bin_config[config-> + num_primary_gie_sub_bins], + paramKey, cfg_file_path); + if (config->primary_gie_sub_bin_config[config->num_primary_gie_sub_bins].enable){ + config->num_primary_gie_sub_bins++; + } + } + else if (paramKey.compare(0, video_template_str.size(), video_template_str) == 0) { + if (config->num_primary_gie_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_video_template_yaml (&config->video_template_sub_bin_config[config-> + num_primary_gie_sub_bins], + paramKey, cfg_file_path); + if (config->video_template_sub_bin_config[config->num_primary_gie_sub_bins].enable){ + config->num_primary_gie_sub_bins++; + } + } + else if (paramKey == "meta-mux") { + parse_err = !parse_metamux_yaml (&config->meta_mux_config, cfg_file_path); + } + else if (paramKey.compare(0, branch_str.size(), branch_str) == 0) { + if(configyml[paramKey]["pgie-id"].as()){ + config->srcids_config[config->num_src_ids_sub_bins].pgie_id = + configyml[paramKey]["pgie-id"].as(); + if(configyml[paramKey]["src-ids"]) { + std::string src_ids = configyml[paramKey]["src-ids"].as(); + int index = config->num_src_ids_sub_bins; + config->srcids_config[index].src_ids = (char*) calloc(sizeof(char) * src_ids.size(), sizeof(char)); + std::strncpy (config->srcids_config[index].src_ids , src_ids.c_str(), src_ids.size()); + g_print("src_ids:%s\n", config->srcids_config[index].src_ids); + } + } + config->num_src_ids_sub_bins++; + } + else if (paramKey.compare(0, tracker_str.size(), tracker_str) == 0) { + if (config->num_tracker_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d tracker", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + + enable = configyml[paramKey]["enable"].as(); + if(configyml[paramKey]["cfg-file-path"]) { + std::string csv_file_path = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + + parse_err = + !parse_tracker_yaml (&config->tracker_config[config-> + num_tracker_sub_bins], abs_csv_path); + g_free(abs_csv_path); + config->tracker_config[config->num_tracker_sub_bins].enable = enable; + config->num_tracker_sub_bins++; + } + } + else if (paramKey.compare(0, sgie_str.size(), sgie_str) == 0) { + if (config->num_secondary_gie_sub_bins == MAX_SECONDARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_SECONDARY_GIE_BINS); + ret = FALSE; + goto done; + } + enable = configyml[paramKey]["enable"].as(); + if( configyml[paramKey]["cfg-file-path"]) { + std::string csv_file_path = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, csv_file_path.c_str(), 1024); + char *abs_csv_path = (char*) malloc(sizeof(char) * 1024); + get_absolute_file_path_yaml (cfg_file_path, str, abs_csv_path); + g_free(str); + NvDsGieConfig* pCfg = config->secondary_gie_sub_bin_config[config-> + num_secondary_gie_sub_bins]; + parse_err = + !parse_sgie_yaml(config, pCfg, + paramKey, abs_csv_path, enable); + } + } + else if (paramKey.compare(0, sink_str.size(), sink_str) == 0) { + if (config->num_sink_sub_bins == MAX_SINK_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sinks", MAX_SINK_BINS); + ret = FALSE; + goto done; + } + parse_err = + !parse_sink_yaml (&config-> + sink_bin_sub_bin_config[config->num_sink_sub_bins], paramKey, cfg_file_path); + if (config-> + sink_bin_sub_bin_config[config->num_sink_sub_bins].enable) { + config->num_sink_sub_bins++; + } + } + else if (paramKey.compare(0, msgcons_str.size(), msgcons_str) == 0) { + if (config->num_message_consumers == MAX_MESSAGE_CONSUMERS) { + NVGSTDS_ERR_MSG_V ("App supports max %d consumers", MAX_MESSAGE_CONSUMERS); + ret = FALSE; + goto done; + } + parse_err = !parse_msgconsumer_yaml ( + &config->message_consumer_config[config->num_message_consumers], + paramKey, cfg_file_path); + + if (config->message_consumer_config[config->num_message_consumers].enable) { + config->num_message_consumers++; + } + } + else if (paramKey == "tiled-display") { + parse_err = !parse_tiled_display_yaml (config, &config->tiled_display_config, cfg_file_path); + } + else if (paramKey == "img-save") { + parse_err = !parse_image_save_yaml (&config->image_save_config , cfg_file_path); + } + + else if (paramKey == "ds-example") { + parse_err = !parse_dsexample_yaml (&config->dsexample_config, cfg_file_path); + } + else if (paramKey == "message-converter") { + parse_err = !parse_msgconv_yaml (&config->msg_conv_config, paramKey, cfg_file_path); + } + else if (paramKey.compare(0, analytics_str.size(), analytics_str) == 0) { + if (config->num_analysis_sub_bins == MAX_PRIMARY_GIE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d analysis", MAX_PRIMARY_GIE_BINS); + ret = FALSE; + goto done; + } + enable = configyml[paramKey]["enable"].as(); + config->dsanalytics_config[config->num_analysis_sub_bins].enable = enable; + if(configyml[paramKey]["cfg-file-path"]) { + std::string temp = configyml[paramKey]["cfg-file-path"].as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + config->dsanalytics_config[config->num_analysis_sub_bins].config_file_path = + (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (cfg_file_path, str, + config->dsanalytics_config[config->num_analysis_sub_bins].config_file_path)) { + g_printerr ("Error: Could not parse config-file in dsanalytics.\n"); + g_free (str); + goto done; + } + g_free (str); + } else { + cout << "[WARNING] Unknown param found in nvds-analytics: " << paramKey << endl; + } + config->num_analysis_sub_bins++; + } + else if (paramKey == "tests") { + parse_err = !parse_tests_yaml (config, cfg_file_path); + } + + if (parse_err) { + cout << "failed parsing" << endl; + goto done; + } + } + /* Updating batch size when source list is enabled */ + /* if (config->source_list_enabled == TRUE) { + // For streammux and pgie, batch size is set to number of sources + config->streammux_config.batch_size = config->num_source_sub_bins; + config->primary_gie_config.batch_size = config->num_source_sub_bins; + if (config->sgie_batch_size != 0) { + for (i = 0; i < config->num_secondary_gie_sub_bins; i++) { + config->secondary_gie_sub_bin_config[i].batch_size = config->sgie_batch_size; + } + } + } */ + + unsigned int i, j, k; + for (i = 0; i < config->num_secondary_gie_sub_bins; i++) { + for(int j = 0; j < config->num_secondary_gie_num[i]; j++ ){ + if (config->secondary_gie_sub_bin_config[i][j].unique_id == + config->primary_gie_sub_bin_config[i].unique_id) { + NVGSTDS_ERR_MSG_V ("Non unique gie ids found"); + ret = FALSE; + goto done; + } + } + } + + for (k = 0; k < config->num_secondary_gie_sub_bins; k++) { + for (i = 0; i < config->num_secondary_gie_num[k]; i++) { + for (j = i + 1; j < config->num_secondary_gie_num[k]; j++) { + if (config->secondary_gie_sub_bin_config[k][i].unique_id == + config->secondary_gie_sub_bin_config[k][j].unique_id) { + NVGSTDS_ERR_MSG_V ("Non unique gie id %d found", + config->secondary_gie_sub_bin_config[k][i].unique_id); + ret = FALSE; + goto done; + } + } + } + } + + for (i = 0; i < config->num_source_sub_bins; i++) { + if (config->multi_source_config[i].type == NV_DS_SOURCE_URI_MULTIPLE) { + if (config->multi_source_config[i].num_sources < 1) { + config->multi_source_config[i].num_sources = 1; + } + for (j = 1; j < config->multi_source_config[i].num_sources; j++) { + if (config->num_source_sub_bins == MAX_SOURCE_BINS) { + NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS); + ret = FALSE; + goto done; + } + memcpy (&config->multi_source_config[config->num_source_sub_bins], + &config->multi_source_config[i], + sizeof (config->multi_source_config[i])); + config->multi_source_config[config->num_source_sub_bins].type = + NV_DS_SOURCE_URI; + config->multi_source_config[config->num_source_sub_bins].uri = + g_strdup_printf (config->multi_source_config[config-> + num_source_sub_bins].uri, j); + config->num_source_sub_bins++; + } + config->multi_source_config[i].type = NV_DS_SOURCE_URI; + config->multi_source_config[i].uri = + g_strdup_printf (config->multi_source_config[i].uri, 0); + } + } + + ret = TRUE; +done: + if (!ret) { + cout << __func__ << " failed" << endl; + } + return ret; +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp new file mode 100644 index 00000000..7c2ff6c8 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/cover_table.hpp @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +class CoverTable +{ +public: + CoverTable(int nrows, int ncols) : nrows(nrows), ncols(ncols) + { + rows.resize(nrows); + cols.resize(ncols); + } + + inline void coverRow(int row) + { + rows[row] = 1; + } + + inline void coverCol(int col) + { + cols[col] = 1; + } + + inline void uncoverRow(int row) + { + rows[row] = 0; + } + + inline void uncoverCol(int col) + { + cols[col] = 0; + } + + inline bool isCovered(int row, int col) const + { + return rows[row] || cols[col]; + } + + inline bool isRowCovered(int row) const + { + return rows[row]; + } + + inline bool isColCovered(int col) const + { + return cols[col]; + } + + inline void clear() + { + for (int i = 0; i < nrows; i++) + { + uncoverRow(i); + } + for (int j = 0; j < ncols; j++) + { + uncoverCol(j); + } + } + + const int nrows; + const int ncols; + +private: + std::vector rows; + std::vector cols; +}; diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp new file mode 100644 index 00000000..2bb73ddf --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/munkres_algorithm.cpp @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pair_graph.hpp" +#include "cover_table.hpp" + +#include +#include +#include +#include +#include + +template +using Vec1D = std::vector; +template +using Vec2D = std::vector>; +template +using Vec3D = std::vector>; + +// Helper method to subtract the minimum row from cost_graph +void subtract_minimum_row(Vec2D &cost_graph, int nrows, int ncols) +{ + for (int i = 0; i < nrows; i++) + { + // Iterate the find the minimum + float min = cost_graph[i][0]; + for (int j = 0; j < ncols; j++) + { + float val = cost_graph[i][j]; + if (val < min) + { + min = val; + } + } + + // Subtract the Minimum + for (int j = 0; j < ncols; j++) + { + cost_graph[i][j] -= min; + } + } +} + +// Helper method to subtract the minimum col from cost_graph +void subtract_minimum_column(Vec2D &cost_graph, int nrows, int ncols) +{ + for (int j = 0; j < ncols; j++) + { + // Iterate and find the minimum + float min = cost_graph[0][j]; + for (int i = 0; i < nrows; i++) + { + float val = cost_graph[i][j]; + if (val < min) + { + min = val; + } + } + + // Subtract the minimum + for (int i = 0; i < nrows; i++) + { + cost_graph[i][j] -= min; + } + } +} + +void munkresStep1(Vec2D &cost_graph, PairGraph &star_graph, int nrows, + int ncols) +{ + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (!star_graph.isRowSet(i) && !star_graph.isColSet(j) && (cost_graph[i][j] == 0)) + { + star_graph.set(i, j); + } + } + } +} + +// Exits if '1' is returned +bool munkresStep2(const PairGraph &star_graph, CoverTable &cover_table) +{ + int k = + star_graph.nrows < star_graph.ncols ? star_graph.nrows : star_graph.ncols; + int count = 0; + for (int j = 0; j < star_graph.ncols; j++) + { + if (star_graph.isColSet(j)) + { + cover_table.coverCol(j); + count++; + } + } + return count >= k; +} + +bool munkresStep3(Vec2D &cost_graph, const PairGraph &star_graph, + PairGraph &prime_graph, CoverTable &cover_table, std::pair &p, + int nrows, int ncols) +{ + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (cost_graph[i][j] == 0 && !cover_table.isCovered(i, j)) + { + prime_graph.set(i, j); + if (star_graph.isRowSet(i)) + { + cover_table.coverRow(i); + cover_table.uncoverCol(star_graph.colForRow(i)); + } + else + { + p.first = i; + p.second = j; + return 1; + } + } + } + } + return 0; +}; + +void munkresStep4(PairGraph &star_graph, PairGraph &prime_graph, + CoverTable &cover_table, std::pair &p) +{ + // This process should be repeated until no star is found in prime's column + while (star_graph.isColSet(p.second)) + { + // First find and reset any star found in the prime's columns + std::pair s = {star_graph.rowForCol(p.second), p.second}; + star_graph.reset(s.first, s.second); + + // Set this prime to a star + star_graph.set(p.first, p.second); + + // Repeat the same process for prime in cleared star's row + p = {s.first, prime_graph.colForRow(s.first)}; + } + star_graph.set(p.first, p.second); + cover_table.clear(); + prime_graph.clear(); +} + +void munkresStep5(Vec2D &cost_graph, const CoverTable &cover_table, + int nrows, int ncols) +{ + bool valid = false; + float min; + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (!cover_table.isCovered(i, j)) + { + if (!valid) + { + min = cost_graph[i][j]; + valid = true; + } + else if (cost_graph[i][j] < min) + { + min = cost_graph[i][j]; + } + } + } + } + + for (int i = 0; i < nrows; i++) + { + if (cover_table.isRowCovered(i)) + { + for (int j = 0; j < ncols; j++) + { + cost_graph[i][j] += min; + } + } + } + for (int j = 0; j < ncols; j++) + { + if (!cover_table.isColCovered(j)) + { + for (int i = 0; i < nrows; i++) + { + cost_graph[i][j] -= min; + } + } + } +} + +void munkres_algorithm(Vec2D &cost_graph, PairGraph &star_graph, int nrows, + int ncols) +{ + PairGraph prime_graph(nrows, ncols); + CoverTable cover_table(nrows, ncols); + prime_graph.clear(); + cover_table.clear(); + star_graph.clear(); + + int step = 0; + if (ncols >= nrows) + { + subtract_minimum_row(cost_graph, nrows, ncols); + } + if (ncols > nrows) + { + step = 1; + } + + std::pair p; + bool done = false; + while (!done) + { + switch (step) + { + case 0: + subtract_minimum_column(cost_graph, nrows, ncols); + case 1: + munkresStep1(cost_graph, star_graph, nrows, ncols); + case 2: + if (munkresStep2(star_graph, cover_table)) + { + done = true; + break; + } + case 3: + if (!munkresStep3(cost_graph, star_graph, prime_graph, cover_table, p, + nrows, ncols)) + { + step = 5; + break; + } + case 4: + munkresStep4(star_graph, prime_graph, cover_table, p); + step = 2; + break; + case 5: + munkresStep5(cost_graph, cover_table, nrows, ncols); + step = 3; + break; + } + } +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp new file mode 100644 index 00000000..3f3c8b38 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/pair_graph.hpp @@ -0,0 +1,128 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +class PairGraph +{ +public: + PairGraph(int nrows, int ncols) : nrows(nrows), ncols(ncols) + { + this->rows.resize(nrows); + this->cols.resize(ncols); + } + + /** + * Returns the column index of the pair matching this row + */ + inline int colForRow(int row) const + { + return this->rows[row]; + } + + /** + * Returns the row index of the pair matching this column + */ + inline int rowForCol(int col) const + { + return this->cols[col]; + } + + /** + * Creates a pair between row and col + */ + inline void set(int row, int col) + { + this->rows[row] = col; + this->cols[col] = row; + } + + inline bool isRowSet(int row) const + { + return rows[row] >= 0; + } + + inline bool isColSet(int col) const + { + return cols[col] >= 0; + } + + inline bool isPair(int row, int col) + { + return rows[row] == col; + } + + /** + * Clears pair between row and col + */ + inline void reset(int row, int col) + { + this->rows[row] = -1; + this->cols[col] = -1; + } + + /** + * Clears all pairs in graph + */ + void clear() + { + for (int i = 0; i < this->nrows; i++) + { + this->rows[i] = -1; + } + for (int j = 0; j < this->ncols; j++) + { + this->cols[j] = -1; + } + } + + int numPairs() + { + int count = 0; + for (int i = 0; i < nrows; i++) + { + if (rows[i] >= 0) + { + count++; + } + } + return count; + } + + std::vector> pairs() + { + std::vector> p(numPairs()); + int count = 0; + for (int i = 0; i < nrows; i++) + { + if (isRowSet(i)) + { + p[count++] = {i, colForRow(i)}; + } + } + return p; + } + + const int nrows; + const int ncols; + +private: + std::vector rows; + std::vector cols; +}; diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp new file mode 100644 index 00000000..11352de6 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/apps/deepstream-parallel-infer/post_process/body_pose/post_process.cpp @@ -0,0 +1,428 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pair_graph.hpp" +#include "cover_table.hpp" +#include "munkres_algorithm.cpp" + +#include +#include +#include + +//#include "gstnvdsmeta.h" +//#include "gstnvdsinfer.h" +#include "nvdsgstutils.h" +#include "nvbufsurface.h" + +#include +#include +#include +#include +#include + +#define EPS 1e-6 + +template +using Vec1D = std::vector; +template +using Vec2D = std::vector>; +template +using Vec3D = std::vector>; + +static const int M = 2; + +static Vec2D topology{ + {0, 1, 15, 13}, + {2, 3, 13, 11}, + {4, 5, 16, 14}, + {6, 7, 14, 12}, + {8, 9, 11, 12}, + {10, 11, 5, 7}, + {12, 13, 6, 8}, + {14, 15, 7, 9}, + {16, 17, 8, 10}, + {18, 19, 1, 2}, + {20, 21, 0, 1}, + {22, 23, 0, 2}, + {24, 25, 1, 3}, + {26, 27, 2, 4}, + {28, 29, 3, 5}, + {30, 31, 4, 6}, + {32, 33, 17, 0}, + {34, 35, 17, 5}, + {36, 37, 17, 6}, + {38, 39, 17, 11}, + {40, 41, 17, 12}}; + +/* Method to find peaks in the output tensor. 'window_size' represents how many pixels we are considering at once to find a maximum value, or a ‘peak’. + Once we find a peak, we mark it using the ‘is_peak’ boolean in the inner loop and assign this maximum value to the center pixel of our window. + This is then repeated until we cover the entire frame. */ +void find_peaks(Vec1D &counts_out, Vec3D &peaks_out, void *cmap_data, + NvDsInferDims &cmap_dims, float threshold, int window_size, int max_count) +{ + int w = window_size / 2; + int width = cmap_dims.d[2]; + int height = cmap_dims.d[1]; + + counts_out.assign(cmap_dims.d[0], 0); + peaks_out.assign(cmap_dims.d[0], Vec2D(max_count, Vec1D(M, + 0))); + + for (unsigned int c = 0; c < cmap_dims.d[0]; c++) + { + int count = 0; + float *cmap_data_c = (float *)cmap_data + c * width * height; + + for (int i = 0; i < height && count < max_count; i++) + { + for (int j = 0; j < width && count < max_count; j++) + { + float value = cmap_data_c[i * width + j]; + + if (value < threshold) + continue; + + int ii_min = i - w; + int jj_min = j - w; + int ii_max = i + w + 1; + int jj_max = j + w + 1; + + if (ii_min < 0) + ii_min = 0; + if (ii_max > height) + ii_max = height; + if (jj_min < 0) + jj_min = 0; + if (jj_max > width) + jj_max = width; + + bool is_peak = true; + for (int ii = ii_min; ii < ii_max; ii++) + { + for (int jj = jj_min; jj < jj_max; jj++) + { + if (cmap_data_c[ii * width + jj] > value) + { + is_peak = false; + } + } + } + + if (is_peak) + { + peaks_out[c][count][0] = i; + peaks_out[c][count][1] = j; + count++; + } + } + } + + counts_out[c] = count; + } +} + +/* Normalize the peaks found in 'find_peaks' and apply non-maximal suppression*/ +Vec3D +refine_peaks(Vec1D &counts, + Vec3D &peaks, void *cmap_data, NvDsInferDims &cmap_dims, + int window_size) +{ + int w = window_size / 2; + int width = cmap_dims.d[2]; + int height = cmap_dims.d[1]; + + Vec3D refined_peaks(peaks.size(), Vec2D(peaks[0].size(), + Vec1D(peaks[0][0].size(), 0))); + + for (unsigned int c = 0; c < cmap_dims.d[0]; c++) + { + int count = counts[c]; + auto &refined_peaks_a_bc = refined_peaks[c]; + auto &peaks_a_bc = peaks[c]; + float *cmap_data_c = (float *)cmap_data + c * width * height; + + for (int p = 0; p < count; p++) + { + auto &refined_peak = refined_peaks_a_bc[p]; + auto &peak = peaks_a_bc[p]; + + int i = peak[0]; + int j = peak[1]; + float weight_sum = 0.0f; + + for (int ii = i - w; ii < i + w + 1; ii++) + { + int ii_idx = ii; + + if (ii < 0) + ii_idx = -ii; + else if (ii >= height) + ii_idx = height - (ii - height) - 2; + + for (int jj = j - w; jj < j + w + 1; jj++) + { + int jj_idx = jj; + + if (jj < 0) + jj_idx = -jj; + else if (jj >= width) + jj_idx = width - (jj - width) - 2; + + float weight = cmap_data_c[ii_idx * width + jj_idx]; + refined_peak[0] += weight * ii; + refined_peak[1] += weight * jj; + weight_sum += weight; + } + } + + refined_peak[0] /= weight_sum; + refined_peak[1] /= weight_sum; + refined_peak[0] += 0.5; + refined_peak[1] += 0.5; + refined_peak[0] /= height; + refined_peak[1] /= width; + } + } + + return refined_peaks; +} + +/* Create a bipartite graph to assign detected body-parts to a unique person in the frame. This method also takes care of finding the line integral to assign scores + to these points */ +Vec3D +paf_score_graph(void *paf_data, NvDsInferDims &paf_dims, + Vec2D &topology, Vec1D &counts, + Vec3D &peaks, int num_integral_samples) +{ + int K = topology.size(); + int H = paf_dims.d[1]; + int W = paf_dims.d[2]; + int max_count = peaks[0].size(); + Vec3D score_graph(K, Vec2D(max_count, Vec1D(max_count, 0))); + + for (int k = 0; k < K; k++) + { + auto &score_graph_nk = score_graph[k]; + auto &paf_i_idx = topology[k][0]; + auto &paf_j_idx = topology[k][1]; + auto &cmap_a_idx = topology[k][2]; + auto &cmap_b_idx = topology[k][3]; + float *paf_i = (float *)paf_data + paf_i_idx * H * W; + float *paf_j = (float *)paf_data + paf_j_idx * H * W; + + auto &counts_a = counts[cmap_a_idx]; + auto &counts_b = counts[cmap_b_idx]; + auto &peaks_a = peaks[cmap_a_idx]; + auto &peaks_b = peaks[cmap_b_idx]; + + for (int a = 0; a < counts_a; a++) + { + // Point A + float pa_i = peaks_a[a][0] * H; + float pa_j = peaks_a[a][1] * W; + + for (int b = 0; b < counts_b; b++) + { + // Point B + float pb_i = peaks_b[b][0] * H; + float pb_j = peaks_b[b][1] * W; + + // Vector from Point A to Point B + float pab_i = pb_i - pa_i; + float pab_j = pb_j - pa_j; + + // Normalized Vector from Point A to Point B + float pab_norm = sqrtf(pab_i * pab_i + pab_j * pab_j) + EPS; + float uab_i = pab_i / pab_norm; + float uab_j = pab_j / pab_norm; + + float integral = 0.0; + float increment = 1.0f / num_integral_samples; + + for (int t = 0; t < num_integral_samples; t++) + { + // Integral Point T + float progress = (float)t / (float)num_integral_samples; + float pt_i = pa_i + progress * pab_i; + float pt_j = pa_j + progress * pab_j; + + // Convert to Integer + int pt_i_int = (int)pt_i; + int pt_j_int = (int)pt_j; + + // Edge cases for if the point is out of bounds, just skip them + if (pt_i_int < 0) + continue; + if (pt_i_int > H) + continue; + if (pt_j_int < 0) + continue; + if (pt_j_int > W) + continue; + + // Vector at integral point + float pt_paf_i = paf_i[pt_i_int * W + pt_j_int]; + float pt_paf_j = paf_j[pt_i_int * W + pt_j_int]; + + // Dot Product Normalized A->B with PAF Vector + float dot = pt_paf_i * uab_i + pt_paf_j * uab_j; + integral += dot; + + progress += increment; + } + + // Normalize the integral with respect to the number of samples + integral /= num_integral_samples; + score_graph_nk[a][b] = integral; + } + } + } + return score_graph; +} + +/* + This method takes care of solving the graph assignment problem using Munkres algorithm. Munkres algorithm is defind in 'munkres_algorithm.cpp' + */ + +Vec3D +assignment(Vec3D &score_graph, + Vec2D &topology, Vec1D &counts, float score_threshold, int max_count) +{ + int K = topology.size(); + Vec3D connections(K, Vec2D(M, Vec1D(max_count, -1))); + + Vec3D cost_graph = score_graph; + for (Vec2D &cg_iter1 : cost_graph) + for (Vec1D &cg_iter2 : cg_iter1) + for (float &cg_iter3 : cg_iter2) + cg_iter3 = -cg_iter3; + auto &cost_graph_out_a = cost_graph; + + for (int k = 0; k < K; k++) + { + int cmap_a_idx = topology[k][2]; + int cmap_b_idx = topology[k][3]; + int nrows = counts[cmap_a_idx]; + int ncols = counts[cmap_b_idx]; + auto star_graph = PairGraph(nrows, ncols); + auto &cost_graph_out_a_nk = cost_graph_out_a[k]; + munkres_algorithm(cost_graph_out_a_nk, star_graph, nrows, ncols); + + auto &connections_a_nk = connections[k]; + auto &score_graph_a_nk = score_graph[k]; + + for (int i = 0; i < nrows; i++) + { + for (int j = 0; j < ncols; j++) + { + if (star_graph.isPair(i, j) && score_graph_a_nk[i][j] > score_threshold) + { + connections_a_nk[0][i] = j; + connections_a_nk[1][j] = i; + } + } + } + } + return connections; +} + +/* This method takes care of connecting all the body parts detected to each other + after finding the relationships between them in the 'assignment' method */ +Vec2D +connect_parts( + Vec3D &connections, Vec2D &topology, Vec1D &counts, + int max_count) +{ + int K = topology.size(); + int C = counts.size(); + + Vec2D visited(C, Vec1D(max_count, 0)); + + Vec2D objects(max_count, Vec1D(C, -1)); + + int num_objects = 0; + for (int c = 0; c < C; c++) + { + if (num_objects >= max_count) + { + break; + } + + int count = counts[c]; + + for (int i = 0; i < count; i++) + { + if (num_objects >= max_count) + { + break; + } + + std::queue> q; + bool new_object = false; + q.push({c, i}); + + while (!q.empty()) + { + auto node = q.front(); + q.pop(); + int c_n = node.first; + int i_n = node.second; + + if (visited[c_n][i_n]) + { + continue; + } + + visited[c_n][i_n] = 1; + new_object = true; + objects[num_objects][c_n] = i_n; + + for (int k = 0; k < K; k++) + { + int c_a = topology[k][2]; + int c_b = topology[k][3]; + + if (c_a == c_n) + { + int i_b = connections[k][0][i_n]; + if (i_b >= 0) + { + q.push({c_b, i_b}); + } + } + + if (c_b == c_n) + { + int i_a = connections[k][1][i_n]; + if (i_a >= 0) + { + q.push({c_a, i_a}); + } + } + } + } + + if (new_object) + { + num_objects++; + } + } + } + + objects.resize(num_objects); + return objects; +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/build.sh b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/build.sh new file mode 100644 index 00000000..cb500606 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/build.sh @@ -0,0 +1,14 @@ + +make + +## enable NVSTREAMMUX_ADAPTIVE_BATCHING +if [ x"$NVSTREAMMUX_ADAPTIVE_BATCHING" != x"yes" ]; then + export NVSTREAMMUX_ADAPTIVE_BATCHING=yes + rm -rf ~/.cache/gstreamer-1.0/ + echo "export NVSTREAMMUX_ADAPTIVE_BATCHING=yes" +fi + +## dict.txt is label file for LPR model +if [ ! -f dict.txt ]; then + wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/lprnet/versions/deployable_v1.0/files/us_lp_characters.txt' -O dict.txt +fi diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt new file mode 100644 index 00000000..4050359d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/US_LPD/config_uslpd_inferserver.txt @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "US_LPD" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/US_LPD/usa_lpd_label.txt" + detection { + num_detected_classes: 1 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + operate_on_gie_id: 3 + process_mode: PROCESS_MODE_CLIP_OBJECTS + secondary_reinfer_interval: 0 + object_control { + bbox_filter { + min_width: 40 + min_height: 30 + } + } +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml new file mode 100644 index 00000000..7b10f9ca --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/dstest5_msgconv_sample_config.yml @@ -0,0 +1,119 @@ +############################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml new file mode 100644 index 00000000..c499a4e2 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml new file mode 100644 index 00000000..0d4f705d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie1.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml new file mode 100644 index 00000000..c0ec3b7a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml new file mode 100644 index 00000000..395de59e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 yolo +#brach2 bodypose +# +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_inferserver.txt + config-file: ../../bodypose2d/config_body2_infer.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv new file mode 100644 index 00000000..213c663c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv new file mode 100644 index 00000000..29a2b484 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv new file mode 100644 index 00000000..2663e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml new file mode 100644 index 00000000..30041242 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml new file mode 100644 index 00000000..a05c3001 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie0.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml new file mode 100644 index 00000000..0d4f705d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie1.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml new file mode 100644 index 00000000..c0ec3b7a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml new file mode 100644 index 00000000..bc777f10 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + #csv-file-path: sources_4.csv + csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_infer.txt + config-file: ../../bodypose2d/config_body2_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 1 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv new file mode 100644 index 00000000..213c663c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv new file mode 100644 index 00000000..f343e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_different_source.csv @@ -0,0 +1,3 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4,2,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv new file mode 100644 index 00000000..2663e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_lpr/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml new file mode 100644 index 00000000..30041242 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml new file mode 100644 index 00000000..c499a4e2 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml new file mode 100644 index 00000000..2157b0fa --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml new file mode 100644 index 00000000..c0ec3b7a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml new file mode 100644 index 00000000..b1dbc79e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 yolo +#brach2 bodypose +# +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + #which source should be showed, -1 means showing all. + show-source: 2 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 400000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../yolov4/config_yolov4_infer.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 0 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + #config-file: ../../bodypose2d/config_body2_inferserver.txt + config-file: ../../bodypose2d/config_body2_infer.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 0 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 0 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux0.txt + + +tests: + file-loop: 0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv new file mode 100644 index 00000000..213c663c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv new file mode 100644 index 00000000..29a2b484 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv new file mode 100644 index 00000000..2663e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/bodypose_yolo_win1/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml new file mode 100644 index 00000000..30041242 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml new file mode 100644 index 00000000..9f33ab5b --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie0.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 0 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_secondary_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 0 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_secondary_vehiclemake.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml new file mode 100644 index 00000000..2157b0fa --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml new file mode 100644 index 00000000..c0ec3b7a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml new file mode 100644 index 00000000..ba873101 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 vehicle detecor + color checing + maker checking + type checking +#brach2 peoplenet + tracker + analytics +#branch3 trafficnet + tracker + lpd + lpr +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 0 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 30 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 1 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 1 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + config-file: ../../peoplenet/config_peoplenet_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 1 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 1 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux1.txt + + +tests: + file-loop: 0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv new file mode 100644 index 00000000..213c663c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv new file mode 100644 index 00000000..29a2b484 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv new file mode 100644 index 00000000..2663e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle0_lpr_analytic/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics0.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics1.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt new file mode 100644 index 00000000..4830f612 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/analytics2.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=-1 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt new file mode 100644 index 00000000..ea5e5b2e --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/config_nvdsanalytics.txt @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-2] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=1 +class-id=0 + +[overcrowding-stream-1] +enable=1 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +#line-crossing-Entry=1072;911;1143;1058;944;1020;1297;1020; +line-crossing-Exit=789;672;1084;900;851;773;1203;732 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=loose + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml new file mode 100644 index 00000000..30041242 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/dstest5_msgconv_sample_config.yml @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sensor0: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor1: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor2: + enable: 1 + type: Camera + id: HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +sensor3: + enable: 1 + type: Camera + id: HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 + location: 45.293701447;-75.8303914499;48.1557479338 + description: Aisle Camera + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 0 + type: intersection/road + name: HWY_20_AND_LOCUST__EBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place1: + enable: 1 + id: 1 + type: intersection/road + name: HWY_20_AND_LOCUST__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place2: + enable: 1 + id: 2 + type: intersection/road + name: HWY_20_AND_DEVON__WBA + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +place3: + enable: 1 + id: 3 + type: intersection/road + name: HWY_20_AND_LOCUST + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: C_127_158 + place-sub-field2: Lane 1 + place-sub-field3: P1 + +analytics0: + enable: 1 + id: XYZ_1 + description: Vehicle Detection and License Plate Recognition + source: OpenALR + version: 1.0 + +analytics1: + enable: 1 + id: XYZ_2 + description: Vehicle Detection and License Plate Recognition 1 + source: OpenALR + version: 1.0 + +analytics2: + enable: 1 + id: XYZ_3 + description: Vehicle Detection and License Plate Recognition 2 + source: OpenALR + version: 1.0 + +analytics3: + enable: 1 + id: XYZ_4 + description: Vehicle Detection and License Plate Recognition 4 + source: OpenALR + version: 1.0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml new file mode 100644 index 00000000..c499a4e2 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie0.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 11 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 12 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_carmake.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml new file mode 100644 index 00000000..2157b0fa --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie1.yml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 0 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 20 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: ../../vehicle/config_infer_secondary_plan_engine_vehicletypes.txt diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml new file mode 100644 index 00000000..c0ec3b7a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/secondary-gie2.yml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +secondary-gie0: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 30 + operate-on-gie-id: 3 + operate-on-class-ids: 0 + config-file: ../../US_LPD/config_uslpd_inferserver.txt + +secondary-gie1: + enable: 1 + ##(0): nvinfer; (1): nvinferserver + plugin-type: 1 + ## nvinferserserver's gpu-id can only set from its own config-file + #gpu-id=0 + batch-size: 16 + gie-unique-id: 31 + operate-on-gie-id: 30 + operate-on-class-ids: 0 + config-file: ../../us_lprnet/config_uslpr_inferserver.txt + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml new file mode 100644 index 00000000..07b1c22b --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/source4_1080p_dec_parallel_infer.yml @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#branch1 vehicle detecor + color checing + maker checking + type checking +#brach2 peoplenet + tracker + analytics +#branch3 trafficnet + tracker + lpd + lpr +# +# +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + ##gie-kitti-output-dir=streamscl + +tiled-display: + enable: 1 + rows: 2 + columns: 2 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_4.csv + #csv-file-path: sources_4_different_source.csv + #csv-file-path: sources_4_rtsp.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 7=nv3dsink (Jetson only) + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 1 + #iframeinterval=10 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 1 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +sink3: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: ;; + topic: + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + +sink4: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 6 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 30 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 10.23.136.84;9092 + topic: dstest + disable-msgconv: 1 + #Optional: + #msg-broker-config: ../../deepstream-test4/cfg_kafka.txt + + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv : 1 +message-converter: + enable: 0 + msg-conv-config: dstest5_msgconv_sample_config.yml + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 0 + # Name of library having custom implementation. + msg-conv-msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + # Id of component in case only selected message to parse. + #msg-conv-comp-id: + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + #value changed + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Serif + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + buffer-pool-size: 4 + batch-size: 4 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + +primary-gie0: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + #interval: 0 + gie-unique-id: 1 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch0: + ## pgie's id + pgie-id: 1 + ## select sources by sourceid + src-ids: 0;1;2 + +tracker0: + enable: 0 + cfg-file-path: tracker0.yml + +nvds-analytics0: + enable: 0 + cfg-file-path: analytics0.txt + +secondary-gie0: + enable: 1 + ##support mulptiple sgie. + cfg-file-path: secondary-gie0.yml + +primary-gie1: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 2 + nvbuf-memory-type: 0 + config-file: ../../peoplenet/config_peoplenet_inferserver.txt + +branch1: + ## pgie's id + pgie-id: 2 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker1: + enable: 1 + cfg-file-path: tracker1.yml + +nvds-analytics1: + enable: 1 + cfg-file-path: analytics1.txt + +secondary-gie1: + enable: 0 + ##supoort multiple sgie + cfg-file-path: secondary-gie1.yml + +primary-gie2: + enable: 1 + #(0): nvinfer; (1): nvinferserver + plugin-type: 1 + gpu-id: 0 + #input-tensor-meta: 1 + batch-size: 4 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + interval: 0 + gie-unique-id: 3 + nvbuf-memory-type: 0 + config-file: ../../trafficcamnet/config_trafficcamnet_inferserver.txt + +branch2: + ## pgie's id + pgie-id: 3 + ## select sources by sourceid + src-ids: 1;2;3 + +tracker2: + enable: 1 + cfg-file-path: tracker2.yml + +nvds-analytics2: + enable: 0 + cfg-file-path: analytics2.txt + +secondary-gie2: + enable: 1 + ##supoort multiple sgie + cfg-file-path: secondary-gie2.yml + + +meta-mux: + enable: 1 + config-file: ../../metamux/config_metamux1.txt + + +tests: + file-loop: 0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv new file mode 100644 index 00000000..213c663c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,4,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv new file mode 100644 index 00000000..29a2b484 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_different_source.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,1,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv new file mode 100644 index 00000000..2663e406 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/sources_4_rtsp.csv @@ -0,0 +1,4 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4,2,0,0 +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4,1,0,0 +1,4,rtsp://10.19.225.227/media/video1,1,0,0 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker0.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker1.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml new file mode 100644 index 00000000..73756a63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/apps/vehicle_lpr_analytic/tracker2.yml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tracker: + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=config_tracker_NvDCF_accuracy.yml + # ll-config-file=config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt new file mode 100644 index 00000000..8a2745b0 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_infer.txt @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[property] +gpu-id=0 +model-engine-file=../../../../tritonserver/models/bodypose2d/1/model.onnx_b4_gpu0_fp16.engine +onnx-file=../../../../tritonserver/models/bodypose2d/1/model.onnx +batch-size=4 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +gie-unique-id=2 +#output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid +#0=Detection 1=Classifier 2=Segmentation +network-type=100 +process-mode=1 +net-scale-factor=0.01743071291615827 +offsets=114.74;114.74;114.74 +#0=RGB 1=BGR 2=GRAY +model-color-format=0 +output-tensor-meta=1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt new file mode 100644 index 00000000..aa007518 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/bodypose2d/config_body2_inferserver.txt @@ -0,0 +1,61 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 4 + backend { + triton { + model_name: "bodypose2d" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01743071291615827 + channel_offsets: [114.74,114.74,114.74] + } + } + + postprocess { + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} + +output_control { + output_tensor_meta: true +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt new file mode 100644 index 00000000..bc8449bc --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux0.txt @@ -0,0 +1,35 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +# sink pad name which data will be pass to src pad. +active-pad=sink_0 +# default pts-tolerance is 60 ms. +pts-tolerance=60000 + +[user-configs] + +[group-0] +# src-ids-model-= +# mux all source if don't set it. +src-ids-model-1=0;1 +src-ids-model-2=1;2 +src-ids-model-3=1;2 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt new file mode 100644 index 00000000..3996397a --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/metamux/config_metamux1.txt @@ -0,0 +1,35 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +# sink pad name which data will be pass to src pad. +active-pad=sink_0 +# default pts-tolerance is 60 ms. +pts-tolerance=60000 + +[user-configs] + +[group-0] +# src-ids-model-= +# mux all source if don't set it. +src-ids-model-1=0;1 +src-ids-model-2=1;2 +src-ids-model-3=0;1 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt new file mode 100644 index 00000000..52892568 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/peoplenet/config_peoplenet_inferserver.txt @@ -0,0 +1,70 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 8 + backend { + triton { + model_name: "peoplenet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/peoplenet/labels.txt" + detection { + num_detected_classes: 4 + + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt new file mode 100644 index 00000000..02f08684 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/trafficcamnet/config_trafficcamnet_inferserver.txt @@ -0,0 +1,70 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 8 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/trafficcamnet/labels.txt" + detection { + num_detected_classes: 4 + + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + interval: 0 +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt new file mode 100644 index 00000000..01ea8b1d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/us_lprnet/config_uslpr_inferserver.txt @@ -0,0 +1,64 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 5 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "us_lprnet" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0.0,0.0,0.0] + } + } + + postprocess { + classification { + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + threshold: 0.5 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 6 + } + custom_lib { + path: "../../gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} +input_control { + operate_on_gie_id: 30 + process_mode: PROCESS_MODE_CLIP_OBJECTS + interval: 0 +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt new file mode 100644 index 00000000..a6501fb2 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_plan_engine_primary.txt @@ -0,0 +1,78 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "conv2d_bbox"}, + {name: "conv2d_cov/Sigmoid"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt new file mode 100644 index 00000000..86c6e40c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_carmake.txt @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 6 + gpu_ids: [0] + max_batch_size: 16 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "Secondary_CarMake" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/Secondary_CarMake/labels.txt" + classification { + threshold: 0.51 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt new file mode 100644 index 00000000..7e080234 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/vehicle/config_infer_secondary_plan_engine_vehicletypes.txt @@ -0,0 +1,66 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: [0] + max_batch_size: 16 + backend { + triton { + model_name: "Secondary_VehicleTypes" + version: -1 + model_repo { + root: "../../../../tritonserver/models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../../../../tritonserver/models/Secondary_VehicleTypes/labels.txt" + classification { + threshold: 0.2 + } + } + +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names new file mode 100644 index 00000000..d70398f0 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +potted plant +bed +dining table +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt new file mode 100644 index 00000000..3aaf797d --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/configs/yolov4/config_yolov4_infer.txt @@ -0,0 +1,80 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=../../../../tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx +model-engine-file=../../../../tritonserver/models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx_b4_gpu0_fp16.engine +labelfile-path=coco.names +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +parse-bbox-func-name=NvDsInferParseCustomYoloV4 +custom-lib-path=../../gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-filter=0 +#scaling-compute-hw=0 + +[class-attrs-all] +nms-iou-threshold=0.6 +pre-cluster-threshold=0.4 diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README new file mode 100644 index 00000000..c07149cf --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvdsmetamux/README @@ -0,0 +1,99 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +Refer to the DeepStream SDK documentation for a description of the plugin. +-------------------------------------------------------------------------------- +Pre-requisites: +- GStreamer-1.0 Development package +- GStreamer-1.0 Base Plugins Development package + +Install using: + sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev +-------------------------------------------------------------------------------- +Compiling and installing the plugin: +Run make and sudo make install + +NOTE: To compile the sources, run make with "sudo" or root permission. + +About: +This plugin performs batch meta mux for the same source and the "same" frame. +The "same" frame is based on the frame PTS. nvdsmetamux will try to find the +nearest frame PTS of the same source. There is PTS diff tolerance between the +PTS of the frame when search the nearest frame. Application can configure the +PTS diff tolerance. +Application can select the sink pad which video frame will be passed to src pad. +Application also can configure to select the source ids which output from one +model. The meta data will be merged if application select it. + + +Properties: + active-pad : Active sink pad which buffer will transfer to src pad + flags: readable, writable + String. Default: null + config-file : Preprocess Config File + flags: readable, writable + String. Default: null + latency : Additional latency in live mode to allow upstream to take longer to produce buffers for the current position (in nanoseconds) + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + min-upstream-latency: When sources with a higher latency are expected to be plugged in dynamically after the aggregator has started playing, this allows overriding the minimum latency reported by the initial source(s). This is only taken into account when larger than the actually reported minimum latency. (nanoseconds) + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + name : The name of the object + flags: readable, writable + String. Default: "nvdsmetamux0" + parent : The parent of the object + flags: readable, writable + Object of type "GstObject" + pts-tolerance : Time diff tolerance when search the same frame of the same source id in microseconds + flags: readable, writable + Integer64. Range: -9223372036854775808 - 9223372036854775807 Default: 60000 + start-time : Start time to use if start-time-selection=set + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 18446744073709551615 + start-time-selection: Decides which start time is output + flags: readable, writable + Enum "GstAggregatorStartTimeSelection" Default: 0, "zero" + (0): zero - Start at 0 running time (default) + (1): first - Start at first observed input running time + (2): set - Set start time with start-time property + + +Configuration properties: + +Group config-key Description +------------------------------------------------------------------------------------------------------------------------------------------------------ +[propert] enable Enable the functions of MetaMux + active-pad Use the source from this pad to synchronize the sources from the branches + pts-tolerance When the difference between the branch source and the base source is larger than tolerance value, + meatamux will not combine the metadata into current output +[group] src-ids-model The source IDs list to be output for specified GIE. The GIE uique-id should be attached as the key + postfix. E.G. "src-ids-model-3: 0;1;3" means to output source 0, source 1 and source 3 inference result + from GIE with unique-id 3. + + +NOTE: +1. nvdsmetamux is alpha quality currently. +2. Please refer test/metamux.sh for more command which been tested. + +Run: +# mux batch meta which from different model. +gst-launch-1.0 nvstreammux name=m batch-size=4 width=1920 height=1080 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_0 nvstreammux name=m2 batch-size=4 width=1920 height=1080 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess_yoloV4.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_infer_primary_yoloV4.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_1 nvdsmetamux config-file=/opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsmetamux/config_metamux.txt name=meta ! nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! queue ! nvvideoconvert ! queue ! x264enc ! queue ! rtph264pay config-interval=10 pt=96 ! udpsink host=10.19.225.205 port=5000 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_3 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m2.sink_3 + +# mux batch meta which from different model with tee. +gst-launch-1.0 nvstreammux name=m batch-size=4 width=1920 height=1080 sync-inputs=0 batched-push-timeout= 100000 live-source=1 ! nvvideoconvert ! tee name=t t. ! nvvideoconvert ! video/x-raw(memory:NVMM),width=1920,height=1082 ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_0 t. ! queue ! nvdspreprocess config-file= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_preprocess_yoloV4.txt ! nvinfer config-file-path= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-preprocess-test/config_infer_primary_yoloV4.txt input-tensor-meta=1 batch-size=4 ! queue ! meta.sink_1 nvdsmetamux config-file=/opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdsmetamux/config_metamux.txt name=meta ! nvmultistreamtiler width=1920 height=1080 ! nvvideoconvert ! nvdsosd ! queue ! nvvideoconvert ! queue ! x264enc ! queue ! rtph264pay config-interval=10 pt=96 ! udpsink host=10.19.225.205 port=5000 uridecodebin3 uri=rtsp://10.19.225.227/media/video1 ! m.sink_0 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_1 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_2 filesrc location = /opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! m.sink_3 + diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile new file mode 100644 index 00000000..676fd0ab --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/Makefile @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CC:= g++ +NVCC:=/usr/local/cuda/bin/nvcc + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I/usr/local/cuda/include + +LIBS:= -lnvinfer_plugin -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas -lstdc++fs +LFLAGS:= -shared -Wl,--start-group $(LIBS) -Wl,--end-group + +INCS:= $(wildcard *.h) +SRCFILES:= nvdsparsebbox_Yolo.cpp + +TARGET_LIB:= libnvdsinfer_custom_impl_Yolo.so + +TARGET_OBJS:= $(SRCFILES:.cpp=.o) +TARGET_OBJS:= $(TARGET_OBJS:.cu=.o) + +all: $(TARGET_LIB) + +%.o: %.cpp $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + $(NVCC) -c -o $@ --compiler-options '-fPIC' $< + +$(TARGET_LIB) : $(TARGET_OBJS) + $(CC) -o $@ $(TARGET_OBJS) $(LFLAGS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_OBJS) $(TARGET_LIB) diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp new file mode 100644 index 00000000..fa72b657 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer_custom_impl.h" + +static const int NUM_CLASSES_YOLO = 80; + +float clamp(const float val, const float minVal, const float maxVal) +{ + assert(minVal <= maxVal); + return std::min(maxVal, std::max(minVal, val)); +} + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList); + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) + { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + const NvDsInferLayerInfo *boxes = nullptr; + const NvDsInferLayerInfo *scores = nullptr; + const NvDsInferLayerInfo *num = nullptr; + const NvDsInferLayerInfo *classes_layer = nullptr; + + for (size_t l = 0; l < outputLayersInfo.size(); l++) { + if (!strcmp(outputLayersInfo[l].layerName, "num_detections")) { + num = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_boxes")) { + boxes = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_scores")) { + scores = &outputLayersInfo[l]; + } + if (!strcmp(outputLayersInfo[l].layerName, "nmsed_classes")) { + classes_layer = &outputLayersInfo[l]; + } + } + + if (!boxes || !scores || !classes_layer) { + std::cerr << "ERROR: Missing required output layers (nmsed_boxes, nmsed_scores, nmsed_classes)" << std::endl; + return false; + } + + const float* bbox_buffer = (const float*)boxes->buffer; + const float* score_buffer = (const float*)scores->buffer; + const float* class_buffer = (const float*)classes_layer->buffer; + + // Get number of detections + uint num_bboxes; + if (num) { + num_bboxes = ((int*)num->buffer)[0]; + } else { + num_bboxes = boxes->inferDims.d[0]; + } + + for (uint n = 0; n < num_bboxes; ++n) { + int class_id = static_cast(class_buffer[n]); + + // Validate class_id + if (class_id < 0 || class_id >= (int)detectionParams.numClassesConfigured) { + continue; + } + + float score = score_buffer[n]; + if (score < detectionParams.perClassPreclusterThreshold[class_id]) { + continue; + } + + // Parse bbox: [x1, y1, x2, y2] normalized coordinates + float bx1 = bbox_buffer[n * 4]; + float by1 = bbox_buffer[n * 4 + 1]; + float bx2 = bbox_buffer[n * 4 + 2]; + float by2 = bbox_buffer[n * 4 + 3]; + + // Convert to pixel coordinates + float x1 = clamp(bx1 * networkInfo.width, 0.0f, (float)networkInfo.width); + float y1 = clamp(by1 * networkInfo.height, 0.0f, (float)networkInfo.height); + float x2 = clamp(bx2 * networkInfo.width, 0.0f, (float)networkInfo.width); + float y2 = clamp(by2 * networkInfo.height, 0.0f, (float)networkInfo.height); + + NvDsInferParseObjectInfo outObj; + outObj.left = x1; + outObj.top = y1; + outObj.width = clamp(x2 - x1, 0.0f, (float)networkInfo.width); + outObj.height = clamp(y2 - y1, 0.0f, (float)networkInfo.height); + outObj.classId = class_id; + outObj.rotation_angle = 0.0f; + outObj.detectionConfidence = score; + + if (outObj.width < 1 || outObj.height < 1) { + continue; + } + + objectList.push_back(outObj); + } + + return true; +} +/* YOLOv4 implementations end*/ + + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV4); diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile new file mode 100644 index 00000000..1e0cd153 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/Makefile @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +CC:= g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes + +LIBS:= -lnvinfer +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvinfer_custom_lpr_parser.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_lpr.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp new file mode 100644 index 00000000..dca1f52c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonclient/sample/gst-plugins/gst-nvinferserver/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer.h" +#include + +using namespace std; +using std::string; +using std::vector; + +static bool dict_ready=false; +std::vector dict_table; + +extern "C" +{ + +bool NvDsInferParseCustomNVPlate(std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, float classifierThreshold, + std::vector &attrList, std::string &attrString) +{ + int *outputStrBuffer = NULL; + float *outputConfBuffer = NULL; + NvDsInferAttribute LPR_attr; + + int seq_len = 0; + + // Get list + vector str_idxes; + int prev = 100; + + // For confidence + double bank_softmax_max[16] = {0.0}; + unsigned int valid_bank_count = 0; + bool do_softmax = false; + ifstream fdict; + + setlocale(LC_CTYPE, ""); + + if(!dict_ready) { + fdict.open("dict.txt"); + if(!fdict.is_open()) + { + cout << "open dictionary file failed." << endl; + return false; + } + while(!fdict.eof()) { + string strLineAnsi; + if ( getline(fdict, strLineAnsi) ) { + dict_table.push_back(strLineAnsi); + } + } + dict_ready=true; + fdict.close(); + } + + int layer_size = outputLayersInfo.size(); + + LPR_attr.attributeConfidence = 1.0; + + seq_len = networkInfo.width/4; + + for( int li=0; li(outputLayersInfo[li].buffer); + } + else if (outputLayersInfo[li].dataType == 3) { + if(!outputStrBuffer) + outputStrBuffer = static_cast(outputLayersInfo[li].buffer); + } + } + } + + for(int seq_id = 0; seq_id < seq_len; seq_id++) { + do_softmax = false; + + int curr_data = outputStrBuffer[seq_id]; + if (seq_id == 0) { + prev = curr_data; + str_idxes.push_back(curr_data); + if ( curr_data != static_cast(dict_table.size()) ) do_softmax = true; + } else { + if (curr_data != prev) { + str_idxes.push_back(curr_data); + if (static_cast(curr_data) != dict_table.size()) do_softmax = true; + } + prev = curr_data; + } + + // Do softmax + if (do_softmax) { + do_softmax = false; + bank_softmax_max[valid_bank_count] = outputConfBuffer[curr_data]; + valid_bank_count++; + } + } + + attrString = ""; + for(unsigned int id = 0; id < str_idxes.size(); id++) { + if (static_cast(str_idxes[id]) != dict_table.size()) { + attrString += dict_table[str_idxes[id]]; + } + } + + //Ignore the short string, it may be wrong plate string + if (valid_bank_count >= 3) { + + LPR_attr.attributeIndex = 0; + LPR_attr.attributeValue = 1; + LPR_attr.attributeLabel = strdup(attrString.c_str()); + for (unsigned int count = 0; count < valid_bank_count; count++) { + LPR_attr.attributeConfidence *= bank_softmax_max[count]; + } + attrList.push_back(LPR_attr); + } + + return true; +} + +}//end of extern "C" diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/build_engine.sh b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/build_engine.sh new file mode 100644 index 00000000..3f5bda63 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/build_engine.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +IS_JETSON_PLATFORM=`uname -i | grep aarch64` + +export PATH=$PATH:/usr/src/tensorrt/bin + +trtexec --fp16 --onnx=./models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx.nms.onnx --saveEngine=./models/yolov4/1/yolov4_-1_3_416_416_dynamic.onnx_b32_gpu0.engine --minShapes=input:1x3x416x416 --optShapes=input:16x3x416x416 --maxShapes=input:32x3x416x416 --shapes=input:16x3x416x416 + +mkdir -p models/trafficcamnet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned.onnx' -O ./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=labels.txt' -O ./models/trafficcamnet/labels.txt +trtexec --onnx=./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx --fp16 \ + --saveEngine=./models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx_b8_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":4x3x544x960 --maxShapes="input_1:0":8x3x544x960 + +mkdir -p models/US_LPD/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=LPDNet_usa_pruned_tao5.onnx' -O models/US_LPD/1/LPDNet_usa_pruned_tao5.onnx +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/lpdnet/versions/pruned_v1.0/files/usa_lpd_label.txt' -O models/US_LPD/usa_lpd_label.txt +trtexec --onnx=models/US_LPD/1/LPDNet_usa_pruned_tao5.onnx --fp16 \ + --saveEngine=models/US_LPD/1//LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x480x640 \ + --optShapes="input_1:0":16x3x480x640 --maxShapes="input_1:0":16x3x480x640 + +mkdir models/us_lprnet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lprnet/deployable_onnx_v1.1/files?redirect=true&path=us_lprnet_baseline18_deployable.onnx' -O models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx +trtexec --onnx=models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx --fp16 \ + --saveEngine=models/us_lprnet/1/us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine --minShapes="image_input":1x3x48x96 \ + --optShapes="image_input":8x3x48x96 --maxShapes="image_input":16x3x48x96 + +mkdir -p models/peoplenet/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=resnet34_peoplenet.onnx' \ + -O models/peoplenet/1/resnet34_peoplenet.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/deployable_quantized_onnx_v2.6.3/files?redirect=true&path=labels.txt' \ + -O models/peoplenet/labels.txt +trtexec --onnx=./models/peoplenet/1/resnet34_peoplenet.onnx --fp16 \ + --saveEngine=./models/peoplenet/1/resnet34_peoplenet.onnx_b8_gpu0_fp16.engine \ + --minShapes="input_1:0":1x3x544x960 --optShapes="input_1:0":8x3x544x960 --maxShapes="input_1:0":8x3x544x960 + +#generate engine for vehicle related models. +echo "Building Model Secondary_CarMake..." +mkdir -p models/Secondary_CarMake/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ + -O models/Secondary_CarMake/1/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ + -O models/Secondary_CarMake/labels.txt +trtexec --onnx=models/Secondary_CarMake/1/resnet18_pruned.onnx --fp16 \ + --saveEngine=models/Secondary_CarMake/1/resnet18_pruned.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":8x3x224x224 --maxShapes="input_1:0":16x3x224x224 + +echo "Building Model Secondary_VehicleTypes..." +mkdir -p models/Secondary_VehicleTypes/1/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ + -O models/Secondary_VehicleTypes/1/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ + -O models/Secondary_VehicleTypes/labels.txt +trtexec --onnx=models/Secondary_VehicleTypes/1/resnet18_pruned.onnx --fp16 \ + --saveEngine=models/Secondary_VehicleTypes/1/resnet18_pruned.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":8x3x224x224 --maxShapes="input_1:0":16x3x224x224 +echo "Finished generating engine files." diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt new file mode 100644 index 00000000..c2d6d0a3 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_CarMake/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "Secondary_CarMake" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "resnet18_pruned.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 224, 224] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [ 20 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt new file mode 100644 index 00000000..011735a7 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/Secondary_VehicleTypes/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "Secondary_VehicleTypes" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "resnet18_pruned.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 224, 224] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [6] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt new file mode 100644 index 00000000..9a7ff7eb --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/config.pbtxt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "US_LPD" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 480, 640] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [4, 30, 40] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [1, 30, 40] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt new file mode 100644 index 00000000..95ffacc9 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/US_LPD/usa_lpd_label.txt @@ -0,0 +1 @@ +lpd diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt new file mode 100644 index 00000000..901d52a4 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/bodypose2d/config.pbtxt @@ -0,0 +1,37 @@ +name: "bodypose2d" +platform: "onnxruntime_onnx" +backend: "onnxruntime" +max_batch_size: 4 +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ + 3, + 224, + 224 + ] + } +] +output [ + { + name: "266" + data_type: TYPE_FP32 + dims: [ + 1 + ] + }, + { + name: "268", + data_type: TYPE_FP32 + dims: [ + 1 + ] + } +] + +instance_group { + count: 1 + gpus: 0 + kind: KIND_GPU +} diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt new file mode 100644 index 00000000..75f713cb --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/peoplenet/config.pbtxt @@ -0,0 +1,48 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "peoplenet" +platform: "tensorrt_plan" +max_batch_size: 8 +default_model_filename: "resnet34_peoplenet.onnx_b8_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 12, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 3, 34, 60 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/peoplenet/labels.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/peoplenet/labels.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt new file mode 100644 index 00000000..dfaa023c --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/config.pbtxt @@ -0,0 +1,49 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "trafficcamnet" +platform: "tensorrt_plan" +max_batch_size: 8 +default_model_filename: "resnet18_trafficcamnet_pruned.onnx_b8_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 544, 960] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [16, 34, 60] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [4, 34, 60] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt new file mode 100644 index 00000000..1a200952 --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/trafficcamnet/labels.txt @@ -0,0 +1,4 @@ +car +bicycle +person +road_sign diff --git a/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt new file mode 100644 index 00000000..548ee9ab --- /dev/null +++ b/src/apps/reference_apps/deepstream_parallel_inference_app/tritonserver/models/us_lprnet/config.pbtxt @@ -0,0 +1,50 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: "us_lprnet" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "image_input" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 48, 96] + } +] +output [ + { + name: "tf_op_layer_ArgMax" + data_type: TYPE_INT32 + dims: [24] + }, + + { + name: "tf_op_layer_Max" + data_type: TYPE_FP32 + dims: [ 24 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/reference_apps/legacy_apps/.gitkeep b/src/apps/reference_apps/legacy_apps/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors.png b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors.png new file mode 100644 index 00000000..baa587eb Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors.png differ diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png new file mode 100644 index 00000000..0761eccc Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/.backtobackdetectors_pipeline.png differ diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/Makefile b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/Makefile new file mode 100644 index 00000000..3e21fac6 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/Makefile @@ -0,0 +1,60 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= back-to-back-detectors + + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/README.md b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/README.md new file mode 100644 index 00000000..583056ad --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/README.md @@ -0,0 +1,89 @@ +# BACK-TO-BACK-DETECTORS REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains Back to Back detector application to show the +capability of Deepstream SDK. + +This sample builds on top of the deepstream-test1 sample to demonstrate how to +add multiple back-to-back detectors in the pipeline. + +Two instances of "nvinfer" or "nvinferserver" element are added to the pipeline serially after +nvstreammux and before the display components. Both the "nvinfer" or "nvinferserver" instances have +their own config files. + +The first "nvinfer" or "nvinferserver" instance (Person/Vehicle/Bicycle/RoadSign) will always act +as primary detector. + +The second "nvinfer" or "nvinferserver" instance (Face Detection) can be configured as +primary(full-frame) / secondary (operating on primary detected objects). By +default it is configured in the secondary mode. To change the second "nvinfer" or "nvinferserver" +instance to primary mode, change the macro `SECOND_DETECTOR_IS_SECONDARY` in the +sources to 0. + +## Prequisites: + +Please follow instructions in the `apps/sample_apps/deepstream-app/README` on how +to install the prequisites for Deepstream SDK, the DeepStream SDK itself and the +apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit the `primary_detector_config.txt` according to the location of the models to be used + +## Steps to download the models: +- To download the models for the second nvinfer, visit: + https://github.com/NVIDIA-AI-IOT/redaction_with_deepstream +- Use the following commands: +``` + $ cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/ + $ cd deepstream_reference_apps/deepstream-app-tao-configs/ + $ sudo cp -a * /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ + $ sudo apt install -y wget zip + $ cd /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ + $ sudo ./download_models.sh + +- Setup Triton model repository: + $ cd /opt/nvidia/deepstream/deepstream/samples/ +- Run prepare_ds_triton_model_repo.sh script to create Primary infer model "PrimaryDetector" + $ ./prepare_ds_triton_model_repo.sh +- Run prepare_ds_triton_tao_model_repo.sh script to create Secondary infer model "FaceNet" + $ ./prepare_ds_triton_tao_model_repo.sh + +``` + +Back to back detectors app pipeline: +![DS Back to back detectors Pipeline](.backtobackdetectors_pipeline.png) + +The result should be like below: +![DS Back to back detectors Screenshot](.backtobackdetectors.png) +## Compilation Steps and Execution: +``` + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make + + $ ./back-to-back-detectors + Ex.: ./back-to-back-detectors /opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.h264 + +Use option "-t inferserver" to select nvinferserver as the inference plugin + $ ./back-to-back-detectors -t inferserver + Ex.: ./back-to-back-detectors -t inferserver /opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.h264 + +NOTE: +- For Jetson, first run the below commands + $ cd /opt/nvidia/deepstream/deepstream/samples/triton_tao_model_repo + $ sudo ln -s ../triton_model_repo/Primary_Detector . +Then run the app. + +``` + +NOTE: +- Run the above commands with sudo. +- Edit the paths in `secondary_detector_config.txt` to the location of the models + downloaded from the above site. +- back-to-back-detectors application does not run inside jetson triton docker. + diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/back_to_back_detectors.c b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/back_to_back_detectors.c new file mode 100644 index 00000000..f4de1523 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/back_to_back_detectors.c @@ -0,0 +1,383 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "gstnvdsmeta.h" +#include + +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +#define SGIE_CLASS_ID_LP 1 +#define SGIE_CLASS_ID_FACE 0 + +/* Change this to 0 to make the 2nd detector act as a primary(full-frame) detector. + * When set to 1, it will act as secondary(operates on primary detected objects). */ +#define SECOND_DETECTOR_IS_SECONDARY 1 + +#define NVINFER_PLUGIN "nvinfer" +#define NVINFERSERVER_PLUGIN "nvinferserver" + +#define INFER_PGIE_CONFIG_FILE "primary_detector_config.txt" +#define INFER_SGIE_CONFIG_FILE "config_infer_primary_yoloV8_face.txt" +#define INFERSERVER_PGIE_CONFIG_FILE "inferserver/primary_detector_config.txt" +#define INFERSERVER_SGIE_CONFIG_FILE "inferserver/secondary_detector_config.txt" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +gint frame_number = 0; +gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", + "Roadsign" +}; + +#define PRIMARY_DETECTOR_UID 1 +#define SECONDARY_DETECTOR_UID 2 + +/* nvvidconv_sink_pad_buffer_probe will extract metadata received on nvvideoconvert sink pad + * and update params for drawing rectangle, object information etc. */ + +static GstPadProbeReturn +nvvidconv_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *) info->data; + NvDsObjectMeta *obj_meta = NULL; + guint vehicle_count = 0; + guint person_count = 0; + guint face_count = 0; + NvDsMetaList * l_frame = NULL; + NvDsMetaList * l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *) (l_obj->data); + + /* Check that the object has been detected by the primary detector + * and that the class id is that of vehicles/persons. */ + if (obj_meta->unique_component_id == PRIMARY_DETECTOR_UID) { + if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) + vehicle_count++; + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) + person_count++; + } + + if (obj_meta->unique_component_id == SECONDARY_DETECTOR_UID) { + if (obj_meta->class_id == SGIE_CLASS_ID_FACE) { + face_count++; + /* Print this info only when operating in secondary model. */ + if (SECOND_DETECTOR_IS_SECONDARY) + g_print ("Face found for parent object %p (type=%s)\n", + obj_meta->parent, pgie_classes_str[obj_meta->parent->class_id]); + } + } + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); + offset += snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); + offset += snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Face = %d ", face_count); + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = "Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + + + g_print ("Frame Number = %d Vehicle Count = %d Person Count = %d" + " Face Count = %d\n", + frame_number, vehicle_count, person_count, + face_count); + frame_number++; + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +usage(const char *bin) +{ + g_printerr ("Usage: %s \n", bin); + g_printerr ("For nvinferserver, Usage: %s -t inferserver \n", bin); +} + +int +main (int argc, char *argv[]) +{ + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, + *decoder = NULL, *streammux = NULL, *sink = NULL, *primary_detector = NULL, + *secondary_detector = NULL, *nvvidconv = NULL, *nvosd = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *nvvidconv_sink_pad = NULL; + gboolean is_nvinfer_server = FALSE; + gchar *input_stream = NULL; + const char *infer_plugin = NVINFER_PLUGIN; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Check input arguments */ + if (argc < 2) { + usage(argv[0]); + return -1; + } + + if (argc >=2 && !strcmp("-t", argv[1])) { + if (!strcmp("inferserver", argv[2])) { + is_nvinfer_server = TRUE; + } else { + usage(argv[0]); + return -1; + } + g_print ("Using nvinferserver as the inference plugin\n"); + } + + if (is_nvinfer_server) { + infer_plugin = NVINFERSERVER_PLUGIN; + } + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("pipeline"); + + /* Source element for reading from the file */ + source = gst_element_factory_make ("filesrc", "file-source"); + + /* Since the data format in the input file is elementary h264 stream, + * we need a h264parser */ + h264parser = gst_element_factory_make ("h264parse", "h264-parser"); + + /* Use nvdec_h264 for hardware accelerated decode on GPU */ + decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + /* Create two nvinfer instances for the two back-to-back detectors */ + primary_detector = gst_element_factory_make (infer_plugin, "primary-nvinference-engine1"); + + secondary_detector = gst_element_factory_make (infer_plugin, "primary-nvinference-engine2"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + /* Finally render the osd output */ + if(prop.integrated) { + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!source || !h264parser || !decoder || !primary_detector || !secondary_detector + || !nvvidconv || !nvosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + /* we set the input filename to the source element */ + if (is_nvinfer_server) { + input_stream = argv[3]; + g_object_set (G_OBJECT (source), "location", argv[3], NULL); + } else { + input_stream = argv[1]; + g_object_set (G_OBJECT (source), "location", argv[1], NULL); + } + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", 1, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set the config files for the two detectors. We demonstrate this by using + * the same detector model twice but making them act as vehicle-only and + * person-only detectors by adjusting the bbox confidence thresholds in the + * two seperate config files. */ + if (is_nvinfer_server) { + g_object_set (G_OBJECT (primary_detector), "config-file-path", INFERSERVER_PGIE_CONFIG_FILE, + "unique-id", PRIMARY_DETECTOR_UID, NULL); + } else { + g_object_set (G_OBJECT (primary_detector), "config-file-path", INFER_PGIE_CONFIG_FILE, + "unique-id", PRIMARY_DETECTOR_UID, NULL); + } + + if (is_nvinfer_server) { + g_object_set (G_OBJECT (secondary_detector), "config-file-path", INFERSERVER_SGIE_CONFIG_FILE, + "unique-id", SECONDARY_DETECTOR_UID, "process-mode", SECOND_DETECTOR_IS_SECONDARY ? 2 : 1, NULL); + } else { + g_object_set (G_OBJECT (secondary_detector), "config-file-path", INFER_SGIE_CONFIG_FILE, + "unique-id", SECONDARY_DETECTOR_UID, "process-mode", SECOND_DETECTOR_IS_SECONDARY ? 2 : 1, NULL); + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), + source, h264parser, decoder, streammux, primary_detector, secondary_detector, + nvvidconv, nvosd, sink, NULL); + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (decoder, pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + /* we link the elements together */ + /* file-source -> h264-parser -> nvh264-decoder -> + * pgie -> nvvidconv -> nvosd -> video-renderer */ + + if (!gst_element_link_many (source, h264parser, decoder, NULL)) { + g_printerr ("Elements could not be linked: 1. Exiting.\n"); + return -1; + } + + if (!gst_element_link_many (streammux, primary_detector, secondary_detector, + nvvidconv, nvosd, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the nvvideoconvert element, since by that time, the buffer would have + * had got all the metadata. */ + nvvidconv_sink_pad = gst_element_get_static_pad (nvvidconv, "sink"); + if (!nvvidconv_sink_pad) + g_print ("Unable to get sink pad\n"); + else + gst_pad_add_probe (nvvidconv_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + nvvidconv_sink_pad_buffer_probe, NULL, NULL); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", input_stream); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + return 0; +} diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt new file mode 100644 index 00000000..a8db3102 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/config_infer_primary_yoloV8_face.txt @@ -0,0 +1,26 @@ +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-color-format=0 +onnx-file=yolov8n-face.onnx +model-engine-file=yolov8n-face.onnx_b1_gpu0_fp32.engine +#int8-calib-file=calib.table +labelfile-path=labels.txt +batch-size=1 +network-mode=0 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +process-mode=1 +network-type=3 +cluster-mode=4 +maintain-aspect-ratio=1 +symmetric-padding=1 +#workspace-size=2000 +parse-bbox-instance-mask-func-name=NvDsInferParseYoloFace +custom-lib-path=nvdsinfer_custom_impl_Yolo_face/libnvdsinfer_custom_impl_Yolo_face.so +output-instance-mask=1 + +[class-attrs-all] +pre-cluster-threshold=0.25 +topk=300 diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt new file mode 100644 index 00000000..85bd20ec --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/primary_detector_config.txt @@ -0,0 +1,74 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 30 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "output_cov/Sigmoid"}, + {name: "output_bbox/BiasAdd"} + ] + triton { + model_name: "Primary_Detector" + version: -1 + model_repo { + root: "/opt/nvidia/deepstream/deepstream-9.0/samples/triton_tao_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream-9.0/samples/models/Primary_Detector/labels.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt new file mode 100644 index 00000000..2fb005ed --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/inferserver/secondary_detector_config.txt @@ -0,0 +1,75 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1" + }] + outputs: [ + {name: "output_bbox/BiasAdd"}, + {name: "output_cov/Sigmoid"} + ] + triton { + model_name: "facenet" + version: -1 + model_repo { + root: "/opt/nvidia/deepstream/deepstream-9.0/samples/triton_tao_model_repo" + strict_model_config: true + } + } + } + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1" + maintain_aspect_ratio: 1 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + channel_offsets: [0, 0, 0] + } + } + postprocess { + labelfile_path: "/opt/nvidia/deepstream/deepstream-9.0/samples/configs/tao_pretrained_models/labels_facenet.txt" + detection { + num_detected_classes: 1 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_class_ids: 2 + operate_on_gie_id: 1 + interval: 0 +} diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/primary_detector_config.txt b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/primary_detector_config.txt new file mode 100644 index 00000000..8b4ff755 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/primary_detector_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt +int8-calib-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/cal_trt.bin +batch-size=1 +network-mode=2 +process-mode=1 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +cluster-mode=2 +offsets=0.0;0.0;0.0 +maintain-aspect-ratio=0 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/src/apps/reference_apps/legacy_apps/back-to-back-detectors/secondary_detector_config.txt b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/secondary_detector_config.txt new file mode 100644 index 00000000..15807938 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/back-to-back-detectors/secondary_detector_config.txt @@ -0,0 +1,84 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +process-mode=2 +net-scale-factor=0.00392156862745098 +tlt-model-key=nvidia_tlt +tlt-encoded-model=/opt/nvidia/deepstream/deepstream-9.0/samples/models/tao_pretrained_models/facenet/model.etlt +labelfile-path=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/tao_pretrained_models/labels_facenet.txt +int8-calib-file=/opt/nvidia/deepstream/deepstream-9.0/samples/models/tao_pretrained_models/facenet/int8_calibration.txt +model-engine-file=/opt/nvidia/deepstream/deepstream-9.0/samples/models/tao_pretrained_models/facenet/model.etlt_b1_gpu0_int8.engine +force-implicit-batch-dim=1 +batch-size=1 +network-mode=0 +num-detected-classes=1 +interval=0 +gie-unique-id=2 +infer-dims=3;416;736 +uff-input-blob-name=input_1 +output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid +input-object-min-width=64 +input-object-min-height=64 +maintain-aspect-ratio=1 +# Person has class-id 2 for the primary detector. This ensures that this secondary +# detector only works on persons. +operate-on-class-ids=2 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/LICENSE.md b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/LICENSE.md new file mode 100644 index 00000000..81908580 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/LICENSE.md @@ -0,0 +1,15 @@ +SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/Makefile b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/Makefile new file mode 100644 index 00000000..68a06dec --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/Makefile @@ -0,0 +1,81 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-test5-analytics + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(DS_VER)/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(DS_VER)/bin/ + +SRCS:= deepstream_test5_app_main.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/deepstream_utc.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/deepstream_app.c +SRCS+= /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/deepstream_app_config_parser.c +SRCS+= $(wildcard /opt/nvidia/deepstream/deepstream/sources/apps/apps-common/src/*.c) +SRCS+= /opt/nvidia/deepstream/deepstream/sources/libs/nvds_msgapi_common_src/nvds_utils.cpp + +INCS= $(wildcard *.h) +INC_DIR=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5 + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(SRCS:.c=.o) +OBJS:= $(OBJS:.cpp=.o) +OBJS+= deepstream_nvdsanalytics_meta.o + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/apps/apps-common/includes -I./includes \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/ \ + -DDS_VERSION_MINOR=1 -DDS_VERSION_MAJOR=5 +CFLAGS+= -I$(INC_DIR) +CFLAGS+= -I/usr/local/cuda-$(CUDA_VER)/include + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper \ + -lnvdsgst_customhelper -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm \ + -lgstrtspserver-1.0 -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) -lnvbufsurface -lnvds_logger -lcrypto +LIBS+= -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart + +CFLAGS+= `pkg-config --cflags $(PKGS)` + +LIBS+= `pkg-config --libs $(PKGS)` + +all: $(APP) + +deepstream_nvdsanalytics_meta.o: deepstream_nvdsanalytics_meta.cpp $(INCS) Makefile + $(CXX) -c -o $@ -Wall -Werror $(CFLAGS) $< + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ -Wall -Werror $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/README.md b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/README.md new file mode 100644 index 00000000..04138d01 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/README.md @@ -0,0 +1,100 @@ +**People count application With Deepstream SDK and Transfer Learning Toolkit** + +* [Description](#description) +* [Prerequisites](#prerequisites) +* [Getting Started](#GettingStarted) +* [Build](#build) +* [Run](#run) +* [Output](#output) +* [References](#references) +

+ +

+ +## Description + + This is a sample application for counting people entering/leaving in a building using NVIDIA Deepstream SDK, Transfer Learning Toolkit (TLT) and pre-trained models. This application can be used to build real-time occupancy analytics application for smart buildings, hospitals, retail, etc. The application is based on deepstream-test5 sample application. + + It takes streaming video as input, counts the number of people crossing a tripwire and sends the live data to the cloud. In this application, you will learn: + + - How to use PeopleNet model from NGC + - How to use NvDsAnalytics plugin to draw line and count people crossing the line + - How to send the analytics data to cloud or another microservice over Kafka + + You can extend this application to change region of interest, use cloud-to-edge messaging to trigger record in the DeepStream application or build analytic dashboard or database to store the metadata. + +To learn how to build this demo step-by-step, check out the on-demand webinar on [Creating Intelligent places using DeepStream SDK](https://info.nvidia.com/iva-occupancy-webinar-reg-page.html?ondemandrgt=yes). + +## Prerequisites + + +- Install Deepstream: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/index.html#page/DeepStream_Development_Guide/deepstream_quick_start.html#] + +- Download PeopleNet model: [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet/files] + +- This application is based on deepstream-test5 application. More about test5 application: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_test5.html] + +- Install Kafka: [https://kafka.apache.org/quickstart] and create the kafka topic: + + `tar -xzf kafka_2.13-3.5.0.tgz` + + `cd kafka_2.13-3.5.0` + + `bin/zookeeper-server-start.sh config/zookeeper.properties` + + `bin/kafka-server-start.sh config/server.properties` + + `bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092` + +## Getting Started + +- Preferably clone the repo in $DS_SDK_ROOT/sources/apps/sample_apps/ +- Download peoplnet model: `cd deepstream-occupancy-analytics/config && ./model.sh` +- For Jetson use: bin/jetson/libnvds_msgconv.so +- For x86 use: bin/x86/libnvds_msgconv.so + +## Build and Configure + +- Set CUDA_VER in the MakeFile as per platform. + + For Jetson, CUDA_VER=11.4 + + For x86, CUDA_VER=11.8 + + `cd deepstream-occupancy-analytics && make` + +- Set **msg-conv-msg2p-lib** at **[sink1]** group in + **dstest_occupancy_analytics.txt** as per platform + + For Jetson + + msg-conv-msg2p-lib=$DEEPSTREAM_SDK_PATH/deepstream-occupancy-analytics/bin/jetson/libnvds_msgconv.so + + For x86 + + msg-conv-msg2p-lib=$DEEPSTREAM_SDK_PATH/deepstream-occupancy-analytics/bin/x86/libnvds_msgconv.so + +## Run + + `./deepstream-test5-analytics -c config/dstest_occupancy_analytics.txt` + + In another terminal run this command to see the kafka messages: + + `bin/kafka-console-consumer.sh --topic quickstart-events --from-beginning --bootstrap-server localhost:9092` + + +## Output + + The output will look like this: + + ![alt-text](images/kafka_messages.gif) + + Where you can see the kafka messages for entry and exit count. + +## References + +- CREATE INTELLIGENT PLACES USING NVIDIA PRE-TRAINED VISION MODELS AND DEEPSTREAM SDK: [https://info.nvidia.com/iva-occupancy-webinar-reg-page.html?ondemandrgt=yes] +- Deepstream SDK: [https://developer.nvidia.com/deepstream-sdk] +- Deepstream Quick Start Guide: [https://docs.nvidia.com/metropolis/deepstream/dev-guide/index.html#page/DeepStream_Development_Guide/deepstream_quick_start.html#] +- Transfer Learning Toolkit: [https://developer.nvidia.com/transfer-learning-toolkit] + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt new file mode 100644 index 00000000..5de13ea3 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/dstest_occupancy_analytics.txt @@ -0,0 +1,209 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 +#gie-kitti-output-dir=streamscl + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +#(0): nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 + +[source1] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=3 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 +# smart record specific fields, valid only for source type=4 +# 0 = disable, 1 = through cloud events, 2 = through cloud + local events +smart-record=2 +# 0 = mp4, 1 = mkv +#smart-rec-container=0 +smart-rec-start-time=1 +smart-rec-start-time=1 +smart-rec-file-prefix=smart_record +#smart-rec-dir-path=/home/monika/record +# cache size in seconds +smart-rec-cache=10 + +[sink0] +enable=1 +type=2 +#1=mp4 2=mkv./bin/kafka-topics --create --bootstrap-server localhost:9092 \ +#--replication-factor 1 --partitions 1 --topic users +container=1 +#1=h264 2=h265 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +#iframeinterval=10 +bitrate=100000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +output-file=resnet.mp4 +source-id=0 + +[sink1] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvoverlaysink 6=MsgConvBroker +type=6 +msg-conv-config=msgconv_sample_config.txt +# Name of library having custom implementation. +# msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-occupancy-analytics/bin/jetson/libnvds_msgconv.so +# msg-conv-msg2p-lib=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-occupancy-analytics/bin/x86/libnvds_msgconv.so +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=0 +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +#Provide your msg-broker-conn-str here +msg-broker-conn-str=localhost;9092;quickstart-events +#topic= +#Optional: +#msg-broker-config=../../deepstream-test4/cfg_kafka.txt + +[sink2] +enable=1 +type=1 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +## only SW mpeg4 is supported right now. +codec=3 +sync=1 +bitrate=2000000 +output-file=out.mp4 +source-id=0 + +# sink type = 6 by default creates msg converter + broker. +# To use multiple brokers use this group for converter and use +# sink type = 6 with disable-msgconv = 1 +[message-converter] +enable=0 +msg-conv-config=msgconv_sample_config.txt +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=0 +# Id of component in case only selected message to parse. +#msg-conv-comp-id= + +# Configure this group to enable cloud message consumer. +[message-consumer0] +enable=0 +proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +conn-str=localhost;9092 +#config-file= +subscribe-topic-list=quickstart-events +# Use this option if message has sensor name as id instead of index (0,1,2 etc.). +sensor-list-file=msgconv_sample_config.txt + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=10 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +batch-size=1 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 +## If set to TRUE, system timestamp will be attached as ntp timestamp +## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached +# attach-sys-ts-as-ntp=1 + +[primary-gie] +enable=1 +gpu-id=0 +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +bbox-border-color0=1;0;0;1 +#bbox-border-color1=0;1;1;1 +#bbox-border-color2=0;1;1;1 +#bbox-border-color3=0;1;0;1 +nvbuf-memory-type=0 +interval=0 +config-file=pgie_peoplenet_tao_config.txt +#infer-raw-output-dir=/opt/nvidia/deepstream/deepstream-9.0/samples/primary_detector_raw_output/ + +[tracker] +enable=1 +tracker-width=640 +tracker-height=384 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_IOU.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvSORT.yml +ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDeepSORT.yml +enable-batch-process=0 + +[nvds-analytics] +enable=1 +config-file=nvdsanalytics_config.txt + +[tests] +file-loop=0 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/model.sh b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/model.sh new file mode 100644 index 00000000..ce752acb --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/model.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +if [ ! -d peoplenet ];then + mkdir peoplenet +fi + +cd peoplenet +if [ ! -e labels.txt ];then + echo "Downloading peoplenet label.... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/labels.txt +fi + +if [ ! -e resnet34_peoplenet_pruned_int8.etlt ];then + echo "Downloading peoplenet etlt model.... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/resnet34_peoplenet_pruned_int8.etlt +fi + +if [ ! -e resnet34_peoplenet_pruned_int8.txt ];then + echo "Downloading peoplenet int8 .... " + wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet/versions/pruned_quantized_v2.3.2/files/resnet34_peoplenet_pruned_int8.txt +fi +cd - diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt new file mode 100644 index 00000000..f27abd06 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/msgconv_sample_config.txt @@ -0,0 +1,1942 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[sensor0] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor1] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor2] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor3] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor4] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor5] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor6] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor7] +enable=1 +type=Camera +id=HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor8] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor9] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor10] +enable=1 +type=Camera +id=HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor11] +enable=1 +type=Camera +id=HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor12] +enable=1 +type=Camera +id=HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor13] +enable=1 +type=Camera +id=HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor14] +enable=1 +type=Camera +id=HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor15] +enable=1 +type=Camera +id=HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor16] +enable=1 +type=Camera +id=HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor17] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor18] +enable=1 +type=Camera +id=HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor19] +enable=1 +type=Camera +id=HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place0] +enable=1 +id=0 +type=intersection/road +name=HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place1] +enable=1 +id=1 +type=intersection/road +name=HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place2] +enable=1 +id=2 +type=intersection/road +name=HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place3] +enable=1 +id=3 +type=intersection/road +name=HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place4] +enable=1 +id=4 +type=intersection/road +name=HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place5] +enable=1 +id=5 +type=intersection/road +name=HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place6] +enable=1 +id=6 +type=intersection/road +name=HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place7] +enable=1 +id=7 +type=intersection/road +name=HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place8] +enable=1 +id=8 +type=intersection/road +name=HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place9] +enable=1 +id=9 +type=intersection/road +name=HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place10] +enable=1 +id=10 +type=intersection/road +name=HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place11] +enable=1 +id=11 +type=intersection/road +name=HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place12] +enable=1 +id=12 +type=intersection/road +name=HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place13] +enable=1 +id=13 +type=intersection/road +name=HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place14] +enable=1 +id=14 +type=intersection/road +name=HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place15] +enable=1 +id=15 +type=intersection/road +name=HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place16] +enable=1 +id=16 +type=intersection/road +name=HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place17] +enable=1 +id=17 +type=intersection/road +name=HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place18] +enable=1 +id=18 +type=intersection/road +name=HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place19] +enable=1 +id=19 +type=intersection/road +name=HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 2 + +[sensor20] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor21] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor22] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor23] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor24] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor25] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor26] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor27] +enable=1 +type=Camera +id=INT2_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor28] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor29] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor30] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor31] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor32] +enable=1 +type=Camera +id=INT2_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor33] +enable=1 +type=Camera +id=INT2_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor34] +enable=1 +type=Camera +id=INT2_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor35] +enable=1 +type=Camera +id=INT2_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor36] +enable=1 +type=Camera +id=INT2_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor37] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor38] +enable=1 +type=Camera +id=INT2_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor39] +enable=1 +type=Camera +id=INT2_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place20] +enable=1 +id=0 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place21] +enable=1 +id=1 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place22] +enable=1 +id=2 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place23] +enable=1 +id=3 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place24] +enable=1 +id=4 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place25] +enable=1 +id=5 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place26] +enable=1 +id=6 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place27] +enable=1 +id=7 +type=intersection/road +name=INT2_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place28] +enable=1 +id=8 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place29] +enable=1 +id=9 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place30] +enable=1 +id=10 +type=intersection/road +name=INT2_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place31] +enable=1 +id=11 +type=intersection/road +name=INT2_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place32] +enable=1 +id=12 +type=intersection/road +name=INT2_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place33] +enable=1 +id=13 +type=intersection/road +name=INT2_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place34] +enable=1 +id=14 +type=intersection/road +name=INT2_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place35] +enable=1 +id=15 +type=intersection/road +name=INT2_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place36] +enable=1 +id=16 +type=intersection/road +name=INT2_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place37] +enable=1 +id=17 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place38] +enable=1 +id=18 +type=intersection/road +name=INT2_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place39] +enable=1 +id=19 +type=intersection/road +name=INT2_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 3 +[sensor40] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor41] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor42] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor43] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor44] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor45] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor46] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor47] +enable=1 +type=Camera +id=INT3_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor48] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor49] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor50] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor51] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor52] +enable=1 +type=Camera +id=INT3_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor53] +enable=1 +type=Camera +id=INT3_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor54] +enable=1 +type=Camera +id=INT3_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor55] +enable=1 +type=Camera +id=INT3_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor56] +enable=1 +type=Camera +id=INT3_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor57] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor58] +enable=1 +type=Camera +id=INT3_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor59] +enable=1 +type=Camera +id=INT3_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place40] +enable=1 +id=0 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place41] +enable=1 +id=1 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place42] +enable=1 +id=2 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place43] +enable=1 +id=3 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place44] +enable=1 +id=4 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place45] +enable=1 +id=5 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place46] +enable=1 +id=6 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place47] +enable=1 +id=7 +type=intersection/road +name=INT3_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place48] +enable=1 +id=8 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place49] +enable=1 +id=9 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place50] +enable=1 +id=10 +type=intersection/road +name=INT3_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place51] +enable=1 +id=11 +type=intersection/road +name=INT3_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place52] +enable=1 +id=12 +type=intersection/road +name=INT3_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place53] +enable=1 +id=13 +type=intersection/road +name=INT3_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place54] +enable=1 +id=14 +type=intersection/road +name=INT3_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place55] +enable=1 +id=15 +type=intersection/road +name=INT3_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place56] +enable=1 +id=16 +type=intersection/road +name=INT3_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place57] +enable=1 +id=17 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place58] +enable=1 +id=18 +type=intersection/road +name=INT3_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place59] +enable=1 +id=19 +type=intersection/road +name=INT3_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 4 +[sensor60] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor61] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor62] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor63] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor64] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor65] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor66] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor67] +enable=1 +type=Camera +id=INT4_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor68] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor69] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor70] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor71] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor72] +enable=1 +type=Camera +id=INT4_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor73] +enable=1 +type=Camera +id=INT4_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor74] +enable=1 +type=Camera +id=INT4_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor75] +enable=1 +type=Camera +id=INT4_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor76] +enable=1 +type=Camera +id=INT4_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor77] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor78] +enable=1 +type=Camera +id=INT4_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor79] +enable=1 +type=Camera +id=INT4_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place60] +enable=1 +id=0 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place61] +enable=1 +id=1 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place62] +enable=1 +id=2 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place63] +enable=1 +id=3 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place64] +enable=1 +id=4 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place65] +enable=1 +id=5 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place66] +enable=1 +id=6 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place67] +enable=1 +id=7 +type=intersection/road +name=INT4_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place68] +enable=1 +id=8 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place69] +enable=1 +id=9 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place70] +enable=1 +id=10 +type=intersection/road +name=INT4_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place71] +enable=1 +id=11 +type=intersection/road +name=INT4_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place72] +enable=1 +id=12 +type=intersection/road +name=INT4_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place73] +enable=1 +id=13 +type=intersection/road +name=INT4_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place74] +enable=1 +id=14 +type=intersection/road +name=INT4_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place75] +enable=1 +id=15 +type=intersection/road +name=INT4_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place76] +enable=1 +id=16 +type=intersection/road +name=INT4_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place77] +enable=1 +id=17 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place78] +enable=1 +id=18 +type=intersection/road +name=INT4_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place79] +enable=1 +id=19 +type=intersection/road +name=INT4_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +## Group 5 +[sensor80] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__EBA__4_11_2018_4_59_59_508_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor81] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__WBA__4_11_2018_4_59_59_379_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor82] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor83] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__4_11_2018_4_59_59_320_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor84] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__WBA__4_11_2018_4_59_59_550_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor85] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__EBA__4_11_2018_4_59_59_543_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor86] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__WBA__4_11_2018_4_59_59_134_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor87] +enable=1 +type=Camera +id=INT5_HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor88] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor89] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EBA__4_11_2018_4_59_59_793_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor90] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__WB__4_11_2018_4_59_59_450_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor91] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__WBA__4_11_2018_4_59_59_860_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + + +[sensor92] +enable=1 +type=Camera +id=INT5_HWY_20_AND_JFK__EB__4_11_2018_4_59_59_872_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor93] +enable=1 +type=Camera +id=INT5_HWY_20_AND_UNIVERSITY__WB__4_11_2018_4_59_59_308_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor94] +enable=1 +type=Camera +id=INT5_HWY_20_AND_UNIVERSITY__EB__4_11_2018_4_59_59_734_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor95] +enable=1 +type=Camera +id=INT5_HWY_20_AND_CENTURY__WB__4_11_2018_5_00_00_072_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor96] +enable=1 +type=Camera +id=INT5_HWY_20_AND_DEVON__EB__4_11_2018_4_59_59_728_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor97] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__WB__4_11_2018_4_59_57_927_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor98] +enable=1 +type=Camera +id=INT5_HWY_20_AND_WACKER__EB__4_11_2018_4_59_59_473_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[sensor99] +enable=1 +type=Camera +id=INT5_HWY_20_AND_LOCUST__HILL_EB_26-76__4_11_2018_4_59_59_433_AM_UTC-07_00 +location=45.293701447;-75.8303914499;48.1557479338 +description=Aisle Camera +coordinate=5.2;10.1;11.2 + +[place80] +enable=1 +id=0 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place81] +enable=1 +id=1 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place82] +enable=1 +id=2 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place83] +enable=1 +id=3 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place84] +enable=1 +id=4 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place85] +enable=1 +id=5 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place86] +enable=1 +id=6 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place87] +enable=1 +id=7 +type=intersection/road +name=INT5_HWY_20_AND_BRYANT__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place88] +enable=1 +id=8 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place89] +enable=1 +id=9 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place90] +enable=1 +id=10 +type=intersection/road +name=INT5_HWY_20_AND_JFK__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place91] +enable=1 +id=11 +type=intersection/road +name=INT5_HWY_20_AND_JFK__WBA +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place92] +enable=1 +id=12 +type=intersection/road +name=INT5_HWY_20_AND_JFK__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place93] +enable=1 +id=13 +type=intersection/road +name=INT5_HWY_20_AND_UNIVERSITY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place94] +enable=1 +id=14 +type=intersection/road +name=INT5_HWY_20_AND_UNIVERSITY__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place95] +enable=1 +id=15 +type=intersection/road +name=INT5_HWY_20_AND_CENTURY__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place96] +enable=1 +id=16 +type=intersection/road +name=INT5_HWY_20_AND_DEVON__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place97] +enable=1 +id=17 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__WB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place98] +enable=1 +id=18 +type=intersection/road +name=INT5_HWY_20_AND_WACKER__EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[place99] +enable=1 +id=19 +type=intersection/road +name=INT5_HWY_20_AND_LOCUST__HILL_EB +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=C_127_158 +place-sub-field2=Lane 1 +place-sub-field3=P1 + +[analytics0] +enable=1 +id=XYZ +#description=Vehicle Detection and License Plate Recognition +source=OpenALR +version=1.0 + +[analytics1] +enable=1 +id=XYZ +#description=Vehicle Detection and License Plate Recognition 1 +source=OpenALR +version=1.0 + + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt new file mode 100644 index 00000000..37e3a50f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/nvdsanalytics_config.txt @@ -0,0 +1,102 @@ +################################################################################ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=-1 + +## Per stream configuration +[roi-filtering-stream-1] +#enable or disable following feature +enable=0 +#ROI to filter select objects, and remove from meta data +roi-RF=295;643;579;634;642;913;56;828 +#remove objects in the ROI +inverse-roi=0 +class-id=0 + + +[overcrowding-stream-1] +enable=0 +roi-OC=0;0;579;400;642;900;0;900 +#no of objects that will trigger OC +object-threshold=3 +class-id=-1 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +line-crossing-Exit=900;1000;850;900;300;1000;1350;800; +line-crossing-Entry=750;670;800;750;300;850;1350;650 +#line_color=0.75;0.25;0;1 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=balanced + +[line-crossing-stream-1] +enable=1 +#Label;direction;lc +line-crossing-Exit=900;1000;850;900;300;1000;1350;800; +line-crossing-Entry=750;670;800;750;300;850;1350;650 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=1 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=balanced + +[direction-detection-stream-0] +enable=0 +#Label;direction; +direction-South=284;840;360;662; +direction-North=1106;622;1312;701; +class-id=0 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt new file mode 100644 index 00000000..48d1e0d8 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/config/pgie_peoplenet_tao_config.txt @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +tlt-model-key=tlt_encode +tlt-encoded-model=peoplenet/resnet34_peoplenet_pruned_int8.etlt +labelfile-path=peoplenet/labels.txt +# model-engine-file=peoplenet/resnet34_peoplenet_pruned_int8.etlt_b1_gpu0_int8.engine +int8-calib-file=peoplenet/resnet34_peoplenet_pruned_int8.txt +input-dims=3;544;960;0 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=3 +cluster-mode=1 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd;output_cov/Sigmoid + +[class-attrs-all] +pre-cluster-threshold=0.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 + +[class-attrs-1] +pre-cluster-threshold=1.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 +[class-attrs-2] +pre-cluster-threshold=1.4 +## Set eps=0.7 and minBoxes for cluster-mode=1(DBSCAN) +eps=0.7 +minBoxes=1 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp new file mode 100644 index 00000000..6ca15c1d --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_nvdsanalytics_meta.cpp @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "nvds_analytics_meta.h" +#include "analytics.h" + +/* custom_parse_nvdsanalytics_meta_data + * and extract nvanalytics metadata */ + extern "C" void +analytics_custom_parse_nvdsanalytics_meta_data (NvDsMetaList *l_user, AnalyticsUserMeta *data) +{ + std::stringstream out_string; + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + /* convert to metadata */ + NvDsAnalyticsFrameMeta *meta = + (NvDsAnalyticsFrameMeta *) user_meta->user_meta_data; + /* Fill the data for entry, exit,occupancy */ + data->lcc_cnt_entry = 0; + data->lcc_cnt_exit = 0; + data->lccum_cnt = 0; + data->lcc_cnt_entry = meta->objLCCumCnt["Entry"]; + data->lcc_cnt_exit = meta->objLCCumCnt["Exit"]; + + if (meta->objLCCumCnt["Entry"]> meta->objLCCumCnt["Exit"]) + data->lccum_cnt = meta->objLCCumCnt["Entry"] - meta->objLCCumCnt["Exit"]; + // g_print("Enter: %d, Exit: %d\n", data->lcc_cnt_entry,data->lcc_cnt_exit); +} + + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c new file mode 100644 index 00000000..f3a343e7 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/deepstream_test5_app_main.c @@ -0,0 +1,1259 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "deepstream_app.h" +#include "deepstream_config_file_parser.h" +#include "nvds_version.h" + +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" + +#include "deepstream_test5_app.h" + +/*Analytics header*/ +#include "analytics.h" + +#define MAX_DISPLAY_LEN (64) +#define MAX_TIME_STAMP_LEN (64) +#define STREAMMUX_BUFFER_POOL_SIZE (16) + +/** @{ + * Macro's below and corresponding code-blocks are used to demonstrate + * nvmsgconv + Broker Metadata manipulation possibility + */ + +/** + * IMPORTANT Note 1: + * The code within the check for model_used == APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE + * is applicable as sample demo code for + * configs that use resnet PGIE model + * with class ID's: {0, 1, 2, 3} for {CAR, BICYCLE, PERSON, ROADSIGN} + * followed by optional Tracker + 3 X SGIEs (Vehicle-Type,Color,Make) + * only! + * Please comment out the code if using any other + * custom PGIE + SGIE combinations + * and use the code as reference to write your own + * NvDsEventMsgMeta generation code in generate_event_msg_meta() + * function + */ +typedef enum +{ + APP_CONFIG_ANALYTICS_MODELS_UNKNOWN = 0, + APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE = 1, +} AppConfigAnalyticsModel; + +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_CAR (0) +#ifdef GENERATE_DUMMY_META_EXT +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_PERSON (2) +#endif +/** @} */ + +/* PERSON ID definition. */ +#define PERSON_ID 0 + +#ifdef EN_DEBUG +#define LOGD(...) printf(__VA_ARGS__) +#else +#define LOGD(...) +#endif + +static TestAppCtx *testAppCtx; +GST_DEBUG_CATEGORY (NVDS_APP); + +/** @{ imported from deepstream-app as is */ + + +#define MAX_INSTANCES 128 +#define APP_TITLE "DeepStreamTest5App" + +#define DEFAULT_X_WINDOW_WIDTH 1920 +#define DEFAULT_X_WINDOW_HEIGHT 1080 + +AppCtx *appCtx[MAX_INSTANCES]; +static guint cintr = FALSE; +static GMainLoop *main_loop = NULL; +static gchar **cfg_files = NULL; +static gchar **input_files = NULL; +static gchar **override_cfg_file = NULL; +static gboolean playback_utc = TRUE; +static gboolean print_version = FALSE; +static gboolean show_bbox_text = TRUE; +static gboolean force_tcp = TRUE; +static gboolean print_dependencies_version = FALSE; +static gboolean quit = FALSE; +static gint return_value = 0; +static guint num_instances; +static guint num_input_files; +static GMutex fps_lock; +static gdouble fps[MAX_SOURCE_BINS]; +static gdouble fps_avg[MAX_SOURCE_BINS]; + +static Display *display = NULL; +static Window windows[MAX_INSTANCES] = { 0 }; + +static GThread *x_event_thread = NULL; +static GMutex disp_lock; + +static guint rrow, rcol, rcfg; +static gboolean rrowsel = FALSE, selecting = FALSE; +static AppConfigAnalyticsModel model_used = APP_CONFIG_ANALYTICS_MODELS_UNKNOWN; + +/** @} imported from deepstream-app as is */ +GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version, + "Print DeepStreamSDK version", NULL} + , + {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text, + "Display Bounding box labels in tiled mode", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version, + "Print DeepStreamSDK and dependencies version", NULL} + , + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files, + "Set the config file", NULL} + , + {"override-cfg-file", 'o', 0, G_OPTION_ARG_FILENAME_ARRAY, &override_cfg_file, + "Set the override config file, used for on-the-fly model update feature", + NULL} + , + {"input-file", 'i', 0, G_OPTION_ARG_FILENAME_ARRAY, &input_files, + "Set the input file", NULL} + , + {"playback-utc", 'p', 0, G_OPTION_ARG_INT, &playback_utc, + "Playback utc; default=true (base UTC from file/rtsp URL); =false (base UTC from file-URL or RTCP Sender Report)", + NULL} + , + {"pgie-model-used", 'm', 0, G_OPTION_ARG_INT, &model_used, + "PGIE Model used; {0 - Unknown [DEFAULT]}, {1: Resnet 4-class [Car, Bicycle, Person, Roadsign]}", + NULL} + , + {"no-force-tcp", 0, G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &force_tcp, + "Do not force TCP for RTP transport", NULL} + , + {NULL} + , +}; + + + static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + + + + static GstClockTime +generate_ts_rfc3339_from_ts (char *buf, int buf_size, GstClockTime ts, + gchar * src_uri, gint stream_id) +{ + time_t tloc; + struct tm tm_log; + char strmsec[6]; //.nnnZ\0 + int ms; + + GstClockTime ts_generated; + + if (playback_utc + || (appCtx[0]->config.multi_source_config[stream_id].type != + NV_DS_SOURCE_RTSP)) { + if (testAppCtx->streams[stream_id].meta_number == 0) { + testAppCtx->streams[stream_id].timespec_first_frame = + extract_utc_from_uri (src_uri); + memcpy (&tloc, + (void *) (&testAppCtx->streams[stream_id].timespec_first_frame. + tv_sec), sizeof (time_t)); + ms = testAppCtx->streams[stream_id].timespec_first_frame.tv_nsec / + 1000000; + testAppCtx->streams[stream_id].gst_ts_first_frame = ts; + ts_generated = + GST_TIMESPEC_TO_TIME (testAppCtx->streams[stream_id]. + timespec_first_frame); + if (ts_generated == 0) { + g_print + ("WARNING; playback mode used with URI [%s] not conforming to timestamp format;" + " check README; using system-time\n", src_uri); + clock_gettime (CLOCK_REALTIME, + &testAppCtx->streams[stream_id].timespec_first_frame); + ts_generated = + GST_TIMESPEC_TO_TIME (testAppCtx->streams[stream_id]. + timespec_first_frame); + } + } else { + GstClockTime ts_current = + GST_TIMESPEC_TO_TIME (testAppCtx-> + streams[stream_id].timespec_first_frame) + (ts - + testAppCtx->streams[stream_id].gst_ts_first_frame); + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC (ts_current, timespec_current); + memcpy (&tloc, (void *) (×pec_current.tv_sec), sizeof (time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts_current; + } + } else { + /** ts itself is UTC Time in ns */ + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC (ts, timespec_current); + memcpy (&tloc, (void *) (×pec_current.tv_sec), sizeof (time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts; + } + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); + LOGD ("ts=%s\n", buf); + + return ts_generated; +} + + + static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + /* + if (srcMeta->sensorStr) { + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + } + */ + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + obj->age = srcObj->age; + + if (srcObj->gender) + obj->gender = g_strdup (srcObj->gender); + if (srcObj->cap) + obj->cap = g_strdup (srcObj->cap); + if (srcObj->hair) + obj->hair = g_strdup (srcObj->hair); + if (srcObj->apparel) + obj->apparel = g_strdup (srcObj->apparel); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + + static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + user_meta->user_meta_data = NULL; + + if (srcMeta->ts) { + g_free (srcMeta->ts); + } + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->sensorStr) { + g_free (srcMeta->sensorStr); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + NvDsVehicleObject *obj = (NvDsVehicleObject *) srcMeta->extMsg; + if (obj->type) + g_free (obj->type); + if (obj->color) + g_free (obj->color); + if (obj->make) + g_free (obj->make); + if (obj->model) + g_free (obj->model); + if (obj->license) + g_free (obj->license); + if (obj->region) + g_free (obj->region); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->gender) + g_free (obj->gender); + if (obj->cap) + g_free (obj->cap); + if (obj->hair) + g_free (obj->hair); + if (obj->apparel) + g_free (obj->apparel); + } + g_free (srcMeta->extMsg); + srcMeta->extMsg = NULL; + srcMeta->extMsgSize = 0; + } + g_free (srcMeta); +} + +#ifdef GENERATE_DUMMY_META_EXT + +#endif /**< GENERATE_DUMMY_META_EXT */ + + static void +generate_event_msg_meta (gpointer data, gint class_id, gboolean useTs, + GstClockTime ts, gchar * src_uri, gint stream_id, guint sensor_id, + AnalyticsUserMeta * obj_params, float scaleW, float scaleH, + NvDsFrameMeta * frame_meta) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + GstClockTime ts_generated = 0; + meta->objType = NVDS_OBJECT_TYPE_UNKNOWN; /**< object unknown */ + meta->frameId = frame_meta->frame_num; + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + //strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + /** INFO: This API is called once for every 30 frames (now) */ + if (useTs && src_uri) { + ts_generated = + generate_ts_rfc3339_from_ts (meta->ts, MAX_TIME_STAMP_LEN, ts, src_uri, + stream_id); + } else { + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + } + + /** tracking ID */ + meta->trackingId = class_id; + + (void) ts_generated; + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PERSON_ID; + meta->occupancy = obj_params->lccum_cnt; + meta->lccum_cnt_entry = obj_params->lcc_cnt_entry; + meta->lccum_cnt_exit = obj_params->lcc_cnt_exit ; + meta->source_id = obj_params->source_id; +// g_print("source id: %d, Enter: %d, Exit: %d\n", meta->source_id, meta->lccum_cnt_entry, meta->lccum_cnt_exit); + +} + +/* + * Access analytics data. + */ +void analytics_custom_parse_nvdsanalytics_meta_data (NvDsMetaList *l_user, AnalyticsUserMeta *data); + +/** + * Callback function to be called once all inferences (Primary + Secondary) + * are done. This is opportunity to modify content of the metadata. + * e.g. Here Person is being replaced with Man/Woman and corresponding counts + * are being maintained. It should be modified according to network classes + * or can be removed altogether if not required. + */ + static void +bbox_generated_probe_after_analytics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + NvDsObjectMeta *obj_meta = NULL; + GstClockTime buffer_pts = 0; + guint32 stream_id = 0; + + + if (!appCtx->config.dsanalytics_config.enable){ + g_print ("Unable to get nvdsanalytics src pad\n"); + return; + } + + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = l_frame->data; + stream_id = frame_meta->source_id; + GstClockTime buf_ntp_time = 0; + if (playback_utc == FALSE) { + /** Calculate the buffer-NTP-time + * derived from this stream's RTCP Sender Report here: + */ + StreamSourceInfo *src_stream = &testAppCtx->streams[stream_id]; + buf_ntp_time = frame_meta->ntp_timestamp; + + if (buf_ntp_time < src_stream->last_ntp_time) { + NVGSTDS_WARN_MSG_V ("Source %d: NTP timestamps are backward in time." + " Current: %lu previous: %lu", stream_id, buf_ntp_time, + src_stream->last_ntp_time); + } + src_stream->last_ntp_time = buf_ntp_time; + } + + GList *l; + NvDsMetaList *l_analyticsuser; + l_analyticsuser = frame_meta->frame_user_meta_list; + + AnalyticsUserMeta *user_data = + (AnalyticsUserMeta *) g_malloc0(sizeof(AnalyticsUserMeta)); + if (l_analyticsuser != NULL) { + analytics_custom_parse_nvdsanalytics_meta_data(l_analyticsuser, user_data); + } + user_data->source_id = stream_id; + + // l_analyticsuser = l_analyticsuser->next; + + /* Code from test5 application */ + for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) { + /* Now using above information we need to form a text that should + * be displayed on top of the bounding box, so lets form it here. */ + obj_meta = (NvDsObjectMeta *) (l->data); + + { + /** + * Enable only if this callback is after tiler + * NOTE: Scaling back code-commented + * now that bbox_generated_probe_after_analytics() is post analytics + * (say pgie, tracker or sgie) + * and before tiler, no plugin shall scale metadata and will be + * corresponding to the nvstreammux resolution + */ + float scaleW = 0; + float scaleH = 0; + /* Frequency of messages to be send will be based on use case. + * Here message is being sent for first object every 30 frames. + */ + buffer_pts = frame_meta->buf_pts; + if (!appCtx->config.streammux_config.pipeline_width + || !appCtx->config.streammux_config.pipeline_height) { + g_print ("invalid pipeline params\n"); + return; + } + LOGD ("stream %d==%d [%d X %d]\n", frame_meta->source_id, + frame_meta->pad_index, frame_meta->source_frame_width, + frame_meta->source_frame_height); + scaleW = + (float) frame_meta->source_frame_width / + appCtx->config.streammux_config.pipeline_width; + scaleH = + (float) frame_meta->source_frame_height / + appCtx->config.streammux_config.pipeline_height; + + if (playback_utc == FALSE) { + /** Use the buffer-NTP-time derived from this stream's RTCP Sender + * Report here: + */ + buffer_pts = buf_ntp_time; + } + /** Generate NvDsEventMsgMeta for every object */ + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + generate_event_msg_meta (msg_meta, PERSON_ID, TRUE, + /**< useTs NOTE: Pass FALSE for files without base-timestamp in URI */ + buffer_pts, + appCtx->config.multi_source_config[stream_id].uri, stream_id, + appCtx->config.multi_source_config[stream_id].camera_id, + user_data, scaleW, scaleH, frame_meta); + testAppCtx->streams[stream_id].meta_number++; + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + /* + * Since generated event metadata has custom objects for + * Vehicle / Person which are allocated dynamically, we are + * setting copy and free function to handle those fields when + * metadata copy happens between two components. + */ + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.batch_meta = batch_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + } + testAppCtx->streams[stream_id].frameCount++; + + g_free(user_data); + } +} + +/** @{ imported from deepstream-app as is */ + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ + static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + cintr = TRUE; +} + +/** + * callback function to print the performance numbers of each stream. + */ + static void +perf_cb (gpointer context, NvDsAppPerfStruct * str) +{ + static guint header_print_cnt = 0; + guint i; + AppCtx *appCtx = (AppCtx *) context; + guint numf = str->num_instances; + + g_mutex_lock (&fps_lock); + for (i = 0; i < numf; i++) { + fps[i] = str->fps[i]; + fps_avg[i] = str->fps_avg[i]; + } + + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < numf; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + printf ("%s", asctime (tm)); + if (num_instances > 1) + g_print ("PERF(%d): ", appCtx->index); + else + g_print ("**PERF: "); + + for (i = 0; i < numf; i++) { + g_print ("%.2f (%.2f)\t", fps[i], fps_avg[i]); + } + g_print ("\n"); + g_mutex_unlock (&fps_lock); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ + static gboolean +check_for_interrupt (gpointer data) +{ + if (quit) { + return FALSE; + } + + if (cintr) { + cintr = FALSE; + + quit = TRUE; + g_main_loop_quit (main_loop); + + return FALSE; + } + return TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ + static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + + static gboolean +kbhit (void) +{ + struct timeval tv; + fd_set rdfs; + + tv.tv_sec = 0; + tv.tv_usec = 0; + + FD_ZERO (&rdfs); + FD_SET (STDIN_FILENO, &rdfs); + + select (STDIN_FILENO + 1, &rdfs, NULL, NULL, &tv); + return FD_ISSET (STDIN_FILENO, &rdfs); +} + +/* + * Function to enable / disable the canonical mode of terminal. + * In non canonical mode input is available immediately (without the user + * having to type a line-delimiter character). + */ + static void +changemode (int dir) +{ + static struct termios oldt, newt; + + if (dir == 1) { + tcgetattr (STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~(ICANON); + tcsetattr (STDIN_FILENO, TCSANOW, &newt); + } else + tcsetattr (STDIN_FILENO, TCSANOW, &oldt); +} + + static void +print_runtime_commands (void) +{ + g_print ("\nRuntime commands:\n" + "\th: Print this help\n" + "\tq: Quit\n\n" "\tp: Pause\n" "\tr: Resume\n\n"); + + if (appCtx[0]->config.tiled_display_config.enable) { + g_print + ("NOTE: To expand a source in the 2D tiled display and view object details," + " left-click on the source.\n" + " To go back to the tiled display, right-click anywhere on the window.\n\n"); + } +} + +/** + * Loop function to check keyboard inputs and status of each pipeline. + */ + static gboolean +event_thread_func (gpointer arg) +{ + guint i; + gboolean ret = TRUE; + + // Check if all instances have quit + for (i = 0; i < num_instances; i++) { + if (!appCtx[i]->quit) + break; + } + + if (i == num_instances) { + quit = TRUE; + g_main_loop_quit (main_loop); + return FALSE; + } + // Check for keyboard input + if (!kbhit ()) { + //continue; + return TRUE; + } + int c = fgetc (stdin); + g_print ("\n"); + + gint source_id; + GstElement *tiler = appCtx[rcfg]->pipeline.tiled_display_bin.tiler; + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (selecting) { + if (rrowsel == FALSE) { + if (c >= '0' && c <= '9') { + rrow = c - '0'; + g_print ("--selecting source row %d--\n", rrow); + rrowsel = TRUE; + } + } else { + if (c >= '0' && c <= '9') { + int tile_num_columns = appCtx[rcfg]->config.tiled_display_config.columns; + rcol = c - '0'; + selecting = FALSE; + rrowsel = FALSE; + source_id = tile_num_columns * rrow + rcol; + g_print ("--selecting source col %d sou=%d--\n", rcol, source_id); + if (source_id >= (gint) appCtx[rcfg]->config.num_source_sub_bins) { + source_id = -1; + } else { + appCtx[rcfg]->show_bbox_text = TRUE; + appCtx[rcfg]->active_source_index = source_id; + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + } + } + } + } + switch (c) { + case 'h': + print_runtime_commands (); + break; + case 'p': + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + case 'r': + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + case 'q': + quit = TRUE; + g_main_loop_quit (main_loop); + ret = FALSE; + break; + case 'c': + if (selecting == FALSE && source_id == -1) { + g_print("--selecting config file --\n"); + c = fgetc(stdin); + if (c >= '0' && c <= '9') { + rcfg = c - '0'; + if (rcfg < num_instances) { + g_print("--selecting config %d--\n", rcfg); + } else { + g_print("--selected config file %d out of bound, reenter\n", rcfg); + rcfg = 0; + } + } + } + break; + case 'z': + if (source_id == -1 && selecting == FALSE) { + g_print ("--selecting source --\n"); + selecting = TRUE; + } else { + if (!show_bbox_text) { + GstElement *nvosd = appCtx[rcfg]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + } + appCtx[rcfg]->active_source_index = -1; + selecting = FALSE; + rcfg = 0; + g_print("--tiled mode --\n"); + } + break; + default: + break; + } + return ret; +} + + static int +get_source_id_from_coordinates (float x_rel, float y_rel, AppCtx *appCtx) +{ + int tile_num_rows = appCtx->config.tiled_display_config.rows; + int tile_num_columns = appCtx->config.tiled_display_config.columns; + + int source_id = (int) (x_rel * tile_num_columns); + source_id += ((int) (y_rel * tile_num_rows)) * tile_num_columns; + + /* Don't allow clicks on empty tiles. */ + if (source_id >= (gint) appCtx->config.num_source_sub_bins) + source_id = -1; + + return source_id; +} + +/** + * Thread to monitor X window events. + */ + static gpointer +nvds_x_event_thread (gpointer data) +{ + g_mutex_lock (&disp_lock); + while (display) { + XEvent e; + guint index; + while (XPending (display)) { + XNextEvent (display, &e); + switch (e.type) { + case ButtonPress: + { + XWindowAttributes win_attr; + XButtonEvent ev = e.xbutton; + gint source_id; + GstElement *tiler; + + XGetWindowAttributes (display, ev.window, &win_attr); + + for (index = 0; index < MAX_INSTANCES; index++) + if (ev.window == windows[index]) + break; + + tiler = appCtx[index]->pipeline.tiled_display_bin.tiler; + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (ev.button == Button1 && source_id == -1) { + source_id = + get_source_id_from_coordinates (ev.x * 1.0 / win_attr.width, + ev.y * 1.0 / win_attr.height, appCtx[index]); + if (source_id > -1) { + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + appCtx[index]->active_source_index = source_id; + appCtx[index]->show_bbox_text = TRUE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", TRUE, NULL); + } + } else if (ev.button == Button3) { + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + appCtx[index]->active_source_index = -1; + if (!show_bbox_text) { + appCtx[index]->show_bbox_text = FALSE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + } + } + } + break; + case KeyRelease: + { + KeySym p, r, q; + guint i; + p = XKeysymToKeycode (display, XK_P); + r = XKeysymToKeycode (display, XK_R); + q = XKeysymToKeycode (display, XK_Q); + if (e.xkey.keycode == p) { + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == r) { + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == q) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + case ClientMessage: + { + Atom wm_delete; + for (index = 0; index < MAX_INSTANCES; index++) + if (e.xclient.window == windows[index]) + break; + + wm_delete = XInternAtom (display, "WM_DELETE_WINDOW", 1); + if (wm_delete != None && wm_delete == (Atom) e.xclient.data.l[0]) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + } + } + g_mutex_unlock (&disp_lock); + g_usleep (G_USEC_PER_SEC / 20); + g_mutex_lock (&disp_lock); + } + g_mutex_unlock (&disp_lock); + return NULL; +} + +/** + * callback function to add application specific metadata. + * Here it demonstrates how to display the URI of source in addition to + * the text generated after inference. + */ + static gboolean +overlay_graphics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + return TRUE; +} + +/** @} imported from deepstream-app as is */ + + int +main (int argc, char *argv[]) +{ + testAppCtx = (TestAppCtx *) g_malloc0 (sizeof (TestAppCtx)); + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + guint i; + + ctx = g_option_context_new ("Nvidia DeepStream Test5"); + group = g_option_group_new ("abc", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + g_print ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + + if (print_version) { + g_print ("deepstream-test5-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (print_dependencies_version) { + g_print ("deepstream-test5-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (cfg_files) { + num_instances = g_strv_length (cfg_files); + } + if (input_files) { + num_input_files = g_strv_length (input_files); + } + + if (!cfg_files || num_instances == 0) { + NVGSTDS_ERR_MSG_V ("Specify config file with -c option"); + return_value = -1; + goto done; + } + + for (i = 0; i < num_instances; i++) { + appCtx[i] = (AppCtx *) g_malloc0 (sizeof (AppCtx)); + appCtx[i]->person_class_id = -1; + appCtx[i]->car_class_id = -1; + appCtx[i]->index = i; + appCtx[i]->active_source_index = -1; + if (show_bbox_text) { + appCtx[i]->show_bbox_text = TRUE; + } + + if (input_files && input_files[i]) { + appCtx[i]->config.multi_source_config[0].uri = + g_strdup_printf ("file://%s", input_files[i]); + g_free (input_files[i]); + } + + if (!parse_config_file (&appCtx[i]->config, cfg_files[i])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[i]); + appCtx[i]->return_value = -1; + goto done; + } + + if (override_cfg_file && override_cfg_file[i]) { + if (!g_file_test (override_cfg_file[i], + G_FILE_TEST_IS_REGULAR | G_FILE_TEST_IS_SYMLINK)) + { + g_print ("Override file %s does not exist, quitting...\n", + override_cfg_file[i]); + appCtx[i]->return_value = -1; + goto done; + } + } + } + + for (i = 0; i < num_instances; i++) { + for (guint j = 0; j < appCtx[i]->config.num_source_sub_bins; j++) { + /** Force the source (applicable only if RTSP) + * to use TCP for RTP/RTCP channels. + * forcing TCP to avoid problems with UDP port usage from within docker- + * container. + * The UDP RTCP channel when run within docker had issues receiving + * RTCP Sender Reports from server + */ + if (force_tcp) + appCtx[i]->config.multi_source_config[j].select_rtp_protocol = 0x04; + } + if (!create_pipeline (appCtx[i], bbox_generated_probe_after_analytics, + NULL, perf_cb, overlay_graphics)) { + NVGSTDS_ERR_MSG_V ("Failed to create pipeline"); + return_value = -1; + goto done; + } + + /* if (appCtx[i]->config.dsanalytics_config.enable){ + GstPad *src_pad = NULL; + GstElement *nvdsanalytics = appCtx[i]->pipeline.common_elements.dsanalytics_bin.elem_dsanalytics; + src_pad = gst_element_get_static_pad (nvdsanalytics, "src"); + if (!src_pad) + g_print ("Unable to get nvdsanalytics src pad\n"); + else + { + gst_pad_add_probe (src_pad, GST_PAD_PROBE_TYPE_BUFFER, + nvdsanalytics_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (src_pad); + } + }*/ + + /** Now add probe to RTPSession plugin src pad */ + for (guint j = 0; j < appCtx[i]->pipeline.multi_src_bin.num_bins; j++) { + testAppCtx->streams[j].id = j; + } + /** In test5 app, as we could have several sources connected + * for a typical IoT use-case, raising the nvstreammux's + * buffer-pool-size to 16 */ + g_object_set (appCtx[i]->pipeline.multi_src_bin.streammux, + "buffer-pool-size", STREAMMUX_BUFFER_POOL_SIZE, NULL); + } + + main_loop = g_main_loop_new (NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, NULL); + + g_mutex_init (&disp_lock); + display = XOpenDisplay (NULL); + for (i = 0; i < num_instances; i++) { + guint j; + + if (!show_bbox_text) { + GstElement *nvosd = appCtx[i]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set(G_OBJECT(nvosd), "display-text", FALSE, NULL); + } + + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) { + NVGSTDS_ERR_MSG_V ("Failed to set pipeline to PAUSED"); + return_value = -1; + goto done; + } + + if (!appCtx[i]->config.tiled_display_config.enable) + continue; + + for (j = 0; j < appCtx[i]->config.num_sink_sub_bins; j++) { + XTextProperty xproperty; + gchar *title; + guint width, height; + XSizeHints hints = {0}; + + if (!GST_IS_VIDEO_OVERLAY (appCtx[i]->pipeline.instance_bins[0].sink_bin. + sub_bins[j].sink)) { + continue; + } + + if (!display) { + NVGSTDS_ERR_MSG_V ("Could not open X Display"); + return_value = -1; + goto done; + } + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width) + width = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width; + else + width = appCtx[i]->config.tiled_display_config.width; + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height) + height = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height; + else + height = appCtx[i]->config.tiled_display_config.height; + + width = (width) ? width : DEFAULT_X_WINDOW_WIDTH; + height = (height) ? height : DEFAULT_X_WINDOW_HEIGHT; + + hints.flags = PPosition | PSize; + hints.x = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_x; + hints.y = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_y; + hints.width = width; + hints.height = height; + + windows[i] = + XCreateSimpleWindow (display, RootWindow (display, + DefaultScreen (display)), hints.x, hints.y, width, height, 2, + 0x00000000, 0x00000000); + + XSetNormalHints(display, windows[i], &hints); + + if (num_instances > 1) + title = g_strdup_printf (APP_TITLE "-%d", i); + else + title = g_strdup (APP_TITLE); + if (XStringListToTextProperty ((char **) &title, 1, &xproperty) != 0) { + XSetWMName (display, windows[i], &xproperty); + XFree (xproperty.value); + } + + XSetWindowAttributes attr = { 0 }; + if ((appCtx[i]->config.tiled_display_config.enable && + appCtx[i]->config.tiled_display_config.rows * + appCtx[i]->config.tiled_display_config.columns == 1) || + (appCtx[i]->config.tiled_display_config.enable == 0 && + appCtx[i]->config.num_source_sub_bins == 1)) { + } else { + attr.event_mask = ButtonPress | KeyRelease; + } + XChangeWindowAttributes (display, windows[i], CWEventMask, &attr); + + Atom wmDeleteMessage = XInternAtom (display, "WM_DELETE_WINDOW", False); + if (wmDeleteMessage != None) { + XSetWMProtocols (display, windows[i], &wmDeleteMessage, 1); + } + XMapRaised (display, windows[i]); + XSync (display, 1); //discard the events for now + gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (appCtx + [i]->pipeline.instance_bins[0].sink_bin.sub_bins[j].sink), + (gulong) windows[i]); + gst_video_overlay_expose (GST_VIDEO_OVERLAY (appCtx[i]->pipeline. + instance_bins[0].sink_bin.sub_bins[j].sink)); + if (!x_event_thread) + x_event_thread = g_thread_new ("nvds-window-event-thread", + nvds_x_event_thread, NULL); + } + } + + /* Dont try to set playing state if error is observed */ + if (return_value != -1) { + for (i = 0; i < num_instances; i++) { + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + + g_print ("\ncan't set pipeline to playing state.\n"); + return_value = -1; + goto done; + } + } + } + + print_runtime_commands (); + + changemode (1); + + g_timeout_add (40, event_thread_func, NULL); + g_main_loop_run (main_loop); + + changemode (0); + +done: + + g_print ("Quitting\n"); + for (i = 0; i < num_instances; i++) { + if (appCtx[i] == NULL) + continue; + + if (appCtx[i]->return_value == -1) + return_value = -1; + + destroy_pipeline (appCtx[i]); + + g_mutex_lock (&disp_lock); + if (windows[i]) + XDestroyWindow (display, windows[i]); + windows[i] = 0; + g_mutex_unlock (&disp_lock); + + g_free (appCtx[i]); + } + + g_mutex_lock (&disp_lock); + if (display) + XCloseDisplay (display); + display = NULL; + g_mutex_unlock (&disp_lock); + g_mutex_clear (&disp_lock); + + if (main_loop) { + g_main_loop_unref (main_loop); + } + + if (ctx) { + g_option_context_free (ctx); + } + + if (return_value == 0) { + g_print ("App run successful\n"); + } else { + g_print ("App run failed\n"); + } + + gst_deinit (); + + return return_value; + + g_free (testAppCtx); + + return 0; +} + + static gchar * +get_first_result_label (NvDsClassifierMeta * classifierMeta) +{ + GList *n; + for (n = classifierMeta->label_info_list; n != NULL; n = n->next) { + NvDsLabelInfo *labelInfo = (NvDsLabelInfo *) (n->data); + if (labelInfo->result_label[0] != '\0') { + return g_strdup (labelInfo->result_label); + } + } + return NULL; +} + + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif new file mode 100644 index 00000000..327ea7b7 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/kafka_messages.gif differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/test.png b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/test.png new file mode 100644 index 00000000..9fca4621 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/images/test.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h new file mode 100644 index 00000000..402f15aa --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/analytics.h @@ -0,0 +1,17 @@ +#ifndef _ANALYTICS_H_ +#define _ANALYTICS_H_ + +#include + +/* User defined */ +typedef struct +{ + + guint32 lcc_cnt_exit; + guint32 lccum_cnt; + guint32 lcc_cnt_entry; + guint32 source_id; + +} AnalyticsUserMeta; + +#endif diff --git a/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h new file mode 100644 index 00000000..23f5a02f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-occupancy-analytics/includes/nvdsmeta_schema.h @@ -0,0 +1,271 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * NVIDIA DeepStream: Metadata Extension Structures + * + * @b Description: This file defines the NVIDIA DeepStream metadata structures + * used to describe metadata objects. + */ + +/** + * @defgroup metadata_extensions Metadata Extension Structures + * + * Defines metadata structures used to describe metadata objects. + * + * @ingroup NvDsMetaApi + * @{ + */ + +#ifndef NVDSMETA_H_ +#define NVDSMETA_H_ + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Defines event type flags. + */ +typedef enum NvDsEventType { + NVDS_EVENT_ENTRY, + NVDS_EVENT_EXIT, + NVDS_EVENT_MOVING, + NVDS_EVENT_STOPPED, + NVDS_EVENT_EMPTY, + NVDS_EVENT_PARKED, + NVDS_EVENT_RESET, + + /** Reserved for future use. Custom events must be assigned values + greater than this. */ + NVDS_EVENT_RESERVED = 0x100, + /** Specifies a custom event. */ + NVDS_EVENT_CUSTOM = 0x101, + NVDS_EVENT_FORCE32 = 0x7FFFFFFF +} NvDsEventType; + +/** + * Defines object type flags. + */ +typedef enum NvDsObjectType { + NVDS_OBJECT_TYPE_VEHICLE, + NVDS_OBJECT_TYPE_PERSON, + NVDS_OBJECT_TYPE_FACE, + NVDS_OBJECT_TYPE_BAG, + NVDS_OBJECT_TYPE_BICYCLE, + NVDS_OBJECT_TYPE_ROADSIGN, + /** Reserved for future use. Custom objects must be assigned values + greater than this. */ + NVDS_OBJECT_TYPE_RESERVED = 0x100, + /** Specifies a custom object. */ + NVDS_OBJECT_TYPE_CUSTOM = 0x101, + /** "object" key will be missing in the schema */ + NVDS_OBJECT_TYPE_UNKNOWN = 0x102, + NVDS_OBEJCT_TYPE_FORCE32 = 0x7FFFFFFF +} NvDsObjectType; + +/** + * Defines payload type flags. + */ +typedef enum NvDsPayloadType { + NVDS_PAYLOAD_DEEPSTREAM, + NVDS_PAYLOAD_DEEPSTREAM_MINIMAL, + /** Reserved for future use. Custom payloads must be assigned values + greater than this. */ + NVDS_PAYLOAD_RESERVED = 0x100, + /** Specifies a custom payload. You must implement the nvds_msg2p_* + interface. */ + NVDS_PAYLOAD_CUSTOM = 0x101, + NVDS_PAYLOAD_FORCE32 = 0x7FFFFFFF +} NvDsPayloadType; + +/** + * Holds a rectangle's position and size. + */ +typedef struct NvDsRect { + float top; /**< Holds the position of rectangle's top in pixels. */ + float left; /**< Holds the position of rectangle's left side in pixels. */ + float width; /**< Holds the rectangle's width in pixels. */ + float height; /**< Holds the rectangle's height in pixels. */ +} NvDsRect; + +/** + * Holds geolocation parameters. + */ +typedef struct NvDsGeoLocation { + gdouble lat; /**< Holds the location's latitude. */ + gdouble lon; /**< Holds the location's longitude. */ + gdouble alt; /**< Holds the location's altitude. */ +} NvDsGeoLocation; + +/** + * Hold a coordinate's position. + */ +typedef struct NvDsCoordinate { + gdouble x; /**< Holds the coordinate's X position. */ + gdouble y; /**< Holds the coordinate's Y position. */ + gdouble z; /**< Holds the coordinate's Z position. */ +} NvDsCoordinate; + +/** + * Holds an object's signature. + */ +typedef struct NvDsObjectSignature { + /** Holds a pointer to an array of signature values. */ + gdouble *signature; + /** Holds the number of signature values in @a signature. */ + guint size; +} NvDsObjectSignature; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObject { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ +} NvDsVehicleObject; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ +} NvDsPersonObject; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ +} NvDsFaceObject; + +/** + * Holds event message meta data. + * + * You can attach various types of objects (vehicle, person, face, etc.) + * to an event by setting a pointer to the object in @a extMsg. + * + * Similarly, you can attach a custom object to an event by setting a pointer to the object in @a extMsg. + * A custom object must be handled by the metadata parsing module accordingly. + */ +typedef struct NvDsEventMsgMeta { + /** Holds the event's type. */ + NvDsEventType type; + /** Holds the object's type. */ + NvDsObjectType objType; + /** Holds the object's bounding box. */ + NvDsRect bbox; + /** Holds the object's geolocation. */ + NvDsGeoLocation location; + /** Holds the object's coordinates. */ + NvDsCoordinate coordinate; + /** Holds the object's signature. */ + NvDsObjectSignature objSignature; + /** Holds the object's class ID. */ + gint objClassId; + /** Holds the ID of the sensor that generated the event. */ + gint sensorId; + /** Holds the ID of the analytics module that generated the event. */ + gint moduleId; + /** Holds the ID of the place related to the object. */ + gint placeId; + /** Holds the ID of the component (plugin) that generated this event. */ + gint componentId; + /** Holds the video frame ID of this event. */ + gint frameId; + /** Holds the confidence level of the inference. */ + gdouble confidence; + /** Holds the object's tracking ID. */ + gint trackingId; + /** Holds a pointer to the generated event's timestamp. */ + gchar *ts; + /** Holds a pointer to the detected or inferred object's ID. */ + gchar *objectId; + + /** Holds a pointer to a string containing the sensor's identity. */ + gchar *sensorStr; + /** Holds a pointer to a string containing other attributes associated with + the object. */ + gchar *otherAttrs; + /** Holds a pointer to the name of the video file. */ + gchar *videoPath; + /** Holds a pointer to event message meta data. This can be used to hold + data that can't be accommodated in the existing fields, or an associated + object (representing a vehicle, person, face, etc.). */ + gpointer extMsg; + /** Holds the size of the custom object at @a extMsg. */ + guint extMsgSize; + + /*My data*/ + guint occupancy; + guint source_id; + guint lccum_cnt_entry; + guint lccum_cnt_exit; +} NvDsEventMsgMeta; + +/** + * Holds event information. + */ +typedef struct _NvDsEvent { + /** Holds the type of event. */ + NvDsEventType eventType; + /** Holds a pointer to event metadata. */ + NvDsEventMsgMeta *metadata; +} NvDsEvent; + +/** + * Holds payload metadata. + */ +typedef struct NvDsPayload { + /** Holds a pointer to the payload. */ + gpointer payload; + /** Holds the size of the payload. */ + guint payloadSize; + /** Holds the ID of the component (plugin) which attached the payload + (optional). */ + guint componentId; +} NvDsPayload; + +#ifdef __cplusplus +} +#endif +#endif /* NVDSMETA_H_ */ + +/** @} */ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/LICENSE.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/LICENSE.md new file mode 100644 index 00000000..06673f95 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/LICENSE.md @@ -0,0 +1,15 @@ +SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/Makefile b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/Makefile new file mode 100644 index 00000000..0532392f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/Makefile @@ -0,0 +1,62 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= ds-retail-iva + + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) -g + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lrt \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -lnvds_yml_parser \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/README.md new file mode 100644 index 00000000..4e538d9a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/README.md @@ -0,0 +1,185 @@ +# Description + +This is a sample application to perform real-time Intelligent Video Analytics (IVA) in a brick and mortar retail environment using NVIDIA DeepStream, TAO, and pre-trained models. DeepStream is used to run DL inference on a video feed inside a store to detect and track customers, and identify whether the detected persons are carrying shopping baskets. The inference output of this Computer Vision (CV) pipeline is streamed, using Kafka, to a Time-Series Database (TSDB) for archival and further processing. A Django app serves a REST-ful API to query insights based on the inference data. We also demonstrate a sample front-end dashboard to quickly visualize the various Key Performance Indicators (KPIs) available through the Django app. + +This application is based on deepstream-test4 and deepstream-test5 sample applications included with DeepStream. The architecture diagram below shows how all the components are connected. + +![](./media/output.gif) + +What is this DeepStream pipeline made of? + +* Primary Detector: PeopleNet Pre-Trained Model (PTM) from NGC +* Secondary Detector: Custom classification model trained using TAO toolkit to classify people with and without shopping baskets +* Object Tracker: NvDCF tracker +* Message Converter: Custom message converter to generate custom payload from inference data +* Message Broker: Message broker to relay inference data to a kafka server + +# Table of Contents +* [Description](#description) +* [Table of Contents](#table-of-contents) +* [Application Architecture](#application-architecture) +* [Prerequisites](#prerequisites) +* [Getting Started](#getting-started) +* [Build](#build) +* [Running the Application](#run-the-application) +* [Output](#output) +* [Advanced](#advanced) + +# Application Architecture + + +
+ + +# Quick Start + +# Prerequisites + +1. Install the latest [NVIDIA drivers](https://www.nvidia.com/download/index.aspx) for your operating system and GPU. + +2. Install Docker and the NVIDIA Container Toolkit - Refer to this [README](docs/install_nvidia_container_toolkit.md). + +3. **OPTIONAL:** Install python and pip. Required for front-end only. Can omit if not using front-end. + +4. Install DeepStream SDK [instructions](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_Quickstart.html) + + * Pull the docker image for DeepStream development + ```bash + docker pull nvcr.io/nvidia/deepstream:6.1-devel + ``` + + * Allow external applications to connect to the host's X display + ```bash + xhost + + ``` + + **Note:** If you are using a remote machine, the above command will not work from an SSH session. It has to be executed from a VNC/RDP connection. + + * Run the container + ```bash + docker run -it --entrypoint /bin/bash --gpus all --rm --network=host -e DISPLAY=:0 -v /tmp/.X11-unix/:/tmp/.X11-unix --privileged -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/deepstream:6.1-devel + ``` + + This command will + * Start the container + * Provide access to all GPUs + * Host the container on the host's network + * Forwards the display of the host to the container along with some other volumes + * Opens an interactive terminal to run commands from within the container + +5. Install git-lfs inside the container + + ```bash + apt install git-lfs + ``` + + +6. We need a kafka message broker and kSQL database. For the purpose of this project, we use [confluent-platform](https://docs.confluent.io/platform/current/quickstart/ce-docker-quickstart.html) to setup these services. So, lets setup confluent-platform: + + **Note:** Bash commands in this section should be run from a separete terminal window and **not from within the DeepStream container**. + + ```bash + wget https://raw.githubusercontent.com/confluentinc/cp-all-in-one/7.2.1-post/cp-all-in-one/docker-compose.yml + docker-compose up -d + ``` + * Verify if all the containers started successfully by running `docker ps` + + ![](./media/docker_container_ps.png) + +7. Create a kafka-topic that will be used to receive messages sent by the DeepStream app + + ```bash + docker exec -it broker /bin/bash + # Within the container + kafka-topics --bootstrap-server "localhost:9092" --topic "detections" --create + ``` + + * You can also create the kafka topic by navigating to the confluent control center > cluster > Topics > Add Topic. + +8. Setup a [kSQL stream](https://docs.ksqldb.io/en/latest/concepts/streams/) based on the topic `detections`: + + a) If you used the above mentioned docker compose file, you can access kSQL CLI by running + ``` + docker exec -it ksqldb-cli ksql http://ksqldb-server:8088 + ``` + b) Once the CLI is active, copy-paste the content from [confluent-platform/stream_creation.sql](confluent-platform/stream_creation.sql) into the CLI to create the stream + + **Note:** You don't have to explicitly create the topic in confluent-kafka. The broker will automatically create a topic once DeepStream sends messages to a new topic. + +# Getting Started + +1. If you are using DeepStream via a docker container as mentioned in the instructions above, execute the following command to open the terminal of the DeepStream docker container if it's not already open. Otherwise, skip this step. + + ```bash + docker exec -it /bin/bash + ``` + + You can locate the container id by running the following command: + + ```bash + docker container ps + ``` + ![](./media/deepstream_container.png) + +2. Clone the repo in $DS_SDK_ROOT/sources/apps/sample_apps/ + ```bash + cd /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps + git clone https://github.com/NVIDIA-AI-IOT/deepstream-retail-analytics.git + cd deepstream-retail-analytics + git lfs pull + ``` + + * Although not necessary, it is recommended to verify the checksum of model and input files to confirm file integrity + ```bash + cd files/ + sha512sum -c checksum.txt + ``` + +3. Download PeopleNet model with the model download script. Download the `.etlt` and `labels.txt` files. + ```bash + bash ./download_models.sh + ``` +4. The custom message converter should be built inside the docker. The build instructions are [custom nvmsgconv library](nvmsgconv/README.md). + +# Build for x86 dGPU system + +Run the following commands from project root + +Modify the below command with the cuda version installed in the docker container. To check the CUDA version inside the docker container you can use `nvcc --version` command. + +```bash +export CUDA_VER= +make -B +``` + +# Run the application + +## Running the DeepStream Application + +```bash +./ds-retail-iva configs/retail_iva.yml --no-display +``` + +The `--no-display` flag in the above command is optional. If the application is running from within a docker container without a display attached, you should use this flag. + +## Running the front-end + +```bash +cd ds-retail-iva-frontend +pip install -r requirements.txt +python3 manage.py runserver 0.0.0.0:8000 +``` + +Open a browser and go to [http://localhost:8000](http://localhost:8000) to visualize the dashboard + +# Output + +**Dashboard** + + + +# Advanced + +[TAO README](./TAO/README.md) - Follow the instructions in this file to create a dataset and train a classification model using TAO toolkit + +[NvMsgConv README](./nvmsgconv/README.md) - Follow this README to build a custom library to modify message payload generated by DeepStream diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/README.md new file mode 100644 index 00000000..b9fdcc89 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/README.md @@ -0,0 +1,109 @@ +# Training a Classification Model using TAO Toolkit + +# Dataset Structure + +TAO Toolkit requirements + +* One folder for training, validation and testing +* Each of the above folders should contain one folder for each class that the model should learn ("hasBasket" and "noBasket" in this case) + +``` +|--dataset_root: + |--train + |--hasBasket: + |--1.jpg + |--2.jpg + |--noBasket: + |--01.jpg + |--02.jpg + |--val + |--hasBasket: + |--3.jpg + |--4.jpg + |--noBasket: + |--03.jpg + |--04.jpg + |--test + |--hasBasket: + |--5.jpg + |--6.jpg + |--noBasket: + |--05.jpg + |--06.jpg +``` + +If your dataset is in KITTI format (object detection) and you would like to convert it to a classification dataset, you can use the [`kitti_to_classification.py`](kitti_to_classification.py) file provided in this directory. + + +# Training a classification model + +## Prerequisites + +* Installation of TAO toolkit +* Download a PTM (pre-trained model) from NGC. We will use [resnet34](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/pretrained_detectnet_v2/files) for this example + +**Note:** You could use NGC command line tool or curl to download the model + +## Steps to train a classification model + +Refer to [TAO documentation](https://docs.nvidia.com/tao/tao-toolkit/text/image_classification.html) for more information on how to modify the spec file for training. + +We need to set a key to encode the model. The same key is to be used when saving/loading the model. In this example the key is set as `nvidia_tlt` + +* Create a directory to store checkpoints + +```bash +mkdir classification_model +``` + +* Train the model + +```bash +tao classification train -e SPECS_classification_train.txt -k nvidia_tlt -r classification_model +``` + +* Evaluate the model + +The path for the evaluation dataset is specified in the `eval_config` in the spec file + +```bash +tao classification evaluate -e SPECS_classification_train.txt -k nvidia_tlt +``` + +* OPTIONAL: Running inference on an image/directory using the model + +```bash +tao classification inference -m -i -k nvidia_tlt -cm -e SPECS_classification_train.txt +``` + +* Exporting the model + +Checkpoints for the model trained are located in the `classification_model/weights` directory as `.tlt` files(The directory to store output is set during the training step). + +Training logs are located in the same folder as JSON and CSV files. + +Pick a model to export depending on the loss/accuracy values + +```bash +tao classification export -m -k nvidia_tlt -o basketClassifier.etlt +``` + +* Deploying the model using DeepStream + +There are two ways to use the above exported model with DeepStream + +**Option 1:** Use the above exported `.etlt` model directly with DeepStream. + +**Option 2:** Use the `tao-converter` and generate a device specific engine file. The generated engine file should also be specified in the config file for the inference engine. Refer to [basket_classifier.yml](../configs/basket_classifier.yml) for an example config file. + +**Generating an engine using tao-converter** + +*Output Nodes:* Since this is a classification model, there is only only one classification node "predictions/Softmax" + +*Dimensions:* Dimensions for the below command should be the same as it was specified in the [SPECS_classification_train.txt](./SPECS_classification_train.txt) + +*input_file:* The input for the below command is the model exported in the previous command + +```bash +tao-converter -k nvidia_tlt -d 3,224,224 -o predictions/Softmax basketClassifier.etlt +``` \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt new file mode 100644 index 00000000..d7bfdf61 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/SPECS_classification_train.txt @@ -0,0 +1,79 @@ +model_config { + # Model Architecture can be chosen from: + # ['resnet', 'vgg', 'googlenet', 'alexnet'] + arch: "resnet" + # for resnet --> n_layers can be [10, 18, 50] + # for vgg --> n_layers can be [16, 19] + n_layers: 34 + use_batch_norm: True + use_bias: False + all_projections: False + use_pooling: True + retain_head: True + resize_interpolation_method: BICUBIC + # if you want to use the pretrained model, + # image size should be "3,224,224" + # otherwise, it can be "3, X, Y", where X,Y >= 16 + input_image_size: "3,224,224" +} +train_config { + val_dataset_path: "/work/basketDetection/classification_data/" + train_dataset_path: "/work/basketDetection/classification_data/" + pretrained_model_path: "/work/basketDetection/resnet_34.hdf5" + # Only ['sgd', 'adam'] are supported for optimizer + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 50 + n_epochs: 150 + # Number of CPU cores for loading data + n_workers: 16 + # regularizer + reg_config { + # regularizer type can be "L1", "L2" or "None". + type: "L2" + # if the type is not "None", + # scope can be either "Conv2D" or "Dense" or both. + scope: "Conv2D,Dense" + # 0 < weight decay < 1 + weight_decay: 0.000015 + } + # learning_rate + lr_config { + cosine { + learning_rate: 0.04 + soft_start: 0.0 + } + } + enable_random_crop: False + enable_center_crop: False + enable_color_augmentation: True + mixup_alpha: 0.2 + label_smoothing: 0.1 + preprocess_mode: "caffe" + image_mean { + key: 'b' + value: 103.9 + } + image_mean { + key: 'g' + value: 116.8 + } + image_mean { + key: 'r' + value: 123.7 + } +} +eval_config { + eval_dataset_path: "/work/basketDetection/classification_data/" + model_path: "/work/basketDetection/classification_model/weights/resnet_150.tlt" + top_k: 3 + batch_size: 256 + n_workers: 8 + enable_center_crop: False +} \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py new file mode 100644 index 00000000..7e4c22cf --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/TAO/kitti_to_classification.py @@ -0,0 +1,46 @@ +import os +from glob import glob +import cv2 +from tqdm import tqdm + + +def read_label_file(filepath): + labels = [] + coordinates = [] + + # Read the file + with open(filepath, "r") as file: + lines = file.readlines() + + for line in lines: + line = line.split() + lab, _, _, _, x1, y1, x2, y2, _, _, _, _, _, _, _ = line + x1, y1, x2, y2 = map(float, [x1, y1, x2, y2]) + x1, y1, x2, y2 = map(int, [x1, y1, x2, y2]) + labels.append(lab) + coordinates.append((x1, y1, x2, y2)) + + return labels, coordinates + + +def crop_image(img, xmin, ymin, xmax, ymax): + return img[ymin:ymax, xmin:xmax :] + + +def draw_bbox(img, xmin, ymin, xmax, ymax): + img = cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 3) + return img + + +if __name__ == "__main__": + images = glob("default/image_2/*.PNG") + + for image in tqdm(images): + label_file, _ = os.path.splitext(os.path.basename(image)) + labels, coordinates = read_label_file(f"default/label_2/{label_file}.txt") + img = cv2.imread(image) + for i, label in enumerate(labels): + img_cropped = crop_image(img, coordinates[i][0], coordinates[i][1], coordinates[i][2], coordinates[i][3]) + # img_annotated = draw_bbox(img, coordinates[i][0], coordinates[i][1], coordinates[i][2], coordinates[i][3]) + cv2.imwrite(f"classification_data/{label}/{label_file}_{i}.png", img_cropped) + # cv2.imwrite(f"test_{i}.png", img_annotated) \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE new file mode 100644 index 00000000..1186e010 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/THIRD_PARTY_LICENSE @@ -0,0 +1,5 @@ +python-ceriti: https://github.com/certifi/python-certifi/blob/master/LICENSE + +This Source Code Form is subject to the terms of the Mozilla Public License, +v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain +one at http://mozilla.org/MPL/2.0/. diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/README.md new file mode 100644 index 00000000..0950c675 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/README.md @@ -0,0 +1,16 @@ +# Config files for the Project + +README.md + +`retail_iva.yml`: Config file for this app. Holds information about input parameters for various plugins + +`pgie_config_peoplenet.yml`: Config file ofr Primary Inference Engine (PeopleNet) + +`basket_classifier.yml`, `basket_classifier.txt`: Config file for Secondary Inference Engine + +`dstest4_msgconv_config.txt`, `dstest4_msgconv_config.yml`: Default config file for message converter + +`dstest4_tracker_config.txt`: Config file for NvDCF tracker + +`dstest4_msgconv_config_1.txt`, `dstest4_msgconv_config_1.yml`: Config file we use in this application for message converter + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt new file mode 100644 index 00000000..c424eb9c --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.txt @@ -0,0 +1,44 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[property] +gpu-id=0 +# preprocessing parameters=These are the same for all classification models generated by TAO Toolkit. +net-scale-factor=1.0 +offsets=103.939;116.779;123.68 +model-color-format=1 + +# Model specific paths. These need to be updated for every classification model. +labelfile-path=/work/basketDetection/classification_model/labels.txt +tlt-encoded-model=/work/basketDetection/basketClassifier.etlt +model-engine-file=/work/basketDetection/basketClassifier.etlt_b1_gpu0_fp32.engine +tlt-model-key=nvidia_tlt +infer-dims=3;224;224 # where c = number of channels, h = height of the model input, w = width of model input +uff-input-blob-name=input_1 +output-blob-names=predictions/Softmax +operate-on-gie-id=1 +# operate-on-class-ids=1 +batch-size=1 + +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=0 +# process-mode=2 - inferences on crops from primary detector, 1 - inferences on whole frame +process-mode=2 +interval=0 +network-type=1 # defines that the model is a classifier. +gie-unique-id=2 +classifier-threshold=0.2 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml new file mode 100644 index 00000000..64f20b2e --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/basket_classifier.yml @@ -0,0 +1,45 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +property: + gpu-id: 0 + # preprocessing parameters: These are the same for all classification models generated by TAO Toolkit. + net-scale-factor: 1.0 + offsets: 103.939;116.779;123.68 + model-color-format: 1 + + # Model specific paths. These need to be updated for every classification model. + labelfile-path: ../files/basket_classifier_labels.txt + tlt-encoded-model: ../files/basketClassifier.etlt + # model-engine-file: ../files/basketClassifier.etlt_b1_gpu0_fp32.engine + tlt-model-key: nvidia_tlt + infer-dims: 3;224;224 # where c = number of channels, h = height of the model input, w = width of model input + uff-input-blob-name: input_1 + output-blob-names: predictions/Softmax + # operate-on-gie-id: 1 + # operate-on-class-ids: 1 + batch-size: 1 + classifier-type: hasBasketClassifier + + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 0 + # process-mode: 2 - inferences on crops from primary detector, 1 - inferences on whole frame + process-mode: 2 + interval: 0 + network-type: 1 # defines that the model is a classifier. + gie-unique-id: 2 + classifier-threshold: 0.5 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml new file mode 100644 index 00000000..671ab807 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source: + # location: /opt/nvidia/deepstream/deepstream-9.0/samples/streams/sample_720p.h264 + location: compressed.h264 + +streammux: + batch-size: 1 + batched-push-timeout: 40000 + width: 1280 + height: 720 + +tracker: + tracker-width: 640 + tracker-height: 384 + gpu-id: 0 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + # ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_DeepSORT.yml + enable-batch-process: 1 + +msgconv: + msg2p-lib: /opt/nvidia/deepstream/deepstream-6.1/sources/libs/nvmsgconv/libnvds_msgconv.so + payload-type: 0 + msg2p-newapi: 0 + frame-interval: 30 + +msgbroker: + proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + conn-str: localhost;9092 + topic: detections + sync: 0 + +sink: + sync: 1 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt new file mode 100644 index 00000000..429890e7 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.txt @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[sensor0] +enable=1 +type=Camera +id=CAMERA_ID +location=45.293701447;-75.8303914499;48.1557479338 +description="Entrance of Garage Right Lane" +coordinate=5.2;10.1;11.2 + +[place0] +enable=1 +id=1 +type=garage +name=XYZ +location=30.32;-40.55;100.0 +coordinate=1.0;2.0;3.0 +place-sub-field1=walsh +place-sub-field2=lane1 +place-sub-field3=P2 + +[place1] +enable=1 +id=1 +type=garage +name=XYZ +location=28.47;47.46;1.53 +coordinate=1.0;2.0;3.0 +place-sub-field1="C-76-2" +place-sub-field2="LEV/EV/CP/ADA" +place-sub-field3=P2 + +[analytics0] +enable=1 +id=XYZ +description="Vehicle Detection and License Plate Recognition" +source=OpenALR +version=1.0 \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml new file mode 100644 index 00000000..0025a851 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +sensor0: + enable: 1 + type: Camera + id: CAMERA_ID + location: 45.293701447;-75.8303914499;48.1557479338 + description: "Entrance of Garage Right Lane" + coordinate: 5.2;10.1;11.2 + +place0: + enable: 1 + id: 1 + type: garage + name: XYZ + location: 30.32;-40.55;100.0 + coordinate: 1.0;2.0;3.0 + place-sub-field1: walsh + place-sub-field2: lane1 + place-sub-field3: P2 + +place1: + enable: 1 + id: 1 + type: garage + name: XYZ + location: 28.47;47.46;1.53 + coordinate: 1.0;2.0;3.0 + place-sub-field1: "C-76-2" + place-sub-field2: "LEV/EV/CP/ADA" + place-sub-field3: P2 + +analytics0: + enable: 1 + id: XYZ + description: "Vehicle Detection and License Plate Recognition" + source: OpenALR + version: 1.0 \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt new file mode 100644 index 00000000..e0144245 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.txt @@ -0,0 +1,16 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml new file mode 100644 index 00000000..5928c51c --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_msgconv_config_1.yml @@ -0,0 +1,16 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt new file mode 100644 index 00000000..adfb62f4 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/dstest4_tracker_config.txt @@ -0,0 +1,34 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Mandatory properties for the tracker: +# tracker-width, tracker-height: needs to be multiple of 32 for NvDCF and DeepSORT +# gpu-id +# ll-lib-file: path to low-level tracker lib +# ll-config-file: required to set different tracker types +# +[tracker] +tracker-width=960 +tracker-height=544 +gpu-id=0 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_IOU.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +enable-batch-process=1 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml new file mode 100644 index 00000000..df687c40 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/pgie_config_peoplenet.yml @@ -0,0 +1,77 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + infer-dims: 3;544;960 + onnx-file: ../models/peoplenet/resnet34_peoplenet_int8.onnx + int8-calib-file: ../models/peoplenet/resnet34_peoplenet_int8.txt + labelfile-path: ../models/peoplenet/labels.txt + model-engine-file: ../models/peoplenet/resnet34_peoplenet_int8.onnx_b1_gpu0_int8.engine + batch-size: 1 + process-mode: 1 + model-color-format: 0 + # 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 0 + num-detected-classes: 3 + interval: 0 + gie-unique-id: 1 + output-blob-names: output_bbox/BiasAdd:0;output_cov/Sigmoid:0 + cluster-mode: 3 + # We are interested only in the person class. Filter out bag and face classes. + filter-out-class-ids: 1;2 # Filter out bag and face class. + +# Use the config params below for NMS clustering mode +class-attrs-all: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.3 + minBoxes: 3 + dbscan-min-score: 1.3 + eps: 0.15 + detected-min-w: 20 + detected-min-h: 20 + threshold: 0.7 + +# [property] +# ## model-specific params like paths to model, engine, label files, etc. are to be added by users + +# gpu-id=0 +# net-scale-factor=0.0039215697906911373 +# input-dims=3;544;960;0 +# uff-input-blob-name=input_1 +# process-mode=1 +# model-color-format=0 +# ## 0=FP32, 1=INT8, 2=FP16 mode +# network-mode=1 +# num-detected-classes=3 +# interval=0 +# gie-unique-id=1 +# output-blob-names=output_cov/Sigmoid;output_bbox/BiasAdd +# ## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +# cluster-mode=3 +# maintain-aspect-ratio=1 + +# [class-attrs-all] +# pre-cluster-threshold=0.3 +# nms-iou-threshold=0.5 +# minBoxes=3 +# dbscan-min-score=1.3 +# eps=0.15 +# detected-min-w=20 +# detected-min-h=20 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml new file mode 100644 index 00000000..ddc2058e --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/configs/retail_iva.yml @@ -0,0 +1,62 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source: + # location: /opt/nvidia/deepstream/deepstream-9.0/samples/streams/sample_720p.h264 + # location: /work/retail_cropped.264 + # location: /work/data/trimmed.h264 + location: ../deepstream-retail-analytics/files/0001_compressed.h264 + # location: rtsp:// + +streammux: + batch-size: 1 + batched-push-timeout: 40000 + width: 1920 + height: 1080 + + +# tracker: +# tracker-width: 640 +# tracker-height: 384 +# gpu-id: 0 +# ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# # ll-config-file required to set different tracker types +# # ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_IOU.yml +# # ll-config-file: /opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file: /opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# # ll-config-file=/opt/nvidia/deepstream/deepstream-9.0/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +# enable-batch-process: 1 + +msgconv: + msg2p-lib: ../nvmsgconv/libnvds_msgconv.so + #msg2p-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_msgconv.so + payload-type: 0 + msg2p-newapi: 0 + frame-interval: 30 + +msgbroker: + proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + conn-str: localhost;9092 + topic: detections + sync: 0 + +sink: + sync: 1 + +filesink: + location: /opt/nvidia/deepstream/deepstream-6.1/sources/apps/sample_apps/deepstream-retail-analytics/retail_output.mp4 + sync: 0 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml new file mode 100644 index 00000000..5d5e6f06 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/docker-compose.yml @@ -0,0 +1,177 @@ +--- +version: '2' +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.1.1 + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker: + image: confluentinc/cp-server:7.1.1 + hostname: broker + container_name: broker + depends_on: + - zookeeper + ports: + - "9092:9092" + - "9101:9101" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'true' + CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' + + schema-registry: + image: confluentinc/cp-schema-registry:7.1.1 + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.5.3-7.1.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: 'broker:29092' + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.1.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR + + control-center: + image: confluentinc/cp-enterprise-control-center:7.1.1 + hostname: control-center + container_name: control-center + depends_on: + - broker + - schema-registry + - connect + - ksqldb-server + ports: + - "9021:9021" + environment: + CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' + CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083' + CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" + CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088" + CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + CONTROL_CENTER_REPLICATION_FACTOR: 1 + CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 + CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 + CONFLUENT_METRICS_TOPIC_REPLICATION: 1 + PORT: 9021 + + ksqldb-server: + image: confluentinc/cp-ksqldb-server:7.1.1 + hostname: ksqldb-server + container_name: ksqldb-server + depends_on: + - broker + - connect + ports: + - "8088:8088" + environment: + KSQL_CONFIG_DIR: "/etc/ksql" + KSQL_BOOTSTRAP_SERVERS: "broker:29092" + KSQL_HOST_NAME: ksqldb-server + KSQL_LISTENERS: "http://0.0.0.0:8088" + KSQL_CACHE_MAX_BYTES_BUFFERING: 0 + KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + KSQL_KSQL_CONNECT_URL: "http://connect:8083" + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1 + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true' + KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true' + + ksqldb-cli: + image: confluentinc/cp-ksqldb-cli:7.1.1 + container_name: ksqldb-cli + depends_on: + - broker + - connect + - ksqldb-server + entrypoint: /bin/sh + tty: true + + ksql-datagen: + image: confluentinc/ksqldb-examples:7.1.1 + hostname: ksql-datagen + container_name: ksql-datagen + depends_on: + - ksqldb-server + - broker + - schema-registry + - connect + command: "bash -c 'echo Waiting for Kafka to be ready... && \ + cub kafka-ready -b broker:29092 1 40 && \ + echo Waiting for Confluent Schema Registry to be ready... && \ + cub sr-ready schema-registry 8081 40 && \ + echo Waiting a few seconds for topic creation to finish... && \ + sleep 11 && \ + tail -f /dev/null'" + environment: + KSQL_CONFIG_DIR: "/etc/ksql" + STREAMS_BOOTSTRAP_SERVERS: broker:29092 + STREAMS_SCHEMA_REGISTRY_HOST: schema-registry + STREAMS_SCHEMA_REGISTRY_PORT: 8081 + + rest-proxy: + image: confluentinc/cp-kafka-rest:7.1.1 + depends_on: + - broker + - schema-registry + ports: + - 8082:8082 + hostname: rest-proxy + container_name: rest-proxy + environment: + KAFKA_REST_HOST_NAME: rest-proxy + KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092' + KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" + KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql new file mode 100644 index 00000000..478cc3a4 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/confluent-platform/stream_creation.sql @@ -0,0 +1,26 @@ +CREATE STREAM DETECTIONS_STREAM ( + messageid varchar, + mdsversion varchar, + timestamp varchar, + object struct< + id varchar, + speed int, + direction int, + orientation int, + detection varchar, + obj_prop struct< + hasBasket varchar, + confidence double>, + bbox struct< + topleftx int, + toplefty int, + bottomrightx int, + bottomrighty int>>, + event_des struct< + id varchar, + type varchar>, + videopath varchar) WITH ( + KAFKA_TOPIC='detections', + VALUE_FORMAT='JSON', + TIMESTAMP='timestamp', + TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''); \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md new file mode 100644 index 00000000..02243743 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/docs/install_nvidia_container_toolkit.md @@ -0,0 +1,50 @@ + +# NVIDIA Container Toolkit + +* To install NVIDIA container toolkit follow these instructions: + + 1. Setup docker repository + ```bash + sudo apt-get update + + sudo apt-get install ca-certificates curl gnupg lsb-release + + sudo mkdir -p /etc/apt/keyrings + + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + ``` + + 2. Install [Docker](https://docs.docker.com/engine/install/) + ```bash + sudo apt-get update + + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin + ``` + + 3. Install [nvidia-docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) + ```bash + distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \ + && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + ``` + + 4. **OPTIONAL:** Post-install instructions to run docker without sudo + + ```bash + sudo groupadd docker + sudo usermod -aG docker $USER + ``` + + ```bash + sudo apt-get update + sudo apt-get install -y nvidia-docker2 + ``` + 5. Restart docker + + ```bash + sudo systemctl restart docker + ``` \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/download_models.sh b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/download_models.sh new file mode 100644 index 00000000..87fa3c11 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/download_models.sh @@ -0,0 +1,24 @@ +#!/bin/sh +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +mkdir -p ./models/peoplenet +cd ./models/peoplenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.3/files?redirect=true&path=labels.txt' -O labels.txt + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md new file mode 100644 index 00000000..9502d6a7 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/README.md @@ -0,0 +1,165 @@ +# DeepStream Retail IVA Frontend + +This repo is a django-based frontend application for [DeepStream Retail IVA](https://gitlab-master.nvidia.com/admantri/ds-retail-iva). This application will be referred to as the parent application henceforth. + +# Prerequisites + +* Working installation of python and pip. Preferred to use conda + + * Installing anaconda/miniconda + + ```bash + wget https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh + chmod +x Miniconda3-py37_4.12.0-Linux-x86_64.sh + ./Miniconda3-py37_4.12.0-Linux-x86_64.sh + ``` + + * You can create a conda environment using the [environment.yml](./environment.yml) file provided in this repo + + ```bash + conda env create -f environment.yml + ``` + + * If you already have python installed and prefer to not use conda, you can setup the required packages by using the [requirements.txt](./requirements.txt) file + + ```bash + pip install -r requirements.txt + ``` + +* This project assumes that you have already setup confluent platform with kSQL database running as instructed in the [prerequisites](https://gitlab-master.nvidia.com/admantri/ds-retail-iva#prerequisites) section of the parent application. + + +# Running the dashboard + +```bash +python manage.py runserver 0.0.0.0:8000 +``` + +# URLs supported by this API + +All the endpoints supported by this application can be found in the [urls.py](./retail_iva/urls.py) file of the project. + +* http://localhost:8000/ - Homepage where a dashboard with all the plots is shown +* http://localhost:8000/num-visitors-region/ - API endpoint to get the number of visitors in a given region. + + Inputs for this endpoint + + * topleftx - x coordinate of the top left corner + * toplefty - y coordinate of the top left corner + * bottomrightx - x coordinate of the bottom right corner + * bottomrighty - y coordinate of the bottom right corner + + Example URL: http://localhost:8000/num-visitors-region?topleftx=50&bottomrightx=1400&toplefty=0&bottomrighty=1440 + +* http://localhost:8000/num-visitors-time/ - API endpoint to get the number of visitors in a given time window + + Inputs for this endpoint + + * start_time: Starting time of the time window + * end_time: Ending time of the time window + + The API returns 0 if start_time > end_time + + This API supports three forms of inputs + + * If both start_time and end_time are provided, the API calculates the number + of visitors that arrived in that time window + * If only start_time is provided, the API calculates the number of visitors + that arrived after that time + * If only end_time is provided, the API calculates the number of visitors + that arrived before that time + * If none are provided, the API returns "-1" + + Example URL: http://localhost:8000/num-visitors-time?start_time=2022-07-29T03:44:41&end_time=2022-07-29T03:44:45 + +* http://localhost:8000/visitor-path - API endpoint to get the path of a visitor in the store. This API renders a path of the visitor on the background of the store. + + Inputs for this endpoint + + * person_id: This is the ID given by the NvDCF tracker + + Example URL: http://localhost:8000/visitor-path?person_id=7 + + +# Config Files + +This project comes with two config files + +## [store_config.ini](./store_config.ini) + +This is a config file that carries information about coordinates of each aisle. This config file is used to generate a bar chart shown on the dashboard with information about how many visitors are present in each aisle. Adding new sections to this file will automatically result in a refreshed bar graph. + +Each section in the bar graph requires 4 coordinates: + +* topleftx - x coordinate of the top left corner +* toplefty - y coordinate of the top left corner +* bottomrightx - x coordinate of the bottom right corner +* bottomrighty - y coordinate of the bottom right corner + + +## [config.py](./config.py) + +This is the configuration file for the Django project. + +`TEST_MODE` - Set this to True to show a dashboard with fake data generated by running [random_message_generator.py](./random_message_generator.py) + +`ksql_server` - URL for the kSQL server. + +`kafka_server` - URL for the kafka server + +`ksql_stream_name` and `kafka_topic` are set depending on `TEST_MODE`. + + +# Generating random data for testing + +* [`random_message_generator.py`](./random_message_generator.py) can be used to generate fake kafka message data and deliver it to the kafka server. + +* Although, kafka server creates the topic as messages are delivered, we have to explicitly create a kSQL stream. It is important to create the stream since the dashboard is entirely dependant on the kSQL stream. Follow the instructions given in the parent application's README to create a stream. + +```SQL +CREATE STREAM TEST_STREAM ( +messageid varchar, +mdsversion varchar, +timestamp varchar, +object struct< + id varchar, + speed int, + direction int, + orientation int, + detection varchar, + obj_prop struct< + hasBasket varchar, + confidence double>, + bbox struct< + topleftx int, + toplefty int, + bottomrightx int, + bottomrighty int>>, +event_des struct< + id varchar, + type varchar>, +videopath varchar) WITH ( + KAFKA_TOPIC='detections', + VALUE_FORMAT='JSON', + TIMESTAMP='timestamp', + TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''); + +``` + +* To check if the stream was created successfully run `list streams` from the ksql-cli + +* Modify [this line](./random_message_generator.py#L69) to change the number of messages generated. + + +* To generate fake data and send messages execute + +```bash +python random_message_generator.py +``` + +* To verify if the messages were stored in the stream, run the below command from ksql-cli + +```SQL +select * from test_stream +``` + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py new file mode 100644 index 00000000..258d4ddc --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AnalyticsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'analytics' diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py new file mode 100644 index 00000000..1c23edd4 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/influxdb_connecter.py @@ -0,0 +1,41 @@ +import sys +import cv2 +import numpy as np +from influxdb import InfluxDBClient as idb +from PIL import Image + +def get_num_visitors_time_window(start_time, end_time, client): + pass + +def get_num_visitors_in_region(topleftx, bottomrightx, toplefty, bottomrighty, client): + client.switch_database("detections") + result = client.query(f'SELECT * FROM detections where \ + topleftx > {topleftx} and bottomrightx < {bottomrightx} and \ + toplefty > {toplefty} and bottomrighty < {bottomrighty}') + + num_results = len(result.raw["series"][0]["values"]) + + return num_results, result + +def get_visitor_path(person_id, client): + client.switch_database("detections") + result = client.query(f'SELECT * FROM detections where id=\'{person_id}\'') + + path = [] + + for val in result.raw["series"][0]["values"]: + bottomrightx = val[1] + bottomrighty = val[2] + topleftx = val[5] + toplefty = val[6] + path.append([(bottomrightx + topleftx)//2, (bottomrighty + toplefty)//2]) + + # img = cv2.imread("frame0.jpg") + img = np.zeros([1080,1920,3],dtype=np.uint8) + img.fill(0) + path = np.array(path) + cv2.drawContours(img, [path], 0, (255,255,255), 2) + cv2.imwrite("testImg.png", img) + + return path, Image.fromarray(img) + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py new file mode 100644 index 00000000..8d0720f2 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/ksqldb_connecter.py @@ -0,0 +1,281 @@ +import sys +sys.path.append("../retail_iva") + +import ast +import datetime +from configparser import ConfigParser +import cv2 +import numpy as np +import pandas as pd +from PIL import Image +from ksql import KSQLAPI +import matplotlib.pyplot as plt +# import imageio +from tqdm import tqdm +import config + + +def query_parser(query): + """ + Utility function to parse the query generator object returned by KSQLAPI + This function returns a list of all rows of the table as retuened by the query + """ + # The first element returned by the generator object is a list containing the column headers + # Skip the first row + next(query) + + # Iterate through the rest of the output + output = [item[:-2] for item in query] + + # The last element returned by the generator object is always "]" + # Remove the last element + output = output[:-1] + return output + + +def get_num_visitors_time_window(client, start_time=None, end_time=None): + """ + This API can perform the following functions based on the inputs provided + + * If both start_time and end_time are provided, the API calculates the number + of visitors that arrived in that time window + * If only start_time is provided, the API calculates the number of visitors + that arrived after that time + * If only end_time is provided, the API calculates the number of visitors + that arrived before that time + """ + + # Both start_time and end_time are not None + if start_time is not None and end_time is not None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime > \'{start_time}\' and rowtime < \'{end_time}\'') + # Only start_time is provided + elif start_time is not None and end_time is None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime > \'{start_time}\'') + # Only end_time is provided + elif start_time is None and end_time is not None: + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + rowtime < \'{end_time}\'') + else: + return -1 + + # Parse the query + output = query_parser(query) + + # Count the number of unique objects and return it + # It is important to convert it to a set to avoid duplicates in counting + items = [item for item in output] + items = set(items) + + return len(items), items + + +def get_num_visitors_in_region(topleftx, bottomrightx, toplefty, bottomrighty, client): + """ + This API is used to fetch the number of visitors in a rectangle defined by its + top left and bottom right corners + """ + topleftx, toplefty = int(topleftx), int(toplefty) + bottomrightx, bottomrighty = int(bottomrightx), int(bottomrighty) + img = cv2.imread("background.png") + cv2.rectangle(img, (topleftx, toplefty), (bottomrightx, bottomrighty), (0, 0, 255), 3) + cv2.imwrite("roi.png", img) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + query = client.query(f'select object->id from {config.ksql_stream_name} where \ + object->bbox->topleftx > {topleftx} and \ + object->bbox->bottomrightx < {bottomrightx} and \ + object->bbox->toplefty > {toplefty} and \ + object->bbox->bottomrighty < {bottomrighty};') + + output = query_parser(query) + + items = [item for item in output] + items = set(items) + + return len(items), items, Image.fromarray(img) + + +def get_visitor_path(person_id, client): + """ + This API gets a visitor's path given the tracker ID. + In order to estimate the visitor's location, we take the midpoint + of the box detected. + """ + query = client.query(f'select \ + object->bbox->topleftx, \ + object->bbox->bottomrightx, \ + object->bbox->toplefty, \ + object->bbox->bottomrighty \ + from {config.ksql_stream_name} \ + where object->id=\'{person_id}\';') + + output = query_parser(query) + + path = [] + + frames = [] + + # Iterate through all the rows and store the coordinates of the bbox + for item in output: + img = cv2.imread("background.png") + item = ast.literal_eval(item) + topleftx = item["row"]["columns"][0] + bottomrightx = item["row"]["columns"][1] + toplefty = item["row"]["columns"][2] + bottomrighty = item["row"]["columns"][3] + center_x, center_y = (bottomrightx + topleftx)//2, bottomrighty + img[center_y-10:center_y+10, center_x-10:center_x+10, :] = 255 + img = cv2.resize(img, dsize=(1280, 720)) + frames.append(img) + path.append([center_x, center_y]) + + # with imageio.get_writer("test.gif", mode="I") as writer: + # for frame in tqdm(frames): + # writer.append_data(frame) + + img = cv2.imread("background.png") + path = np.array(path) + cv2.drawContours(img, [path], 0, (0,0,255), 5) + img = cv2.resize(img, (1280, 720)) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + cv2.imwrite("testImg.png", img) + + return path, Image.fromarray(img) + + +def get_multiple_visitor_path(person_ids, client): + """ + API to fetch the paths of multiple visitors. + Example use case: + To track a parent and child in the store and identify where the parent and child split up + """ + img = cv2.imread("background.png") + colors = [(0, 0, 255), (0, 255, 0), (255, 0, 0)] + for id in person_ids: + path, _ = get_visitor_path(id, client) + cv2.drawContours(img, [path], 0, (0,0,255), 5) + img = cv2.resize(img, (1280, 720)) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + cv2.imwrite("multiple.png", img) + return + + +def get_store_heatmap(client, start_time=None, end_time=None): + """ + Function to get a heatmap of the store. + """ + img = plt.imread("background.png") + fig, ax = plt.subplots(2, 1) + ax[0].imshow(img, extent=(0, 2560, 0, 1440)) + query = client.query(f'select \ + object->bbox->topleftx, \ + object->bbox->bottomrightx, \ + object->bbox->toplefty, \ + object->bbox->bottomrighty \ + from {config.ksql_stream_name}') + + output = query_parser(query) + + coordinates = [] + + for item in output: + item = ast.literal_eval(item) + topleftx = item["row"]["columns"][0] + bottomrightx = item["row"]["columns"][1] + toplefty = item["row"]["columns"][2] + bottomrighty = item["row"]["columns"][3] + coordinates.append([(bottomrightx + topleftx)//2, (bottomrighty + toplefty)//2]) + + coordinates = np.array(coordinates) + ax[1].hist2d(coordinates[:, 0], coordinates[:, 1], range=[[0, 2560], [0, 1440]]) + # plt.colorbar() + fig.savefig("heatmap.png", dpi=100) + return + + +def get_basket_pie(client): + """ + Function to get a pie chart of the number of people holding baskets vs no basket + """ + query = client.query(f'select object->obj_prop->hasBasket, \ + object->id \ + from {config.ksql_stream_name};') + output = query_parser(query) + + counts = {} + + for item in output: + item = ast.literal_eval(item) + id = item["row"]["columns"][1] + basket_class = item["row"]["columns"][0] + counts.__setitem__(id, basket_class) + + results = {'hasBasket':0, 'noBasket':0} + + for value in counts.values(): + results[value] += 1 + + return results + + +def get_time_plot(client): + timeframe = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d") + + query = client.query(f'select timestamp, object->id from {config.ksql_stream_name} \ + where rowtime > \'{timeframe}\'') + results = [] + + output = query_parser(query) + + for item in output: + item = ast.literal_eval(item) + timestamp = pd.to_datetime(item["row"]["columns"][0]).round('1h').strftime('%Y-%m-%d %H:%M:%S') + id = item["row"]["columns"][1] + results.append({"TIMESTAMP":timestamp, "ID":id}) + + df = pd.DataFrame(results, columns=["TIMESTAMP", "ID"]) + + target_df = ( + df.groupby('TIMESTAMP') + .agg(COUNT_PERSONID=('ID', 'nunique')) + .reset_index() + ) + return target_df.to_dict('records') + + +def get_aisle_counts(client): + cfg = ConfigParser() + cfg.read("store_config.ini") + + # Dictionary to store the number of visitors in each aisle + results = [] + + # Read all the sections from the config file + for section in cfg.sections(): + topleftx = cfg.getint(section, "topleftx") + bottomrightx = cfg.getint(section, "bottomrightx") + toplefty = cfg.getint(section, "toplefty") + bottomrighty = cfg.getint(section, "bottomrighty") + num_people, _, _ = get_num_visitors_in_region(topleftx, bottomrightx, + toplefty, bottomrighty, client) + results.append({"aisle": section, "count": num_people}) + + return results + + + +if __name__ == "__main__": + client = KSQLAPI(config.ksql_server) + # print(client.ksql("list streams;")) + # print(get_num_visitors_in_region(50, 1250, 0, 1440, client)) + # get_visitor_path(7, client) + # print(get_num_visitors_time_window(client, "2022-07-13", "2022-07-16")) + # print(get_num_visitors_time_window(client, start_time="2022-07-13")) + # print(get_num_visitors_time_window(client, end_time="2022-07-20")) + # get_store_heatmap(client) + # get_multiple_visitor_path([3, 17], client) + # print(get_basket_pie(client)) + # print(get_time_plot(client)) + # print(get_aisle_counts(client)) \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/migrations/__init__.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py new file mode 100644 index 00000000..71a83623 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html new file mode 100644 index 00000000..92b6955c --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/dashboard.html @@ -0,0 +1,269 @@ + + + + + + + + + + + + + + + + + + + + + +
Number of Visitors Today
+ +
+ + + + + + + + + + + {%block cust_count%} + + {%endblock cust_count%} + +
People w & w/o basketsAisle CountsNumber of Customers in the last hour

{{customer_count}}

+ + + + + + + + \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html new file mode 100644 index 00000000..6b853e9e --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/path_view.html @@ -0,0 +1,9 @@ + + + {%block content %} +
+ +
+ {%endblock content%} + + \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html new file mode 100644 index 00000000..dc696bf4 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/region_view.html @@ -0,0 +1,18 @@ + + + {%block content %} +
+

Number of visitors: {{num_visitors}}

+
+ +
+

Visitor IDs: {{visitors}}

+
+ +
+

Region of Interest

+ +
+ {%endblock content%} + + \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html new file mode 100644 index 00000000..1d97064f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/templates/time_view.html @@ -0,0 +1,13 @@ + + + {%block content %} +
+

{{num_visitors}}

+
+ +
+

Visitor IDs: {{visitors}}

+
+ {%endblock content%} + + \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py new file mode 100644 index 00000000..47bca3f3 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/analytics/views.py @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import datetime +from io import BytesIO +from django.http import HttpResponse +from django.shortcuts import render +from .ksqldb_connecter import get_num_visitors_in_region, get_visitor_path +from .ksqldb_connecter import get_num_visitors_time_window +from .ksqldb_connecter import get_basket_pie +from .ksqldb_connecter import get_time_plot, get_aisle_counts +from ksql import KSQLAPI + +import config +client = KSQLAPI(config.ksql_server) + + +def img_to_base64_str(img): + """ + Helper function to convert PIL image to base64 for rendering + on the webpage + """ + buffered = BytesIO() + img.save(buffered, format="PNG") + buffered.seek(0) + img_byte = buffered.getvalue() + img_str = "data:image/png;base64," + base64.b64encode(img_byte).decode() + return img_str + + +def home(request): + """ + Function to return the dashboard view. + This function calls multiple APIs that return data + * to generate a pie showing people w & w/o baskets + * to generate a plot showing how many people were in the store vs time of the day + * to generate a bar showing how many people were in each aisle + """ + now = datetime.datetime.now() + delta = (now - datetime.timedelta(hours=10)).strftime("%Y-%m-%dT%H:%M:%S") + now = now.strftime("%Y-%m-%dT%H:%M:%S") + basket_counts = get_basket_pie(client) + time_bar = get_time_plot(client) + aisle_counts = get_aisle_counts(client) + customer_count, _ = get_num_visitors_time_window(client, + start_time=delta, + end_time=now) + return render(request, "dashboard.html", context={"basket_counts":json.dumps(basket_counts), + "time_bar": json.dumps(time_bar), + "aisle_counts": aisle_counts, + "customer_count": customer_count}) + + +def num_visitors_in_region_view(request): + if request.method == "GET": + topleftx = request.GET.get("topleftx") + toplefty = request.GET.get("toplefty") + bottomrightx = request.GET.get("bottomrightx") + bottomrighty = request.GET.get("bottomrighty") + coordinates = [topleftx, toplefty, bottomrightx, bottomrighty] + if None in coordinates: + return HttpResponse("Unsupported request format. Use GET request with \ + topleftx, toplefty, bottomrightx, bottomrighty params") + + num_visitors, visitor_ids, roi = get_num_visitors_in_region(topleftx, + bottomrightx, + toplefty, + bottomrighty, client) + + print(visitor_ids) + + roi = img_to_base64_str(roi) + + # return HttpResponse(num_visitors) + return render(request, "region_view.html", context={"num_visitors":num_visitors, + "visitors":visitor_ids, + "img":roi}) + + return HttpResponse("Unsupported request format. Use GET request with \ + topleftx, toplefty, bottomrightx, bottomrighty params") + + +def visitor_path_view(request): + """ + Function to generate a visitor's path given the tracker ID + """ + if request.method == "GET": + person_id = request.GET.get("person_id") + path, img = get_visitor_path(person_id, client) + img = img_to_base64_str(img) + return render(request, "path_view.html", context={"img":img}) + + return HttpResponse("Invalid request format") + + +def num_visitors_in_time_window_view(request): + if request.method == "GET": + start_time = request.GET.get("start_time") + end_time = request.GET.get("end_time") + num_visitors, visitor_ids = get_num_visitors_time_window(client, start_time, end_time) + # return HttpResponse(num_visitors) + return render(request, "time_view.html", context={"num_visitors":num_visitors, + "visitors":visitor_ids}) + + return HttpResponse("Unsupported request format. Use GET request with at least \ + start_time and end_time") diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png new file mode 100644 index 00000000..b8c3bb6f Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/background.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py new file mode 100644 index 00000000..40b9610e --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/config.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set TEST_MODE to false to show dashboard metrics from actual data +TEST_MODE = True + +ksql_server = "http://localhost:8088" +kafka_server = "localhost:9092" + +# test_stream is a dummy stream created to read/write generated data +# test_topic is a dummy kafka topic created to read/write generated data +# This is done to separate actual data from generated data + +if TEST_MODE: + ksql_stream_name = "test_stream" + kafka_topic = "test_topic" +else: + ksql_stream_name = "detections_stream" + kafka_topic = "detections" diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml new file mode 100644 index 00000000..7120549a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/environment.yml @@ -0,0 +1,59 @@ +name: web-dev +channels: + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - ca-certificates=2022.07.19=h06a4308_0 + - certifi=2022.6.15=py38h06a4308_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.3=he6710b0_2 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - ncurses=6.3=h5eee18b_3 + - openssl=1.1.1q=h7f8727e_0 + - pip=22.1.2=py38h06a4308_0 + - python=3.8.13=h12debd9_0 + - readline=8.1.2=h7f8727e_1 + - setuptools=61.2.0=py38h06a4308_0 + - sqlite=3.39.2=h5082296_0 + - tk=8.6.12=h1ccaba5_0 + - wheel=0.37.1=pyhd3eb1b0_0 + - xz=5.2.5=h7f8727e_1 + - zlib=1.2.12=h7f8727e_2 + - pip: + - anyio==3.6.1 + - asgiref==3.5.2 + - backports-zoneinfo==0.2.1 + - charset-normalizer==2.1.0 + - confluent-kafka==1.9.2 + - cycler==0.11.0 + - django==4.1 + - fonttools==4.34.4 + - h11==0.12.0 + - h2==4.1.0 + - hpack==4.0.0 + - httpcore==0.15.0 + - httpx==0.23.0 + - hyperframe==6.0.1 + - idna==3.3 + - kiwisolver==1.4.4 + - matplotlib==3.5.2 + - numpy==1.23.1 + - opencv-python==4.6.0.66 + - packaging==21.3 + - pandas==1.4.3 + - pillow==9.2.0 + - pyksql==0.11.0 + - pyparsing==3.0.9 + - python-dateutil==2.8.2 + - pytz==2022.1 + - requests==2.28.1 + - rfc3986==1.5.0 + - six==1.16.0 + - sniffio==1.2.0 + - sqlparse==0.4.2 + - tqdm==4.64.0 + - urllib3==1.26.11 +prefix: /work/anaconda3/envs/web-dev diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py new file mode 100644 index 00000000..cf86fe2a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py new file mode 100644 index 00000000..e2a89c43 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/random_message_generator.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from random import randrange, choice +import datetime +import json +from uuid import uuid4 +from confluent_kafka import Producer +from config import kafka_topic, kafka_server +from tqdm import tqdm + +def generate_random_timestamp(start): + ts = (start - datetime.timedelta(hours=randrange(12), + minutes=randrange(60), + seconds=randrange(60))).strftime("%Y-%m-%dT%H:%M:%S.%f") + return ts[:-3] + 'Z' + + +def generate_random_bbox(max_width=2560, max_height=1440): + bottomrightx = randrange(max_width) + topleftx = randrange(bottomrightx) + bottomrighty = randrange(max_height) + toplefty = randrange(bottomrighty) + return {"topleftx": topleftx, + "toplefty": toplefty, + "bottomrightx": bottomrightx, + "bottomrighty": bottomrighty} + + +def generate_random_obj_id(max_id=10): + return str(randrange(max_id)) + + +def generate_random_basket_des(classes=["hasBasket", "noBasket"]): + return choice(classes) + + +def generate_kafka_message(): + message = {} + # Set messageid and mdsversion + message.__setitem__("messageid", str(uuid4())) + message.__setitem__("mdsversion", "1.0") + + # Set timestamp + message.__setitem__("timestamp", generate_random_timestamp(datetime.datetime.now())) + + # Create and set object + object = {"id":generate_random_obj_id(), + "speed": 0, + "direction": 0, + "orientation": 0, + "detection": "person", + "obj_prop": {"hasBasket": generate_random_basket_des(), + "confidence": 0.99}, + "bbox": generate_random_bbox()} + message.__setitem__("object", object) + + # Set event_des and videopath + message.__setitem__("event_des", {"id": str(uuid4()), "type": "entry"}) + message.__setitem__("videopath", "") + return message + + +if __name__ == "__main__": + if kafka_topic == "detections": + print("WARNING: Writing messages to main topic. Set TEST_MODE to True in config.py to write messages to dummy topic") + + # Define a producer for the topic + prd = Producer({"bootstrap.servers":kafka_server}) + + # Write messages to the topic + for _ in tqdm(range(1)): + msg = generate_kafka_message() + print(msg) + prd.produce(topic=kafka_topic, value=json.dumps(msg, indent=4)) + + # Block until messages are sent + prd.poll(10000) + prd.flush() diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt new file mode 100644 index 00000000..c4bceba5 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/requirements.txt @@ -0,0 +1,35 @@ +anyio==3.6.1 +asgiref==3.5.2 +backports.zoneinfo==0.2.1 +charset-normalizer==2.1.0 +cycler==0.11.0 +distlib==0.3.4 +Django==4.1 +filelock==3.7.0 +fonttools==4.34.4 +h11==0.12.0 +h2==4.1.0 +hpack==4.0.0 +httpcore==0.15.0 +httpx==0.23.0 +hyperframe==6.0.1 +idna==3.3 +kiwisolver==1.4.4 +matplotlib==3.5.2 +numpy==1.23.1 +opencv-python==4.6.0.66 +packaging==21.3 +pandas==1.4.3 +Pillow==9.2.0 +platformdirs==2.5.2 +pykSQL==0.11.0 +pyparsing==3.0.9 +python-dateutil==2.8.2 +pytz==2022.1 +requests==2.28.1 +rfc3986==1.5.0 +six==1.16.0 +sniffio==1.2.0 +sqlparse==0.4.2 +tqdm==4.64.0 +urllib3==1.26.11 diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py new file mode 100644 index 00000000..90d60168 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for retail_iva project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + +application = get_asgi_application() diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py new file mode 100644 index 00000000..29e92799 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/settings.py @@ -0,0 +1,124 @@ +""" +Django settings for retail_iva project. + +Generated by 'django-admin startproject' using Django 4.0.5. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.0/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-y%*c((yh(%l_$&7z+$ei^40l4l&nkuhdxfdb^=mfabl53bc&*=' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ["*"] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'analytics' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'retail_iva.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'retail_iva.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py new file mode 100644 index 00000000..3829e495 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/urls.py @@ -0,0 +1,26 @@ +"""retail_iva URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path +from analytics import views + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', views.home), + path('num-visitors-region/', views.num_visitors_in_region_view), + path('num-visitors-time/', views.num_visitors_in_time_window_view), + path('visitor-path/', views.visitor_path_view), +] diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py new file mode 100644 index 00000000..1fa1c8ce --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/retail_iva/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for retail_iva project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'retail_iva.settings') + +application = get_wsgi_application() diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini new file mode 100644 index 00000000..2e00b7ec --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/ds-retail-iva-frontend/store_config.ini @@ -0,0 +1,17 @@ +[DEFAULT] +topleftx = 0 +bottomrightx = 0 +toplefty = 0 +bottomrighty = 0 + +[Aisle1] +topleftx = 50 +bottomrightx = 1400 +toplefty = 0 +bottomrighty = 1440 + +[Aisle2] +topleftx = 1401 +bottomrightx = 2560 +toplefty = 0 +bottomrighty = 1440 \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/README.md new file mode 100644 index 00000000..10d99a1f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/README.md @@ -0,0 +1,21 @@ +# Description of files + +* [0001_compressed.h264](./0001_compressed.h264) - Input file for running the pipeline +* [basketClassifier.etlt](./basketClassifier.etlt) - Model file for classifying people with and without baskets +* [basket_classifier_labels.txt](./basket_classifier_labels.txt) - Label file for the above model + +# RECOMMENDED: Verify checksum after cloning files using git LFS + +LFS files are not cloned by default when you clone the repository + +Clone model and input files by running + +```bash +git lfs pull +``` + +Verify the checksum of files after cloning + +```bash +sha512sum -c checksum.txt +``` \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt new file mode 100644 index 00000000..c28aaeb0 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basketClassifier.etlt differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt new file mode 100644 index 00000000..897557e1 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/basket_classifier_labels.txt @@ -0,0 +1 @@ +hasBasket;noBasket \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/checksum.txt b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/checksum.txt new file mode 100644 index 00000000..d7f8d76a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/files/checksum.txt @@ -0,0 +1,2 @@ +73563263b7b11b3c3355ea4af76733f783f75abcd2936d463829fee8ece33dece1c477ddcc437e0eb499f35ecd7bfbd87f8c96a3e41d0b6940f96eabe1b30984 0001_compressed.h264 +1dc119bea04488b095eae0e2c7c963daed2504276ff489c215bb2cb60bb4e9e82a335e2f3b29c17ee3dd5a15403733fd5d5e67fd67f4e9025cc4c86c700fadea basketClassifier.etlt diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/arch.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/arch.png new file mode 100644 index 00000000..3a76aec4 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/arch.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/dashboard.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/dashboard.png new file mode 100644 index 00000000..58797965 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/dashboard.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png new file mode 100644 index 00000000..47056b8d Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/deepstream_container.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png new file mode 100644 index 00000000..679bede2 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/docker_container_ps.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/output.gif b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/output.gif new file mode 100644 index 00000000..059c9986 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/output.gif differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png new file mode 100644 index 00000000..cf848886 Binary files /dev/null and b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/media/retail-iva-arch.png differ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h new file mode 100644 index 00000000..14f99f3e --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvdsmeta_schema.h @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * NVIDIA Corporation and its licensors retain all intellectual property + * and proprietary rights in and to this software, related documentation + * and any modifications thereto. Any use, reproduction, disclosure or + * distribution of this software and related documentation without an express + * license agreement from NVIDIA Corporation is strictly prohibited. + * + */ + +/** + * @file + * NVIDIA DeepStream: Metadata Extension Structures + * + * @b Description: This file defines the NVIDIA DeepStream metadata structures + * used to describe metadata objects. + */ + +/** + * @defgroup metadata_extensions Metadata Extension Structures + * + * Defines metadata structures used to describe metadata objects. + * + * @ingroup NvDsMetaApi + * @{ + */ + +#ifndef NVDSMETA_H_ +#define NVDSMETA_H_ + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Defines event type flags. + */ +typedef enum NvDsEventType { + NVDS_EVENT_ENTRY, + NVDS_EVENT_EXIT, + NVDS_EVENT_MOVING, + NVDS_EVENT_STOPPED, + NVDS_EVENT_EMPTY, + NVDS_EVENT_PARKED, + NVDS_EVENT_RESET, + + /** Reserved for future use. Custom events must be assigned values + greater than this. */ + NVDS_EVENT_RESERVED = 0x100, + /** Specifies a custom event. */ + NVDS_EVENT_CUSTOM = 0x101, + NVDS_EVENT_FORCE32 = 0x7FFFFFFF +} NvDsEventType; + +/** + * Defines object type flags. + */ +typedef enum NvDsObjectType { + NVDS_OBJECT_TYPE_VEHICLE, + NVDS_OBJECT_TYPE_PERSON, + NVDS_OBJECT_TYPE_FACE, + NVDS_OBJECT_TYPE_BAG, + NVDS_OBJECT_TYPE_BICYCLE, + NVDS_OBJECT_TYPE_ROADSIGN, + NVDS_OBJECT_TYPE_VEHICLE_EXT, + NVDS_OBJECT_TYPE_PERSON_EXT, + NVDS_OBJECT_TYPE_FACE_EXT, + /** Reserved for future use. Custom objects must be assigned values + greater than this. */ + NVDS_OBJECT_TYPE_RESERVED = 0x100, + /** Specifies a custom object. */ + NVDS_OBJECT_TYPE_CUSTOM = 0x101, + /** "object" key will be missing in the schema */ + NVDS_OBJECT_TYPE_UNKNOWN = 0x102, + NVDS_OBEJCT_TYPE_FORCE32 = 0x7FFFFFFF +} NvDsObjectType; + +/** + * Defines payload type flags. + */ +typedef enum NvDsPayloadType { + NVDS_PAYLOAD_DEEPSTREAM, + NVDS_PAYLOAD_DEEPSTREAM_MINIMAL, + /** Reserved for future use. Custom payloads must be assigned values + greater than this. */ + NVDS_PAYLOAD_RESERVED = 0x100, + /** Specifies a custom payload. You must implement the nvds_msg2p_* + interface. */ + NVDS_PAYLOAD_CUSTOM = 0x101, + NVDS_PAYLOAD_FORCE32 = 0x7FFFFFFF +} NvDsPayloadType; + +/** + * Holds a rectangle's position and size. + */ +typedef struct NvDsRect { + float top; /**< Holds the position of rectangle's top in pixels. */ + float left; /**< Holds the position of rectangle's left side in pixels. */ + float width; /**< Holds the rectangle's width in pixels. */ + float height; /**< Holds the rectangle's height in pixels. */ +} NvDsRect; + +/** + * Holds geolocation parameters. + */ +typedef struct NvDsGeoLocation { + gdouble lat; /**< Holds the location's latitude. */ + gdouble lon; /**< Holds the location's longitude. */ + gdouble alt; /**< Holds the location's altitude. */ +} NvDsGeoLocation; + +/** + * Hold a coordinate's position. + */ +typedef struct NvDsCoordinate { + gdouble x; /**< Holds the coordinate's X position. */ + gdouble y; /**< Holds the coordinate's Y position. */ + gdouble z; /**< Holds the coordinate's Z position. */ +} NvDsCoordinate; + +/** + * Holds an object's signature. + */ +typedef struct NvDsObjectSignature { + /** Holds a pointer to an array of signature values. */ + gdouble *signature; + /** Holds the number of signature values in @a signature. */ + guint size; +} NvDsObjectSignature; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObject { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ +} NvDsVehicleObject; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + // Modified the stock NvDsPersonObject that comes with DeepStream 6.1 + gchar *hasBasket; +} NvDsPersonObject; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ +} NvDsFaceObject; + +/** + * Holds a vehicle object's parameters. + */ +typedef struct NvDsVehicleObjectExt { + gchar *type; /**< Holds a pointer to the type of the vehicle. */ + gchar *make; /**< Holds a pointer to the make of the vehicle. */ + gchar *model; /**< Holds a pointer to the model of the vehicle. */ + gchar *color; /**< Holds a pointer to the color of the vehicle. */ + gchar *region; /**< Holds a pointer to the region of the vehicle. */ + gchar *license; /**< Holds a pointer to the license number of the vehicle.*/ + + GList *mask; /**< Holds a list of polygons for vehicle mask. */ +} NvDsVehicleObjectExt; + +/** + * Holds a person object's parameters. + */ +typedef struct NvDsPersonObjectExt { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + + GList *mask; /**< Holds a list of polygons for person mask. */ +} NvDsPersonObjectExt; + +/** + * Holds a face object's parameters. + */ +typedef struct NvDsFaceObjectWithExt { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person + is wearing, if any. */ + gchar *glasses; /**< Holds a pointer to the type of glasses the person + is wearing, if any. */ + gchar *facialhair;/**< Holds a pointer to the person's facial hair color. */ + gchar *name; /**< Holds a pointer to the person's name. */ + gchar *eyecolor; /**< Holds a pointer to the person's eye color. */ + guint age; /**< Holds the person's age. */ + + GList *mask; /**< Holds a list of polygons for face mask. */ +} NvDsFaceObjectExt; + +/** + * Holds event message meta data. + * + * You can attach various types of objects (vehicle, person, face, etc.) + * to an event by setting a pointer to the object in @a extMsg. + * + * Similarly, you can attach a custom object to an event by setting a pointer to the object in @a extMsg. + * A custom object must be handled by the metadata parsing module accordingly. + */ +typedef struct NvDsEventMsgMeta { + /** Holds the event's type. */ + NvDsEventType type; + /** Holds the object's type. */ + NvDsObjectType objType; + /** Holds the object's bounding box. */ + NvDsRect bbox; + /** Holds the object's geolocation. */ + NvDsGeoLocation location; + /** Holds the object's coordinates. */ + NvDsCoordinate coordinate; + /** Holds the object's signature. */ + NvDsObjectSignature objSignature; + /** Holds the object's class ID. */ + gint objClassId; + /** Holds the ID of the sensor that generated the event. */ + gint sensorId; + /** Holds the ID of the analytics module that generated the event. */ + gint moduleId; + /** Holds the ID of the place related to the object. */ + gint placeId; + /** Holds the ID of the component (plugin) that generated this event. */ + gint componentId; + /** Holds the video frame ID of this event. */ + gint frameId; + /** Holds the confidence level of the inference. */ + gdouble confidence; + /** Holds the object's tracking ID. */ + guint64 trackingId; + /** Holds a pointer to the generated event's timestamp. */ + gchar *ts; + /** Holds a pointer to the detected or inferred object's ID. */ + gchar *objectId; + + /** Holds a pointer to a string containing the sensor's identity. */ + gchar *sensorStr; + /** Holds a pointer to a string containing other attributes associated with + the object. */ + gchar *otherAttrs; + /** Holds a pointer to the name of the video file. */ + gchar *videoPath; + /** Holds a pointer to event message meta data. This can be used to hold + data that can't be accommodated in the existing fields, or an associated + object (representing a vehicle, person, face, etc.). */ + gpointer extMsg; + /** Holds the size of the custom object at @a extMsg. */ + guint extMsgSize; +} NvDsEventMsgMeta; + +/** + * Holds event information. + */ +typedef struct _NvDsEvent { + /** Holds the type of event. */ + NvDsEventType eventType; + /** Holds a pointer to event metadata. */ + NvDsEventMsgMeta *metadata; +} NvDsEvent; + +/** + * Holds data for any user defined custom message to be attached to the payload + * message : custom message to be attached + * size : size of the custom message + */ +typedef struct _NvDsCustomMsgInfo { + void *message; + guint size; +}NvDsCustomMsgInfo; + +/** + * Holds payload metadata. + */ +typedef struct NvDsPayload { + /** Holds a pointer to the payload. */ + gpointer payload; + /** Holds the size of the payload. */ + guint payloadSize; + /** Holds the ID of the component (plugin) which attached the payload + (optional). */ + guint componentId; +} NvDsPayload; + +#ifdef __cplusplus +} +#endif +#endif /* NVDSMETA_H_ */ + +/** @} */ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile new file mode 100644 index 00000000..9d47b479 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/Makefile @@ -0,0 +1,48 @@ +############################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +CC:= g++ + +PKGS:= glib-2.0 gobject-2.0 json-glib-1.0 uuid + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I./deepstream_schema + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -lyaml-cpp + +SRCFILES:= nvmsgconv.cpp \ + deepstream_schema/eventmsg_payload_peoplenet.cpp \ + deepstream_schema/dsmeta_payload.cpp \ + deepstream_schema/deepstream_schema.cpp +TARGET_LIB:= libnvds_msgconv.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LIBS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md new file mode 100644 index 00000000..5004b786 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/README.md @@ -0,0 +1,124 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +Refer to the DeepStream SDK documentation for a description of the plugin. + +-------------------------------------------------------------------------------- +Pre-requisites: +- glib-2.0 +- json-glib-1.0 +- uuid +- yaml-cpp + +Install using: + sudo apt-get install libglib2.0-dev libjson-glib-dev uuid-dev libyaml-cpp-dev + +-------------------------------------------------------------------------------- +Compiling and installing the plugin: +Run make and sudo make install + +NOTE: To compile the sources, run make with "sudo" or root permission. + +# How to Modify NvMsgConv Plugin? + +The schema for the kafka messages sent by the DS app is defined in the [eventmsg_payload.cpp](./deepstream_schema/eventmsg_payload.cpp) file. + +In this example, we will demonstrate how to build a custom message converter library to add custom message information to the message payload. + +**Why should we modify the default library file?** + +* Notice the [`NvDsPersonObject`](https://docs.nvidia.com/metropolis/deepstream/sdk-api/structNvDsPersonObject.html) struct provided in the `nvdsmeta_schema.h` file. The struct by default has the ability to carry only information about gender, age etc. + +* The message schema for this project is different from the default schema that comes with DeepStream SDK. In order to generate this new schema, we need to modify the + +## Step 1 - Modify NvDsPersonObject + +For this use case, we would like to modify the object so that it has an additional attribute [`hasBasket`](../nvdsmeta_schema.h#L159). To achieve this we modify the existing struct in `/opt/nvidia/deepstream/deepstream/sources/includes/nvdsmeta_schema.h` as below. + +```c +typedef struct NvDsPersonObject { + gchar *gender; /**< Holds a pointer to the person's gender. */ + gchar *hair; /**< Holds a pointer to the person's hair color. */ + gchar *cap; /**< Holds a pointer to the type of cap the person is + wearing, if any. */ + gchar *apparel; /**< Holds a pointer to a description of the person's + apparel. */ + guint age; /**< Holds the person's age. */ + // Modified the stock NvDsPersonObject that comes with DeepStream 6.1 + gchar *hasBasket; +} NvDsPersonObject; +``` + +## Step 2 - Modify library file for NvMsgConv + +Original file that comes with DeepStream SDK - [eventmsg_payload.cpp](./deepstream_schema/eventmsg_payload.cpp) + +Modified file for current use case - [eventmsg_payload_peoplenet.cpp](./deepstream_schema/eventmsg_payload_peoplenet.cpp) + +It is a good exercise to compare the two files to understand how to modify the original library file. + +Sample message generated by the new message library + +```json +{ + "messageid": "50e5a5f5-0568-4d45-9b97-4bcd5ee695be", + "mdsversion": "1.0", + "timestamp": "2022-08-17T04:11:15.074Z", + "object": { + "id": "8", + "speed": 0, + "direction": 0, + "orientation": 0, + "detection": "person", + "obj_prop": { + "hasBasket": "hasBasket", + "confidence": 0.99 + }, + "bbox": { + "topleftx": 1877, + "toplefty": 52, + "bottomrightx": 2524, + "bottomrighty": 101 + } + }, + "event_des": { + "id": "87ad972c-175b-4eb9-9012-11999314a7e7", + "type": "entry" + }, + "videopath": "" +} +``` + +* The default library message converter plugin has fields for sensor, place, analytics data. Since we are not using any of those keys for the current project, we can remove the functions that generate those objects in the payload. + +* We are concerned only with `NvDsPersonObject` for this project. We can safely remove the message creation functions for all other classes. + +* You can set the newly added attribute `hasBasket` by adding the following line to [`generate_object_object`](./deepstream_schema/eventmsg_payload_peoplenet.cpp#L72) function + +```cpp +json_object_set_string_member (jobject, "hasBasket", dsObj->hasBasket); +``` + +## Step 3 - Compiling the C++ file to generate the library file + +* Modify the path in [`Makefile`](./Makefile) to reflect the path of the new C++ file + +* Run `make -B` to compile all the codes and generate the new library + +* You can find the new library file in the same directory - [libnvds_msgconv.so](./libnvds_msgconv.so) + +* Add the path for the newly compiled library in the [`msgconv`](../configs/retail_iva.yml#L49) section of the main config file \ No newline at end of file diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp new file mode 100644 index 00000000..25077fde --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.cpp @@ -0,0 +1,789 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +#include "deepstream_schema.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +static void +get_csv_tokens (const string &text, vector &tokens) +{ + /* This is based on assumption that fields and their locations + * are fixed in CSV file. This should be updated accordingly if + * that is not the case. + */ + gint count = 0; + + gchar **csv_tokens = g_strsplit (text.c_str(), ",", -1); + gchar **temp = csv_tokens; + gchar *token; + + while (*temp && count < DEFAULT_CSV_FIELDS) { + token = *temp++; + tokens.push_back (string(g_strstrip(token))); + count++; + } + g_strfreev (csv_tokens); +} + +static bool +nvds_msg2p_parse_sensor (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject sensorObj; + gint sensorId; + gchar *keyVal; + + + if (sscanf (group, CONFIG_GROUP_SENSOR "%u", &sensorId) < 1) { + cout << "Wrong sensor group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + keyVal = NULL; + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + sensorObj.id = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_TYPE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_TYPE, &error); + sensorObj.type = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_DESCRIPTION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_DESCRIPTION, &error); + sensorObj.desc = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_LOCATION)) { + gsize length; + gdouble *location = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_LOCATION, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + g_free (location); + goto done; + } + + memcpy (sensorObj.location, location, length * sizeof (gdouble)); + g_free (location); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_COORDINATE)) { + gsize length; + gdouble *coordinate = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_COORDINATE, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + g_free (coordinate); + goto done; + } + + memcpy (sensorObj.coordinate, coordinate, length * sizeof (gdouble)); + g_free (coordinate); + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + + if (keyVal) + g_free (keyVal); + } + + privObj->sensorObj.insert (make_pair (sensorId, sensorObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +static bool +nvds_msg2p_parse_place (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject placeObj; + gint placeId; + gchar *keyVal; + + if (sscanf (group, CONFIG_GROUP_PLACE "%u", &placeId) < 1) { + cout << "Wrong place group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->placeObj.find (placeId); + if (idMap != privObj->placeObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + placeObj.id = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_TYPE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_TYPE, &error); + placeObj.type = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_NAME)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_NAME, &error); + placeObj.name = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_LOCATION)) { + gsize length; + gdouble *location = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_LOCATION, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + g_free (location); + goto done; + } + + memcpy (placeObj.location, location, length * sizeof (gdouble)); + g_free (location); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_COORDINATE)) { + gsize length; + gdouble *coordinate = g_key_file_get_double_list (key_file, group, + CONFIG_KEY_COORDINATE, + &length, &error); + if (length != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + g_free (coordinate); + goto done; + } + + memcpy (placeObj.coordinate, coordinate, length * sizeof (gdouble)); + g_free (coordinate); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD1)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD1, &error); + placeObj.subObj.field1 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD2)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD2, &error); + placeObj.subObj.field2 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_PLACE_SUB_FIELD3)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_PLACE_SUB_FIELD3, &error); + placeObj.subObj.field3 = keyVal; + g_free (keyVal); + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + } + + privObj->placeObj.insert (pair (placeId, placeObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +static bool +nvds_msg2p_parse_analytics (void *privData, GKeyFile *key_file, gchar *group) +{ + bool ret = false; + bool isEnabled = false; + gchar **keys = NULL; + gchar **key = NULL; + GError *error = NULL; + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + gint moduleId; + gchar *keyVal; + + if (sscanf (group, CONFIG_GROUP_ANALYTICS "%u", &moduleId) < 1) { + cout << "Wrong analytics module group name " << group << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (moduleId); + if (idMap != privObj->analyticsObj.end()) { + cout << "Duplicate entries for " << group << endl; + return ret; + } + + isEnabled = g_key_file_get_boolean (key_file, group, CONFIG_KEY_ENABLE, + &error); + if (!isEnabled) { + // Not enabled, skip the parsing of keys. + ret = true; + goto done; + } else { + g_key_file_remove_key (key_file, group, CONFIG_KEY_ENABLE, + &error); + CHECK_ERROR (error); + } + + keys = g_key_file_get_keys (key_file, group, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + keyVal = NULL; + if (!g_strcmp0 (*key, CONFIG_KEY_ID)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_ID, &error); + analyticsObj.id = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_SOURCE)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_SOURCE, &error); + analyticsObj.source = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_DESCRIPTION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_DESCRIPTION, &error); + analyticsObj.desc = keyVal; + CHECK_ERROR (error); + } else if (!g_strcmp0 (*key, CONFIG_KEY_VERSION)) { + keyVal = g_key_file_get_string (key_file, group, + CONFIG_KEY_VERSION, &error); + analyticsObj.version = keyVal; + CHECK_ERROR (error); + } else { + cout << "Unknown key " << *key << " for group [" << group <<"]\n"; + } + + if (keyVal) + g_free (keyVal); + } + + privObj->analyticsObj.insert (make_pair (moduleId, analyticsObj)); + + ret = true; + +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + + return ret; +} + +/* Separate a config file entry with delimiters + * into strings. */ +static std::vector +split_string (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ';') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +static bool +nvds_msg2p_parse_sensor_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject sensorObj; + gint sensorId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_SENSOR "%u", &sensorId) < 1) { + cout << "Wrong sensor group format " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + sensorObj.id = itr->second.as(); + } else if (paramKey == "type") { + sensorObj.type = itr->second.as(); + } else if (paramKey == "description") { + sensorObj.desc = itr->second.as(); + } else if (paramKey == "location") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + sensorObj.location[i] = std::stod(vec[i]); + } + } else if (paramKey == "coordinate") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + sensorObj.coordinate[i] = std::stod(vec[i]); + } + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str << "]\n"; + } + } + privObj->sensorObj.insert (make_pair (sensorId, sensorObj)); + ret = true; + +done: + return ret; +} + +static bool +nvds_msg2p_parse_place_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject placeObj; + gint placeId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_PLACE "%u", &placeId) < 1) { + cout << "Wrong place group name " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->placeObj.find (placeId); + if (idMap != privObj->placeObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + placeObj.id = itr->second.as(); + } else if (paramKey == "type") { + placeObj.type = itr->second.as(); + } else if (paramKey == "name") { + placeObj.name = itr->second.as(); + } else if (paramKey == "location") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like lat;lon;alt" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + placeObj.location[i] = std::stod(vec[i]); + } + } else if (paramKey == "coordinate") { + std::string str = itr->second.as(); + std::vector vec = split_string (str); + if (vec.size() != 3) { + cout << "Wrong values provided, it should be like x;y;z" << endl; + goto done; + } + for(int i = 0; i < 3; i++) { + placeObj.coordinate[i] = std::stod(vec[i]); + } + } else if (paramKey == "place-sub-field1") { + placeObj.subObj.field1 = itr->second.as(); + } else if (paramKey == "place-sub-field2") { + placeObj.subObj.field2 = itr->second.as(); + } else if (paramKey == "place-sub-field3") { + placeObj.subObj.field3 = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str <<"]\n"; + } + } + privObj->placeObj.insert (pair (placeId, placeObj)); + ret = true; + +done: + return ret; +} + +static bool +nvds_msg2p_parse_analytics_yaml (void *privData, gchar *cfg_file_path, std::string group_str) +{ + bool ret = false; + bool isEnabled = false; + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + gint moduleId; + + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + if (sscanf (group_str.c_str(), CONFIG_GROUP_ANALYTICS "%u", &moduleId) < 1) { + cout << "Wrong analytics module group name " << group_str << endl; + return ret; + } + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (moduleId); + if (idMap != privObj->analyticsObj.end()) { + cout << "Duplicate entries for " << group_str << endl; + return ret; + } + + if (configyml[group_str]["enable"]) { + isEnabled = configyml[group_str]["enable"].as(); + if(isEnabled == FALSE) { + ret = true; + goto done; + } + } else { + ret = true; + goto done; + } + + for(YAML::const_iterator itr = configyml[group_str].begin(); + itr != configyml[group_str].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enable") { + continue; + } else if (paramKey == "id") { + analyticsObj.id = itr->second.as(); + } else if (paramKey == "source") { + analyticsObj.source = itr->second.as(); + } else if (paramKey == "description") { + analyticsObj.desc = itr->second.as(); + } else if (paramKey == "version") { + analyticsObj.version = itr->second.as(); + } else { + cout << "Unknown key " << paramKey << " for group [" << group_str <<"]\n"; + } + } + privObj->analyticsObj.insert (make_pair (moduleId, analyticsObj)); + ret = true; + +done: + return ret; +} + +bool nvds_msg2p_parse_csv (void *privData, const gchar *file) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject analyticsObj; + NvDsSensorObject sensorObj; + NvDsPlaceObject placeObj; + bool retVal = true; + bool firstRow = true; + string line; + gint i, index = 0; + + ifstream inputFile (file); + if (!inputFile.is_open()) { + cout << "Couldn't open CSV file " << file << endl; + return false; + } + + privObj = (NvDsPayloadPriv *) privData; + + try { + + while (getline (inputFile, line)) { + + if (firstRow) { + // Discard first row as it will have header fields. + firstRow = false; + continue; + } + + vector tokens; + get_csv_tokens (line, tokens); + // Ignore first cameraId field. + i = 1; + + // sensor object fields + sensorObj.id = tokens.at(i++); + sensorObj.type = "Camera"; + sensorObj.desc = tokens.at(i++); + + //Hard coded values but can be read from CSV file. + sensorObj.location[0] = 0; //atof (tokens.at(i++).c_str ()); + sensorObj.location[1] = 0; + sensorObj.location[2] = 0; + sensorObj.coordinate[0] = 0; + sensorObj.coordinate[1] = 0; + sensorObj.coordinate[2] = 0; + + // place object fields + placeObj.id = "Id"; + placeObj.type = "building/garage"; + placeObj.name = "endeavor"; + placeObj.location[0] = 0; + placeObj.location[1] = 0; + placeObj.location[2] = 0; + placeObj.coordinate[0] = 0; + placeObj.coordinate[1] = 0; + placeObj.coordinate[2] = 0; + //Ignore cameraIDstring + i++; + placeObj.subObj.field1 = tokens.at(i++); + placeObj.subObj.field2 = tokens.at(i++); + placeObj.subObj.field3 = tokens.at(i++); + + // analytics object fields + // hard coded values but can be read from CSV file. + analyticsObj.id = ""; + analyticsObj.source = ""; + analyticsObj.desc = ""; + analyticsObj.version = "1.0"; + + privObj->sensorObj.insert (make_pair (index, sensorObj)); + privObj->placeObj.insert (make_pair (index, placeObj)); + privObj->analyticsObj.insert (make_pair (index, analyticsObj)); + + index++; + } + } catch (const std::out_of_range& oor) { + std::cerr << "Out of Range error: " << oor.what() << '\n'; + retVal = false; + } + + inputFile.close (); + return retVal; +} + +bool nvds_msg2p_parse_yaml (void *privData, const gchar *file) +{ + bool retVal = true; + YAML::Node configyml = YAML::LoadFile(file); + std::string sensor_str = "sensor"; + std::string place_str = "place"; + std::string analytics_str = "analytics"; + gchar *cfg_file = (gchar *) malloc(sizeof(gchar *)); + cfg_file = (gchar *) file; + + for(YAML::const_iterator itr = configyml.begin(); itr != configyml.end(); ++itr) + { + std::string paramKey = itr->first.as(); + + if (paramKey.compare(0, sensor_str.size(), sensor_str) == 0) { + retVal = nvds_msg2p_parse_sensor_yaml (privData, cfg_file, paramKey); + } else if (paramKey.compare(0, place_str.size(), place_str) == 0) { + retVal = nvds_msg2p_parse_place_yaml (privData, cfg_file, paramKey); + } else if (paramKey.compare(0, analytics_str.size(), analytics_str) == 0) { + retVal = nvds_msg2p_parse_analytics_yaml (privData, cfg_file, paramKey); + } else { + cout << "Unknown group " << paramKey << endl; + } + + if (!retVal) { + cout << "Failed to parse group " << paramKey << endl; + goto done; + } + } + +done: + return retVal; +} + +bool nvds_msg2p_parse_key_value (void *privData, const gchar *file) +{ + bool retVal = true; + GKeyFile *cfgFile = NULL; + GError *error = NULL; + gchar **groups = NULL; + gchar **group; + + cfgFile = g_key_file_new (); + if (!g_key_file_load_from_file (cfgFile, file, G_KEY_FILE_NONE, &error)) { + g_message ("Failed to load file: %s", error->message); + retVal = false; + goto done; + } + + groups = g_key_file_get_groups (cfgFile, NULL); + + for (group = groups; *group; group++) { + if (!strncmp (*group, CONFIG_GROUP_SENSOR, strlen (CONFIG_GROUP_SENSOR))) { + retVal = nvds_msg2p_parse_sensor (privData, cfgFile, *group); + } else if (!strncmp (*group, CONFIG_GROUP_PLACE, strlen (CONFIG_GROUP_PLACE))) { + retVal = nvds_msg2p_parse_place (privData, cfgFile, *group); + } else if (!strncmp (*group, CONFIG_GROUP_ANALYTICS, strlen (CONFIG_GROUP_ANALYTICS))) { + retVal = nvds_msg2p_parse_analytics (privData, cfgFile, *group); + } else { + cout << "Unknown group " << *group << endl; + } + + if (!retVal) { + cout << "Failed to parse group " << *group << endl; + goto done; + } + } + +done: + if (groups) + g_strfreev (groups); + + if (cfgFile) + g_key_file_free (cfgFile); + + return retVal; +} + +void *create_deepstream_schema_ctx() { + return (void *) new NvDsPayloadPriv; +} + +void destroy_deepstream_schema_ctx(void *ptr) { + NvDsPayloadPriv *privObj = (NvDsPayloadPriv *) ptr; + delete privObj; +} diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h new file mode 100644 index 00000000..e537a246 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/deepstream_schema.h @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @file + * NVIDIA DeepStream: Message Schema payload Generation + * + * @b Description: This file specifies the functions used to generate payload + * based on NVIDIA Deepstream message schema either using eventMsg metadata + * or the NvDSFrame(obj) metadata + */ + +#ifndef NVEVENTMSGCONV_H_ +#define NVEVENTMSGCONV_H_ + +#include "nvdsmeta.h" +#include "nvdsmeta_schema.h" +#include +#include +#include + +using namespace std; + +#define CONFIG_GROUP_SENSOR "sensor" +#define CONFIG_GROUP_PLACE "place" +#define CONFIG_GROUP_ANALYTICS "analytics" + +#define CONFIG_KEY_COORDINATE "coordinate" +#define CONFIG_KEY_DESCRIPTION "description" +#define CONFIG_KEY_ENABLE "enable" +#define CONFIG_KEY_ID "id" +#define CONFIG_KEY_LANE "lane" +#define CONFIG_KEY_LEVEL "level" +#define CONFIG_KEY_LOCATION "location" +#define CONFIG_KEY_NAME "name" +#define CONFIG_KEY_SOURCE "source" +#define CONFIG_KEY_TYPE "type" +#define CONFIG_KEY_VERSION "version" + + +#define CONFIG_KEY_PLACE_SUB_FIELD1 "place-sub-field1" +#define CONFIG_KEY_PLACE_SUB_FIELD2 "place-sub-field2" +#define CONFIG_KEY_PLACE_SUB_FIELD3 "place-sub-field3" + +#define DEFAULT_CSV_FIELDS 10 + + +#define CHECK_ERROR(error) \ + if (error) { \ + cout << "Error: " << error->message << endl; \ + goto done; \ + } + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Store data parsed from the config file in these structures + */ +struct NvDsPlaceSubObject { + string field1; + string field2; + string field3; +}; + +struct NvDsSensorObject { + string id; + string type; + string desc; + gdouble location[3]; + gdouble coordinate[3]; +}; + +struct NvDsPlaceObject { + string id; + string name; + string type; + gdouble location[3]; + gdouble coordinate[3]; + NvDsPlaceSubObject subObj; +}; + +struct NvDsAnalyticsObject { + string id; + string desc; + string source; + string version; +}; + +struct NvDsPayloadPriv { + unordered_map sensorObj; + unordered_map placeObj; + unordered_map analyticsObj; +}; + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta); +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size); +gchar* generate_dsmeta_message (void *privData, void *frameMeta, void *objMeta); +gchar* generate_dsmeta_message_minimal (void *privData, void *frameMeta); +void *create_deepstream_schema_ctx(); +void destroy_deepstream_schema_ctx(void *privData); +bool nvds_msg2p_parse_key_value (void *privData, const gchar *file); +bool nvds_msg2p_parse_csv (void *privData, const gchar *file); +bool nvds_msg2p_parse_yaml (void *privData, const gchar *file); + +#ifdef __cplusplus +} +#endif +#endif /* NVEVENTMSGCONV_H_ */ + diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp new file mode 100644 index 00000000..78a69f7a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/dsmeta_payload.cpp @@ -0,0 +1,502 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +using namespace std; + +#define MAX_TIME_STAMP_LEN (64) + +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +static JsonObject* generate_place_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (frame_meta->source_id); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // place sub object (user to provide the name for sub place ex: parkingSpot/aisle/entrance..etc + jobject = json_object_new (); + + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "place-sub-field", jobject); + + // coordinates for place sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* generate_sensor_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (frame_meta->source_id); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* generate_analytics_module_object (void *privData, NvDsFrameMeta *frame_meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (frame_meta->source_id); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << frame_meta->source_id + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + gchar tracking_id[64]; + //GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", obj_meta->object_id) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + jobject = json_object_new (); + json_object_set_double_member (jobject, "confidence", obj_meta->confidence); + + //Fetch object classifiers detected + for(NvDsClassifierMetaList *cl = obj_meta->classifier_meta_list; cl ; cl=cl->next) { + NvDsClassifierMeta *cl_meta = (NvDsClassifierMeta*) cl->data; + + for(NvDsLabelInfoList *ll = cl_meta->label_info_list; ll ; ll=ll->next) { + NvDsLabelInfo *ll_meta = (NvDsLabelInfo*) ll->data; + if(cl_meta->classifier_type != NULL && strcmp("", cl_meta->classifier_type)) + json_object_set_string_member (jobject, cl_meta->classifier_type, ll_meta->result_label); + } + } + json_object_set_object_member (objectObj, obj_meta->obj_label , jobject); + + // bbox sub object + float scaleW = (float) frame_meta->source_frame_width / + (frame_meta->pipeline_width == 0) ? 1:frame_meta->pipeline_width; + float scaleH = (float) frame_meta->source_frame_height / + (frame_meta->pipeline_height == 0) ? 1:frame_meta->pipeline_height; + + float left = obj_meta->rect_params.left * scaleW; + float top = obj_meta->rect_params.top * scaleH; + float width = obj_meta->rect_params.width * scaleW; + float height = obj_meta->rect_params.height * scaleH; + + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", left); + json_object_set_int_member (jobject, "toplefty", top); + json_object_set_int_member (jobject, "bottomrightx", left + width); + json_object_set_int_member (jobject, "bottomrighty", top + height); + json_object_set_object_member (objectObj, "bbox", jobject); + + // location sub object + jobject = json_object_new (); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +static JsonObject* generate_event_object (NvDsObjectMeta *obj_meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + json_object_set_string_member (eventObj, "type", ""); + return eventObj; +} + +gchar* generate_dsmeta_message (void *privData, void *frameMeta, void *objMeta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)objMeta; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, frame_meta); + + // sensor object + sensorObj = generate_sensor_object (privData, frame_meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, frame_meta); + + // object object + objectObj = generate_object_object (privData, frame_meta, obj_meta); + // event object + eventObj = generate_event_object (obj_meta); + + char ts[MAX_TIME_STAMP_LEN + 1]; + generate_ts_rfc3339 (ts, MAX_TIME_STAMP_LEN); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + json_object_set_string_member (rootObj, "videoPath", ""); + + //Search for any custom message blob within frame usermeta list + JsonArray *jArray = json_array_new (); + for (NvDsUserMetaList *l = frame_meta->frame_user_meta_list; l; l = l->next) { + NvDsUserMeta *frame_usermeta = (NvDsUserMeta *) l->data; + if(frame_usermeta && frame_usermeta->base_meta.meta_type == NVDS_CUSTOM_MSG_BLOB) { + NvDsCustomMsgInfo *custom_blob = (NvDsCustomMsgInfo *) frame_usermeta->user_meta_data; + string msg = string((const char *) custom_blob->message, custom_blob->size); + json_array_add_string_element (jArray, msg.c_str()); + } + } + if(json_array_get_length(jArray) > 0) + json_object_set_array_member (rootObj, "customMessage", jArray); + else + json_array_unref(jArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; + +} + +gchar* generate_dsmeta_message_minimal (void *privData, void *frameMeta) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) frameMeta; + for (NvDsObjectMetaList *obj_l = frame_meta->obj_meta_list; obj_l; obj_l = obj_l->next) { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) obj_l->data; + if (obj_meta == NULL) { + // Ignore Null object. + continue; + } + + // bbox sub object + float scaleW = (float) frame_meta->source_frame_width / + (frame_meta->pipeline_width == 0) ? 1:frame_meta->pipeline_width; + float scaleH = (float) frame_meta->source_frame_height / + (frame_meta->pipeline_height == 0) ? 1:frame_meta->pipeline_height; + + float left = obj_meta->rect_params.left * scaleW; + float top = obj_meta->rect_params.top * scaleH; + float width = obj_meta->rect_params.width * scaleW; + float height = obj_meta->rect_params.height * scaleH; + + ss.str(""); + ss.clear(); + ss << obj_meta->object_id << "|" << left << "|" << top + << "|" << left + width << "|" << top + height + << "|" << obj_meta->obj_label; + + if(g_list_length(obj_meta->classifier_meta_list) > 0) { + ss << "|#"; + //Add classifiers for the object, if any + for(NvDsClassifierMetaList *cl = obj_meta->classifier_meta_list; cl ; cl=cl->next) { + NvDsClassifierMeta *cl_meta = (NvDsClassifierMeta*) cl->data; + for(NvDsLabelInfoList *ll = cl_meta->label_info_list; ll ; ll=ll->next) { + NvDsLabelInfo *ll_meta = (NvDsLabelInfo*) ll->data; + ss<< "|" << ll_meta->result_label; + } + } + ss << "|" << obj_meta->confidence; + } + json_array_add_string_element (jArray, ss.str().c_str()); + } + + //generate timestamp + char ts[MAX_TIME_STAMP_LEN + 1]; + generate_ts_rfc3339 (ts, MAX_TIME_STAMP_LEN); + + //fetch sensor id + string sensorId="0"; + NvDsPayloadPriv *privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (frame_meta->source_id); + if (idMap != privObj->sensorObj.end()) { + NvDsSensorObject &obj = privObj->sensorObj[frame_meta->source_id]; + sensorId = obj.id; + } + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(frame_meta->frame_num).c_str()); + json_object_set_string_member (jobject, "@timestamp", ts); + json_object_set_string_member (jobject, "sensorId", sensorId.c_str()); + + json_object_set_array_member (jobject, "objects", jArray); + + JsonArray *custMsgjArray = json_array_new (); + //Search for any custom message blob within frame usermeta list + for (NvDsUserMetaList *l = frame_meta->frame_user_meta_list; l; l = l->next) { + NvDsUserMeta *frame_usermeta = (NvDsUserMeta *) l->data; + if(frame_usermeta && frame_usermeta->base_meta.meta_type == NVDS_CUSTOM_MSG_BLOB) { + NvDsCustomMsgInfo *custom_blob = (NvDsCustomMsgInfo *) frame_usermeta->user_meta_data; + string msg = string((const char *) custom_blob->message, custom_blob->size); + json_array_add_string_element (custMsgjArray, msg.c_str()); + } + } + if(json_array_get_length(custMsgjArray) > 0) + json_object_set_array_member (jobject, "customMessage", custMsgjArray); + else + json_array_unref(custMsgjArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp new file mode 100644 index 00000000..a218ee1f --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload.cpp @@ -0,0 +1,839 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + + +static JsonObject* +generate_place_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsPlaceObject *dsPlaceObj = NULL; + JsonObject *placeObj; + JsonObject *jobject; + JsonObject *jobject2; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->placeObj.find (meta->placeId); + + if (idMap != privObj->placeObj.end()) { + dsPlaceObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_PLACE << meta->placeId + << " in configuration file" << endl; + return NULL; + } + + /* place object + * "place": + { + "id": "string", + "name": "endeavor", + “type”: “garage”, + "location": { + "lat": 30.333, + "lon": -40.555, + "alt": 100.00 + }, + "entrance/aisle": { + "name": "walsh", + "lane": "lane1", + "level": "P2", + "coordinate": { + "x": 1.0, + "y": 2.0, + "z": 3.0 + } + } + } + */ + + placeObj = json_object_new (); + json_object_set_string_member (placeObj, "id", dsPlaceObj->id.c_str()); + json_object_set_string_member (placeObj, "name", dsPlaceObj->name.c_str()); + json_object_set_string_member (placeObj, "type", dsPlaceObj->type.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsPlaceObj->location[0]); + json_object_set_double_member (jobject, "lon", dsPlaceObj->location[1]); + json_object_set_double_member (jobject, "alt", dsPlaceObj->location[2]); + json_object_set_object_member (placeObj, "location", jobject); + + // parkingSpot / aisle /entrance sub object + jobject = json_object_new (); + + switch (meta->type) { + case NVDS_EVENT_MOVING: + case NVDS_EVENT_STOPPED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + break; + case NVDS_EVENT_EMPTY: + case NVDS_EVENT_PARKED: + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "type", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "parkingSpot", jobject); + break; + case NVDS_EVENT_ENTRY: + case NVDS_EVENT_EXIT: + if (meta->objType == NVDS_OBJECT_TYPE_VEHICLE) { + json_object_set_string_member (jobject, "id", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "aisle", jobject); + } else { + json_object_set_string_member (jobject, "name", dsPlaceObj->subObj.field1.c_str()); + json_object_set_string_member (jobject, "lane", dsPlaceObj->subObj.field2.c_str()); + json_object_set_string_member (jobject, "level", dsPlaceObj->subObj.field3.c_str()); + json_object_set_object_member (placeObj, "entrance", jobject); + } + break; + default: + cout << "Event type not implemented " << endl; + break; + } + + // coordinate sub sub object + jobject2 = json_object_new (); + json_object_set_double_member (jobject2, "x", dsPlaceObj->coordinate[0]); + json_object_set_double_member (jobject2, "y", dsPlaceObj->coordinate[1]); + json_object_set_double_member (jobject2, "z", dsPlaceObj->coordinate[2]); + json_object_set_object_member (jobject, "coordinate", jobject2); + + return placeObj; +} + +static JsonObject* +generate_sensor_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsSensorObj = NULL; + JsonObject *sensorObj; + JsonObject *jobject; + + privObj = (NvDsPayloadPriv *) privData; + auto idMap = privObj->sensorObj.find (meta->sensorId); + + if (idMap != privObj->sensorObj.end()) { + dsSensorObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << meta->sensorId + << " in configuration file" << endl; + return NULL; + } + + /* sensor object + * "sensor": { + "id": "string", + "type": "Camera/Puck", + "location": { + "lat": 45.99, + "lon": 35.54, + "alt": 79.03 + }, + "coordinate": { + "x": 5.2, + "y": 10.1, + "z": 11.2 + }, + "description": "Entrance of Endeavor Garage Right Lane" + } + */ + + // sensor object + sensorObj = json_object_new (); + json_object_set_string_member (sensorObj, "id", dsSensorObj->id.c_str()); + json_object_set_string_member (sensorObj, "type", dsSensorObj->type.c_str()); + json_object_set_string_member (sensorObj, "description", dsSensorObj->desc.c_str()); + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", dsSensorObj->location[0]); + json_object_set_double_member (jobject, "lon", dsSensorObj->location[1]); + json_object_set_double_member (jobject, "alt", dsSensorObj->location[2]); + json_object_set_object_member (sensorObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", dsSensorObj->coordinate[0]); + json_object_set_double_member (jobject, "y", dsSensorObj->coordinate[1]); + json_object_set_double_member (jobject, "z", dsSensorObj->coordinate[2]); + json_object_set_object_member (sensorObj, "coordinate", jobject); + + return sensorObj; +} + +static JsonObject* +generate_analytics_module_object (void *privData, NvDsEventMsgMeta *meta) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsAnalyticsObject *dsObj = NULL; + JsonObject *analyticsObj; + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->analyticsObj.find (meta->moduleId); + + if (idMap != privObj->analyticsObj.end()) { + dsObj = &idMap->second; + } else { + cout << "No entry for " CONFIG_GROUP_ANALYTICS << meta->moduleId + << " in configuration file" << endl; + return NULL; + } + + /* analytics object + * "analyticsModule": { + "id": "string", + "description": "Vehicle Detection and License Plate Recognition", + "confidence": 97.79, + "source": "OpenALR", + "version": "string" + } + */ + + // analytics object + analyticsObj = json_object_new (); + json_object_set_string_member (analyticsObj, "id", dsObj->id.c_str()); + json_object_set_string_member (analyticsObj, "description", dsObj->desc.c_str()); + json_object_set_string_member (analyticsObj, "source", dsObj->source.c_str()); + json_object_set_string_member (analyticsObj, "version", dsObj->version.c_str()); + + return analyticsObj; +} + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: + // vehicle sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "type", dsObj->type); + json_object_set_string_member (jobject, "make", dsObj->make); + json_object_set_string_member (jobject, "model", dsObj->model); + json_object_set_string_member (jobject, "color", dsObj->color); + json_object_set_string_member (jobject, "licenseState", dsObj->region); + json_object_set_string_member (jobject, "license", dsObj->license); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No vehicle object in meta data. Attach empty vehicle sub object. + json_object_set_string_member (jobject, "type", ""); + json_object_set_string_member (jobject, "make", ""); + json_object_set_string_member (jobject, "model", ""); + json_object_set_string_member (jobject, "color", ""); + json_object_set_string_member (jobject, "licenseState", ""); + json_object_set_string_member (jobject, "license", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "vehicle", jobject); + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "apparel", dsObj->apparel); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "apparel", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "person", jobject); + break; + case NVDS_OBJECT_TYPE_FACE_EXT: + // face sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + json_object_set_int_member (jobject, "age", dsObj->age); + json_object_set_string_member (jobject, "gender", dsObj->gender); + json_object_set_string_member (jobject, "hair", dsObj->hair); + json_object_set_string_member (jobject, "cap", dsObj->cap); + json_object_set_string_member (jobject, "glasses", dsObj->glasses); + json_object_set_string_member (jobject, "facialhair", dsObj->facialhair); + json_object_set_string_member (jobject, "name", dsObj->name); + json_object_set_string_member (jobject, "eyecolor", dsObj->eyecolor); + json_object_set_double_member (jobject, "confidence", meta->confidence); + + objectMask = dsObj->mask; + } + } else { + // No face object in meta data. Attach empty face sub object. + json_object_set_int_member (jobject, "age", 0); + json_object_set_string_member (jobject, "gender", ""); + json_object_set_string_member (jobject, "hair", ""); + json_object_set_string_member (jobject, "cap", ""); + json_object_set_string_member (jobject, "glasses", ""); + json_object_set_string_member (jobject, "facialhair", ""); + json_object_set_string_member (jobject, "name", ""); + json_object_set_string_member (jobject, "eyecolor", ""); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_object_member (objectObj, "face", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + default: + cout << "Object type not implemented" << endl; + } + + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + if (objectMask) { + GList *l; + JsonArray *maskArray = json_array_sized_new (g_list_length(objectMask)); + + for (l = objectMask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + JsonArray *polygonArray = json_array_sized_new (polygon->len); + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + + json_array_add_double_element (polygonArray, value); + } + + json_array_add_array_element (maskArray, polygonArray); + } + + json_object_set_array_member (objectObj, "maskoutline", maskArray); + } + + // signature sub array + if (meta->objSignature.size) { + JsonArray *jArray = json_array_sized_new (meta->objSignature.size); + + for (i = 0; i < meta->objSignature.size; i++) { + json_array_add_double_element (jArray, meta->objSignature.signature[i]); + } + json_object_set_array_member (objectObj, "signature", jArray); + } + + // location sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "lat", meta->location.lat); + json_object_set_double_member (jobject, "lon", meta->location.lon); + json_object_set_double_member (jobject, "alt", meta->location.alt); + json_object_set_object_member (objectObj, "location", jobject); + + // coordinate sub object + jobject = json_object_new (); + json_object_set_double_member (jobject, "x", meta->coordinate.x); + json_object_set_double_member (jobject, "y", meta->coordinate.y); + json_object_set_double_member (jobject, "z", meta->coordinate.z); + json_object_set_object_member (objectObj, "coordinate", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *placeObj; + JsonObject *sensorObj; + JsonObject *analyticsObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // place object + placeObj = generate_place_object (privData, meta); + + // sensor object + sensorObj = generate_sensor_object (privData, meta); + + // analytics object + analyticsObj = generate_analytics_module_object (privData, meta); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "@timestamp", meta->ts); + json_object_set_object_member (rootObj, "place", placeObj); + json_object_set_object_member (rootObj, "sensor", sensorObj); + json_object_set_object_member (rootObj, "analyticsModule", analyticsObj); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(events[0].metadata->frameId).c_str()); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp new file mode 100644 index 00000000..8f948e06 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/deepstream_schema/eventmsg_payload_peoplenet.cpp @@ -0,0 +1,423 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "deepstream_schema.h" + +static JsonObject* +generate_event_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *eventObj; + uuid_t uuid; + gchar uuidStr[37]; + + /* + * "event": { + "id": "event-id", + "type": "entry / exit" + } + */ + + uuid_generate_random (uuid); + uuid_unparse_lower(uuid, uuidStr); + + eventObj = json_object_new (); + json_object_set_string_member (eventObj, "id", uuidStr); + + switch (meta->type) { + case NVDS_EVENT_ENTRY: + json_object_set_string_member (eventObj, "type", "entry"); + break; + case NVDS_EVENT_EXIT: + json_object_set_string_member (eventObj, "type", "exit"); + break; + case NVDS_EVENT_MOVING: + json_object_set_string_member (eventObj, "type", "moving"); + break; + case NVDS_EVENT_STOPPED: + json_object_set_string_member (eventObj, "type", "stopped"); + break; + case NVDS_EVENT_PARKED: + json_object_set_string_member (eventObj, "type", "parked"); + break; + case NVDS_EVENT_EMPTY: + json_object_set_string_member (eventObj, "type", "empty"); + break; + case NVDS_EVENT_RESET: + json_object_set_string_member (eventObj, "type", "reset"); + break; + default: + cout << "Unknown event type " << endl; + break; + } + + return eventObj; +} + +static JsonObject* +generate_object_object (void *privData, NvDsEventMsgMeta *meta) +{ + JsonObject *objectObj; + JsonObject *jobject; + guint i; + gchar tracking_id[64]; + GList *objectMask = NULL; + + // object object + objectObj = json_object_new (); + if (snprintf (tracking_id, sizeof(tracking_id), "%lu", meta->trackingId) + >= (int) sizeof(tracking_id)) + g_warning("Not enough space to copy trackingId"); + json_object_set_string_member (objectObj, "id", tracking_id); + json_object_set_double_member (objectObj, "speed", 0); + json_object_set_double_member (objectObj, "direction", 0); + json_object_set_double_member (objectObj, "orientation", 0); + + switch (meta->objType) { + case NVDS_OBJECT_TYPE_PERSON: + // person sub object + jobject = json_object_new (); + + if (meta->extMsgSize) { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + json_object_set_string_member (jobject, "hasBasket", dsObj->hasBasket); + json_object_set_double_member (jobject, "confidence", meta->confidence); + } + } else { + // No person object in meta data. Attach empty person sub object. + json_object_set_string_member (jobject, "hasBasket", "NoBasket"); + json_object_set_double_member (jobject, "confidence", 1.0); + } + json_object_set_string_member (objectObj, "detection", "person"); + json_object_set_object_member (objectObj, "obj_prop", jobject); + break; + case NVDS_OBJECT_TYPE_UNKNOWN: + if(!meta->objectId) { + break; + } + /** No information to add; object type unknown within NvDsEventMsgMeta */ + jobject = json_object_new (); + json_object_set_object_member (objectObj, meta->objectId, jobject); + break; + default: + cout << "Object type not implemented" << endl; + } + + // bbox sub object + jobject = json_object_new (); + json_object_set_int_member (jobject, "topleftx", meta->bbox.left); + json_object_set_int_member (jobject, "toplefty", meta->bbox.top); + json_object_set_int_member (jobject, "bottomrightx", meta->bbox.left + meta->bbox.width); + json_object_set_int_member (jobject, "bottomrighty", meta->bbox.top + meta->bbox.height); + json_object_set_object_member (objectObj, "bbox", jobject); + + return objectObj; +} + +gchar* generate_event_message (void *privData, NvDsEventMsgMeta *meta) +{ + JsonNode *rootNode; + JsonObject *rootObj; + JsonObject *eventObj; + JsonObject *objectObj; + gchar *message; + + uuid_t msgId; + gchar msgIdStr[37]; + + uuid_generate_random (msgId); + uuid_unparse_lower(msgId, msgIdStr); + + // object object + objectObj = generate_object_object (privData, meta); + + // event object + eventObj = generate_event_object (privData, meta); + + // root object + rootObj = json_object_new (); + json_object_set_string_member (rootObj, "messageid", msgIdStr); + json_object_set_string_member (rootObj, "mdsversion", "1.0"); + json_object_set_string_member (rootObj, "timestamp", meta->ts); + json_object_set_object_member (rootObj, "object", objectObj); + json_object_set_object_member (rootObj, "event_des", eventObj); + + if (meta->videoPath) + json_object_set_string_member (rootObj, "videoPath", meta->videoPath); + else + json_object_set_string_member (rootObj, "videoPath", ""); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, rootObj); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (rootObj); + + return message; +} + +static const gchar* +object_enum_to_str (NvDsObjectType type, gchar* objectId) +{ + switch (type) { + case NVDS_OBJECT_TYPE_VEHICLE: + return "Vehicle"; + case NVDS_OBJECT_TYPE_FACE: + return "Face"; + case NVDS_OBJECT_TYPE_PERSON: + return "Person"; + case NVDS_OBJECT_TYPE_BAG: + return "Bag"; + case NVDS_OBJECT_TYPE_BICYCLE: + return "Bicycle"; + case NVDS_OBJECT_TYPE_ROADSIGN: + return "RoadSign"; + case NVDS_OBJECT_TYPE_CUSTOM: + return "Custom"; + case NVDS_OBJECT_TYPE_UNKNOWN: + return objectId ? objectId : "Unknown"; + default: + return "Unknown"; + } +} + +static const gchar* +to_str (gchar* cstr) +{ + return reinterpret_cast(cstr) ? cstr : ""; +} + +static const gchar * +sensor_id_to_str (void *privData, gint sensorId) +{ + NvDsPayloadPriv *privObj = NULL; + NvDsSensorObject *dsObj = NULL; + + g_return_val_if_fail (privData, NULL); + + privObj = (NvDsPayloadPriv *) privData; + + auto idMap = privObj->sensorObj.find (sensorId); + if (idMap != privObj->sensorObj.end()) { + dsObj = &idMap->second; + return dsObj->id.c_str(); + } else { + cout << "No entry for " CONFIG_GROUP_SENSOR << sensorId + << " in configuration file" << endl; + return NULL; + } +} + +static void +generate_mask_array (NvDsEventMsgMeta *meta, JsonArray *jArray, GList *mask) +{ + unsigned int i; + GList *l; + stringstream ss; + bool started = false; + + ss << meta->trackingId << "|" << g_list_length(mask); + + for (l = mask; l != NULL; l = l->next) { + GArray *polygon = (GArray *) l->data; + + if (started) + ss << "|#"; + + started = true; + + for (i = 0; i < polygon->len; i++) { + gdouble value = g_array_index (polygon, gdouble, i); + ss << "|" << value; + } + } + json_array_add_string_element (jArray, ss.str().c_str()); +} + +gchar* generate_event_message_minimal (void *privData, NvDsEvent *events, guint size) +{ + /* + The JSON structure of the frame + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + ".......object-1 attributes...........", + ".......object-2 attributes...........", + ".......object-3 attributes..........." + ] + } + */ + + /* + An example object with Vehicle object-type + { + "version": "4.0", + "id": "frame-id", + "@timestamp": "2018-04-11T04:59:59.828Z", + "sensorId": "sensor-id", + "objects": [ + "957|1834|150|1918|215|Vehicle|#|sedan|Bugatti|M|blue|CA 444|California|0.8", + "..........." + ] + } + */ + + JsonNode *rootNode; + JsonObject *jobject; + JsonArray *jArray; + JsonArray *maskArray = NULL; + guint i; + stringstream ss; + gchar *message = NULL; + + jArray = json_array_new (); + + for (i = 0; i < size; i++) { + GList *objectMask = NULL; + + ss.str(""); + ss.clear(); + + NvDsEventMsgMeta *meta = events[i].metadata; + ss << meta->trackingId << "|" << meta->bbox.left << "|" << meta->bbox.top + << "|" << meta->bbox.left + meta->bbox.width << "|" << meta->bbox.top + meta->bbox.height + << "|" << object_enum_to_str (meta->objType, meta->objectId); + + if (meta->extMsg && meta->extMsgSize) { + // Attach secondary inference attributes. + switch (meta->objType) { + case NVDS_OBJECT_TYPE_VEHICLE: { + NvDsVehicleObject *dsObj = (NvDsVehicleObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON: { + NvDsPersonObject *dsObj = (NvDsPersonObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_FACE: { + NvDsFaceObject *dsObj = (NvDsFaceObject *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + } + } + break; + case NVDS_OBJECT_TYPE_VEHICLE_EXT: { + NvDsVehicleObjectExt *dsObj = (NvDsVehicleObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->type) << "|" << to_str(dsObj->make) << "|" + << to_str(dsObj->model) << "|" << to_str(dsObj->color) << "|" << to_str(dsObj->license) + << "|" << to_str(dsObj->region) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_PERSON_EXT: { + NvDsPersonObjectExt *dsObj = (NvDsPersonObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->apparel) + << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + case NVDS_OBJECT_TYPE_FACE_EXT: { + NvDsFaceObjectExt *dsObj = (NvDsFaceObjectExt *) meta->extMsg; + if (dsObj) { + ss << "|#|" << to_str(dsObj->gender) << "|" << dsObj->age << "|" + << to_str(dsObj->hair) << "|" << to_str(dsObj->cap) << "|" << to_str(dsObj->glasses) + << "|" << to_str(dsObj->facialhair) << "|" << to_str(dsObj->name) << "|" + << "|" << to_str(dsObj->eyecolor) << "|" << meta->confidence; + + if (dsObj->mask) + objectMask = dsObj->mask; + } + } + break; + default: + cout << "Object type (" << meta->objType << ") not implemented" << endl; + break; + } + } + + if (objectMask) { + if (maskArray == NULL) + maskArray = json_array_new (); + generate_mask_array (meta, maskArray, objectMask); + } + + json_array_add_string_element (jArray, ss.str().c_str()); + } + + // It is assumed that all events / objects are associated with same frame. + // Therefore ts / sensorId / frameId of first object can be used. + + jobject = json_object_new (); + json_object_set_string_member (jobject, "version", "4.0"); + json_object_set_string_member (jobject, "id", to_string(events[0].metadata->frameId).c_str()); + json_object_set_string_member (jobject, "@timestamp", events[0].metadata->ts); + if (events[0].metadata->sensorStr) { + json_object_set_string_member (jobject, "sensorId", events[0].metadata->sensorStr); + } else if ((NvDsPayloadPriv *) privData) { + json_object_set_string_member (jobject, "sensorId", + to_str((gchar *) sensor_id_to_str (privData, events[0].metadata->sensorId))); + } else { + json_object_set_string_member (jobject, "sensorId", "0"); + } + + json_object_set_array_member (jobject, "objects", jArray); + if (maskArray && json_array_get_length (maskArray) > 0) + json_object_set_array_member (jobject, "masks", maskArray); + + rootNode = json_node_new (JSON_NODE_OBJECT); + json_node_set_object (rootNode, jobject); + + message = json_to_string (rootNode, TRUE); + json_node_free (rootNode); + json_object_unref (jobject); + + return message; +} diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp new file mode 100644 index 00000000..8bd10d2a --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.cpp @@ -0,0 +1,268 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nvmsgconv.h" +#include "deepstream_schema.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + + +NvDsMsg2pCtx* nvds_msg2p_ctx_create (const gchar *file, NvDsPayloadType type) +{ + NvDsMsg2pCtx *ctx = NULL; + string str; + bool retVal = true; + + /* + * Need to parse configuration / CSV files to get static properties of + * components (e.g. sensor, place etc.) in case of full deepstream schema. + */ + if (type == NVDS_PAYLOAD_DEEPSTREAM) { + g_return_val_if_fail (file, NULL); + + ctx = new NvDsMsg2pCtx; + ctx->privData = create_deepstream_schema_ctx(); + + if (g_str_has_suffix (file, ".csv")) { + retVal = nvds_msg2p_parse_csv (ctx->privData, file); + } else if (g_str_has_suffix (file, ".yml") || + g_str_has_suffix (file, ".yaml")) { + retVal = nvds_msg2p_parse_yaml (ctx->privData, file); + } else { + retVal = nvds_msg2p_parse_key_value (ctx->privData, file); + } + } else { + ctx = new NvDsMsg2pCtx; + /* If configuration file is provided for minimal schema, + * parse it for static values. + */ + if (file) { + ctx->privData = create_deepstream_schema_ctx(); + if (g_str_has_suffix (file, ".yml") || + g_str_has_suffix (file, ".yaml")) { + retVal = nvds_msg2p_parse_yaml (ctx->privData, file); + } else { + retVal = nvds_msg2p_parse_key_value (ctx->privData, file); + } + } else { + ctx->privData = nullptr; + retVal = true; + } + } + + ctx->payloadType = type; + + if (!retVal) { + cout << "Error in creating instance" << endl; + + if (ctx && ctx->privData) + destroy_deepstream_schema_ctx(ctx->privData); + + if (ctx) { + delete ctx; + ctx = NULL; + } + } + return ctx; +} + +void nvds_msg2p_ctx_destroy (NvDsMsg2pCtx *ctx) +{ + destroy_deepstream_schema_ctx(ctx->privData); + ctx->privData = nullptr; + delete ctx; +} + +NvDsPayload** +nvds_msg2p_generate_multiple (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint eventSize, + guint *payloadCount) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload **payloads = NULL; + *payloadCount = 0; + //Set how many payloads are being sent back to the plugin + payloads = (NvDsPayload **) g_malloc0 (sizeof (NvDsPayload*) * 1); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_event_message (ctx->privData, events->metadata); + if (message) { + payloads[*payloadCount]= (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_event_message_minimal (ctx->privData, events, eventSize); + if (message) { + len = strlen (message); + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + payloads[*payloadCount]->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payloads[*payloadCount]->payloadSize = strlen ((char *)payloads[*payloadCount]->payload) + 1; + ++(*payloadCount); + } else + payloads = NULL; + + return payloads; +} + +NvDsPayload* +nvds_msg2p_generate (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload *payload = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_event_message (ctx->privData, events->metadata); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_event_message_minimal (ctx->privData, events, size); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payload->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payload->payloadSize = strlen ((char *)payload->payload) + 1; + } else + payload->payload = NULL; + + return payload; +} + +NvDsPayload* +nvds_msg2p_generate_new (NvDsMsg2pCtx *ctx, void *metadataInfo) +{ + gchar *message = NULL; + gint len = 0; + NvDsMsg2pMetaInfo *meta_info = (NvDsMsg2pMetaInfo *) metadataInfo; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta*) meta_info->frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) meta_info->objMeta; + + NvDsPayload *payload = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_dsmeta_message(ctx->privData, frame_meta, obj_meta); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } + else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_dsmeta_message_minimal (ctx->privData, frame_meta); + if (message) { + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payload->payload = g_memdup (message, len); + payload->payloadSize = len; + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payload->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payload->payloadSize = strlen ((char *)payload->payload) + 1; + } else + payload->payload = NULL; + + return payload; +} + +NvDsPayload** +nvds_msg2p_generate_multiple_new (NvDsMsg2pCtx *ctx, void *metadataInfo, guint *payloadCount) +{ + gchar *message = NULL; + gint len = 0; + NvDsPayload **payloads = NULL; + *payloadCount = 0; + //Set how many payloads are being sent back to the plugin + payloads = (NvDsPayload **) g_malloc0 (sizeof (NvDsPayload*) * 1); + + NvDsMsg2pMetaInfo *meta_info = (NvDsMsg2pMetaInfo *) metadataInfo; + NvDsFrameMeta *frame_meta = (NvDsFrameMeta*) meta_info->frameMeta; + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) meta_info->objMeta; + + if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM) { + message = generate_dsmeta_message(ctx->privData, frame_meta, obj_meta); + if (message) { + payloads[*payloadCount]= (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + len = strlen (message); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_DEEPSTREAM_MINIMAL) { + message = generate_dsmeta_message_minimal (ctx->privData, frame_meta); + if (message) { + len = strlen (message); + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + // Remove '\0' character at the end of string and just copy the content. + payloads[*payloadCount]->payload = g_memdup (message, len); + payloads[*payloadCount]->payloadSize = len; + ++(*payloadCount); + g_free (message); + } + } else if (ctx->payloadType == NVDS_PAYLOAD_CUSTOM) { + payloads[*payloadCount] = (NvDsPayload *) g_malloc0 (sizeof (NvDsPayload)); + payloads[*payloadCount]->payload = (gpointer) g_strdup ("CUSTOM Schema"); + payloads[*payloadCount]->payloadSize = strlen ((char *)payloads[*payloadCount]->payload) + 1; + ++(*payloadCount); + } else + payloads = NULL; + + return payloads; +} + +void +nvds_msg2p_release (NvDsMsg2pCtx *ctx, NvDsPayload *payload) +{ + g_free (payload->payload); + g_free (payload); +} diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h new file mode 100644 index 00000000..246ba897 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/nvmsgconv/nvmsgconv.h @@ -0,0 +1,170 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * @file + * NVIDIA DeepStream: Message Schema Generation Library Interface + * + * @b Description: This file specifies the NVIDIA DeepStream message schema generation + * library interface. + */ + +#ifndef NVMSGCONV_H_ +#define NVMSGCONV_H_ + +#include "nvdsmeta_schema.h" +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + +/** + * @ref NvDsMsg2pCtx is structure for library context. + */ +typedef struct NvDsMsg2pCtx { + /** type of payload to be generated. */ + NvDsPayloadType payloadType; + + /** private to component. Don't change this field. */ + gpointer privData; +} NvDsMsg2pCtx; + +/** + * @ref NvDsMsg2pMetaInfo is structure to hold + * the NvDs metadata related information + to be processed to generate payloads + */ + +typedef struct { + /** Holds the object metadata */ + void *objMeta; + /** Holds the frame metadata */ + void *frameMeta; + /** media type: (ex: audio, video) */ + gchar *mediaType; +} NvDsMsg2pMetaInfo; + +/** + * This function initializes the library with user defined options mentioned + * in the file and returns the handle to the context. + * Static fields which should be part of message payload can be added to + * file instead of frame metadata. + * + * @param[in] file name of file to read static properties from. + * @param[in] type type of payload to be generated. + * + * @return pointer to library context created. This context should be used in + * other functions of library and should be freed with + * @ref nvds_msg2p_ctx_destroy + */ +NvDsMsg2pCtx* nvds_msg2p_ctx_create (const gchar *file, NvDsPayloadType type); + +/** + * Release the resources allocated during context creation. + * + * @param[in] ctx pointer to library context. + */ +void nvds_msg2p_ctx_destroy (NvDsMsg2pCtx *ctx); + +/** + * This function will parse the @ref NvDsEventMsgMeta and will generate message + * payload. Payload will be combination of static values read from + * configuration file and dynamic values received in meta. + * Payload will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] events pointer to array of event objects. + * @param[in] size number of objects in array. + * + * @return pointer to @ref NvDsPayload generated or NULL in case of error. + * This payload should be freed with @ref nvds_msg2p_release + */ +NvDsPayload* +nvds_msg2p_generate (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size); + +/** + * This function will parse the @ref NvDsEventMsgMeta and will generate multiple + * message payloads. Payloads will be combination of static values read from + * configuration file and dynamic values received in meta. + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] events pointer to array of event objects. + * @param[in] size number of objects in array. + * @param[out] payloadCount number of payloads being returned by the function. + * + * @return pointer to @ref array of NvDsPayload pointers generated or NULL in + * case of error. The number of payloads in the array is returned through + * payloadCount. This pointer should be freed by calling g_free() and the + * individual payloads should be freed with @ref nvds_msg2p_release + */ +NvDsPayload** +nvds_msg2p_generate_multiple (NvDsMsg2pCtx *ctx, NvDsEvent *events, guint size, guint *payloadCount); + +/** + * This function will parse the @ref NvDsMsg2pMetaInfo and will generate + * message payloads. Payloads will be combination of static values read from + * configuration file and the deepstream metadata fields passed @ref NvDsMsg2pMetaInfo + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] pointer to type NvDsMsg2pMetaInfo + * + * @return pointer to @ref NvDsPayload generated or NULL in case of error. + * This payload should be freed with @ref nvds_msg2p_release + */ +NvDsPayload* +nvds_msg2p_generate_new (NvDsMsg2pCtx *ctx, void *metadataInfo); + +/** + * This function will parse the @ref NvDsMsg2pMetaInfo and will generate multiple + * message payloads. Payloads will be combination of static values read from + * configuration file and the deepstream metadata fields passed @ref NvDsMsg2pMetaInfo + * Payloads will be generated based on the @ref NvDsPayloadType type provided + * in context creation (e.g. Deepstream, Custom etc.). + * + * @param[in] ctx pointer to library context. + * @param[in] pointer to type NvDsMsg2pMetaInfo + * @param[out] payloadCount number of payloads being returned by the function. + * + * @return pointer to @ref array of NvDsPayload pointers generated or NULL in + * case of error. The number of payloads in the array is returned through + * payloadCount. This pointer should be freed by calling g_free() and the + * individual payloads should be freed with @ref nvds_msg2p_release + */ +NvDsPayload** +nvds_msg2p_generate_multiple_new (NvDsMsg2pCtx *ctx, void *metadataInfo, guint *payloadCount); + +/** + * This function should be called to release memory allocated for payload. + * + * @param[in] ctx pointer to library context. + * @param[in] payload pointer to object that needs to be released. + */ +void nvds_msg2p_release (NvDsMsg2pCtx *ctx, NvDsPayload *payload); + +#ifdef __cplusplus +} +#endif +#endif /* NVMSGCONV_H_ */ diff --git a/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/retail_iva.c b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/retail_iva.c new file mode 100644 index 00000000..cc7978a5 --- /dev/null +++ b/src/apps/reference_apps/legacy_apps/deepstream-retail-analytics/retail_iva.c @@ -0,0 +1,899 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" +#include "nvds_yml_parser.h" + +#define MAX_DISPLAY_LEN 64 +#define MAX_TIME_STAMP_LEN 32 + +// Config files for detectors, tracker and message broker +#define PGIE_CONFIG_FILE "configs/pgie_config_peoplenet.txt" +#define SGIE_CONFIG_FILE "configs/basket_classifier.txt" +#define TRACKER_CONFIG_FILE "configs/dstest4_tracker_config.txt" +#define MSCONV_CONFIG_FILE "configs/dstest4_msgconv_config.txt" + +// Primary detector class IDs +#define PGIE_CLASS_ID_PERSON 0 +#define PGIE_CLASS_ID_FACE 1 +#define PGIE_CLASS_ID_BAG 2 + +// Properties of nvstreammux +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 + +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +// Keys to read tracker config file +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" +#define CONFIG_GPU_ID "gpu-id" + +// Define variables to store config parameters +static gchar *cfg_file = NULL; +static gchar *input_file = NULL; +static gchar *topic = NULL; +static gchar *conn_str = NULL; +static gchar *proto_lib = NULL; +static gint schema_type = 0; +static gint msg2p_meta = 0; +static gint frame_interval = 15; +static gboolean display_off = FALSE; + +// Array to store classes detected by PGIE +gchar pgie_classes_str[3][32] = {"Person", "Bag", "Face"}; + +GOptionEntry entries[] = { + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME, &cfg_file, + "Set the adaptor config file. Optional if connection string has relevant details.", + NULL}, + {"input-file", 'i', 0, G_OPTION_ARG_FILENAME, &input_file, + "Set the input H264 file", NULL}, + {"topic", 't', 0, G_OPTION_ARG_STRING, &topic, + "Name of message topic. Optional if it is part of connection string or config file.", + NULL}, + {"conn-str", 0, 0, G_OPTION_ARG_STRING, &conn_str, + "Connection string of backend server. Optional if it is part of config file.", + NULL}, + {"proto-lib", 'p', 0, G_OPTION_ARG_STRING, &proto_lib, + "Absolute path of adaptor library", NULL}, + {"schema", 's', 0, G_OPTION_ARG_INT, &schema_type, + "Type of message schema (0=Full, 1=minimal), default=0", NULL}, + {"msg2p-meta", 0, 0, G_OPTION_ARG_INT, &msg2p_meta, + "msg2payload generation metadata type (0=Event Msg meta, 1=nvds meta), default=0", + NULL}, + {"frame-interval", 0, 0, G_OPTION_ARG_INT, &frame_interval, + "Frame interval at which payload is generated , default=30", NULL}, + {"no-display", 0, 0, G_OPTION_ARG_NONE, &display_off, "Disable display", + NULL}, + {NULL} +}; + +// Function to check if an input file is a YAML file +#define IS_YAML(file) (g_str_has_suffix (file, ".yml") || g_str_has_suffix (file, ".yaml")) + +gint frame_number = 0; + +// Function to generate timestamp for kafka message +static void +generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime (CLOCK_REALTIME, &ts); + memcpy (&tloc, (void *) (&ts.tv_sec), sizeof (time_t)); + gmtime_r (&tloc, &tm_log); + strftime (buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf (strmsec, sizeof (strmsec), ".%.3dZ", ms); + strncat (buf, strmsec, buf_size); +} + +// Callback function for deep-copying NvDsEventMsgMeta struct +static gpointer +meta_copy_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = g_memdup (srcMeta, sizeof (NvDsEventMsgMeta)); + + if (srcMeta->ts) + dstMeta->ts = g_strdup (srcMeta->ts); + + if (srcMeta->sensorStr) + dstMeta->sensorStr = g_strdup (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = g_memdup (srcMeta->objSignature.signature, + srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *) srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + + if (srcObj->hasBasket) + obj->hasBasket = g_strdup (srcObj->hasBasket); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof (NvDsPersonObject); + } + } + + return dstMeta; +} + +// Callback function to free NvDsEventMsgMeta struct +static void +meta_free_func (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *) user_meta->user_meta_data; + + g_free (srcMeta->ts); + g_free (srcMeta->sensorStr); + + if (srcMeta->objSignature.size > 0) { + g_free (srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free (srcMeta->objectId); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *) srcMeta->extMsg; + + if (obj->hasBasket) + g_free (obj->hasBasket); + } + g_free (srcMeta->extMsg); + srcMeta->extMsgSize = 0; + } + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +// Function to return label from classifier metadata +static gchar * +get_first_result_label (NvDsClassifierMeta * classifierMeta) +{ + GList *n; + // Iterate through all the secondary labels stored in classifierMeta + // Refer to deepstream-test5 + for (n = classifierMeta->label_info_list; n != NULL; n = n->next) { + NvDsLabelInfo *labelInfo = (NvDsLabelInfo *) (n->data); + if (labelInfo->result_label[0] != '\0') { + return g_strdup (labelInfo->result_label); + } + } + return NULL; +} + +// Function to generate metadata for person object type +static void +generate_person_meta (gpointer data) +{ + NvDsPersonObject *obj = (NvDsPersonObject *) data; +} + +// Function to generate event message metadata from object metadata +static void +generate_event_msg_meta (gpointer data, gint class_id, + NvDsObjectMeta * obj_params) +{ + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *) data; + meta->sensorId = 0; + meta->placeId = 0; + meta->moduleId = 0; + meta->sensorStr = g_strdup ("sensor-0"); + + meta->ts = (gchar *) g_malloc0 (MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *) g_malloc0 (MAX_LABEL_SIZE); + + strncpy (meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + generate_ts_rfc3339 (meta->ts, MAX_TIME_STAMP_LEN); + + // * This demonstrates how to attach custom objects. + // * Any custom object as per requirement can be generated and attached + // * like NvDsVehicleObject / NvDsPersonObject. Then that object should + // * be handled in payload generator library (nvmsgconv.cpp) accordingly. + + // Attach the secondary label to the person object detected + + if (class_id == PGIE_CLASS_ID_PERSON) { + meta->type = NVDS_EVENT_ENTRY; + meta->objType = NVDS_OBJECT_TYPE_PERSON; + meta->objClassId = PGIE_CLASS_ID_PERSON; + + NvDsPersonObject *obj = + (NvDsPersonObject *) g_malloc0 (sizeof (NvDsPersonObject)); + generate_person_meta (obj); + + GList *l; + for (l = obj_params->classifier_meta_list; l!= NULL; l = l->next) { + NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *) (l->data); + obj->hasBasket = get_first_result_label(classifierMeta); + } + + meta->extMsg = obj; + meta->extMsgSize = sizeof (NvDsPersonObject); + } +} + +// Probe function to generate OSD data +static GstPadProbeReturn +osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + // Uncomment lines containing is_first_object send kafka messages + // for only the first detection in the frame. + + GstBuffer *buf = (GstBuffer *) info->data; + NvDsFrameMeta *frame_meta = NULL; + NvOSD_TextParams *txt_params = NULL; + guint person_count = 0; + // gboolean is_first_object = TRUE; + NvDsMetaList *l_frame, *l_obj; + gchar *sgie_label=NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + if (!batch_meta) { + // No batch meta attached. + return GST_PAD_PROBE_OK; + } + + for (l_frame = batch_meta->frame_meta_list; l_frame; l_frame = l_frame->next) { + frame_meta = (NvDsFrameMeta *) l_frame->data; + + if (frame_meta == NULL) { + // Ignore Null frame meta. + continue; + } + + // is_first_object = TRUE; + + for (l_obj = frame_meta->obj_meta_list; l_obj; l_obj = l_obj->next) { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) l_obj->data; + + if (obj_meta == NULL) { + // Ignore Null object. + continue; + } + + GList *l; + sgie_label = "NULL"; + for (l = obj_meta->classifier_meta_list; l!= NULL; l = l->next) { + NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *) (l->data); + sgie_label = get_first_result_label(classifierMeta); + } + + txt_params = &(obj_meta->text_params); + if (txt_params->display_text) + g_free (txt_params->display_text); + + txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); + + g_snprintf (txt_params->display_text, MAX_DISPLAY_LEN, "%s %ld %s", + pgie_classes_str[obj_meta->class_id], obj_meta->object_id, sgie_label); /* Person 12 hasBasket */ + + person_count++; + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = obj_meta->rect_params.left; + txt_params->y_offset = obj_meta->rect_params.top - 25; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = "Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + // * Ideally NVDS_EVENT_MSG_META should be attached to buffer by the + // * component implementing detection / recognition logic. + // * Here it demonstrates how to use / attach that meta data. + + if (/*is_first_object && */ !(frame_number % frame_interval)) { + /* Frequency of messages to be send will be based on use case. + * Here message is being sent for first object every frame_interval(default=15). + */ + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *) g_malloc0 (sizeof (NvDsEventMsgMeta)); + msg_meta->bbox.top = obj_meta->rect_params.top; + msg_meta->bbox.left = obj_meta->rect_params.left; + msg_meta->bbox.width = obj_meta->rect_params.width; + msg_meta->bbox.height = obj_meta->rect_params.height; + msg_meta->frameId = frame_number; + msg_meta->trackingId = obj_meta->object_id; + msg_meta->confidence = obj_meta->confidence; + generate_event_msg_meta (msg_meta, obj_meta->class_id, obj_meta); + + NvDsUserMeta *user_event_meta = + nvds_acquire_user_meta_from_pool (batch_meta); + if (user_event_meta) { + user_event_meta->user_meta_data = (void *) msg_meta; + user_event_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + user_event_meta->base_meta.copy_func = + (NvDsMetaCopyFunc) meta_copy_func; + user_event_meta->base_meta.release_func = + (NvDsMetaReleaseFunc) meta_free_func; + nvds_add_user_meta_to_frame (frame_meta, user_event_meta); + } else { + g_print ("Error in attaching event meta to buffer\n"); + } + } + // is_first_object = FALSE; + } + } + } + g_print ("Frame Number = %d " + "Person Count = %d\n", + frame_number, person_count); + frame_number++; + + return GST_PAD_PROBE_OK; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static gchar * +get_absolute_file_path (gchar *cfg_file_path, gchar *file_path) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + if (file_path && file_path[0] == '/') { + return file_path; + } + + if (!realpath (cfg_file_path, abs_cfg_path)) { + g_free (file_path); + return NULL; + } + + // Return absolute path of config file if file_path is NULL. + if (!file_path) { + abs_file_path = g_strdup (abs_cfg_path); + return abs_file_path; + } + + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL); + g_free (file_path); + + return abs_file_path; +} + +// Function to read tracker config file and set properties of the GstElement +static gboolean +set_tracker_properties (GstElement *nvtracker) +{ + gboolean ret = FALSE; + GError *error = NULL; + gchar **keys = NULL; + gchar **key = NULL; + GKeyFile *key_file = g_key_file_new (); + + if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE, + &error)) { + g_printerr ("Failed to load config file: %s\n", error->message); + return FALSE; + } + + keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) { + gint width = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_WIDTH, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) { + gint height = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_HEIGHT, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) { + guint gpu_id = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GPU_ID, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { + char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { + char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS)) { + gboolean enable_batch_process = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "enable_batch_process", + enable_batch_process, NULL); + } else { + g_printerr ("Unknown key '%s' for group [%s]", *key, + CONFIG_GROUP_TRACKER); + } + } + + ret = TRUE; + done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + if (!ret) { + g_printerr ("%s failed", __func__); + } + return ret; +} + +static void +check_gst_element_creation_success(GstElement *element, gchar *component_name) +{ + if (!element) { + g_printerr("%s element could not be created\n", component_name); + } +} + +int +main (int argc, char *argv[]) +{ + // Initialize elements of the DS pipeline + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, + *decoder = NULL, *sink = NULL, *pgie = NULL, *sgie = NULL, + *nvvidconv = NULL, *nvosd = NULL, *nvstreammux = NULL, *nvtracker = NULL; + GstElement *msgconv = NULL, *msgbroker = NULL, *tee = NULL; + GstElement *queue1 = NULL, *queue2 = NULL; + GstElement *transform = NULL; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *osd_sink_pad = NULL; + GstPad *tee_render_pad = NULL; + GstPad *tee_msg_pad = NULL; + GstPad *sink_pad = NULL; + GstPad *src_pad = NULL; + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + + int current_device = -1; + cudaGetDevice (¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + ctx = g_option_context_new("DeepStream Retail IVA"); + group = g_option_group_new("RetailIVA", NULL, NULL, NULL, NULL); + g_option_group_add_entries(group, entries); + + g_option_context_set_main_group(ctx, group); + g_option_context_add_group(ctx, gst_init_get_option_group()); + + if (!g_option_context_parse(ctx, &argc, &argv, &error)) { + g_option_context_free (ctx); + g_printerr("%s", error->message); + return -1; + } + + if (!proto_lib || !input_file) { + if (argc > 1 && !IS_YAML (argv[1])) { + g_printerr ("missing arguments\n"); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr + ("Usage: %s -i -p --conn-str=\n", + argv[0]); + return -1; + } else if (!argv[1]) { + g_printerr ("missing arguments\n"); + g_printerr ("Usage: %s \n", argv[0]); + g_printerr + ("Usage: %s -i -p --conn-str=\n", + argv[0]); + return -1; + } + } + + loop = g_main_loop_new (NULL, FALSE); + + // Create gstreamer elements + // Create pipeline element that will hold connection of all other elements + pipeline = gst_pipeline_new ("retail-iva-pipeline"); + + // Source element for reading the input file + source = gst_element_factory_make ("filesrc", "file-source"); + + // h264parser to parse input file + h264parser = gst_element_factory_make("h264parse", "h264-parser"); + + // nvdec_h264 for hardware accelerated decoding on GPU + decoder = gst_element_factory_make("nvv4l2decoder", "nvv4l2-decoder"); + + nvstreammux = gst_element_factory_make ("nvstreammux", "nvstreammux"); + + // nvinfer to run inferencing on decoder's output - PGIE + pgie = gst_element_factory_make("nvinfer", "primary-inference-engine"); + + // nvinfer to run inferencing on PGIE's output - SGIE + sgie = gst_element_factory_make("nvinfer", "secondary-inference-engine"); + + // tracker to track objects detected by PGIE + nvtracker = gst_element_factory_make("nvtracker", "tracker"); + + // converter to convert from NV12 to RGBA + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter"); + + // osd to draw on converted RGBA buffer + nvosd = gst_element_factory_make("nvdsosd", "nv-onscreendisplay"); + + // Create message converter to generate payload from buffer metadata + msgconv = gst_element_factory_make("nvmsgconv", "nvmsg-converter"); + + // Create message broker to send payload to kafka server + msgbroker = gst_element_factory_make("nvmsgbroker", "nvmsg-broker"); + + // Create teeo to render buffer and send messages simultaneously + tee = gst_element_factory_make("tee", "nvsink-tee"); + + // Create queues + queue1 = gst_element_factory_make("queue", "nvtee-que1"); + queue2 = gst_element_factory_make("queue", "nvtee-que2"); + + // Finally render the osd output + if (display_off) { + sink = gst_element_factory_make("fakesink", "nvvideo-renderer"); + + } else { + sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); + if (prop.integrated) { + transform = + gst_element_factory_make("nvegltransform", "nvegl-transform"); + if (!transform) { + g_printerr("nvegltransform element could not be created. Exiting\n"); + return -1; + } + } + } + + + // Check if the pipeline and all elements are created + if (!pipeline || !source || !h264parser || !decoder || !nvstreammux || !pgie || + !sgie || !nvtracker || !nvvidconv || !nvosd || !msgconv || !msgbroker || + !tee || !queue1 || !queue2 || !sink) { + // Check which element was not created + check_gst_element_creation_success(pipeline, "pipeline"); + check_gst_element_creation_success(source, "source"); + check_gst_element_creation_success(h264parser, "h264parser"); + check_gst_element_creation_success(decoder, "decoder"); + check_gst_element_creation_success(nvstreammux, "nvstreammux"); + check_gst_element_creation_success(pgie, "pgie"); + check_gst_element_creation_success(sgie, "sgie"); + check_gst_element_creation_success(nvtracker, "nvtracker"); + check_gst_element_creation_success(nvvidconv, "nvvidconv"); + check_gst_element_creation_success(nvosd, "nvosd"); + check_gst_element_creation_success(msgconv, "msgconv"); + check_gst_element_creation_success(msgbroker, "msgbroker"); + check_gst_element_creation_success(tee, "tee"); + check_gst_element_creation_success(queue1, "queue1"); + check_gst_element_creation_success(queue2, "queue2"); + check_gst_element_creation_success(sink, "sink"); + g_printerr("One above element could not be created. Exiting \n"); + return -1; + } + + if (argc > 1 && IS_YAML (argv[1])) { + nvds_parse_file_source (source, argv[1], "source"); + nvds_parse_streammux (nvstreammux, argv[1], "streammux"); + + g_object_set (G_OBJECT (pgie), + "config-file-path", "configs/pgie_config_peoplenet.yml", NULL); + + g_object_set (G_OBJECT(sgie), + "config-file-path", "configs/basket_classifier.yml", NULL); + + g_object_set (G_OBJECT (msgconv), "config", "configs/dstest4_msgconv_config.yml", + NULL); + nvds_parse_msgconv (msgconv, argv[1], "msgconv"); + + nvds_parse_msgbroker (msgbroker, argv[1], "msgbroker"); + + if (display_off) + nvds_parse_file_sink (sink, argv[1], "sink"); + else + nvds_parse_egl_sink (sink, argv[1], "sink"); + + } else { + /* we set the input filename to the source element */ + g_object_set (G_OBJECT (source), "location", input_file, NULL); + + g_object_set (G_OBJECT (nvstreammux), "batch-size", 1, NULL); + + g_object_set (G_OBJECT (nvstreammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + + g_object_set (G_OBJECT (msgconv), "config", MSCONV_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (msgconv), "payload-type", schema_type, NULL); + g_object_set (G_OBJECT (msgconv), "msg2p-newapi", msg2p_meta, NULL); + g_object_set (G_OBJECT (msgconv), "frame-interval", frame_interval, NULL); + + g_object_set (G_OBJECT (msgbroker), "proto-lib", proto_lib, + "conn-str", conn_str, "sync", FALSE, NULL); + + if (topic) { + g_object_set (G_OBJECT (msgbroker), "topic", topic, NULL); + } + + if (cfg_file) { + g_object_set (G_OBJECT (msgbroker), "config", cfg_file, NULL); + } + gchar *filepath; + filepath = g_strconcat("output",".mp4",NULL); + g_object_set (G_OBJECT (sink), "sync", TRUE, NULL); + } + + if (!set_tracker_properties(nvtracker)) { + g_printerr ("Failed to set tracker properties. Exiting \n"); + return -1; + } + + // Adding a message handler + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + // Setup a pipeline + // Add all elements to the pipeline + gst_bin_add_many(GST_BIN(pipeline), + source, h264parser, decoder, nvstreammux, pgie, sgie, nvtracker, + nvvidconv, nvosd, tee, queue1, queue2, msgconv, msgbroker, sink, NULL); + + if (prop.integrated) { + if (!display_off) + gst_bin_add (GST_BIN(pipeline), transform); + } + + // Link elements together + /* + file-source -> h264-parser -> nvh264-decoder -> nvstreammux -> + nvinfer -> nvtracker -> nvvidconv -> nvosd -> tee -> video-renderer + | + | -> msgconv -> msgbroker + */ + sink_pad = gst_element_get_request_pad(nvstreammux, "sink_0"); + if (!sink_pad) { + g_printerr("Streammux request sink pad failed. Exiting \n"); + return -1; + } + + src_pad = gst_element_get_static_pad(decoder, "src"); + if (!src_pad) { + g_printerr("Decoder request src pad failed. Exiting \n"); + return -1; + } + + if (gst_pad_link(src_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Failed to link decoder to stream muxer. Exiting \n"); + return -1; + } + + gst_object_unref(sink_pad); + gst_object_unref(src_pad); + + if (!gst_element_link_many(source, h264parser, decoder, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + + if (!gst_element_link_many(nvstreammux, pgie, nvtracker, sgie, nvvidconv, nvosd, tee, NULL)) { + g_printerr ("Elements could not be linked. Exiting \n"); + return -1; + } + + if (!gst_element_link_many(queue1, msgconv, msgbroker, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + + if (prop.integrated) { + if (!display_off) { + if(!gst_element_link_many(queue2, transform, sink, NULL)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + } else { + if (!gst_element_link(queue2, sink)) { + g_printerr("Elements could not be linked. Exiting \n"); + return -1; + } + } + } else { + if (!gst_element_link (queue2, sink)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + } + + sink_pad = gst_element_get_static_pad (queue1, "sink"); + tee_msg_pad = gst_element_get_request_pad(tee, "src_%u"); + tee_render_pad = gst_element_get_request_pad(tee, "src_%u"); + + if (!tee_msg_pad || !tee_render_pad) { + g_printerr("Unable to request pads \n"); + return -1; + } + + if (gst_pad_link(tee_msg_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Unable to link tee and message converter. \n"); + gst_object_unref(sink_pad); + return -1; + } + + gst_object_unref(sink_pad); + + sink_pad = gst_element_get_static_pad(queue2, "sink"); + if (gst_pad_link(tee_render_pad, sink_pad) != GST_PAD_LINK_OK) { + g_printerr("Unable to link tee and render pad. \n"); + gst_object_unref (sink_pad); + return -1; + } + + gst_object_unref(sink_pad); + + // Adding a probe to get informed of the meta data generated. + // We add a probe to the sink pad of the OSD element since by + // that time the buffer would have had got all the metadata + + osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); + if (!osd_sink_pad) { + g_print("Unable to get sink pad\n"); + } else { + if (msg2p_meta == 0) { + gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, NULL, NULL); + } + } + gst_object_unref(osd_sink_pad); + + // Set pipeline to playing state + if (argc > 1 && IS_YAML(argv[1])) { + g_print ("Using file %s \n", argv[1]); + } else { + g_print("Now playing %s\n", input_file); + } + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + // Wait till the pipeline encounters an error or End of Stream (EOS) + g_print("Running ...\n"); + g_main_loop_run(loop); + + // Out of the main loop. Perform clean up + g_print ("Returned, stopped playback\n"); + + g_free (cfg_file); + g_free(input_file); + g_free(topic); + g_free(conn_str); + g_free(proto_lib); + + // Release the request pads from tee and unfer them + gst_element_release_request_pad(tee, tee_msg_pad); + gst_element_release_request_pad(tee, tee_render_pad); + gst_object_unref(tee_msg_pad); + gst_object_unref(tee_render_pad); + + gst_element_set_state(pipeline, GST_STATE_NULL); + g_print("Deleting pipeline \n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + return 0; +} \ No newline at end of file diff --git a/src/apps/reference_apps/pyservicemaker_sample_apps/README.md b/src/apps/reference_apps/pyservicemaker_sample_apps/README.md new file mode 100644 index 00000000..b888082d --- /dev/null +++ b/src/apps/reference_apps/pyservicemaker_sample_apps/README.md @@ -0,0 +1,14 @@ +## Introduction +The apps in this directory are additional samples demonstrating usage of the Python API for DeepStream Service Maker, either by flow API or by pipeline API. See the [Python Service Maker documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_service_maker_python.html) for details. + +Other sample reference apps for pyservicemaker can be found at `/opt/nvidia/deepstream/deepstream/service-maker/sources/apps/python/`. Pysm sample apps using TAO pre-trained models can be found in the [deepstream_tao_apps](../../tao_apps/pysm-apps) repo. + +## Prerequisites +* torchvision +``` +pip3 install torchvision +``` +* pyyaml +``` +pip3 install pyyaml +``` \ No newline at end of file diff --git a/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md b/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md new file mode 100644 index 00000000..306af9bf --- /dev/null +++ b/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/README.md @@ -0,0 +1,17 @@ +## Purpose + +The sample app demonstrates how to simplify [deepstream_nvds_analytics_test_app](../../pipeline_api/deepstream_nvdsanalytics_test_app) +using Flow API. +Flow APIs effectively abstract away the underlying pipeline details, allowing +developers to focus solely on the goals of their specific tasks in a pythonic style. + +## Usage +``` +$ python3 deepstream_nvdsanalytics.py [uri2] ... [uriN] +``` + +For URI(s) with special characters like @,& etc, you need to pass the uri within quotes + +``` +$ python3 deepstream_nvdsanalytics.py 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' +``` \ No newline at end of file diff --git a/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py b/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py new file mode 100644 index 00000000..28c79440 --- /dev/null +++ b/src/apps/reference_apps/pyservicemaker_sample_apps/flow_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py @@ -0,0 +1,100 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from multiprocessing import Process +import sys +import platform +import os + +PIPELINE_NAME = "deepstream-nvdsanalytics-test" +CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/nvdsanalytics_pgie_config.txt" +ANALYTICS_CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/config_nvdsanalytics.txt" +TRACKER_LL_CONFIG_FILE = "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml" +TRACKER_LL_LIB_FILE = "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so" +BATCHED_PUSH_TIMEOUT = 33000 +MUXER_WIDTH = 1920 +MUXER_HEIGHT = 1080 +TILER_WIDTH = 1280 +TILER_HEIGHT = 720 + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + vehicle_count = 0 + person_count = 0 + for object_meta in frame_meta.object_items: + class_id = object_meta.class_id + if class_id == 0: + vehicle_count += 1 + elif class_id == 2: + person_count += 1 + for user_meta in object_meta.nvdsanalytics_obj_items: + nvdsanalytics_obj_info = user_meta.as_nvdsanalytics_obj() + print("Object {0} moving in direction: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.dir_status)) + print("Object {0} line crossing status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.lc_status)) + print("Object {0} overcrowding status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.oc_status)) + print("Object {0} ROI status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.roi_status)) + print("Object {0} status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.obj_status)) + print("Object {0} unique ID: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.unique_id)) + for user_meta in frame_meta.nvdsanalytics_frame_items: + nvdsanalytics_frame_meta = user_meta.as_nvdsanalytics_frame() + print("Frame {0} overcrowding status: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.oc_status)) + print("Frame {0} object in ROI count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_in_roi_cnt)) + print("Frame {0} object line crossing current count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_curr_cnt)) + print("Frame {0} object line crossing cumulative count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_cum_cnt)) + print("Frame {0} unique ID: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.unique_id)) + print("Frame {0} object count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_cnt)) + print(f"Object Counter: Pad Idx={frame_meta.pad_index}," + f"Frame Number={frame_meta.frame_number}," + f"Vehicle Count={vehicle_count}, Person Count={person_count}") + display_text = f"Person={person_count},Vehicle={vehicle_count}" + display_meta = batch_meta.acquire_display_meta() + text = osd.Text() + text.display_text = display_text.encode('ascii') + text.x_offset = 10 + text.y_offset = 12 + text.font.name = osd.FontFamily.Serif + text.font.size = 12 + text.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + text.set_bg_color = True + text.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(text) + frame_meta.append(display_meta) + +def deepstream_nvdsanalytics_test_app(stream_file_path_list): + pipeline = Pipeline("deepstream-nvdsanalytics-test") + flow = Flow(pipeline).batch_capture(stream_file_path_list).infer(CONFIG_FILE_PATH) + flow = flow.track(ll_config_file=TRACKER_LL_CONFIG_FILE, ll_lib_file=TRACKER_LL_LIB_FILE) + flow = flow.analyze(ANALYTICS_CONFIG_FILE_PATH) + flow.attach(what=Probe("counter", ObjectCounterMarker())).render()() + +if __name__ == '__main__': + # Check input arguments + if len(sys.argv) < 2: + sys.stderr.write("usage: %s [uri2] ... [uriN]\n" % sys.argv[0]) + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_nvdsanalytics_test_app, args=(sys.argv[1:],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + print("\nCtrl+C detected. Terminating process...") + process.terminate() \ No newline at end of file diff --git a/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md b/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md new file mode 100644 index 00000000..8df9fe25 --- /dev/null +++ b/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/README.md @@ -0,0 +1,31 @@ +## Introduction + +This document describes the sample deepstream_nvdsanalytics_test application. + +This sample builds on top of the service-maker deepstream_test1 sample at at `/opt/nvidia/deepstream/deepstream/service-maker/sources/apps/python/pipeline_api/deepstream_test1_app` to demonstrate how to: + +* Use multiple sources in the pipeline. +* Use a uridecodebin so that any type of input (e.g. RTSP/File), any GStreamer + supported container format, and any codec can be used as input. +* Configure the stream-muxer to generate a batch of frames and infer on the + batch for better resource utilization. +* Perform analytics on metadata attached by nvinfer and nvtracker using nvdsanalytics plugin +* Extract the stream metadata, which contains useful information about the + frames in the batched buffer. Extract the analytics object level and frame level metadata, + which contains information about region of interest filtering, overcrowding detection, direction + detection, and line crossing. + +Refer to the service-maker deepstream_test1 sample documentation for an example of a +single-stream inference, bounding-box overlay, and rendering. + +## Usage + +Run with the uri(s). + +``` +$ python3 deepstream_nvdsanalytics.py [uri2] ... [uriN] +e.g. +$ python3 deepstream_nvdsanalytics.py file:///home/ubuntu/video1.mp4 file:///home/ubuntu/video2.mp4 +For URI(s) with special characters like @,& etc, you need to pass the uri within quotes +$ python3 deepstream_nvdsanalytics.py 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' 'rtsp://user@ip/cam/realmonitor?channel=1&subtype=0' +``` diff --git a/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py b/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py new file mode 100644 index 00000000..dd2e050b --- /dev/null +++ b/src/apps/reference_apps/pyservicemaker_sample_apps/pipeline_api/deepstream_nvdsanalytics_test_app/deepstream_nvdsanalytics.py @@ -0,0 +1,110 @@ +################################################################################################### +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################################### + +from pyservicemaker import Pipeline, Probe, BatchMetadataOperator, osd +from multiprocessing import Process +import sys +import platform +import os + +PIPELINE_NAME = "deepstream-nvdsanalytics-test" +CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/nvdsanalytics_pgie_config.txt" +ANALYTICS_CONFIG_FILE_PATH = "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-nvdsanalytics-test/config_nvdsanalytics.txt" +TRACKER_LL_CONFIG_FILE = "/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml" +TRACKER_LL_LIB_FILE = "/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so" +BATCHED_PUSH_TIMEOUT = 33000 +MUXER_WIDTH = 1920 +MUXER_HEIGHT = 1080 +TILER_WIDTH = 1280 +TILER_HEIGHT = 720 + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + vehicle_count = 0 + person_count = 0 + for object_meta in frame_meta.object_items: + class_id = object_meta.class_id + if class_id == 0: + vehicle_count += 1 + elif class_id == 2: + person_count += 1 + for user_meta in object_meta.nvdsanalytics_obj_items: + nvdsanalytics_obj_info = user_meta.as_nvdsanalytics_obj() + print("Object {0} moving in direction: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.dir_status)) + print("Object {0} line crossing status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.lc_status)) + print("Object {0} overcrowding status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.oc_status)) + print("Object {0} ROI status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.roi_status)) + print("Object {0} status: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.obj_status)) + print("Object {0} unique ID: {1}".format(object_meta.object_id, nvdsanalytics_obj_info.unique_id)) + for user_meta in frame_meta.nvdsanalytics_frame_items: + nvdsanalytics_frame_meta = user_meta.as_nvdsanalytics_frame() + print("Frame {0} overcrowding status: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.oc_status)) + print("Frame {0} object in ROI count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_in_roi_cnt)) + print("Frame {0} object line crossing current count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_curr_cnt)) + print("Frame {0} object line crossing cumulative count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_lc_cum_cnt)) + print("Frame {0} unique ID: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.unique_id)) + print("Frame {0} object count: {1}".format(frame_meta.frame_number, nvdsanalytics_frame_meta.obj_cnt)) + print(f"Object Counter: Pad Idx={frame_meta.pad_index}," + f"Frame Number={frame_meta.frame_number}," + f"Vehicle Count={vehicle_count}, Person Count={person_count}") + display_text = f"Person={person_count},Vehicle={vehicle_count}" + display_meta = batch_meta.acquire_display_meta() + text = osd.Text() + text.display_text = display_text.encode('ascii') + text.x_offset = 10 + text.y_offset = 12 + text.font.name = osd.FontFamily.Serif + text.font.size = 12 + text.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + text.set_bg_color = True + text.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(text) + frame_meta.append(display_meta) + +def main(file_path): + if isinstance(file_path, list): + file_list = file_path if isinstance(file_path, list) else [file_path] + pipeline = Pipeline(PIPELINE_NAME) + pipeline.add("nvstreammux", "mux", {"batch-size": len(file_list), "batched-push-timeout": BATCHED_PUSH_TIMEOUT, "width": MUXER_WIDTH, "height": MUXER_HEIGHT, "compute-hw": 1, "nvbuf-memory-type": 2}) + for i, file in enumerate(file_list): + pipeline.add("uridecodebin", f"src_{i}", {"uri": file}) + pipeline.link((f"src_{i}", "mux"), ("", "sink_%u")) + pipeline.add("nvinfer", "infer", {"config-file-path": CONFIG_FILE_PATH, "batch-size": len(file_list)}) + pipeline.add("nvtracker", "tracker", {"ll-config-file": TRACKER_LL_CONFIG_FILE, "ll-lib-file": TRACKER_LL_LIB_FILE}) + pipeline.add("nvdsanalytics", "analytics", {"config-file": ANALYTICS_CONFIG_FILE_PATH}) + pipeline.add("nvmultistreamtiler", "tiler", {"width": TILER_WIDTH, "height": TILER_HEIGHT}) + pipeline.add("nvosdbin", "osd").add("nv3dsink" if platform.processor() == "aarch64" else "nveglglessink", "sink") + pipeline.link("mux", "infer", "tracker", "analytics", "tiler", "osd", "sink") + pipeline.attach("tiler", Probe("counter", ObjectCounterMarker())) + pipeline.start().wait() + +if __name__ == '__main__': + # Check input arguments + if len(sys.argv) < 2: + sys.stderr.write("usage: %s [uri2] ... [uriN]\n" % sys.argv[0]) + sys.exit(1) + + # pipeline.wait() in the main function is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=main, args=(sys.argv[1:],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + print("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/reference_apps/runtime_source_add_delete/Makefile b/src/apps/reference_apps/runtime_source_add_delete/Makefile new file mode 100644 index 00000000..6bb817d7 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/Makefile @@ -0,0 +1,62 @@ +################################################################################# +# SPDX-FileCopyrightText: Copyright (c) 2020-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +APP:= deepstream-test-rt-src-add-del + +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +DS_SDK_ROOT:=/opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DS_SDK_ROOT)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= -I$(DS_SDK_ROOT)/sources/includes \ + -I /usr/local/cuda-$(CUDA_VER)/include + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS:= $(shell pkg-config --libs $(PKGS)) + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lm -lnvdsgst_meta \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +.o: .c $(INCS) Makefile + $(CC) -c $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) diff --git a/src/apps/reference_apps/runtime_source_add_delete/README.md b/src/apps/reference_apps/runtime_source_add_delete/README.md new file mode 100644 index 00000000..8a3cd37d --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/README.md @@ -0,0 +1,45 @@ +# RUNTIME SOURCE ADDITION DELETION REFERENCE APP USING DEEPSTREAMSDK 9.0 + +## Introduction +The project contains Runtime source addition/deletion application to show the +capability of Deepstream SDK. + +## Prerequisites: +DeepStream SDK installed which is available at http://developer.nvidia.com/deepstream-sdk +Please follow instructions in the apps/sample_apps/deepstream-app/README on how +to install the prequisites for Deepstream SDK apps. + +## Getting Started + +- Preferably clone the app in + `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/` + +- Edit all the inference models config files according to the location of the models to be used + +## Compilation Steps and Execution: +``` + $ Set CUDA_VER in the MakeFile as per platform. + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 + $ sudo make + + $ ./deepstream-test-rt-src-add-del + $ ./deepstream-test-rt-src-add-del file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 0 nveglglessink 1 #dGPU - nveglglessink Jetson - nv3dsink + $ ./deepstream-test-rt-src-add-del rtsp://127.0.0.1/video 0 nveglglessink 1 #dGPU +``` + +The application demonstrates following pipeline for single source + +uridecodebin -> nvstreammux -> nvinfer -> nvtracker -> nvtiler -> nvvideoconvert -> nvdsosd -> displaysink + +- At runtime after a timeout a source will be added periodically. All the components + are reconfigured during addition/deletion +- After reaching of `MAX_NUM_SOURCES`, each source is deleted periodically till single + source is present in the pipeline +- The app exits, when final source End of Stream is reached or if the last source is deleted. +- filesink and nv3dsink (only Jetson) are also supported. + + + + + diff --git a/src/apps/reference_apps/runtime_source_add_delete/deepstream_test_rt_src_add_del.c b/src/apps/reference_apps/runtime_source_add_delete/deepstream_test_rt_src_add_del.c new file mode 100644 index 00000000..3d368679 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/deepstream_test_rt_src_add_del.c @@ -0,0 +1,725 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "gst-nvmessage.h" +#include "nvdsmeta.h" +#include + +#define MAX_DISPLAY_LEN 64 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 +#define SET_GPU_ID(object, gpu_id) g_object_set (G_OBJECT (object), "gpu-id", gpu_id, NULL); +#define SET_MEMORY(object, mem_id) g_object_set (G_OBJECT (object), "nvbuf-memory-type", mem_id, NULL); + +GMainLoop *loop = NULL; +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 + +#define TILED_OUTPUT_WIDTH 1280 +#define TILED_OUTPUT_HEIGHT 720 +#define GPU_ID 0 +#define MAX_NUM_SOURCES 4 +#define PGIE_CONFIG_FILE "dstest_pgie_config.txt" +#define TRACKER_CONFIG_FILE "dstest_tracker_config.txt" +#define SGIE1_CONFIG_FILE "dstest_sgie1_config.txt" +#define SGIE2_CONFIG_FILE "dstest_sgie2_config.txt" + + +#define CONFIG_GPU_ID "gpu-id" +#define CONFIG_GROUP_TRACKER "tracker" +#define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" +#define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" +#define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" +#define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" +#define CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS "enable-batch-process" + +gint g_num_sources = 0; +gint g_source_id_list[MAX_NUM_SOURCES]; +gboolean g_eos_list[MAX_NUM_SOURCES]; +gboolean g_source_enabled[MAX_NUM_SOURCES]; +GstElement **g_source_bin_list = NULL; +GMutex eos_lock; +gboolean g_run_forever = FALSE; + +/* Assuming Resnet 10 model packaged in DS SDK */ +gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", + "Roadsign" +}; + +GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, + *sgie1 = NULL, *sgie2 = NULL, + *nvvideoconvert = NULL, *nvosd = NULL, *tiler = NULL, *tracker = NULL, *queue = NULL; + +gchar *uri = NULL; + +static gboolean add_sources (gpointer data); + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("decodebin child added %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (g_strrstr (name, "nvv4l2decoder") == name) { + if(prop.integrated) { + g_object_set (object, "enable-max-performance", TRUE, NULL); + g_object_set (object, "bufapi-version", TRUE, NULL); + g_object_set (object, "drop-frame-interval", 0, NULL); + g_object_set (object, "num-extra-surfaces", 0, NULL); + } else { + g_object_set (object, "gpu-id", GPU_ID, NULL); + } + } +} + +static gchar * +get_absolute_file_path (gchar *cfg_file_path, gchar *file_path) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + if (file_path && file_path[0] == '/') { + return file_path; + } + + if (!realpath (cfg_file_path, abs_cfg_path)) { + g_free (file_path); + return NULL; + } + + /* Return absolute path of config file if file_path is NULL. */ + if (!file_path) { + abs_file_path = g_strdup (abs_cfg_path); + return abs_file_path; + } + + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL); + g_free (file_path); + + return abs_file_path; +} + + +static void +cb_newpad (GstElement * decodebin, GstPad * pad, gpointer data) +{ + GstCaps *caps = gst_pad_query_caps (pad, NULL); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + + g_print ("decodebin new pad %s\n", name); + if (!strncmp (name, "video", 5)) { + gint source_id = (*(gint *) data); + gchar pad_name[16] = { 0 }; + GstPad *sinkpad = NULL; + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_request_pad_simple (streammux, pad_name); + if (gst_pad_link (pad, sinkpad) != GST_PAD_LINK_OK) { + g_print ("Failed to link decodebin to pipeline\n"); + } else { + g_print ("Decodebin linked to pipeline\n"); + } + gst_object_unref (sinkpad); + } +} + +static GstElement * +create_uridecode_bin (guint index, gchar * filename) +{ + GstElement *bin = NULL; + gchar bin_name[16] = { }; + + g_print ("creating uridecodebin for [%s]\n", filename); + g_source_id_list[index] = index; + g_snprintf (bin_name, 15, "source-bin-%02d", index); + bin = gst_element_factory_make ("uridecodebin", bin_name); + g_object_set (G_OBJECT (bin), "uri", filename, NULL); + g_signal_connect (G_OBJECT (bin), "pad-added", + G_CALLBACK (cb_newpad), &g_source_id_list[index]); + g_signal_connect (G_OBJECT (bin), "child-added", + G_CALLBACK (decodebin_child_added), &g_source_id_list[index]); + g_source_enabled[index] = TRUE; + + return bin; +} + +static void +stop_release_source (gint source_id) +{ + GstStateChangeReturn state_return; + gchar pad_name[16]; + GstPad *sinkpad = NULL; + state_return = + gst_element_set_state (g_source_bin_list[source_id], GST_STATE_NULL); + switch (state_return) { + case GST_STATE_CHANGE_SUCCESS: + g_print ("STATE CHANGE SUCCESS\n\n"); + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_get_static_pad (streammux, pad_name); + gst_pad_send_event (sinkpad, gst_event_new_eos ()); + gst_pad_send_event (sinkpad, gst_event_new_flush_stop (FALSE)); + gst_element_release_request_pad (streammux, sinkpad); + g_print ("STATE CHANGE SUCCESS %p\n\n", sinkpad); + gst_object_unref (sinkpad); + gst_bin_remove (GST_BIN (pipeline), g_source_bin_list[source_id]); + source_id--; + g_num_sources--; + break; + case GST_STATE_CHANGE_FAILURE: + g_print ("STATE CHANGE FAILURE\n\n"); + break; + case GST_STATE_CHANGE_ASYNC: + g_print ("STATE CHANGE ASYNC\n\n"); + g_snprintf (pad_name, 15, "sink_%u", source_id); + sinkpad = gst_element_get_static_pad (streammux, pad_name); + gst_pad_send_event (sinkpad, gst_event_new_eos ()); + gst_pad_send_event (sinkpad, gst_event_new_flush_stop (FALSE)); + gst_element_release_request_pad (streammux, sinkpad); + g_print ("STATE CHANGE ASYNC %p\n\n", sinkpad); + gst_object_unref (sinkpad); + gst_bin_remove (GST_BIN (pipeline), g_source_bin_list[source_id]); + source_id--; + g_num_sources--; + break; + case GST_STATE_CHANGE_NO_PREROLL: + g_print ("STATE CHANGE NO PREROLL\n\n"); + break; + default: + break; + } + + +} + +static gboolean +delete_sources (gpointer data) +{ + gint source_id; + g_mutex_lock (&eos_lock); + for (source_id = 0; source_id < MAX_NUM_SOURCES; source_id++) { + if (g_eos_list[source_id] == TRUE && g_source_enabled[source_id] == TRUE) { + g_source_enabled[source_id] = FALSE; + stop_release_source (source_id); + } + } + g_mutex_unlock (&eos_lock); + + if (g_num_sources == 0) { + if (g_run_forever==FALSE){ + g_main_loop_quit (loop); + g_print ("All sources Stopped quitting\n"); + } + else { + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + } + return FALSE; + } + + do { + source_id = rand () % MAX_NUM_SOURCES; + } while (!g_source_enabled[source_id]); + g_source_enabled[source_id] = FALSE; + g_print ("Calling Stop %d \n", source_id); + stop_release_source (source_id); + + if (g_num_sources == 0) { + if (g_run_forever==FALSE){ + g_main_loop_quit (loop); + g_print ("All sources Stopped quitting\n"); + } + else { + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + } + return FALSE; + } + + return TRUE; +} + +static gboolean +add_sources (gpointer data) +{ + gint source_id = g_num_sources; + GstElement *source_bin; + GstStateChangeReturn state_return; + + do { + /* Generating random source id between 0 - MAX_NUM_SOURCES - 1, + * which has not been enabled + */ + source_id = rand () % MAX_NUM_SOURCES; + } while (g_source_enabled[source_id]); + g_source_enabled[source_id] = TRUE; + + g_print ("Calling Start %d \n", source_id); + source_bin = create_uridecode_bin (source_id, uri); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + g_source_bin_list[source_id] = source_bin; + gst_bin_add (GST_BIN (pipeline), source_bin); + state_return = + gst_element_set_state (g_source_bin_list[source_id], GST_STATE_PLAYING); + switch (state_return) { + case GST_STATE_CHANGE_SUCCESS: + g_print ("STATE CHANGE SUCCESS\n\n"); + source_id++; + break; + case GST_STATE_CHANGE_FAILURE: + g_print ("STATE CHANGE FAILURE\n\n"); + break; + case GST_STATE_CHANGE_ASYNC: + g_print ("STATE CHANGE ASYNC\n\n"); + state_return = + gst_element_get_state (g_source_bin_list[source_id], NULL, NULL, + GST_CLOCK_TIME_NONE); + source_id++; + break; + case GST_STATE_CHANGE_NO_PREROLL: + g_print ("STATE CHANGE NO PREROLL\n\n"); + break; + default: + break; + } + g_num_sources++; + + + if (g_num_sources == MAX_NUM_SOURCES) { + /* We have reached MAX_NUM_SOURCES to be added, no stop calling this function + * and enable calling delete sources + */ + g_timeout_add_seconds (5, delete_sources, (gpointer) g_source_bin_list); + return FALSE; + } + + return TRUE; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + if (g_run_forever==FALSE){ + g_print ("End of stream\n"); + g_main_loop_quit (loop); + } + break; + case GST_MESSAGE_WARNING: + { + gchar *debug; + GError *error; + gst_message_parse_warning (msg, &error, &debug); + g_printerr ("WARNING from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + g_free (debug); + g_printerr ("Warning: %s\n", error->message); + g_error_free (error); + break; + } + case GST_MESSAGE_ERROR: + { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + case GST_MESSAGE_ELEMENT: + { + if (gst_nvmessage_is_stream_eos (msg)) { + guint stream_id; + if (gst_nvmessage_parse_stream_eos (msg, &stream_id)) { + g_print ("Got EOS from stream %d\n", stream_id); + g_mutex_lock (&eos_lock); + g_eos_list[stream_id] = TRUE; + g_mutex_unlock (&eos_lock); + } + } + break; + } + default: + break; + } + return TRUE; +} + + +/* Tracker config parsing */ + +#define CHECK_ERROR(error) \ + if (error) { \ + g_printerr ("Error while parsing config file: %s\n", error->message); \ + goto done; \ + } + +static gboolean +set_tracker_properties (GstElement *nvtracker) +{ + gboolean ret = FALSE; + GError *error = NULL; + gchar **keys = NULL; + gchar **key = NULL; + GKeyFile *key_file = g_key_file_new (); + + if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE, + &error)) { + if (error) { + g_printerr ("Failed to load config file: %s\n", error->message); + g_error_free (error); + } else { + g_printerr ("Failed to load config file.\n"); + } + return FALSE; + } + + keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error); + CHECK_ERROR (error); + + for (key = keys; *key; key++) { + if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) { + gint width = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_WIDTH, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) { + gint height = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_HEIGHT, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) { + guint gpu_id = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GPU_ID, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { + char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL); + g_free(ll_config_file); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { + char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE, + g_key_file_get_string (key_file, + CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL); + g_free(ll_lib_file); + } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS)) { + gboolean enable_batch_process = + g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, + CONFIG_GROUP_TRACKER_ENABLE_BATCH_PROCESS, &error); + CHECK_ERROR (error); + g_object_set (G_OBJECT (nvtracker), "enable_batch_process", + enable_batch_process, NULL); + } else { + g_printerr ("Unknown key '%s' for group [%s]", *key, + CONFIG_GROUP_TRACKER); + } + } + + ret = TRUE; +done: + if (error) { + g_error_free (error); + } + if (keys) { + g_strfreev (keys); + } + if (key_file) { + g_key_file_free (key_file); + } + if (!ret) { + g_printerr ("%s failed", __func__); + } + return ret; +} + +int +main (int argc, char *argv[]) +{ + GstBus *bus = NULL; + guint bus_watch_id; + guint i, num_sources; + guint tiler_rows, tiler_columns; + guint pgie_batch_size; + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + gboolean sync = TRUE; + gboolean display = TRUE; + + gboolean enc_hw_support = TRUE; + + if (prop.integrated) { + FILE* ptr; + char device_name[50]; + ptr = fopen("/proc/device-tree/model", "r"); + + if(ptr){ + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } + + /* Check input arguments */ + if ((argc != 5)) { + g_printerr ("Usage: %s \n", argv[0]); + g_printerr (" : 0 or 1 \n"); + g_printerr (" : filesink (generates test.mkv) or nveglglessink (dGPU) or nv3dsink (Jetson)\n"); + g_printerr (" : 0 or 1 \n\n"); + g_printerr ("example: %s file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 0 filesink 1\n", argv[0]); + return -1; + } + if (!strcmp(argv[3],"filesink")){ + display = FALSE; + } + else if (!strcmp(argv[3],"nveglglessink") || !strcmp(argv[3],"nv3dsink") ){ + display = TRUE; + } + else { + g_printerr ("Error: set correct sink: filesink, nveglglessink or nv3dsink\n"); + return -1; + } + + num_sources = 1; + g_run_forever = atoi(argv[2]); + sync = atoi(argv[4]); + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + g_mutex_init (&eos_lock); + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("dstest-pipeline"); + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + g_object_set (G_OBJECT (streammux), "batched-push-timeout", 25000, NULL); + g_object_set (G_OBJECT (streammux), "batch-size", 30, NULL); + g_object_set (G_OBJECT (streammux), "drop-pipeline-eos", g_run_forever, NULL); + SET_GPU_ID (streammux, GPU_ID); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), streammux); + g_object_set (G_OBJECT (streammux), "live-source", 1, NULL); + + g_source_bin_list = g_malloc0 (sizeof (GstElement *) * MAX_NUM_SOURCES); + uri = g_strdup (argv[1]); + for (i = 0; i < /*num_sources */ 1; i++) { + GstElement *source_bin = create_uridecode_bin (i, argv[i + 1]); + if (!source_bin) { + g_printerr ("Failed to create source bin. Exiting.\n"); + return -1; + } + g_source_bin_list[i] = source_bin; + gst_bin_add (GST_BIN (pipeline), source_bin); + } + + g_num_sources = num_sources; + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file + */ + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + + /* Use nvtiler to stitch o/p from upstream components */ + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvideoconvert = + gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + tracker = gst_element_factory_make ("nvtracker", "nvtracker"); + + sgie1 = gst_element_factory_make ("nvinfer", "secondary-nvinference-engine1"); + sgie2 = gst_element_factory_make ("nvinfer", "secondary-nvinference-engine2"); + queue = gst_element_factory_make ("queue", "queue"); + + if (display){ + /* Finally render the osd output */ + if (prop.integrated) { + sink = gst_element_factory_make ("nv3dsink", "nv3dsink"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nv3dsink"); +#else + sink = gst_element_factory_make ("nveglglessink", "nveglglessink"); +#endif + } + } + else { + sink = gst_element_factory_make ("nvvideoencfilesinkbin","sink"); + g_object_set (G_OBJECT(sink), "container", 2, "output-file", "test.mkv", NULL); + if (!enc_hw_support){ + g_object_set(G_OBJECT(sink), "enc-type", 1, NULL); + } + } + + if (!pgie || !sgie1 || !sgie2 || !tiler || !nvvideoconvert || !nvosd + || !sink || !tracker) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (sgie1), "config-file-path", SGIE1_CONFIG_FILE, NULL); + g_object_set (G_OBJECT (sgie2), "config-file-path", SGIE2_CONFIG_FILE, NULL); + + /* Set necessary properties of the tracker element. */ + if (!set_tracker_properties(tracker)) { + g_printerr ("Failed to set tracker properties. Exiting.\n"); + return -1; + } + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size < MAX_NUM_SOURCES) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + g_object_set (G_OBJECT (pgie), "batch-size", MAX_NUM_SOURCES, NULL); + } + + /* Set GPU ID of elements */ + SET_GPU_ID (pgie, GPU_ID); + SET_GPU_ID (sgie1, GPU_ID); + SET_GPU_ID (sgie2, GPU_ID); + + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + /* we set the osd properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_columns, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + SET_GPU_ID (tiler, GPU_ID); + SET_GPU_ID (nvvideoconvert, GPU_ID); + SET_GPU_ID (nvosd, GPU_ID); + if(!prop.integrated) { +#ifndef __aarch64__ + SET_GPU_ID (sink, GPU_ID); +#endif + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), pgie, tracker, sgie1, sgie2, + tiler, nvvideoconvert, nvosd, queue, sink, NULL); + + /* we link the elements together */ + /* file-source -> h264-parser -> nvh264-decoder -> + * nvinfer -> nvvideoconvert -> nvosd -> video-renderer */ + if (!gst_element_link_many (streammux, pgie, tracker, sgie1, sgie2, queue, + tiler, nvvideoconvert, nvosd, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + g_object_set (G_OBJECT (sink), "sync", sync, "qos", FALSE, NULL); + + gst_element_set_state (pipeline, GST_STATE_PAUSED); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", argv[1]); + if (gst_element_set_state (pipeline, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + g_printerr ("Failed to set pipeline to playing. Exiting.\n"); + return -1; + } + //GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipeline), GST_DEBUG_GRAPH_SHOW_ALL, "ds-app-playing"); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_timeout_add_seconds (15, add_sources, (gpointer) g_source_bin_list); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + g_free (g_source_bin_list); + g_free (uri); + g_mutex_clear (&eos_lock); + return 0; +} + diff --git a/src/apps/reference_apps/runtime_source_add_delete/dstest_pgie_config.txt b/src/apps/reference_apps/runtime_source_add_delete/dstest_pgie_config.txt new file mode 100644 index 00000000..7edcef9e --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/dstest_pgie_config.txt @@ -0,0 +1,93 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.00392156862745098 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx +model-engine-file=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Primary_Detector/labels.txt +batch-size=30 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +#parse-bbox-func-name=NvDsInferParseCustomResnet +#custom-lib-path=/path/to/libnvdsparsebbox.so +#enable-dbscan=1 +cluster-mode=2 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 +#minBoxes=3 +roi-top-offset=0 +roi-bottom-offset=0 +detected-min-w=0 +detected-min-h=0 +detected-max-w=0 +detected-max-h=0 + +## Per class configuration +#[class-attrs-2] +#pre-cluster-threshold=0.6 +#eps=0.5 +#group-threshold=3 +#roi-top-offset=20 +#roi-bottom-offset=10 +#detected-min-w=40 +#detected-min-h=40 +#detected-max-w=400 +#detected-max-h=800 diff --git a/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie1_config.txt b/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie1_config.txt new file mode 100644 index 00000000..fc2a14b5 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie1_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=1 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleMake/resnet18_vehiclemakenet_pruned.onnx +model-engine-file=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleMake/resnet18_vehiclemakenet_pruned.onnx_b16_gpu0_fp16.engine +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleMake/labels.txt +batch-size=16 +# 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +input-object-min-width=64 +input-object-min-height=64 +process-mode=2 +model-color-format=1 +gpu-id=0 +gie-unique-id=2 +operate-on-gie-id=1 +operate-on-class-ids=0 +is-classifier=1 +classifier-async-mode=0 +classifier-threshold=0.51 +process-mode=2 diff --git a/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie2_config.txt b/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie2_config.txt new file mode 100644 index 00000000..c102a620 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/dstest_sgie2_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path, +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=1 +onnx-file=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleTypes/resnet18_vehicletypenet_pruned.onnx +model-engine-file=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleTypes/resnet18_vehicletypenet_pruned.onnx_b16_gpu0_fp16.engine +labelfile-path=/opt/nvidia/deepstream/deepstream/samples/models/Secondary_VehicleTypes/labels.txt +batch-size=16 +# 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +input-object-min-width=64 +input-object-min-height=64 +process-mode=2 +model-color-format=1 +gpu-id=0 +gie-unique-id=3 +operate-on-gie-id=1 +operate-on-class-ids=0 +is-classifier=1 +classifier-async-mode=0 +classifier-threshold=0.51 +process-mode=2 diff --git a/src/apps/reference_apps/runtime_source_add_delete/dstest_tracker_config.txt b/src/apps/reference_apps/runtime_source_add_delete/dstest_tracker_config.txt new file mode 100644 index 00000000..324eada9 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/dstest_tracker_config.txt @@ -0,0 +1,38 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Mandatory properties for the tracker: +# tracker-width +# tracker-height: needs to be multiple of 32 for NvDCF +# gpu-id +# ll-lib-file: path to low-level tracker lib +# ll-config-file: required for NvDCF, optional for KLT and IOU +# +[tracker] +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=config_tracker_IOU.yml +ll-config-file=config_tracker_NvDCF_perf.yml +# ll-config-file=config_tracker_NvDCF_accuracy.yml +# ll-config-file=config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 diff --git a/src/apps/reference_apps/runtime_source_add_delete/tracker_config.yml b/src/apps/reference_apps/runtime_source_add_delete/tracker_config.yml new file mode 100644 index 00000000..4c03f7a7 --- /dev/null +++ b/src/apps/reference_apps/runtime_source_add_delete/tracker_config.yml @@ -0,0 +1,36 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +%YAML:1.0 + +NvDCF: + maxTargetsPerStream: 30 # Max number of targets to track per stream. Recommended to set >10. Note: this value should account for the targets being tracked in shadow mode as well. Max value depends on the GPU memory capacity + + filterLr: 0.11 # learning rate for DCF filter in exponential moving average. Valid Range: [0.0, 1.0] + gaussianSigma: 0.75 # Standard deviation for Gaussian for desired response when creating DCF filter + + minDetectorConfidence: 0.0 # If the confidence of a detector bbox is lower than this, then it won't be considered for tracking + minTrackerConfidence: 0.6 # If the confidence of an object tracker is lower than this on the fly, then it will be tracked in shadow mode. Valid Range: [0.0, 1.0] + + featureImgSizeLevel: 1 # Size of a feature image. Valid range: {1, 2, 3, 4, 5}, from the smallest to the largest + SearchRegionPaddingScale: 3 # Search region size. Determines how large the search region should be scaled from the target bbox. Valid range: {1, 2, 3}, from the smallest to the largest + + maxShadowTrackingAge: 9 # Max length of shadow tracking (the shadow tracking age is incremented when (1) there's detector input yet no match or (2) tracker confidence is lower than minTrackerConfidence). Once reached, the tracker will be terminated. + probationAge: 0 # Once the tracker age (incremented at every frame) reaches this, the tracker is considered to be valid + earlyTerminationAge: 0 # Early termination age (in terms of shadow tracking age) during the probation period + + minVisibiilty4Tracking: 0.5 # If the visibility of the bbox of a tracker gets lower, then it will be terminated diff --git a/src/apps/tao_apps/.gitattributes b/src/apps/tao_apps/.gitattributes new file mode 100644 index 00000000..31334c95 --- /dev/null +++ b/src/apps/tao_apps/.gitattributes @@ -0,0 +1 @@ +sample/streams/retail_object_h264.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/src/apps/tao_apps/CLA.md b/src/apps/tao_apps/CLA.md new file mode 100644 index 00000000..389a4746 --- /dev/null +++ b/src/apps/tao_apps/CLA.md @@ -0,0 +1,58 @@ +## Individual Contributor License Agreement (CLA) + +**Thank you for submitting your contributions to this project.** + +By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions +to the project. + +### License. + +You hereby represent that all present, past and future contributions are governed by the +[MIT License](https://opensource.org/licenses/MIT) +copyright statement. + +This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights +of the code or documents you contribute to the project itself or its maintainers. +Furthermore you also represent that you have the authority to perform the above waiver +with respect to the entirety of you contributions. + +### Moral Rights. + +To the fullest extent permitted under applicable law, you hereby waive, and agree not to +assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. + +### Third Party Content. + +If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, +specifications, documentation, data, materials, feedback, information or other works of authorship that were not +authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary +rights associated with your Contribution (“Third Party Rights”), +then you agree to include with the submission of your Contribution full details respecting such Third Party +Content and Third Party Rights, including, without limitation, identification of which aspects of your +Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the +Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable +third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater +certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights +do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. + +### Representations. + +You represent that, other than the Third Party Content and Third Party Rights identified by +you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled +to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were +created in the course of your employment with your past or present employer(s), you represent that such +employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer +(s) has waived all of their right, title or interest in or to your Contributions. + +### Disclaimer. + +To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" +basis, without any warranties or conditions, express or implied, including, without limitation, any implied +warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not +required to provide support for your Contributions, except to the extent you desire to provide support. + +### No Obligation. + +You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions +into the project. The decision to use or incorporate your contributions into the project will be made at the +sole discretion of the maintainers or their authorized delegates. diff --git a/src/apps/tao_apps/LICENSE.md b/src/apps/tao_apps/LICENSE.md new file mode 100644 index 00000000..c3e46255 --- /dev/null +++ b/src/apps/tao_apps/LICENSE.md @@ -0,0 +1,7 @@ +Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/apps/tao_apps/Makefile b/src/apps/tao_apps/Makefile new file mode 100644 index 00000000..d873dca3 --- /dev/null +++ b/src/apps/tao_apps/Makefile @@ -0,0 +1,39 @@ +################################################################################ +# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +.PHONY: all install clean + +all: + $(MAKE) -C post_processor + $(MAKE) -C apps + +install: + $(MAKE) -C post_processor install + $(MAKE) -C apps install + +clean: + $(MAKE) -C post_processor clean + $(MAKE) -C apps clean + +%: + $(MAKE) -C post_processor $@ + $(MAKE) -C apps $@ diff --git a/src/apps/tao_apps/README.md b/src/apps/tao_apps/README.md new file mode 100644 index 00000000..2b538823 --- /dev/null +++ b/src/apps/tao_apps/README.md @@ -0,0 +1,294 @@ +# Integrate TAO model with DeepStream SDK + +- [Integrate TAO model with DeepStream SDK](#integrate-tao-model-with-deepstream-sdk) + - [Description](#description) + - [Prerequisites](#prerequisites) + - [Download](#download) + - [1. Download Source Code with SSH or HTTPS](#1-download-source-code-with-ssh-or-https) + - [2. Download Models](#2-download-models) + - [Triton Inference Server](#triton-inference-server) + - [Build](#build) + - [Build Sample Application](#build-sample-application) + - [Run](#run) + - [Information for Customization](#information-for-customization) + - [TAO Models](#tao-models) + - [Label Files](#label-files) + - [DeepStream configuration file](#deepstream-configuration-file) + - [Model Outputs](#model-outputs) + - [1. CitySemSegFormer](#1-citysemsegformer) + - [2. PoseClassification](#2-poseclassification) + - [3~4. Retail Object Detection / PeopleNet Transformer](#34-retail-object-detection--peoplenet-transformer) + - [5~6. Re-Identification / Retail Item Recognition](#56-re-identification--retail-item-recognition) + - [7. BodyPose3DNet](#7-bodypose3dnet) + - [8~9. OCDNet / OCRNet](#89-ocdnet--ocrnet) + - [10~11. LPDNet / LPRNet](#1011-lpdnet--lprnet) + - [12. Mask2Former](#12-mask2former) + - [FAQ](#faq) + - [Measure The Inference Perf](#measure-the-inference-perf) + - [About misc folder](#about-misc-folder) + - [Others Models](#others-models) + - [TAO pyservicemaker samples](#tao-pyservicemaker-samples) + - [Known issues](#known-issues) + +## Description + +This repository provides a DeepStream sample application based on [NVIDIA DeepStream SDK](https://developer.nvidia.com/deepstream-sdk) to run twelve TAO models ( **Peoplenet Transformer** / **CitySemSegformer** / **Mask2Former** /**Reidentification** / **Bodypose3dnet** / **Pose Classifier** / **OCDNet**/ **OCRNet**/ **LPDNet** / **LPRNet** / **Retail Detector**/ **Retail Object Recognition** ) with below files: + +- **apps**: sample application for detection models and segmentation models +- **configs**: DeepStream nvinfer configure file and label files +- **post_processor**: include inference postprocessor for the models +- **models**: The models which will be used as samples. +- **TRT-OSS**: The OSS nvinfer plugin build and download instructions. The OSS plugins are needed for some models with DeepStream 7.1 GA. + +The pipeline of the sample: + +``` + |-->filesink(save the output in local dir) + |--> encode --> + |-->fakesink(use -f option) +uridecoderbin -->streammux-->nvinfer(detection)-->nvosd--> + |--> display +``` + +## Prerequisites + +* [DeepStream SDK 9.0 GA](https://developer.nvidia.com/deepstream-sdk) + + Make sure deepstream-test1 sample can run successful to verify your installation. According to the + [document](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html), + please run below command to install additional audio video packages. + + ``` + /opt/nvidia/deepstream/deepstream/user_additional_install.sh + ``` +* Eigen development packages + ``` + sudo apt install libeigen3-dev + cd /usr/include + sudo ln -sf eigen3/Eigen Eigen + ``` + +## Download + +### 1. Download Source Code with SSH or HTTPS + +``` +sudo apt update +sudo apt install git-lfs +git lfs install --skip-repo +git clone https://github.com/NVIDIA/deepstream.git +//download submoudle +git submodule update --init --recursive +``` +### 2. Download Models +Run below script to download models except multi_task and YoloV5 models. + +``` +sudo ./download_models.sh # (sudo not required in case of docker containers) +``` + +### 3. Download Pre-built TensorRT OSS nvinfer plugin library + +Please download the TensorRT OSS plugin according to your platforms + +[x86 platform TRT OSS plugin download instruction](TRT-OSS/x86/README.md) + +[Jetson platform TRT OSS plugin download instruction](TRT-OSS/Jetson/README.md) + +## Triton Inference Server + +The sample provides three inferencing methods. For the TensorRT based gst-nvinfer inferencing, please skip this part. + +The DeepStream sample application can work as Triton client with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), one of the following two methods can be used to set up the Triton Inference Server before starting a gst-nvinferserver inferncing DeepStream application. + + - Native Triton Inference Server, please refer to [Triton Server](triton_server.md) + - Stand-alone Triton Inference server, please refer to [Triton grpc server](triton_server_grpc.md) + +For the TAO sample applications, please enable Triton or Triton gRPC inferencing with the app YAML configurations. + +E.G. With apps/tao_detection/ds-tao-detection, the "primary-gie" part in configs/app/det_app_config.yml can be modified as following: + +``` +primary-gie: + #0:nvinfer, 1:nvinfeserver + plugin-type: 1 + + #peoplenet-transformer + #config-file-path: ../nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + config-file-path: ../triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + #config-file-path: ../triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + +``` +And then run the app with the command: + +``` +./apps/tao_detection/ds-tao-detection configs/app/det_app_config.yml +``` + +## Build + +### Build Sample Application + +``` +export CUDA_MODULE_LOADING=LAZY +$ export CUDA_VER= + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 +make +``` +## Run + +``` + +1.Usage: ds-tao-detection -c pgie_config_file -i [-b BATCH] [-d] [-f] [-l] + -h: print help info + -c: pgie config file, e.g. pgie_frcnn_tao_config.txt + -i: uri of the input file, start with the file:///, e.g. file:///.../video.mp4 + -b: batch size, this will override the value of "batch-size" in pgie config file + -d: enable display, otherwise it will dump to output MP4 or JPEG file without -f option + -f: use fakesink mode + -l: use loop mode + +2.Usage: ds-tao-detection + e.g. + ./apps/tao_detection/ds-tao-detection configs/app/det_app_config.yml + + +note: If you want use multi-source, you can input multi -i input(e.g., -i uri -i uri...) + Only YAML configurations support Triton and Triton gRPC inferencing. +``` +For detailed model information, pleasing refer to the following table: + +note: + The default $DS_SRC_PATH is /opt/nvidia/deepstream/deepstream + +|Model Type|Tao Model|Demo| +|-----------|----------|----| +|detector|peoplenet_transformer, retail_detector_binary|./apps/tao_detection/ds-tao-detection -c configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt -i file:///$DS_SRC_PATH/samples/streams/sample_720p.mp4
or
./apps/tao_detection/ds-tao-detection configs/app/det_app_config.yml| +|segmentation|citySemSegFormer|./apps/tao_segmentation/ds-tao-segmentation -c configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.txt -i file:///$DS_SRC_PATH/samples/streams/sample_720p.mp4 -w 1820 -e 1024
or
./apps/tao_segmentation/ds-tao-segmentation configs/app/seg_app_config.yml| +|instance segmentation|Mask2Former|export SHOW_MASK=1; ./apps/tao_detection/ds-tao-detection -c configs/nvinfer/mask2former_tao/pgie_mask2former_tao_config.yml -i file:///$DS_SRC_PATH/samples/streams/sample_720p.mp4
or
export SHOW_MASK=1; ./apps/tao_detection/ds-tao-detection configs/app/ins_seg_app.yml| +|others|Re-identification, Retail Object Recognition, PoseClassificationNet, OCDNet, OCRNet, LPDNet, LPRNet, BodyPose3DNet|refer detailed [README](apps/tao_others/README.md) for how to configure and run the model| + +Building the TensorRT engine of citySemSegFormer consumes a lot of device memory. Please `export CUDA_MODULE_LOADING=LAZY` to reduce device memory consumption. Please read [CUDA Environment Variables](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars) for details. + +## Information for Customization + +If you want to do some customization, such as training your own TAO models, running the model in other DeepStream pipeline, you should read below sections. +### TAO Models + +Refer [TAO Doc](https://docs.nvidia.com/tao/tao-toolkit/text/overview.html) for how to train the models, after training finishes, run `tao-export` to generate an ONNX model. This ONNX model can be deployed into DeepStream for fast inference as this sample shows. +This DeepStream sample app also supports the TensorRT engine(plan) file generated by running the `trtexec` tool on the ONNX model. +The TensorRT engine file is hardware dependent, while the ONNX model is not. You may specify either a TensorRT engine file or a ONNX model in the DeepStream configuration file. +### Label Files + +The label file includes the list of class names for a model, which content varies for different models. +User can find the detailed label information for the MODEL in the README.md and the label file under *configs/$(MODEL)_tao/*, e.g. ssd label informantion under *configs/peoplenet_transformer_tao/* + +### DeepStream configuration file + +The DeepStream configuration file includes some runtime parameters for DeepStream **nvinfer** plugin or **nvinferserver** plugin, such as model path, label file path, TensorRT inference precision, input and output node names, input dimensions and so on. +In this sample, each model has its own DeepStream configuration file, e.g. pgie_peoplenet_transformer_tao_config.txt for peoplenet transformer model. +Please refer to [DeepStream Development Guide](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinfer.html#gst-nvinfer-file-configuration-specifications) for detailed explanations of those parameters. + +### Model Outputs + +#### 1. CitySemSegFormer + +- **argmax_1/output**: A [batchSize, H, W, 1] tensor containing the class id per pixel location + +#### 2. PoseClassification + +- refer detailed [README](apps/tao_others/README.md) for how to configure and run the model + +#### 3~4. Retail Object Detection / PeopleNet Transformer + +The model has the following two outputs: + +- **pred_logits**: This is a [batch_size, num_queries, num_classes] tensor of data type float32. The +tensor contains probability values of each class. +- **pred_boxes**: This is a [batch_size, num_queries, 4] tensor of data type float32. The tensor +represents the 2D bounding box coordinates in the format of [center_x, center_y, width, height]. + +#### 5~6. Re-Identification / Retail Item Recognition + +These models are trained to extract the embedding vector from an image. The image is the cropped area of a +bounding box from a primary-gie task, like people detection by `PeopleNet Transformer` or retail item detection +by `Retail Object Detection`. These embedding extraction models are typically arranged +as the secondary GIE module in a Deepstream pipeline. + +##### Re-Identification uses ResNet50 backbone. +The output layer is: +- **fc_pred**: This is a [batch_size, embedding_size] tensor of data type float32. The tensor +contains the embedding vector of size `embedding_size = 256`. + +##### Retail Item Recognition uses ResNet101 backbone. +The output layer is: +- **outputs**: This is a [batch_size, 2048] tensor of data type float32. The tensor contains the embedding +vector of size `2048`. + +#### 7. BodyPose3DNet +The BodyPose3DNet models are used for 3D human pose estimation network, which aims to predict the skeleton for every person in a given input image which consists of keypoints and the connections between them. + +The model's output layers: +- **pose2d**: The two dimension coordinates and probabilities of the keypoints which are in the [batch_size, keypoint_num, 3] tensor. +- **pose2d_org_img**: The two dimension coordinates and probabilities of the keypoints in the original image which are in the [batch_size, keypoint_num, 3] tensor. +- **pose25d**: The 2.5 dimension coordinates and probabilities of the keypoints in the original image which are in the [batch_size, keypoint_num, 4] tensor. +- **pose3d**: The 3 dimension coordinates of the keypoints which are in the [batch_size, keypoint_num, 3] tensor. + +#### 8~9. OCDNet / OCRNet + +##### OCDNet output layer +- **pred**:This is a [batchSize, H, W , 1] float tensor containing the probability of a pixel belongs to text +##### OCRNet output layer +- **output_prob**:This is a [batchSize, W/4] float tensor containing the confidence of each character in the text +- **output_id**:This is a [batchSize, W/4] integer tensor containing each character index of the text. This index can be mapped to character through the OCRNet charater list + +#### 10~11. LPDNet / LPRNet + +##### LPDNet output +- Category labels (lpd) and bounding-box coordinates for each detected license plate in the input image. + +##### LPRNet output +- characters id sequence. (DeepStream post-process plugin is needed to get the final license plate) + +#### 12. Mask2Former +The model has the following three outputs: + +- **pred_classes**: A [100] tensor containing the prediction classes +- **pred_masks**: A [100x800x800] tensor containing the prediction masks +- **pred_scores**: A [100] tensor containing the prediction scores +## FAQ + +### Measure The Inference Perf + +```CQL +# 1. Build TensorRT Engine through this smample, for example, build YoloV3 with batch_size=2 +./apps/tao_detection/ds-tao-detection -c configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt -i file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 -b 2 +## after this is done, it will generate the TRT engine file under models/$(MODEL), e.g. models/peoplenet_transformer/ for above command. +# 2. Measure the Inference Perf with trtexec, following above example +cd models/peoplenet_transformer/ +/usr/src/tensorrt/bin/trtexec --useSpinWait --loadEngine=resnet50_peoplenet_transformer_op17.onnx_b2_gpu0_fp16.engine +## then you can find the per *BATCH* inference time in the trtexec output log +``` + +### About misc folder + +```CQL +# The files in the folder are used by TAO dev blogs: +## 1. Training State-Of-The-Art Models for Classification and Object Detection with NVIDIA TAO Toolkit +## 2. Real time vehicle license plate detection and recognition using NVIDIA TAO Toolkit +``` +## Others Models + +There are some special models which are not exactly detector, classifier or segmetation. The sample application of these special models are put in apps/tao_others. These samples should run on DeepStream 6.1 or above versions. Please refer to apps/tao_others/README.md document for details. + +## TAO pyservicemaker samples + +PyServiceMaker is a python framework that aims to streamline the development process for Deepstream applications. It provides a convenient and efficient means to rapidly construct application prototypes, enabling developers to expedite the prototyping phase and focus on the core functionality of their projects. + +Please refer to [REAME](pysm-apps/README.md) file for more details. + +## Known issues + +1. Currently the citySemSegFormer model only supports batch-size 1. +2. If the segmentation results can't overlay the entire frame, please set SEG_OUTPUT_WIDTH/SEG_OUTPUT_HEIGHT to the model's width/height. diff --git a/src/apps/tao_apps/TRT-OSS/Jetson/README.md b/src/apps/tao_apps/TRT-OSS/Jetson/README.md new file mode 100644 index 00000000..1273c564 --- /dev/null +++ b/src/apps/tao_apps/TRT-OSS/Jetson/README.md @@ -0,0 +1,66 @@ +# Build Jetson TensorRT OSS Plugin + +For DeepStream 6.1.1 GA and 6.2 GA, the TensorRT OSS plugin is not needed. + +Below are the steps to build [TensorRT OSS](https://github.com/NVIDIA/TensorRT) on Jetson device for Jetson libnvinfer_plugin.so. For cross-compiling, refer to TensorRT OSS README. + +## libnvinfer_plugin.so.8.6.2 Provided Here + + **libnvinfer_plugin.so.8.6.2** provided in this folder was built with: + +> Jetson Xavier +> Jetpack GA (CUDA-12.2, cuDNN v8.6, TensorRT8.6.1.2 ) + +**Note** + +You can get the prebuild lib by using "wget https://nvidia.box.com/shared/static/1ih810ui4z52nvwznfk1xaxvdej3mauw -O libnvinfer_plugin.so.8.6.2" if you met some LFS issue. + + +## Build TensorRT OSS Plugin - libnvinfer_plugin.so + +The TensorRT OSS source code does not contains the patch for 8.6 branch +### Get TensorRT OSS Plugin Library + +| DeepStream Release | Jetpack Version | TRT Version | TRT_OSS_CHECKOUT_TAG | +| ------------------- | --------------- | --------------- | --------------------- | +| 5.0 | 4.4 GA - 4.5 | TRT 7.1.3 | release/7.1 | +| 5.0.1 | 4.4 GA - 4.5 | TRT 7.1.3 | release/7.1 | +| 5.1 | 4.5.1 | TRT 7.1.3 | release/7.1 | +| 6.0 EA | 4.5.1 | TRT 7.1.3 | release/7.1 | +| 6.0 GA | 4.6 | TRT 8.0.1 | release/8.0 | +| 6.0.1 | 4.6.1 / 4.6.2 | TRT 8.2.1 | release/8.2 | +| 6.1 | 5.0.1 | TRT 8.4.0.11 | release/8.4 | +| 6.1.1 | 5.0.2 | TRT 8.4.1 | OSS not needed | +| 6.2 | 5.1 | TRT 8.5.1 | OSS not needed | +| 6.3 | 5.1.2 | TRT 8.5.3 | OSS not needed | +| 6.4 | 6.0 | TRT 8.6.1.2 | binary only | +| 7.0 | 6.0 | TRT 8.6.1.2 | binary only | +``` + +### Replace "libnvinfer_plugin.so*" + +``` +sudo mv /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8.x.y ${HOME}/libnvinfer_plugin.so.8.x.y.bak // backup original libnvinfer_plugin.so.x.y +sudo cp $DOWNLOAD_PATH/libnvinfer_plugin.so.8.m.n /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8.x.y +sudo ldconfig +``` + +## GPU_ARCHS + +1. GPU_ARCHs value can be got from "deviceQuery" CUDA sample + +``` +cd /usr/local/cuda/samples/1_Utilities/deviceQuery +sudo make +./deviceQuery +``` + +2. Can also find GPU_ARCHs from below table + +| Jetson Platform | GPU_ARCHS | +| --------------- | --------- | +| TX1 / NANO | 53 | +| TX2 | 62 | +| Xavier / NX | 72 | +| Orin | 87 | + diff --git a/src/apps/tao_apps/TRT-OSS/Jetson/TRT7.1/README.md b/src/apps/tao_apps/TRT-OSS/Jetson/TRT7.1/README.md new file mode 100644 index 00000000..6d52c6b1 --- /dev/null +++ b/src/apps/tao_apps/TRT-OSS/Jetson/TRT7.1/README.md @@ -0,0 +1,58 @@ +# Build Jetson TensorRT OSS Plugin + +Below are the steps to build [TensorRT OSS](https://github.com/NVIDIA/TensorRT) on Jetson device for Jetson libnvinfer_plugin.so. For cross-compiling, refer to TensorRT OSS README. + +## libnvinfer_plugin.so.7.1.3 Provided Here + + **libnvinfer_plugin.so.7.1.3** provided in this folder was built with: + +> Jetson NX +> Jetpack4.4GA (CUDA-10.2, cuDNN v8.0, TensorRT 7.1.3) + +**Note** + +You can get teh prebuild lib using `wget https://nvidia.box.com/shared/static/ezrjriq08q8fy8tvqcswgi0u6yn0bomg.1 -O libnvinfer_plugin.so.7.0.0.1` if you met some LFS issue. + +## Build TensorRT OSS Plugin - libnvinfer_plugin.so + +### 1. Upgrade Cmake + +TensorRT OSS requires cmake \>= v3.13, while the default cmake on Jetson/UBuntu 18.04 is cmake 3.10.2, so upgrade it by + +``` +sudo dpkg --force-all -r cmake +wget https://github.com/Kitware/CMake/releases/download/v3.19.4/cmake-3.19.4.tar.gz +tar xvf cmake-3.19.4.tar.gz +cd cmake-3.19.4/ +./configure +make -j$(nproc) +sudo make install +sudo ln -s /usr/local/bin/cmake /usr/bin/cmake +``` + +### 2. Build TensorRT OSS Plugin + +``` +git clone -b release/7.1 https://github.com/nvidia/TensorRT +cd TensorRT/ +git submodule update --init --recursive +export TRT_SOURCE=`pwd` +cd $TRT_SOURCE +mkdir -p build && cd build +/usr/local/bin/cmake .. -DGPU_ARCHS="53 62 72" -DTRT_LIB_DIR=/usr/lib/aarch64-linux-gnu/ -DCMAKE_C_COMPILER=/usr/bin/gcc -DTRT_BIN_DIR=`pwd`/out +make nvinfer_plugin -j$(nproc) +``` + +After building ends successfully, libnvinfer_plugin.so* will be generated under `pwd`/out/. + +### 3. Replace "libnvinfer_plugin.so*" + +``` +sudo mv /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.7.x.y ${HOME}/libnvinfer_plugin.so.7.x.y.bak // backup original libnvinfer_plugin.so.x.y +sudo cp `pwd`/out/libnvinfer_plugin.so.7.m.n /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.7.x.y +sudo ldconfig +``` + +### Known issues + +If you ran into "[ERROR] IPluginV2DynamicExt requires network without implicit batch dimension" while running yolov3 model, please try to use the libnvinfer_plugin.so.7.1.3_nano_tx2_xavier_nx_for_yolov3 diff --git a/src/apps/tao_apps/TRT-OSS/x86/README.md b/src/apps/tao_apps/TRT-OSS/x86/README.md new file mode 100644 index 00000000..0dd568aa --- /dev/null +++ b/src/apps/tao_apps/TRT-OSS/x86/README.md @@ -0,0 +1,105 @@ +# Build x86 TensorRT OSS Plugin + +For DeepStream 6.1.1 GA, 6.2 GA and 6.3 GA, the TensorRT OSS plugin is not needed. + +Below are the steps to build [TensorRT OSS](https://github.com/NVIDIA/TensorRT) for x86 libnvinfer_plugin.so. For cross-compiling, refer to TensorRT OSS README. + +## libnvinfer_plugin.so.8.6.2 provided Here + + **libnvinfer_plugin.so.8.6.2** provided in this folder was built with: + +> Ubuntu 22.04 LTS +> cuda-12.1 +> cuDNN 8.9.3 +> TensorRT 8.6.2 + +**Note** + +You can get the prebuild lib using `wget https://nvidia.box.com/shared/static/eyqxd1g5kya51wk76i3st5e3m3xhyyfq libnvinfer_plugin.so.8.6.2` if you met some LFS issue. + +If the environment is different from above, you **MUST** build the TRT OSS plugin by yourself. + +## Build TensorRT OSS Plugin - libnvinfer_plugin.so + +Please refer to the Build Guidance under https://github.com/NVIDIA/TensorRT + +*Note:* +*Make sure the GPU_ARCHS of the GPU you are using is in TensorRT OSS [CMakeLists.txt](https://github.com/NVIDIA/TensorRT/blob/master/CMakeLists.txt#L84). If not, you need to specify "GPU_ARCHS" in the build command.* + +### 1. Installl Cmake (>= 3.13) + +For TensorRT8.6, please ignore this step. + +TensorRT OSS requires cmake >= v3.13, so install cmake 3.13 if your cmake version is lower than 3.13 + +``` +wget https://github.com/Kitware/CMake/releases/download/v3.19.4/cmake-3.19.4.tar.gz +tar xvf cmake-3.19.4.tar.gz +cd cmake-3.19.4/ +mkdir $HOME/install +./configure --prefix=$HOME/install +make -j$(nproc) +sudo make install +``` + +### 2. Build TensorRT OSS Plugin +| DeepStream Release | TRT Version | TRT_OSS_CHECKOUT_TAG | +| ------------------- | --------------- | --------------------- | +| 5.0 | TRT 7.0.0 | release/7.0 | +| 5.0.1 | TRT 7.0.0 | release/7.0 | +| 5.1 | TRT 7.2.X | 21.03 | +| 6.0 EA | TRT 7.2.2 | 21.03 | +| 6.0 GA | TRT 8.0.1 | release/8.0 | +| 6.0.1 | TRT 8.2.1 | release/8.2 | +| 6.1 | TRT 8.2.5.1 | release/8.2 | +| 6.1.1 GA | TRT 8.4.1.11 | no OSS plugin is needed | +| 6.2 GA | TRT 8.5.1 | no OSS plugin is needed | +| 6.3 | TRT 8.5.3 | no OSS plugin is needed | +| 6.4 | TRT 8.6.2 | binary plugin only | +| 7.0 | TRT 8.6.2 | binary plugin only | + + +### 3. Replace "libnvinfer_plugin.so*" + +``` +// backup original libnvinfer_plugin.so.x.y, e.g. libnvinfer_plugin.so.8.0.0 +sudo mv /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8.6.1 ${HOME}/libnvinfer_plugin.so.8.6.1.bak +// only replace the real file, don't touch the link files, e.g. libnvinfer_plugin.so, libnvinfer_plugin.so.8 +sudo cp TRT-OSS/x86/TRT8.6/libnvinfer_plugin.so.8.6.2 /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8.6.1 +sudo rm /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so +sudo rm /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8 +sudo ln -s /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8.6.1 /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8 +sudo ln -s /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8 /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so +sudo ldconfig +``` + +## How to Get GPU_ARCHS + +Can use either method to get GPU_ARCHs +1. GPU_ARCHS value can be got by "deviceQuery" CUDA sample + +``` +cd /usr/local/cuda/samples/1_Utilities/deviceQuery +sudo make +./deviceQuery +``` + +2. If there is not "/usr/local/cuda/samples" in your system, you could use the deviceQuery.cpp in this folder, + +``` +nvcc deviceQuery.cpp -o deviceQuery +./deviceQuery +``` + +There will be output like below, which indicates the "GPU_ARCHS" is **75**. + +``` +./deviceQuery + +Detected 2 CUDA Capable device(s) + +Device 0: "Tesla T4" + CUDA Driver Version / Runtime Version 10.2 / 10.2 + CUDA Capability Major/Minor version number: 7.5 +``` + diff --git a/src/apps/tao_apps/TRT-OSS/x86/deviceQuery.cpp b/src/apps/tao_apps/TRT-OSS/x86/deviceQuery.cpp new file mode 100644 index 00000000..6da5ce6e --- /dev/null +++ b/src/apps/tao_apps/TRT-OSS/x86/deviceQuery.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +int main(void) +{ + int deviceCount = 0; + cudaError_t error_id = cudaGetDeviceCount(&deviceCount); + + if (error_id != cudaSuccess) { + printf("cudaGetDeviceCount returned %d\n-> %s\n", + static_cast(error_id), cudaGetErrorString(error_id)); + printf("Result = FAIL\n"); + exit(EXIT_FAILURE); + } + + // This function call returns 0 if there are no CUDA capable devices. + if (deviceCount == 0) { + printf("There are no available device(s) that support CUDA\n"); + } else { + printf("Detected %d CUDA Capable device(s)\n", deviceCount); + } + + int dev, driverVersion = 0, runtimeVersion = 0; + + for (dev = 0; dev < deviceCount; ++dev) { + cudaSetDevice(dev); + cudaDeviceProp deviceProp; + cudaGetDeviceProperties(&deviceProp, dev); + + printf("\nDevice %d: \"%s\"\n", dev, deviceProp.name); + + // Console log + cudaDriverGetVersion(&driverVersion); + cudaRuntimeGetVersion(&runtimeVersion); + printf(" CUDA Driver Version / Runtime Version %d.%d / %d.%d\n", + driverVersion / 1000, (driverVersion % 100) / 10, + runtimeVersion / 1000, (runtimeVersion % 100) / 10); + printf(" CUDA Capability Major/Minor version number: %d.%d\n", + deviceProp.major, deviceProp.minor); + } + + return 0; +} diff --git a/src/apps/tao_apps/apps/Makefile b/src/apps/tao_apps/apps/Makefile new file mode 100644 index 00000000..b4a9bf70 --- /dev/null +++ b/src/apps/tao_apps/apps/Makefile @@ -0,0 +1,45 @@ +################################################################################ +# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +.PHONY: all install clean + +all: + $(MAKE) -C tao_detection + $(MAKE) -C tao_segmentation + $(MAKE) -C tao_classifier + $(MAKE) -C tao_others + +install: + $(MAKE) -C tao_detection install + $(MAKE) -C tao_segmentation install + +clean: + $(MAKE) -C tao_detection clean + $(MAKE) -C tao_segmentation clean + $(MAKE) -C tao_classifier clean + $(MAKE) -C tao_others clean + +%: + $(MAKE) -C tao_detection $@ + $(MAKE) -C tao_segmentation $@ + $(MAKE) -C tao_classifier $@ + $(MAKE) -C tao_others $@ diff --git a/src/apps/tao_apps/apps/tao_classifier/Makefile b/src/apps/tao_apps/apps/tao_classifier/Makefile new file mode 100644 index 00000000..c3eb4a40 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_classifier/Makefile @@ -0,0 +1,69 @@ +################################################################################ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= ds-tao-classifier + +CC:=g++ + +VERBOSE?=0 +ifeq ($(VERBOSE), 1) +AT= +else +AT=@ +endif + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes -I/usr/local/cuda/include +LIB_INSTALL_DIR?=$(DS_SRC_PATH)/lib/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= `pkg-config --cflags $(PKGS)` +CFLAGS+= -std=c++14 + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lyaml-cpp -Wl,-rpath,$(LIB_INSTALL_DIR) -lnvds_yml_parser + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/src/apps/tao_apps/apps/tao_classifier/deepstream_clas_app.c b/src/apps/tao_apps/apps/tao_classifier/deepstream_clas_app.c new file mode 100644 index 00000000..a03ffa8b --- /dev/null +++ b/src/apps/tao_apps/apps/tao_classifier/deepstream_clas_app.c @@ -0,0 +1,925 @@ +/* + * Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "nvds_yml_parser.h" +#include "cuda_runtime_api.h" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 600 +#define MUXER_OUTPUT_HEIGHT 800 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 4000000 + +#define TILED_OUTPUT_WIDTH 600 +#define TILED_OUTPUT_HEIGHT 800 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +#define MAX_SOURCE_BINS 1024 + +static gboolean fileLoop; + +typedef struct +{ + gdouble fps[MAX_SOURCE_BINS]; + gdouble fps_avg[MAX_SOURCE_BINS]; + guint num_instances; +}PerfStruct; + +typedef struct +{ + guint buffer_cnt; + guint total_buffer_cnt; + struct timeval total_fps_time; + struct timeval start_fps_time; + struct timeval last_fps_time; + struct timeval last_sample_fps_time; +}InstancePerfStruct; + +typedef struct +{ + guint num_instances; + GMutex struct_lock; + GstPad *sink_bin_pad; + InstancePerfStruct instance_str[MAX_SOURCE_BINS]; +} PerfStructInt; + +typedef struct _DsSourceBin +{ + GstElement *source_bin; + GstElement *uri_decode_bin; + GstElement *vidconv; + GstElement *nvvidconv; + GstElement *capsfilt; + GstElement *capsraw; + gint index; + gboolean is_imagedec; + gboolean is_streaming; + guint64 accumulated_base; + guint64 prev_accumulated_base; +}DsSourceBinStruct; + +static const char* dgpus_unsupport_hw_enc[] = { + "NVIDIA A100", + "NVIDIA A30", + "NVIDIA H100", // NVIDIA H100 SXM, NVIDIA H100 PCIe, NVIDIA H100 NVL + "NVIDIA T500", + "GeForce MX570 A", + "DGX A100" +}; + +static void +parse_tests_yaml (gint *file_loop, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["tests"].begin(); + itr != configyml["tests"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "file-loop") { + *file_loop = itr->second.as(); + } else { + *file_loop = 0; + } + } +} + +static void +parse_filesink_yaml (gint *enc_type, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["filesink"].begin(); + itr != configyml["filesink"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enc-type") { + int value = itr->second.as(); + if(value == 0 || value == 1){ + *enc_type = value; + } + } else { + *enc_type = 0; + } + } + g_print("enc_type:%d\n", *enc_type); +} + +/* + * Function to seek the source stream to start. + * It is required to play the stream in loop. + */ +static gboolean +seek_decode (gpointer data) +{ + DsSourceBinStruct *bin = (DsSourceBinStruct *) data; + gboolean ret = TRUE; + + gst_element_set_state (bin->source_bin, GST_STATE_PAUSED); + + ret = gst_element_seek (bin->source_bin, 1.0, GST_FORMAT_TIME, + (GstSeekFlags) (GST_SEEK_FLAG_KEY_UNIT | GST_SEEK_FLAG_FLUSH), + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); + + if (!ret) + GST_WARNING ("Error in seeking pipeline"); + + gst_element_set_state (bin->source_bin, GST_STATE_PLAYING); + + return FALSE; +} + +/** + * Probe function to drop certain events to support custom + * logic of looping of each source stream. + */ +static GstPadProbeReturn +restart_stream_buf_prob (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstEvent *event = GST_EVENT (info->data); + DsSourceBinStruct *bin = (DsSourceBinStruct *) u_data; + + if ((info->type & GST_PAD_PROBE_TYPE_BUFFER)) { + GST_BUFFER_PTS(GST_BUFFER(info->data)) += bin->prev_accumulated_base; + } + if ((info->type & GST_PAD_PROBE_TYPE_EVENT_BOTH)) { + if (GST_EVENT_TYPE (event) == GST_EVENT_EOS) { + g_timeout_add (1, seek_decode, bin); + } + + if (GST_EVENT_TYPE (event) == GST_EVENT_SEGMENT) { + GstSegment *segment; + + gst_event_parse_segment (event, (const GstSegment **) &segment); + segment->base = bin->accumulated_base; + bin->prev_accumulated_base = bin->accumulated_base; + bin->accumulated_base += segment->stop; + } + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + /* QOS events from downstream sink elements cause decoder to drop + * frames after looping the file since the timestamps reset to 0. + * We should drop the QOS events since we have custom logic for + * looping individual sources. */ + case GST_EVENT_QOS: + case GST_EVENT_SEGMENT: + case GST_EVENT_FLUSH_START: + case GST_EVENT_FLUSH_STOP: + return GST_PAD_PROBE_DROP; + default: + break; + } + } + return GST_PAD_PROBE_OK; +} + +static gboolean +perf_measurement_callback (gpointer data) +{ + PerfStructInt *str = (PerfStructInt *) data; + guint buffer_cnt[MAX_SOURCE_BINS]; + PerfStruct perf_struct; + struct timeval current_fps_time; + guint i; + static guint header_print_cnt = 0; + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < str->num_instances; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + g_print ("%s", asctime (tm)); + + g_mutex_lock (&str->struct_lock); + + for (i = 0; i < str->num_instances; i++) { + buffer_cnt[i] = + str->instance_str[i].buffer_cnt; + str->instance_str[i].buffer_cnt = 0; + } + + perf_struct.num_instances = str->num_instances; + gettimeofday (¤t_fps_time, NULL); + + g_print ("**PERF: "); + for (i = 0; i < str->num_instances; i++) { + InstancePerfStruct *str1 = &str->instance_str[i]; + gdouble time1 = + (str1->total_fps_time.tv_sec + + str1->total_fps_time.tv_usec / 1000000.0) + + (current_fps_time.tv_sec + current_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + + gdouble time2; + + if (str1->last_sample_fps_time.tv_sec == 0 && + str1->last_sample_fps_time.tv_usec == 0) { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + } else { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->last_sample_fps_time.tv_sec + + str1->last_sample_fps_time.tv_usec / 1000000.0); + } + str1->total_buffer_cnt += buffer_cnt[i]; + perf_struct.fps[i] = buffer_cnt[i] / time2; + if (isnan (perf_struct.fps[i])) + perf_struct.fps[i] = 0; + + perf_struct.fps_avg[i] = str1->total_buffer_cnt / time1; + if (isnan (perf_struct.fps_avg[i])) + perf_struct.fps_avg[i] = 0; + + str1->last_sample_fps_time = str1->last_fps_time; + + g_print ("%.2f(%.2f)\t", perf_struct.fps[i], perf_struct.fps_avg[i]); + } + + g_print("\n"); + + g_mutex_unlock (&str->struct_lock); + + return TRUE; +} + +/** + * Buffer probe function on element. + */ +static GstPadProbeReturn +buf_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + PerfStructInt *str = (PerfStructInt *) u_data; + NvDsBatchMeta *batch_meta = + gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data)); + + if (!batch_meta) + return GST_PAD_PROBE_OK; + + g_mutex_lock (&str->struct_lock); + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + InstancePerfStruct *str1 = &str->instance_str[frame_meta->pad_index]; + gettimeofday (&str1->last_fps_time, NULL); + if (str1->start_fps_time.tv_sec == 0 && str1->start_fps_time.tv_usec == 0) { + str1->start_fps_time = str1->last_fps_time; + } else { + str1->buffer_cnt++; + } + } + g_mutex_unlock (&str->struct_lock); + return GST_PAD_PROBE_OK; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad to videoconvert if no hardware decoder is used */ + if (bin_struct->vidconv) { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->vidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + if (!gst_element_link_many (bin_struct->vidconv, bin_struct->capsraw, + bin_struct->nvvidconv, NULL)) { + g_printerr ("Failed to link videoconvert to nvvideoconvert\n"); + } + } else { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->nvvidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + } + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + g_print ("###Decodebin pick nvidia decoder plugin.\n"); + } else { + /* Get the source bin ghost pad */ + g_print ("###Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + DsSourceBinStruct *src_bin_st = (DsSourceBinStruct *) user_data; + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + + if (g_strstr_len (name, -1, "nvv4l2decoder") == name) { + if (fileLoop && !src_bin_st->is_streaming) { + g_print ("loop model: %s\n", name); + GstPad *gstpad = gst_element_get_static_pad (GST_ELEMENT(object), "sink"); + gst_pad_add_probe(gstpad, (GstPadProbeType) (GST_PAD_PROBE_TYPE_EVENT_BOTH | + GST_PAD_PROBE_TYPE_EVENT_FLUSH | GST_PAD_PROBE_TYPE_BUFFER), restart_stream_buf_prob, user_data, NULL); + gst_object_unref (gstpad); + } + } +} + +static bool +create_source_bin (DsSourceBinStruct *ds_source_struct, gchar * uri) +{ + gchar bin_name[16] = { }; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + ds_source_struct->nvvidconv = NULL; + ds_source_struct->capsfilt = NULL; + ds_source_struct->source_bin = NULL; + ds_source_struct->uri_decode_bin = NULL; + + g_snprintf (bin_name, 15, "source-bin-%02d", ds_source_struct->index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + ds_source_struct->source_bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + ds_source_struct->uri_decode_bin = gst_element_factory_make ("uridecodebin", + "uri-decode-bin"); + ds_source_struct->nvvidconv = gst_element_factory_make ("nvvideoconvert", + "source_nvvidconv"); + ds_source_struct->capsfilt = gst_element_factory_make ("capsfilter", + "source_capset"); + + if (!ds_source_struct->source_bin || !ds_source_struct->uri_decode_bin + || !ds_source_struct->nvvidconv + || !ds_source_struct->capsfilt) { + g_printerr ("One element in source bin could not be created.\n"); + return false; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (ds_source_struct->uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), ds_source_struct); + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), ds_source_struct); + + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (ds_source_struct->capsfilt), "caps", caps, NULL); + + gst_bin_add_many (GST_BIN (ds_source_struct->source_bin), + ds_source_struct->uri_decode_bin, ds_source_struct->nvvidconv, + ds_source_struct->capsfilt, NULL); + + if (!gst_element_link (ds_source_struct->nvvidconv, + ds_source_struct->capsfilt)) { + g_printerr ("Could not link vidconv and capsfilter\n"); + return false; + } + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + GstPad *gstpad = gst_element_get_static_pad (ds_source_struct->capsfilt, + "src"); + if (!gstpad) { + g_printerr ("Could not find srcpad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + return false; + } + if(!gst_element_add_pad (ds_source_struct->source_bin, + gst_ghost_pad_new("src", gstpad))) { + g_printerr ("Could not add ghost pad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + } + gst_object_unref (gstpad); + + return true; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) { + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR: { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static bool +is_enc_hw_support() { + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + bool enc_hw_support = TRUE; + if (prop.integrated) { + char device_name[50]; + FILE* ptr = fopen("/proc/device-tree/model", "r"); + + if (ptr) { + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } else { + for (int i = 0; i < sizeof(dgpus_unsupport_hw_enc)/sizeof(dgpus_unsupport_hw_enc[0]); i++) { + if (!strncasecmp(prop.name, dgpus_unsupport_hw_enc[i], strlen(dgpus_unsupport_hw_enc[i]))) { + enc_hw_support = FALSE; + break; + } + } + } + return enc_hw_support; +} + +/* Check for parsing error. */ +#define RETURN_ON_PARSER_ERROR(parse_expr) \ + if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ + g_printerr("Error in parsing configuration file.\n"); \ + return -1; \ + } + +static void printUsage(const char* cmd) { + g_printerr ("\tUsage: %s -c pgie_config_file -i [-b BATCH]" + " [-d]\n\tOR\n\t %s yml_config_file\n", cmd, cmd); + g_printerr ("-h: \n\tprint help info \n"); + g_printerr ("-c: \n\tpgie config file, e.g. pgie_frcnn_tao_config.txt \n"); + g_printerr ("-i: \n\tH264 or JPEG input file \n"); + g_printerr ("-b: \n\tbatch size, this will override the value of \"batch-size\" in pgie config file \n"); + g_printerr ("-d: \n\tenable display, otherwise dump to output H264 or JPEG file \n"); + g_printerr ("-f: \n\tuse fake_sink to test the performace\n"); + g_printerr ("-l: \n\tloop mode for the pipeline"); + g_printerr ("yml_config_file: \n\tYAML config file, e.g. multi_task_tao_config.yml \n"); +} +int +main (int argc, char *argv[]) { + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source_bin = NULL, + *streammux = NULL, *sink = NULL, + *pgie = NULL, *nvvidconv = NULL, *nvdsosd = NULL, + *parser1 = NULL, *nvvidconv1 = NULL, *enc = NULL, + *tiler = NULL, *mux = NULL; + + DsSourceBinStruct source_struct[128]; + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *osd_sink_pad = NULL; + + gboolean isImage = FALSE; + gboolean useDisplay = FALSE; + gboolean useFakeSink = FALSE; + guint tiler_rows, tiler_cols; + guint batchSize = 0; + guint pgie_batch_size; + guint c; + const char* optStr = "b:c:dhfli:"; + std::string pgie_config; + gboolean showMask = FALSE; + gboolean isYAML = FALSE; + GList* g_list = NULL; + GList* iterator = NULL; + static guint src_cnt = 0; + PerfStructInt str; + fileLoop = FALSE; + int enc_type = 0; + NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER; + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (argc==2 && (g_str_has_suffix(argv[1], ".yml") || + g_str_has_suffix(argv[1], ".yaml"))) { + isYAML = TRUE; + if (NVDS_YAML_PARSER_SUCCESS != nvds_parse_source_list(&g_list, argv[1], "source-list")) { + g_printerr ("No source is found. Exiting.\n"); + return -1; + } + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&pgie_type, argv[1], + "primary-gie")); + printf("pgie_type:%d\n", pgie_type); + + parse_tests_yaml(&fileLoop, argv[1]); + } else { + while ((c = getopt(argc, argv, optStr)) != -1) { + switch (c) { + case 'b': + batchSize = std::atoi(optarg); + batchSize = batchSize == 0 ? 1:batchSize; + break; + case 'c': + pgie_config.assign(optarg); + break; + case 'd': + useDisplay = TRUE; + break; + case 'f': + useFakeSink = TRUE; + break; + case 'i': + g_list = g_list_append(g_list, optarg); + break; + case 'l': + fileLoop = TRUE; + break; + case 'h': + default: + printUsage(argv[0]); + return -1; + } + } + } + + /* Check input arguments */ + if (argc == 1) { + printUsage(argv[0]); + return -1; + } + + if(useDisplay) useFakeSink = FALSE; + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("ds-custom-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), streammux); + + for (iterator = g_list, src_cnt=0; iterator; iterator = iterator->next,src_cnt++) { + /* Source element for reading from the file */ + source_struct[src_cnt].index = src_cnt; + + if (g_strrstr ((gchar *)iterator->data, ".jpg") || g_strrstr ((gchar *)iterator->data, ".jpeg") + || g_strrstr ((gchar *)iterator->data, ".png")) + isImage = TRUE; + else + isImage = FALSE; + if (g_strrstr ((gchar *)iterator->data, "rtsp://") || g_strrstr ((gchar *)iterator->data, "v4l2://") + || g_strrstr ((gchar *)iterator->data, "http://") || g_strrstr ((gchar *)iterator->data, "rtmp://")) { + source_struct[src_cnt].is_streaming = TRUE; + } else { + source_struct[src_cnt].is_streaming = FALSE; + } + if (!create_source_bin (&(source_struct[src_cnt]), (gchar *)iterator->data)) + { + g_printerr ("Source bin could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_struct[src_cnt].source_bin); + + g_snprintf (pad_name_sink, 64, "sink_%d", src_cnt); + sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); + g_print("Request %s pad from streammux\n",pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_struct[src_cnt].source_bin, + pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + } + + str.num_instances = src_cnt; + g_timeout_add (5000, perf_measurement_callback, &str); + + const char* batch_size = std::getenv("BATCH_SIZE"); + if(batch_size != NULL ) { + batchSize = std::stoi(batch_size); + g_printerr("batch size is %d \n", batchSize); + } + + const char* show_mask = std::getenv("SHOW_MASK"); + if(show_mask != NULL && std::stoi(show_mask) == 1) { + showMask= TRUE; + } + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + if (pgie_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + pgie = gst_element_factory_make ("nvinferserver", "primary-nvinference-engine"); + } else { + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + } + + /* Use convertor to convert from NV12 to RGBA as required by nvdsosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvdsosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Finally render the osd output */ + if (isYAML) { + GstElement *eglsink = gst_element_factory_make ("nveglglessink", "test-egl-sink"); + GstElement *filesink = gst_element_factory_make ("filesink", "test-file-sink"); + GstElement *fakesink = gst_element_factory_make("fakesink", "test-fake-sink"); + if(NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "eglsink")){ + useDisplay = TRUE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_file_sink(filesink, argv[1], "filesink")){ + useDisplay = FALSE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "fakesink")){ + useDisplay = FALSE; + useFakeSink = TRUE; + } else { + g_printerr ("No sink is configured. Exiting.\n"); + return -1; + } + g_object_unref(eglsink); + g_object_unref(filesink); + g_object_unref(fakesink); + } + + const char* use_display = std::getenv("USE_DISPLAY"); + if (use_display != NULL && std::stoi(use_display) == 1) { + useDisplay = true; + } + + if (useDisplay == FALSE) { + if (isImage == FALSE) { + parser1 = gst_element_factory_make ("h264parse", "h264-parser1"); + if (isYAML) { + parse_filesink_yaml(&enc_type, argv[1]); + } else { + // 0: HW 1: SW + enc_type = is_enc_hw_support() ? 0 : 1; + } + if (enc_type == 0) { + enc = gst_element_factory_make ("nvv4l2h264enc", "h264-enc"); + } else { + enc = gst_element_factory_make ("x264enc", "h264-enc"); + } + if (!useFakeSink) { + mux = gst_element_factory_make ("qtmux", "mp4-mux"); + if (!mux) { + g_printerr ("Failed to create mp4-mux"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), mux); + } + } else { + parser1 = gst_element_factory_make ("jpegparse", "jpeg-parser1"); + enc = gst_element_factory_make ("jpegenc", "jpeg-enc"); + } + nvvidconv1 = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter1"); + if(!useFakeSink) { + sink = gst_element_factory_make ("filesink", "file-sink"); + } else { + sink = gst_element_factory_make("fakesink", "file-sink"); + } + if (!pgie + || !tiler || !nvvidconv || !nvvidconv1 || !nvdsosd || !enc || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + //save the file to local dir + if (isImage == FALSE) + g_object_set (G_OBJECT (sink), "location", "./out.mp4", NULL); + else + g_object_set (G_OBJECT (sink), "location", "./out.jpg", NULL); + } else { + if(prop.integrated) + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + else +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + if (!pgie + || !tiler || !nvvidconv || !nvdsosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + } + + + if(isYAML) { + nvds_parse_streammux(streammux, argv[1], "streammux"); + if(!batchSize) { + g_object_get(G_OBJECT (streammux), "batch-size", &batchSize, NULL); + } + } + + if(!batchSize) { batchSize = src_cnt; } + + g_print ("batchSize %d...\n", batchSize); + + if(source_struct[0].is_streaming == TRUE) + g_object_set (G_OBJECT (streammux), "live-source", true, NULL); + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", batchSize, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + if(isYAML) { + nvds_parse_gie (pgie, argv[1], "primary-gie"); + } else { + g_object_set (G_OBJECT (pgie), + "config-file-path", pgie_config.c_str(), NULL); + } + + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != batchSize) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, batchSize); + g_object_set (G_OBJECT (pgie), "batch-size", batchSize, NULL); + } + + tiler_rows = (guint) sqrt (batchSize); + tiler_cols = (guint) ceil (1.0 * batchSize / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_cols, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + if(showMask) + g_object_set (G_OBJECT (nvdsosd), "display-mask", 1, "display-bbox", 0, "display-text", 0, "process-mode", 0, NULL); + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + if(useDisplay == FALSE) { + gst_bin_add_many (GST_BIN (pipeline), + pgie, tiler, + nvvidconv, nvdsosd, nvvidconv1, enc, parser1, sink, NULL); + } else { + gst_bin_add_many (GST_BIN (pipeline), + pgie, + tiler, nvvidconv, nvdsosd, sink, NULL); + } + + /* We link the elements together */ + /* uridocoderbin -> + * nvinfer -> nvvideoconvert -> nvdsosd -> video-renderer */ + if (useDisplay == FALSE) { + if(isImage == FALSE && !useFakeSink) { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, nvvidconv1, enc, parser1, mux, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } else { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, nvvidconv1, enc, parser1, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + } else { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + + /*Performance measurement video fps*/ + GstPad *streammux_src_pad = gst_element_get_static_pad (streammux, "src"); + if (!streammux_src_pad) + g_print ("Unable to get streammux src pad\n"); + else + gst_pad_add_probe(streammux_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + buf_probe, &str, NULL); + gst_object_unref (streammux_src_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", pgie_config.c_str()); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + return 0; +} diff --git a/src/apps/tao_apps/apps/tao_detection/Makefile b/src/apps/tao_apps/apps/tao_detection/Makefile new file mode 100644 index 00000000..54f5435f --- /dev/null +++ b/src/apps/tao_apps/apps/tao_detection/Makefile @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= ds-tao-detection + +CC:=g++ + +VERBOSE?=0 +ifeq ($(VERBOSE), 1) +AT= +else +AT=@ +endif + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes -I /usr/local/cuda/include +LIB_INSTALL_DIR?=$(DS_SRC_PATH)/lib/ +APP_INSTALL_DIR?=$(DS_SRC_PATH)/bin/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= `pkg-config --cflags $(PKGS)` +CFLAGS+= -std=c++14 + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper\ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lnvds_yml_parser -lyaml-cpp -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + @mkdir -p $(APP_INSTALL_DIR) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) diff --git a/src/apps/tao_apps/apps/tao_detection/deepstream_det_app.c b/src/apps/tao_apps/apps/tao_detection/deepstream_det_app.c new file mode 100644 index 00000000..d0c4e39f --- /dev/null +++ b/src/apps/tao_apps/apps/tao_detection/deepstream_det_app.c @@ -0,0 +1,920 @@ +/* + * Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "nvds_yml_parser.h" +#include "cuda_runtime_api.h" + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1920 +#define MUXER_OUTPUT_HEIGHT 1080 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 4000000 + +#define TILED_OUTPUT_WIDTH 1920 +#define TILED_OUTPUT_HEIGHT 1080 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +#define MAX_SOURCE_BINS 1024 + +static gboolean fileLoop; + +typedef struct +{ + gdouble fps[MAX_SOURCE_BINS]; + gdouble fps_avg[MAX_SOURCE_BINS]; + guint num_instances; +}PerfStruct; + +typedef struct +{ + guint buffer_cnt; + guint total_buffer_cnt; + struct timeval total_fps_time; + struct timeval start_fps_time; + struct timeval last_fps_time; + struct timeval last_sample_fps_time; +}InstancePerfStruct; + +typedef struct +{ + guint num_instances; + GMutex struct_lock; + GstPad *sink_bin_pad; + InstancePerfStruct instance_str[MAX_SOURCE_BINS]; +} PerfStructInt; + +typedef struct _DsSourceBin +{ + GstElement *source_bin; + GstElement *uri_decode_bin; + GstElement *vidconv; + GstElement *nvvidconv; + GstElement *capsfilt; + GstElement *capsraw; + gint index; + gboolean is_imagedec; + gboolean is_streaming; + guint64 accumulated_base; + guint64 prev_accumulated_base; +}DsSourceBinStruct; + +static const char* dgpus_unsupport_hw_enc[] = { + "NVIDIA A100", + "NVIDIA A30", + "NVIDIA H100", // NVIDIA H100 SXM, NVIDIA H100 PCIe, NVIDIA H100 NVL + "NVIDIA T500", + "GeForce MX570 A", + "DGX A100" +}; + +static void +parse_tests_yaml (gint *file_loop, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["tests"].begin(); + itr != configyml["tests"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "file-loop") { + *file_loop = itr->second.as(); + } else { + *file_loop = 0; + } + } +} + +static void +parse_filesink_yaml (gint *enc_type, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["filesink"].begin(); + itr != configyml["filesink"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enc-type") { + int value = itr->second.as(); + if(value == 0 || value == 1){ + *enc_type = value; + } + } else { + *enc_type = 0; + } + } + g_print("enc_type:%d\n", *enc_type); +} + +/* + * Function to seek the source stream to start. + * It is required to play the stream in loop. + */ +static gboolean +seek_decode (gpointer data) +{ + DsSourceBinStruct *bin = (DsSourceBinStruct *) data; + gboolean ret = TRUE; + + gst_element_set_state (bin->source_bin, GST_STATE_PAUSED); + + ret = gst_element_seek (bin->source_bin, 1.0, GST_FORMAT_TIME, + (GstSeekFlags) (GST_SEEK_FLAG_KEY_UNIT | GST_SEEK_FLAG_FLUSH), + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); + + if (!ret) + GST_WARNING ("Error in seeking pipeline"); + + gst_element_set_state (bin->source_bin, GST_STATE_PLAYING); + + return FALSE; +} + +/** + * Probe function to drop certain events to support custom + * logic of looping of each source stream. + */ +static GstPadProbeReturn +restart_stream_buf_prob (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstEvent *event = GST_EVENT (info->data); + DsSourceBinStruct *bin = (DsSourceBinStruct *) u_data; + + if ((info->type & GST_PAD_PROBE_TYPE_BUFFER)) { + GST_BUFFER_PTS(GST_BUFFER(info->data)) += bin->prev_accumulated_base; + } + if ((info->type & GST_PAD_PROBE_TYPE_EVENT_BOTH)) { + if (GST_EVENT_TYPE (event) == GST_EVENT_EOS) { + g_timeout_add (1, seek_decode, bin); + } + + if (GST_EVENT_TYPE (event) == GST_EVENT_SEGMENT) { + GstSegment *segment; + + gst_event_parse_segment (event, (const GstSegment **) &segment); + segment->base = bin->accumulated_base; + bin->prev_accumulated_base = bin->accumulated_base; + bin->accumulated_base += segment->stop; + } + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + /* QOS events from downstream sink elements cause decoder to drop + * frames after looping the file since the timestamps reset to 0. + * We should drop the QOS events since we have custom logic for + * looping individual sources. */ + case GST_EVENT_QOS: + case GST_EVENT_SEGMENT: + case GST_EVENT_FLUSH_START: + case GST_EVENT_FLUSH_STOP: + return GST_PAD_PROBE_DROP; + default: + break; + } + } + return GST_PAD_PROBE_OK; +} + +static gboolean +perf_measurement_callback (gpointer data) +{ + PerfStructInt *str = (PerfStructInt *) data; + guint buffer_cnt[MAX_SOURCE_BINS]; + PerfStruct perf_struct; + struct timeval current_fps_time; + guint i; + static guint header_print_cnt = 0; + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < str->num_instances; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + g_print ("%s", asctime (tm)); + + g_mutex_lock (&str->struct_lock); + + for (i = 0; i < str->num_instances; i++) { + buffer_cnt[i] = + str->instance_str[i].buffer_cnt; + str->instance_str[i].buffer_cnt = 0; + } + + perf_struct.num_instances = str->num_instances; + gettimeofday (¤t_fps_time, NULL); + + g_print ("**PERF: "); + for (i = 0; i < str->num_instances; i++) { + InstancePerfStruct *str1 = &str->instance_str[i]; + gdouble time1 = + (str1->total_fps_time.tv_sec + + str1->total_fps_time.tv_usec / 1000000.0) + + (current_fps_time.tv_sec + current_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + + gdouble time2; + + if (str1->last_sample_fps_time.tv_sec == 0 && + str1->last_sample_fps_time.tv_usec == 0) { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + } else { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->last_sample_fps_time.tv_sec + + str1->last_sample_fps_time.tv_usec / 1000000.0); + } + str1->total_buffer_cnt += buffer_cnt[i]; + perf_struct.fps[i] = buffer_cnt[i] / time2; + if (isnan (perf_struct.fps[i])) + perf_struct.fps[i] = 0; + + perf_struct.fps_avg[i] = str1->total_buffer_cnt / time1; + if (isnan (perf_struct.fps_avg[i])) + perf_struct.fps_avg[i] = 0; + + str1->last_sample_fps_time = str1->last_fps_time; + + g_print ("%.2f(%.2f)\t", perf_struct.fps[i], perf_struct.fps_avg[i]); + } + + g_print("\n"); + + g_mutex_unlock (&str->struct_lock); + + return TRUE; +} + +/** + * Buffer probe function on element. + */ +static GstPadProbeReturn +buf_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + PerfStructInt *str = (PerfStructInt *) u_data; + NvDsBatchMeta *batch_meta = + gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data)); + + if (!batch_meta) + return GST_PAD_PROBE_OK; + + g_mutex_lock (&str->struct_lock); + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + InstancePerfStruct *str1 = &str->instance_str[frame_meta->pad_index]; + gettimeofday (&str1->last_fps_time, NULL); + if (str1->start_fps_time.tv_sec == 0 && str1->start_fps_time.tv_usec == 0) { + str1->start_fps_time = str1->last_fps_time; + } else { + str1->buffer_cnt++; + } + } + g_mutex_unlock (&str->struct_lock); + return GST_PAD_PROBE_OK; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad to videoconvert if no hardware decoder is used */ + if (bin_struct->vidconv) { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->vidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + if (!gst_element_link_many (bin_struct->vidconv, bin_struct->capsraw, + bin_struct->nvvidconv, NULL)) { + g_printerr ("Failed to link videoconvert to nvvideoconvert\n"); + } + } else { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->nvvidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + } + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + g_print ("###Decodebin pick nvidia decoder plugin.\n"); + } else { + /* Get the source bin ghost pad */ + g_print ("###Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + DsSourceBinStruct *src_bin_st = (DsSourceBinStruct *) user_data; + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + + if (g_strstr_len (name, -1, "nvv4l2decoder") == name) { + if (fileLoop && !src_bin_st->is_streaming) { + g_print ("loop model: %s\n", name); + GstPad *gstpad = gst_element_get_static_pad (GST_ELEMENT(object), "sink"); + gst_pad_add_probe(gstpad, (GstPadProbeType) (GST_PAD_PROBE_TYPE_EVENT_BOTH | + GST_PAD_PROBE_TYPE_EVENT_FLUSH | GST_PAD_PROBE_TYPE_BUFFER), restart_stream_buf_prob, user_data, NULL); + gst_object_unref (gstpad); + } + } +} + +static bool +create_source_bin (DsSourceBinStruct *ds_source_struct, gchar * uri) +{ + gchar bin_name[16] = { }; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + ds_source_struct->nvvidconv = NULL; + ds_source_struct->capsfilt = NULL; + ds_source_struct->source_bin = NULL; + ds_source_struct->uri_decode_bin = NULL; + + g_snprintf (bin_name, 15, "source-bin-%02d", ds_source_struct->index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + ds_source_struct->source_bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + ds_source_struct->uri_decode_bin = gst_element_factory_make ("uridecodebin", + "uri-decode-bin"); + ds_source_struct->nvvidconv = gst_element_factory_make ("nvvideoconvert", + "source_nvvidconv"); + ds_source_struct->capsfilt = gst_element_factory_make ("capsfilter", + "source_capset"); + + if (!ds_source_struct->source_bin || !ds_source_struct->uri_decode_bin + || !ds_source_struct->nvvidconv + || !ds_source_struct->capsfilt) { + g_printerr ("One element in source bin could not be created.\n"); + return false; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (ds_source_struct->uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), ds_source_struct); + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), ds_source_struct); + + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (ds_source_struct->capsfilt), "caps", caps, NULL); + + gst_bin_add_many (GST_BIN (ds_source_struct->source_bin), + ds_source_struct->uri_decode_bin, ds_source_struct->nvvidconv, + ds_source_struct->capsfilt, NULL); + + if (!gst_element_link (ds_source_struct->nvvidconv, + ds_source_struct->capsfilt)) { + g_printerr ("Could not link vidconv and capsfilter\n"); + return false; + } + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + GstPad *gstpad = gst_element_get_static_pad (ds_source_struct->capsfilt, + "src"); + if (!gstpad) { + g_printerr ("Could not find srcpad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + return false; + } + if(!gst_element_add_pad (ds_source_struct->source_bin, + gst_ghost_pad_new("src", gstpad))) { + g_printerr ("Could not add ghost pad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + } + gst_object_unref (gstpad); + + return true; +} + +static gboolean +det_bus_call (GstBus * bus, GstMessage * msg, gpointer data) { + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR: { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static bool +is_enc_hw_support() { + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + bool enc_hw_support = TRUE; + if (prop.integrated) { + char device_name[50]; + FILE* ptr = fopen("/proc/device-tree/model", "r"); + + if (ptr) { + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } else { + for (int i = 0; i < sizeof(dgpus_unsupport_hw_enc)/sizeof(dgpus_unsupport_hw_enc[0]); i++) { + if (!strncasecmp(prop.name, dgpus_unsupport_hw_enc[i], strlen(dgpus_unsupport_hw_enc[i]))) { + enc_hw_support = FALSE; + break; + } + } + } + return enc_hw_support; +} + +/* Check for parsing error. */ +#define RETURN_ON_PARSER_ERROR(parse_expr) \ + if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ + g_printerr("Error in parsing configuration file.\n"); \ + return -1; \ + } + +static void printUsage(const char* cmd) { + g_printerr ("\tUsage: %s -c pgie_config_file -i [-b BATCH] [-d]\n", cmd); + g_printerr ("\tOR\n\t %s yaml_config_file\n", cmd); + g_printerr ("-h: \n\tprint help info \n"); + g_printerr ("-c: \n\tpgie config file, e.g. pgie_frcnn_tao_config.txt \n"); + g_printerr ("-i: \n\tH264 or JPEG input file \n"); + g_printerr ("-b: \n\tbatch size, this will override the value of \"batch-size\" in pgie config file \n"); + g_printerr ("-d: \n\tenable display, otherwise dump to output H264 or JPEG file \n"); + g_printerr ("-f: \n\tuse fake_sink to test the performace\n"); + g_printerr ("-l: \n\tloop mode for the pipeline"); + g_printerr ("yml_config_file: \n\tYAML config file, e.g. seg_app_unet.yml \n"); +} +int +main (int argc, char *argv[]) { + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source_bin = NULL, *streammux = NULL, *sink = NULL, + *pgie = NULL, *nvvidconv = NULL, *nvdsosd = NULL, + *parser1 = NULL, *nvvidconv1 = NULL, *enc = NULL, + *tiler = NULL, *mux = NULL; + DsSourceBinStruct source_struct[128]; + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + GstBus *bus = NULL; + guint bus_watch_id; + + gboolean isImage = FALSE; + gboolean useDisplay = FALSE; + gboolean useFakeSink = FALSE; + guint tiler_rows, tiler_cols; + guint batchSize = 0; + guint pgie_batch_size; + guint c; + const char* optStr = "b:c:dhfli:"; + std::string pgie_config; + gboolean showMask = FALSE; + gboolean isYAML = FALSE; + GList* g_list = NULL; + GList* iterator = NULL; + static guint src_cnt = 0; + PerfStructInt str; + fileLoop = FALSE; + int enc_type = 0; + NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER; + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (argc==2 && (g_str_has_suffix(argv[1], ".yml") || + g_str_has_suffix(argv[1], ".yaml"))) { + isYAML = TRUE; + if (NVDS_YAML_PARSER_SUCCESS != nvds_parse_source_list(&g_list, argv[1], "source-list")) { + g_printerr ("No source is found. Exiting.\n"); + return -1; + } + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&pgie_type, argv[1], + "primary-gie")); + printf("pgie_type:%d\n", pgie_type); + parse_tests_yaml(&fileLoop, argv[1]); + } else { + while ((c = getopt(argc, argv, optStr)) != -1) { + switch (c) { + case 'b': + batchSize = std::atoi(optarg); + batchSize = batchSize == 0 ? 1:batchSize; + break; + case 'c': + pgie_config.assign(optarg); + break; + case 'd': + useDisplay = TRUE; + break; + case 'f': + useFakeSink = TRUE; + break; + case 'i': + g_list = g_list_append(g_list, optarg); + break; + case 'l': + fileLoop = TRUE; + break; + case 'h': + default: + printUsage(argv[0]); + return -1; + } + } + } + + /* Check input arguments */ + if (argc == 1) { + printUsage(argv[0]); + return -1; + } + + if(useDisplay) useFakeSink = FALSE; + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("ds-custom-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), streammux); + + for (iterator = g_list, src_cnt=0; iterator; iterator = iterator->next,src_cnt++) { + /* Source element for reading from the file */ + source_struct[src_cnt].index = src_cnt; + + if (g_strrstr ((gchar *)iterator->data, ".jpg") || g_strrstr ((gchar *)iterator->data, ".jpeg") + || g_strrstr ((gchar *)iterator->data, ".png")) + isImage = TRUE; + else + isImage = FALSE; + if (g_strrstr ((gchar *)iterator->data, "rtsp://") || g_strrstr ((gchar *)iterator->data, "v4l2://") + || g_strrstr ((gchar *)iterator->data, "http://") || g_strrstr ((gchar *)iterator->data, "rtmp://")) { + source_struct[src_cnt].is_streaming = TRUE; + } else { + source_struct[src_cnt].is_streaming = FALSE; + } + if (!create_source_bin (&(source_struct[src_cnt]), (gchar *)iterator->data)) + { + g_printerr ("Source bin could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_struct[src_cnt].source_bin); + + g_snprintf (pad_name_sink, 64, "sink_%d", src_cnt); + sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); + g_print("Request %s pad from streammux\n",pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_struct[src_cnt].source_bin, + pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + } + + str.num_instances = src_cnt; + g_timeout_add (5000, perf_measurement_callback, &str); + + const char* batch_size = std::getenv("BATCH_SIZE"); + if(batch_size != NULL ) { + batchSize = std::stoi(batch_size); + g_printerr("batch size is %d \n", batchSize); + } + + const char* show_mask = std::getenv("SHOW_MASK"); + if(show_mask != NULL && std::stoi(show_mask) == 1) { + showMask= TRUE; + } + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + if (pgie_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + pgie = gst_element_factory_make ("nvinferserver", "primary-nvinference-engine"); + } else { + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + } + + /* Use convertor to convert from NV12 to RGBA as required by nvdsosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvdsosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Finally render the osd output */ + + if (isYAML) { + GstElement *eglsink = gst_element_factory_make ("nveglglessink", "test-egl-sink"); + GstElement *filesink = gst_element_factory_make ("filesink", "test-file-sink"); + GstElement *fakesink = gst_element_factory_make("fakesink", "test-fake-sink"); + if(NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "eglsink")){ + useDisplay = TRUE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_file_sink(filesink, argv[1], "filesink")){ + useDisplay = FALSE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "fakesink")){ + useDisplay = FALSE; + useFakeSink = TRUE; + } else { + g_printerr ("No sink is configured. Exiting.\n"); + return -1; + } + g_object_unref(eglsink); + g_object_unref(filesink); + g_object_unref(fakesink); + } + + const char* use_display = std::getenv("USE_DISPLAY"); + if (use_display != NULL && std::stoi(use_display) == 1) { + useDisplay = true; + } + + if (useDisplay == FALSE) { + if(isImage == FALSE){ + parser1 = gst_element_factory_make ("h264parse", "h264-parser1"); + if (isYAML) { + parse_filesink_yaml(&enc_type, argv[1]); + } else { + // 0: HW 1: SW + enc_type = is_enc_hw_support() ? 0 : 1; + } + if(enc_type == 0){ + enc = gst_element_factory_make ("nvv4l2h264enc", "h264-enc"); + } else { + enc = gst_element_factory_make ("x264enc", "h264-enc"); + } + if(!useFakeSink) { + mux = gst_element_factory_make ("qtmux", "mp4-mux"); + if (!mux) { + g_printerr ("Failed to create mp4-mux"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), mux); + } + } else { + parser1 = gst_element_factory_make ("jpegparse", "jpeg-parser1"); + enc = gst_element_factory_make ("jpegenc", "jpeg-enc"); + } + nvvidconv1 = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter1"); + if(!useFakeSink) { + sink = gst_element_factory_make ("filesink", "file-sink"); + } else { + sink = gst_element_factory_make("fakesink", "file-sink"); + } + if (!pgie + || !tiler || !nvvidconv || !nvvidconv1 || !nvdsosd || !enc || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + //save the file to local dir + if(isImage == FALSE) + g_object_set (G_OBJECT (sink), "location", "./out.mp4", NULL); + else + g_object_set (G_OBJECT (sink), "location", "./out.jpg", NULL); + } else { + if(prop.integrated) + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + else +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + if (!pgie + || !tiler || !nvvidconv || !nvdsosd || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + } + + if(isYAML) { + nvds_parse_streammux(streammux, argv[1], "streammux"); + if(!batchSize) { + g_object_get(G_OBJECT (streammux), "batch-size", &batchSize, NULL); + } + } + + if(!batchSize) { batchSize = src_cnt; } + + g_print ("batchSize %d...\n", batchSize); + + if(source_struct[0].is_streaming == TRUE) + g_object_set (G_OBJECT (streammux), "live-source", true, NULL); + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", batchSize, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + if(isYAML) { + nvds_parse_gie (pgie, argv[1], "primary-gie"); + } else { + g_object_set (G_OBJECT (pgie), + "config-file-path", pgie_config.c_str(), NULL); + } + + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != batchSize) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, batchSize); + g_object_set (G_OBJECT (pgie), "batch-size", batchSize, NULL); + } + + tiler_rows = (guint) sqrt (batchSize); + tiler_cols = (guint) ceil (1.0 * batchSize / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_cols, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + if(showMask) + g_object_set (G_OBJECT (nvdsosd), "display-mask", 1, "display-bbox", 0, "display-text", 0, "process-mode", 0, NULL); + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, det_bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + if(useDisplay == FALSE) { + gst_bin_add_many (GST_BIN (pipeline), + pgie, tiler, nvvidconv, nvdsosd, nvvidconv1, enc, parser1, + sink, NULL); + } else { + gst_bin_add_many (GST_BIN (pipeline), + pgie, tiler, nvvidconv, nvdsosd, sink, NULL); + } + + /* We link the elements together */ + /* uridecoderbin -> + * nvinfer -> nvvideoconvert -> nvdsosd -> video-renderer */ + if (useDisplay == FALSE) { + if(isImage == FALSE && !useFakeSink) { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, nvvidconv1, enc, parser1, mux, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } else { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, nvvidconv1, enc, parser1, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + } else { + if (!gst_element_link_many (streammux, pgie, tiler, + nvvidconv, nvdsosd, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + + /*Performance measurement video fps*/ + GstPad *streammux_src_pad = gst_element_get_static_pad (streammux, "src"); + if (!streammux_src_pad) + g_print ("Unable to get streammux src pad\n"); + else + gst_pad_add_probe(streammux_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + buf_probe, &str, NULL); + gst_object_unref (streammux_src_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", pgie_config.c_str()); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + return 0; +} diff --git a/src/apps/tao_apps/apps/tao_others/Makefile b/src/apps/tao_apps/apps/tao_others/Makefile new file mode 100644 index 00000000..34b03add --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/Makefile @@ -0,0 +1,12 @@ +all: + @make -C deepstream-mdx-perception-app + #@make -C deepstream-nvocdr-app + @make -C deepstream-pose-classification + @make -C deepstream_lpr_app + @make -C deepstream_custom_preprocessing_app +clean: + @make clean -C deepstream-mdx-perception-app + #@make clean -C deepstream-nvocdr-app + @make clean -C deepstream-pose-classification + @make clean -C deepstream_lpr_app + @make clean -C deepstream_custom_preprocessing_app diff --git a/src/apps/tao_apps/apps/tao_others/README.md b/src/apps/tao_apps/apps/tao_others/README.md new file mode 100644 index 00000000..cfbe4f8c --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/README.md @@ -0,0 +1,137 @@ +## Description +This sample is to show the following TAO models runing with DeepStream + +* Retail Object Detection Binary +* Retail Object Recognition +* PeopleNet Transformer +* ReIdentificationNet +* PoseClassificationNet +* OCDNet +* OCRNet +* LPDNet +* LPRNet + +## Prerequisition + +* [DeepStream SDK 7.1 GA or above](https://developer.nvidia.com/deepstream-sdk) + + Make sure deepstream-test1 sample can run successful to verify your DeepStream installation + +* [TAO models](https://docs.nvidia.com/tao/tao-toolkit/text/overview.html) + + Nvidia has provides all the trainable models in NGC. + The models used in the sample application are pre-trained models provided by TAO: + +| Model name | NGC link | Version | +|------------|-----------|---------| +| Retail Object Detection Binary | [link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_detection)|deployable_retail_object_detection_binary_v2.2.2.3| +|PeopleNet Transformer | [link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer)|deployable_v1.1| +|Retail Object Recognition|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_recognition)|deployable_v2.0| +|ReIdentificationNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/reidentificationnet)|deployable_v1.2| +|PoseClassificationNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/poseclassificationnet)|deployable_onnx_v1.0| +|OCDNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocdnet)|deployable_onnx_v2.4| +|OCRNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocrnet)|deployable_v2.1.1| +|LPDNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lpdnet)|pruned_v2.3.1| +|LPRNet|[link](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/lprnet)|deployable_onnx_v1.1| + + ## Triton Server Settings + + The DeepStream sample applications can work as Triton client with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), one of the following two methods can be used to set up the Triton Inference Server before starting a gst-nvinferserver inferncing DeepStream application. + + - Native Triton Inference Server, please refer to [Triton Server](../../triton_server.md) + - Stand-alone Triton Inference server, please refer to [Triton grpc server](../../triton_server_grpc.md). + +Please enable Triton or Triton gRPC inferencing with the app YAML configurations. + +## Download + +1. Download Project with HTTPS +``` + sudo apt update + sudo apt install git-lfs + git lfs install --skip-repo + git clone https://github.com/NVIDIA/deepstream.git +``` +2. Prepare Models and TensorRT engine + +There are pre-trained TAO models available in [NGC](https://ngc.nvidia.com/catalog/models). +Please run the following script to download pre-trained models. + +``` + cd deepstream/src/apps/tao_apps + chmod 755 download_models.sh + ./download_models.sh +``` + +## Build and Run +Build the application +``` + make + cd apps/tao_others +``` + +Start to run the nvocdr application +Please prepare the nvocdr libs first, you can refer to [NVOCDR_README](./deepstream-nvocdr-app/README.md) +``` + cd deepstream-nvocdr-app + ./deepstream-nvocdr-app +``` + +Start to run the pose classification application +``` + cd deepstream-pose-classification + ./deepstream-pose-classification-app +``` + +Start to run the mdx perception application +``` + cd deepstream-mdx-perception-app + ./deepstream-mdx-perception-app -c +OR + ./deepstream-mdx-perception-app +``` + +Start to run the car license plate recognition sample application +```shell + cd deepstream_lpr_app + ##For US car plate recognition + cp dict_us.txt dict.txt + ##For Chinese car plate recognition + cp dict_ch.txt dict.txt + ##Run the sample app + ./deepstream-lpr-app/deepstream-lpr-app <1:US car plate model|2: Chinese car plate model> \ + <1: output as mp4 file| 2:fakesink| 3:display output> <0:ROI disable|1:ROI enable> \ + ... + ##OR + ./deepstream-lpr-app/deepstream-lpr-app +``` + +Start to run the custom preprocessing sample application +```shell + cd deepstream_custom_preprocessing_app + #Run "nvdspreprocess + pgie" pipeline + gst-launch-1.0 filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 nvstreammux name=mux batch-size=2 width=1920 height=1080 ! nvdspreprocess config-file=config_preprocess_frame.txt ! nvinfer input-tensor-meta=true config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.yml ! nvmultistreamtiler ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdsosd ! nvvideoconvert ! nvv4l2h264enc ! h264parse ! qtmux ! filesink location=out.mp4 +``` + +A sample of mdx perception: + +`./deepstream-mdx-perception-app -c ../../../configs/app/peoplenet_reidentification.yml` + +A sample of nvocdr: +Please prepare the nvocdr libs first, you can refer to [NVOCDR_README](./deepstream-nvocdr-app/README.md) + +`./deepstream-nvocdr-app nvocdr_app_config.yml` + +A sample of pose classification: + +`./deepstream-pose-classification-app ../../../configs/app/deepstream_pose_classification_config.yaml` + +A sample of the car license plate recognition: + +`./deepstream-lpr-app/deepstream-lpr-app 1 1 1 infer /opt/nvidia/deepstream/deepstream/samples/streams/sample_qHD.mp4 out.mp4` + +or +`./deepstream-lpr-app/deepstream-lpr-app ../../../configs/app/lpr_app_infer_us_config.yml` + +## Known Issue +1.For the deepstream-nvocdr-app, if the DeepStream version is lower than 6.4, the TensorRT OSS plugin is needed. Please refer to [NVOCDR_README](./deepstream-nvocdr-app/README.md). To avoid affecting the results of other apps, please replace the TensorRT plugin with the original one after running this app. diff --git a/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.cpp b/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.cpp new file mode 100644 index 00000000..7c539be4 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.cpp @@ -0,0 +1,806 @@ +/* + * Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include "ds_yml_parse.h" +#include "nvds_yml_parser.h" + +static gchar *get_absolute_file_path(gchar *cfg_file_path, const gchar *file_path) { + gchar abs_cfg_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + if (file_path && file_path[0] == '/') { + return (gchar *)file_path; + } + + if (!realpath(cfg_file_path, abs_cfg_path)) { + return NULL; + } + + /* Return absolute path of config file if file_path is NULL. */ + if (!file_path) { + abs_file_path = g_strdup(abs_cfg_path); + return abs_file_path; + } + + delim = g_strrstr(abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + abs_file_path = g_strconcat(abs_cfg_path, file_path, NULL); + return abs_file_path; +} + +NvDsYamlParserStatus +ds_parse_rtsp_output(GstElement * sink, + GstRTSPServer *server, GstRTSPMediaFactory *factory, + gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + + YAML::const_iterator itr = docs[i].begin(); + std::string group_name = itr->first.as(); + docs_indx_umap[group_name] = i; + docs_indx_vec.push_back(i); + + } + } + + if (!sink || !server || !factory) + { + std::cerr << "[ERROR] Pass element does not exist!" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } else { + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(sink)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "udpsink")) { + std::cerr << "[ERROR] Passed element is not udpsink" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + } + + int docs_indx_vec_size = docs_indx_vec.size(); + + int docs_indx_umap_size = docs_indx_umap.size(); + + if (docs_indx_umap_size != docs_indx_vec_size) { + std::cerr << "[ERROR] Duplicate group names in the config file : " << group << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + + for (int i = 0; i< docs_indx_vec_size; i++) + { + int indx = docs_indx_vec [i]; + for(YAML::const_iterator itr = docs[indx][group].begin(); itr != docs[indx][group].end(); ++itr) + { + paramKey = itr->first.as(); + + if(paramKey == "udpport") { + char udpsrc_pipeline[512]; + g_object_set (G_OBJECT (sink), "host", "224.224.255.255", "port", + itr->second.as(), "async", FALSE, "sync", 0, NULL); + sprintf (udpsrc_pipeline, + "( udpsrc name=pay0 port=%s buffer-size=65536 caps=\"application/x-rtp, media=video, " + "clock-rate=90000, encoding-name=H264, payload=96 \" )", + itr->second.as().c_str()); + gst_rtsp_media_factory_set_launch (factory, udpsrc_pipeline); + } + else if(paramKey == "rtspport") { + g_object_set (G_OBJECT(server), "service", itr->second.as().c_str(), NULL); + g_print("Please reach RTSP with rtsp://ip:%s/ds-out-avc\n", itr->second.as().c_str()); + } + else { + std::cerr << "!! [WARNING] Unknown param found : " << paramKey << std::endl; + } + } + } + + return NVDS_YAML_PARSER_SUCCESS; +} + +NvDsYamlParserStatus +ds_parse_enc_config(GstElement *encoder, + gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + + YAML::const_iterator itr = docs[i].begin(); + std::string group_name = itr->first.as(); + docs_indx_umap[group_name] = i; + docs_indx_vec.push_back(i); + + } + } + + if (!encoder) + return NVDS_YAML_PARSER_ERROR; + else { + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(encoder)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvv4l2h264enc") + && g_strcmp0(GST_OBJECT_NAME(factory), "nvv4l2h265enc")) { + std::cerr << "[ERROR] Passed element is not encoder" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + } + + int docs_indx_vec_size = docs_indx_vec.size(); + + int docs_indx_umap_size = docs_indx_umap.size(); + + if (docs_indx_umap_size != docs_indx_vec_size) { + std::cerr << "[ERROR] Duplicate group names in the config file : " << group << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + + for (int i = 0; i< docs_indx_vec_size; i++) + { + int indx = docs_indx_vec [i]; + for(YAML::const_iterator itr = docs[indx][group].begin(); itr != docs[indx][group].end(); ++itr) + { + paramKey = itr->first.as(); + + if(paramKey == "bitrate") { + g_object_set(G_OBJECT(encoder), "bitrate", + itr->second.as(), NULL); + } + else if(paramKey == "iframeinterval") { + g_object_set(G_OBJECT(encoder), "iframeinterval", + itr->second.as(), NULL); + } + else { + std::cerr << "!! [WARNING] Unknown param found : " << paramKey << std::endl; + } + } + } + + return NVDS_YAML_PARSER_SUCCESS; +} + +NvDsYamlParserStatus +ds_parse_videotemplate_config(GstElement *vtemplate, + gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + + YAML::const_iterator itr = docs[i].begin(); + std::string group_name = itr->first.as(); + docs_indx_umap[group_name] = i; + docs_indx_vec.push_back(i); + + } + } + + if (!vtemplate) + return NVDS_YAML_PARSER_ERROR; + else { + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(vtemplate)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvdsvideotemplate")) { + std::cerr << "[ERROR] Passed element is not nvdsvideotemplate" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + } + + int docs_indx_vec_size = docs_indx_vec.size(); + + int docs_indx_umap_size = docs_indx_umap.size(); + + if (docs_indx_umap_size != docs_indx_vec_size) { + std::cerr << "[ERROR] Duplicate group names in the config file : " << group << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + + for (int i = 0; i< docs_indx_vec_size; i++) + { + int indx = docs_indx_vec [i]; + for(YAML::const_iterator itr = docs[indx][group].begin(); itr != docs[indx][group].end(); ++itr) + { + paramKey = itr->first.as(); + + if(paramKey == "customlib-name") { + g_object_set(G_OBJECT(vtemplate), "customlib-name", + itr->second.as().c_str(), NULL); + } + else if(paramKey == "customlib-props") { + g_object_set(G_OBJECT(vtemplate), "customlib-props", + itr->second.as().c_str(), NULL); + } + else { + std::cerr << "!! [WARNING] Unknown param found : " << paramKey << std::endl; + } + } + } + + return NVDS_YAML_PARSER_SUCCESS; +} + +NvDsYamlParserStatus +ds_parse_ocdr_videotemplate_config(GstElement *vtemplate, + gchar *cfg_file_path, const char* group) +{ + if (!vtemplate) + return NVDS_YAML_PARSER_ERROR; + else { + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(vtemplate)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvdsvideotemplate")) { + std::cerr << "[ERROR] Passed element is not nvdsvideotemplate" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + } + + YAML::Node node = YAML::LoadFile(cfg_file_path); + + YAML::Node docs = node[group]; + + if(docs["customlib-name"]) { + std::string libPath = docs["customlib-name"].as(); + g_object_set(G_OBJECT(vtemplate), "customlib-name", + libPath.c_str(), NULL); + } + + if(docs["customlib-props"]) { + auto listNode = docs["customlib-props"]; + for(uint32_t i = 0; i < listNode.size(); i++) { + std::string tmpProb = listNode[i].as(); + g_object_set(G_OBJECT(vtemplate), "customlib-props", + tmpProb.c_str(), NULL); + } + } + + return NVDS_YAML_PARSER_SUCCESS; +} + + +guint +ds_parse_group_type(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + guint val = 0; + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + + if (docs[i][group]["type"]) { + val= docs[i][group]["type"].as(); + return val; + } + } + } + return 0; +} + +NvDsYamlParserStatus ds_parse_nvdsanalytics(GstElement *element, gchar *cfg_file_path, const char* group) +{ + NvDsYamlParserStatus ret = NVDS_YAML_PARSER_SUCCESS; + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(element)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvdsanalytics")) { + std::cerr << "[ERROR] Passed element is not nvdsanalytics" << std::endl; + ret = NVDS_YAML_PARSER_ERROR; + return ret; + } + + std::string paramKey = ""; + auto docs = YAML::LoadAllFromFile(cfg_file_path); + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + int total_docs = docs.size(); + + for (int i =0; i < total_docs;i++) + { + if (docs[i][group].Type() != YAML::NodeType::Null) { + if (docs[i][group]["enable"]) { + gboolean val= docs[i][group]["enable"].as(); + if(val == FALSE) { + g_print("!! [WARNING] \"analytics\" group not enabled.\n"); + g_object_set(G_OBJECT(element), "enable", val, NULL); + } + } + + YAML::const_iterator itr = docs[i].begin(); + std::string group_name = itr->first.as(); + docs_indx_umap[group_name] = i; + docs_indx_vec.push_back(i); + } + } + + int docs_indx_vec_size = docs_indx_vec.size(); + int docs_indx_umap_size = docs_indx_umap.size(); + if (docs_indx_umap_size != docs_indx_vec_size) { + std::cerr << "[ERROR] Duplicate group names in the config file : " << group << std::endl; + ret = NVDS_YAML_PARSER_ERROR; + return ret; + } + + for (int i = 0; i< docs_indx_vec_size; i++) + { + int indx = docs_indx_vec [i]; + for(YAML::const_iterator itr = docs[indx][group].begin(); itr != docs[indx][group].end(); ++itr) + { + paramKey = itr->first.as(); + + if (paramKey == "enable" && docs[indx][group]["enable"].as() == TRUE) { + continue; + } else if(paramKey == "config-file") { + std::string temp = itr->second.as(); + if (temp.empty()) { + g_printerr ("Error: Could not parse config-file-path in %s.\n", group); + return NVDS_YAML_PARSER_ERROR; + } + char *config_file_path = get_absolute_file_path(cfg_file_path, temp.c_str()); + if (!config_file_path) { + g_printerr ("Error: Could not get absolute path for config-file-path in %s.\n", group); + return NVDS_YAML_PARSER_ERROR; + } + g_print("Setting config-file for nvdsanalytics: %s\n", config_file_path); + g_object_set(G_OBJECT(element), "config-file", config_file_path, NULL); + g_free(config_file_path); + } else { + std::cerr << "!! [WARNING] Unknown param found for nvdsanalytics: " << paramKey << std::endl; + } + } + } + + return ret; +} + +static const char* dgpus_unsupport_hw_enc[] = { + "NVIDIA A100", + "NVIDIA A30", + "NVIDIA H100", // NVIDIA H100 SXM, NVIDIA H100 PCIe, NVIDIA H100 NVL + "NVIDIA T500", + "GeForce MX570 A", + "DGX A100" +}; + +static bool +is_enc_hw_support() { + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + bool enc_hw_support = TRUE; + if (prop.integrated) { + char device_name[50]; + FILE* ptr = fopen("/proc/device-tree/model", "r"); + + if (ptr) { + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } else { + for (uint32_t i = 0; i < sizeof(dgpus_unsupport_hw_enc)/sizeof(dgpus_unsupport_hw_enc[0]); i++) { + if (!strncasecmp(prop.name, dgpus_unsupport_hw_enc[i], strlen(dgpus_unsupport_hw_enc[i]))) { + enc_hw_support = FALSE; + break; + } + } + } + return enc_hw_support; +} + +guint +ds_parse_enc_type(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + guint val = 0; + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + if (docs[i][group]["enc-type"]) { + val = docs[i][group]["enc-type"].as(); + // If hardware encoding is configured but the hardware does not support it, + // fallback to software encoding + if (val == 0 && !is_enc_hw_support()) { + g_print("** WARN: hardware encoding is not supported, fallback to software encoding \n"); + val = 1; + } + return val; + } + } + } + return 0; +} + +guint +ds_parse_enc_codec(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + guint val = 0; + + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + + if (docs[i][group]["codec"]) { + val= docs[i][group]["codec"].as(); + return val; + } + } + } + return 0; +} + +GString * +ds_parse_file_name(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + + int total_docs = docs.size(); + GString *str = NULL; + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + if (docs[i][group]["filename"]) { + std::string temp = docs[i][group]["filename"].as(); + str = g_string_new(temp.c_str()); + return str; + } + } + } + return NULL; +} + +GString * +ds_parse_config_yml_filepath(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + + int total_docs = docs.size(); + GString *str = NULL; + + g_print("total %d item\n",total_docs); + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + g_print("group %s found %d\n", group, !(docs[i][group]["config-file-path"])); + if (docs[i][group]["config-file-path"]) { + std::string temp = docs[i][group]["config-file-path"].as(); + str = g_string_new(temp.c_str()); + return str; + } + } + } + return NULL; +} + +/* this is only for video encoding. */ +void +create_video_encoder(bool isH264, int enc_type, GstElement** conv_capfilter, + GstElement** outenc, GstElement** encparse, GstElement** rtppay) +{ + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + g_print("in create_video_encoder, isH264:%d, enc_type:%d\n", isH264, enc_type); + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, + "I420", NULL); + if(enc_type == ENCODER_TYPE_HW) { + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + } + g_object_set (G_OBJECT (*conv_capfilter), "caps", caps, NULL); + + if(isH264) { + if(enc_type == ENCODER_TYPE_HW) { + *outenc = gst_element_factory_make ("nvv4l2h264enc" ,"nvvideo-h264enc"); + } else { + *outenc = gst_element_factory_make ("x264enc" ,"x264enc"); + } + *encparse = gst_element_factory_make ("h264parse", "encparse"); + if(rtppay) + *rtppay = gst_element_factory_make ("rtph264pay", "rtppay"); + } else { + if(enc_type == ENCODER_TYPE_HW) { + *outenc = gst_element_factory_make ("nvv4l2h265enc" ,"nvvideo-h265enc"); + } else { + *outenc = gst_element_factory_make ("x265enc" ,"x265enc"); + } + *encparse = gst_element_factory_make ("h265parse", "encparse"); + if(rtppay) + *rtppay = gst_element_factory_make ("rtph265pay", "rtppay"); + } +} + +guint +ds_parse_group_enable(gchar *cfg_file_path, const char* group) +{ + std::string paramKey = ""; + std::vector docs = YAML::LoadAllFromFile(cfg_file_path); + std::vector docs_indx_vec; + std::unordered_map docs_indx_umap; + int total_docs = docs.size(); + guint val = 0; + for (int i =0; i < total_docs;i++) + { + if (!docs[i][group].IsNull()) { + if (docs[i][group]["enable"]) { + val= docs[i][group]["enable"].as(); + return val; + } + } + } + return 0; +} + +/** Function to get the absolute path of a file.*/ +gboolean +get_absolute_file_path_yaml ( + const gchar * cfg_file_path, const gchar * file_path, + char *abs_path_str) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar abs_real_file_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + /* Absolute path. No need to resolve further. */ + if (file_path[0] == '/') { + /* Check if the file exists, return error if not. */ + if (!realpath (file_path, abs_real_file_path)) { + /* Ignore error if file does not exist and use the unresolved path. */ + if (errno != ENOENT) + return FALSE; + } + g_strlcpy (abs_path_str, abs_real_file_path, _PATH_MAX); + return TRUE; + } + + /* Get the absolute path of the config file. */ + if (!realpath (cfg_file_path, abs_cfg_path)) { + return FALSE; + } + + /* Remove the file name from the absolute path to get the directory of the + * config file. */ + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + /* Get the absolute file path from the config file's directory path and + * relative file path. */ + abs_file_path = g_strconcat (abs_cfg_path, file_path, nullptr); + + /* Resolve the path.*/ + if (realpath (abs_file_path, abs_real_file_path) == nullptr) { + /* Ignore error if file does not exist and use the unresolved path. */ + if (errno == ENOENT) + g_strlcpy (abs_real_file_path, abs_file_path, _PATH_MAX); + else + return FALSE; + } + + g_free (abs_file_path); + + g_strlcpy (abs_path_str, abs_real_file_path, _PATH_MAX); + return TRUE; +} + +NvDsYamlParserStatus +nvds_parse_postprocess (GstElement *element, gchar* app_cfg_file_path, const char* group) +{ + NvDsYamlParserStatus ret = NVDS_YAML_PARSER_SUCCESS; + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(element)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvdspostprocess")) { + std::cerr << "[ERROR] Passed element is not nvdspostprocess" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + + if (!app_cfg_file_path) { + printf("Config file not provided.\n"); + return NVDS_YAML_PARSER_ERROR; + } + + YAML::Node configyml = YAML::LoadFile(app_cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "config-file-path") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + char *path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (app_cfg_file_path, str, + path)) { + ret = NVDS_YAML_PARSER_ERROR; + } + g_object_set(G_OBJECT(element), "postprocesslib-config-file", + path, NULL); + printf("postprocesslib-config-file:%s\n", path); + g_free (str); + g_free(path); + } else if (paramKey == "lib-name") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + char *path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (app_cfg_file_path, str, + path)) { + ret = NVDS_YAML_PARSER_ERROR; + } + g_object_set(G_OBJECT(element), "postprocesslib-name", + path, NULL); + printf("postprocesslib-name:%s\n", path); + g_free (str); + g_free(path); + } else { + printf("[WARNING] Unknown param found in postprocess: %s\n", paramKey.c_str()); + } + } + return ret; +} + +NvDsYamlParserStatus +nvds_parse_preprocess (GstElement *element, gchar* app_cfg_file_path, const char* group) +{ + NvDsYamlParserStatus ret = NVDS_YAML_PARSER_SUCCESS; + GstElementFactory *factory = GST_ELEMENT_GET_CLASS(element)->elementfactory; + if (g_strcmp0(GST_OBJECT_NAME(factory), "nvdspreprocess")) { + std::cerr << "[ERROR] Passed element is not nvdspreprocess" << std::endl; + return NVDS_YAML_PARSER_ERROR; + } + + if (!app_cfg_file_path) { + printf("Config file not provided.\n"); + return NVDS_YAML_PARSER_ERROR; + } + + YAML::Node configyml = YAML::LoadFile(app_cfg_file_path); + for(YAML::const_iterator itr = configyml[group].begin(); + itr != configyml[group].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "config-file-path") { + std::string temp = itr->second.as(); + char* str = (char*) malloc(sizeof(char) * 1024); + std::strncpy (str, temp.c_str(), 1024); + char *config_file_path = (char*) malloc(sizeof(char) * 1024); + if (!get_absolute_file_path_yaml (app_cfg_file_path, str, + config_file_path)) { + ret = NVDS_YAML_PARSER_ERROR; + } + g_object_set(G_OBJECT(element), "config-file", + config_file_path, NULL); + printf("config_file_path:%s\n", config_file_path); + g_free (str); + g_free(config_file_path); + } else { + printf("[WARNING] Unknown param found in preprocess: %s\n", paramKey.c_str()); + } + } + return ret; +} + +void +parse_streammux_width_height_yaml (gint *width, gint *height, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["streammux"].begin(); + itr != configyml["streammux"].end(); ++itr) { + std::string paramKey = itr->first.as(); + if (paramKey == "width") { + *width = itr->second.as(); + } else if(paramKey == "height"){ + *height = itr->second.as(); + } + } +} + +void +parse_sink_type_yaml (gint *type, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["sink"].begin(); + itr != configyml["sink"].end(); ++itr) { + std::string paramKey = itr->first.as(); + if (paramKey == "sink-type") { + *type = itr->second.as(); + } + } +} + +void +parse_sink_enc_type_yaml (gint *enc_type, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["sink"].begin(); + itr != configyml["sink"].end(); ++itr) { + std::string paramKey = itr->first.as(); + if (paramKey == "enc-type") { + int value = itr->second.as(); + if(value == 0 || value == 1){ + *enc_type = value; + } + } + } +} \ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.h b/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.h new file mode 100644 index 00000000..f3ef5cbe --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/common/ds_yml_parse.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DS_YAML_PARSER_H_ +#define _DS_YAML_PARSER_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include +#include +#include "nvds_yml_parser.h" +#define _PATH_MAX 1024 + +typedef enum +{ + ENCODER_TYPE_HW, + ENCODER_TYPE_SW +} EncHwSwType; + +NvDsYamlParserStatus +ds_parse_rtsp_output(GstElement *sink, + GstRTSPServer *server, GstRTSPMediaFactory *factory, + gchar *cfg_file_path, const char* group); + +NvDsYamlParserStatus +ds_parse_enc_config(GstElement *encoder, + gchar *cfg_file_path, const char* group); + +guint +ds_parse_group_type(gchar *cfg_file_path, const char* group); + +guint +ds_parse_enc_type(gchar *cfg_file_path, const char* group); + +guint +ds_parse_enc_codec(gchar *cfg_file_path, const char* group); + +GString * +ds_parse_file_name(gchar *cfg_file_path, const char* group); + +GString * +ds_parse_config_yml_filepath(gchar *cfg_file_path, const char* group); + +NvDsYamlParserStatus +ds_parse_videotemplate_config(GstElement *vtemplate, + gchar *cfg_file_path, const char* group); + +NvDsYamlParserStatus +ds_parse_ocdr_videotemplate_config(GstElement *vtemplate, + gchar *cfg_file_path, const char* group); + +NvDsYamlParserStatus +ds_parse_nvdsanalytics(GstElement *element, gchar *cfg_file_path, const char* group); + +void +create_video_encoder(bool isH264, int enc_type, GstElement** conv_capfilter, + GstElement** outenc, GstElement** encparse, GstElement** rtppay); + + /** Function to get the absolute path of a file.*/ +gboolean +get_absolute_file_path_yaml ( + const gchar * cfg_file_path, const gchar * file_path, + char *abs_path_str); + +/** Parse preprocess configurations. */ +NvDsYamlParserStatus +nvds_parse_preprocess (GstElement *element, gchar* app_cfg_file_path, const char* group); + +/** Parse postprocess configurations. */ +NvDsYamlParserStatus +nvds_parse_postprocess (GstElement *element, gchar* app_cfg_file_path, const char* group); + +/** Parse width and height of nvstreammux. */ +void +parse_streammux_width_height_yaml (gint *width, gint *height, gchar *cfg_file_path); + +/** Parse type of sink. */ +void +parse_sink_type_yaml (gint *type, gchar *cfg_file_path); + +/** Parse enc type of sink. */ +void +parse_sink_enc_type_yaml (gint *enc_type, gchar *cfg_file_path); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/Makefile new file mode 100644 index 00000000..52846cc1 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/Makefile @@ -0,0 +1,78 @@ +################################################################################ +# Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= deepstream-mdx-perception-app + +CXX=g++ + + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= deepstream_mdx_perception_app.c + +SRCS+= ../../../../../apps/sample_apps/deepstream-app/deepstream_app.c +SRCS+= ../../../../../apps/sample_apps/deepstream-app/deepstream_app_config_parser.c +SRCS+= ../../../../../apps/sample_apps/deepstream-app/deepstream_app_config_parser_yaml.cpp +SRCS+= $(wildcard ../../../../../apps/common/src/*.c) +SRCS+= $(wildcard ../../../../../apps/common/src/deepstream-yaml/*.cpp) +SRCS+= $(wildcard ../../../../../utils/nvds_msgapi/common_src/*.cpp) +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(SRCS:.c=.o) +OBJS:= $(OBJS:.cpp=.o) + +CFLAGS+= -I../../../../../apps/apps-common/includes \ + -I../../../../../apps/sample_apps/deepstream-app/ \ + -I../../../../../../includes/ \ + -I../../../../common/includes/ \ + -I/usr/local/cuda/include +# -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=5 \ + +LIBS:= -L/usr/local/cuda/lib64/ -lcudart +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lnvdsgst_customhelper \ + -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm -lnvds_logger -lcrypto \ + -lyaml-cpp -lcuda -lgstrtspserver-1.0 -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/README.md b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/README.md new file mode 100644 index 00000000..004f0b9c --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/README.md @@ -0,0 +1,134 @@ +## Description +The MDX perception sample application drives two Deepstream pipelines, i.e. retail +item recognition and people ReID. Retail item recognition pipeline detects retail +items from a video and extracts the embedding vector out of every detection bounding box. +The embedding vector can form a query to a database of embedding vectors and find +the closest match. +People ReID detects people from a video and extracts the embedding vector out of +every detection bounding box. The pipelines have a primary GIE module detecting +the objects of interest from a video frame. The secondary GIE module extracts an +embedding vector from the primary GIE result. + +The TAO 4.0 pretrained models used in this sample application: + +* [Retail Object Detection Binary](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_detection) +* [Retail Object Recognition](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/retail_object_recognition) +* [ReIdentificationNet Model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/reidentificationnet) +* [PeopleNet Transformer Model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer) + +## Prerequisition + +* DeepStream SDK 6.2 GA and above + +* NvDsInferParseCustomDDETRTAO +The custom post-processing plugin for Deformable DETR. The source code of tis plugin +is included in `post_processor/nvdsinfer_custombboxparser_tao.cpp`. + +* NvDsInferParseCustomEfficientDetTAO +The custom post-processing plugin for EfficientDet architecture. The source code +of tis plugin is included in `post_processor/nvdsinfer_custombboxparser_tao.cpp`. + +## Application Pipeline +The application pipeline graph + +![MDX perception application pipeline](mdx_perception_pipeline.png) + +## Build And Run +The application can be build and run seperately. Download the pre-trained models if haven't. +``` +export DS_TAO_APPS_HOME= +cd $DS_TAO_APPS_HOME +download_models.sh +``` + +Build the applications and run to inference one picture. +``` +cd $DS_TAO_APPS_HOME/apps/tao_others/deepstream-mdx-perception-app +make +./deepstream-mdx-perception-app -c ../../../configs/app/peoplenet_reidentification.yml -m 2 --tiledtext +./deepstream-mdx-perception-app -c ../../../configs/app/retail_object_detection_recognition.yml -m 3 --tiledtext +``` + +Uses can optionally modify `configs/app/retail_object_detection_recognition.yml` to enable sink2 which allows us to send metadata over Kafka, it can be modified to send metadata to any other broker such as Redis/Azure etc.., that DeepStream supports. By making use of a simple consumer script such as the one below + +``` +# Contents of consumer.py +#!/usr/bin/python +import time +from kafka import KafkaConsumer +from json import loads +import uuid + +consumer = KafkaConsumer( + 'test', + bootstrap_servers='127.0.0.1:9092', + auto_offset_reset='latest', + enable_auto_commit=True, + group_id=str(uuid.uuid1()), + value_deserializer=lambda x: loads(x.decode('utf-8')) +) + +# do a dummy poll to retrieve some message +consumer.poll() + +# go to end of the stream +consumer.seek_to_end() + +for event in consumer: + event_data = event.value + print(event_data) +``` + +The above script requries user to have a running Kafka broker. Here is a sample way on starting a kafka server using docker compose. Please refer to https://kafka.apache.org/quickstart for more details/options on using Kafka. + +``` +# docker-compose.yml +version: "3" +services: + zookeeper: + image: confluentinc/cp-zookeeper:5.4.3 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka: + image: confluentinc/cp-kafka:5.4.3 + depends_on: + - zookeeper + ports: + - 9092:9092 + environment: + KAFKA_BROKER_ID: 0 + KAFKA_LOG_DIRS: /kafka/logs + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://127.0.0.1:9092 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'true' + CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' +``` + +User can start the kafka server with the following command + +``` +docker-compose up -d +``` + +User can run the script on a separate terminal with the kafka broker running the following way (by default the ds app is configured to publish metadata to kafka running at port 9092) +``` +python3 consumer.py + +# Sample output metadata that is being sent from the DS pipeline (when sink2 is enabled) for the retail_object_detection_recognition_pipeline with PTM v2.0 + +{'version': '4.0', 'id': '367', '@timestamp': '1970-01-01T00:00:00.000Z', 'sensorId': 'stream1', 'objects': ['2|469.249|229.068|1183.21|523.846|Product|#|lipton_noodle_soup|||0.90813|#|embedding|-0.0956719,....,0.434932|']}\ + +``` \ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.c b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.c new file mode 100644 index 00000000..36f70930 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.c @@ -0,0 +1,1852 @@ +/* + * Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "deepstream_app.h" +#include "deepstream_config_file_parser.h" +#include "nvds_version.h" + +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" +#include +#include "deepstream_mdx_perception_app.h" + +#define MAX_DISPLAY_LEN (64) +#define MAX_TIME_STAMP_LEN (64) +#define STREAMMUX_BUFFER_POOL_SIZE (16) + +#define INOTIFY_EVENT_SIZE (sizeof (struct inotify_event)) +#define INOTIFY_EVENT_BUF_LEN (1024 * ( INOTIFY_EVENT_SIZE + 16)) + +#define IS_YAML(file) (g_str_has_suffix(file, ".yml") || g_str_has_suffix(file, ".yaml")) +#define NVDS_OBJECT_USER_EMBEDDING (nvds_get_user_meta_type("NVIDIA.NVINFER.EMBEDDING")) +/** @{ + * Macro's below and corresponding code-blocks are used to demonstrate + * nvmsgconv + Broker Metadata manipulation possibility + */ + +/** + * IMPORTANT Note 1: + * The code within the check for model_used == APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE + * is applicable as sample demo code for + * configs that use resnet PGIE model + * with class ID's: {0, 1, 2, 3} for {CAR, BICYCLE, PERSON, ROADSIGN} + * followed by optional Tracker + 3 X SGIEs (Vehicle-Type,Color,Make) + * only! + * Please comment out the code if using any other + * custom PGIE + SGIE combinations + * and use the code as reference to write your own + * NvDsEventMsgMeta generation code in generate_event_msg_meta() + * function + */ +typedef enum +{ + APP_CONFIG_ANALYTICS_MODELS_UNKNOWN = 0, + APP_CONFIG_ANALYTICS_RETAIL = 1, + APP_CONFIG_ANALYTICS_PEOPLE_REID = 2, + APP_CONFIG_ANALYTICS_RETAIL_DUAL_HEAD_SGIE = 3, +} AppConfigAnalyticsModel; + +#define MAX_LINES_IN_LABEL_FILE 320 +#define MAX_CHAR_LENGTH_PER_LINE 100 + +// Variable to store labels for FSL dual head model +char labels[MAX_LINES_IN_LABEL_FILE][MAX_CHAR_LENGTH_PER_LINE]; +guint dual_head_classes = 315; + +/** + * IMPORTANT Note 2: + * GENERATE_DUMMY_META_EXT macro implements code + * that assumes APP_CONFIG_ANALYTICS_RESNET_PGIE_3SGIE_TYPE_COLOR_MAKE + * case discussed above, and generate dummy metadata + * for other classes like Person class + * + * Vehicle class schema meta (NvDsVehicleObject) is filled + * in properly from Classifier-Metadata; + * see in-code documentation and usage of + * schema_fill_sample_sgie_vehicle_metadata() + */ +#define GENERATE_DUMMY_META_EXT + +/** Following class-ID's + * used for demonstration code + * assume an ITS detection model + * which outputs CLASS_ID=0 for Vehicle class + * and CLASS_ID=2 for Person class + * and SGIEs X 3 same as the sample DS config for test5-app: + * configs/test5_config_file_src_infer_tracker_sgie.txt + */ + +#define SECONDARY_GIE_VEHICLE_TYPE_UNIQUE_ID (4) +#define SECONDARY_GIE_VEHICLE_COLOR_UNIQUE_ID (5) +#define SECONDARY_GIE_VEHICLE_MAKE_UNIQUE_ID (6) +#define SECONDARY_GIE_MTMC_PERSON (0) + +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_CAR (0) +#ifdef GENERATE_DUMMY_META_EXT +#define RESNET10_PGIE_3SGIE_TYPE_COLOR_MAKECLASS_ID_PERSON (2) +#endif +/** @} */ + +#ifdef EN_DEBUG +#define LOGD(...) printf(__VA_ARGS__) +#else +#define LOGD(...) +#endif + +static TestAppCtx *testAppCtx; +GST_DEBUG_CATEGORY (NVDS_APP); + +/** @{ imported from deepstream-app as is */ + + +#define MAX_INSTANCES 128 +#define APP_TITLE "DeepStreamMDXPerception" + +#define DEFAULT_X_WINDOW_WIDTH 1920 +#define DEFAULT_X_WINDOW_HEIGHT 1080 + +AppCtx *appCtx[MAX_INSTANCES]; +static guint cintr = FALSE; +static GMainLoop *main_loop = NULL; +static gchar **cfg_files = NULL; +static gchar **input_files = NULL; +static gchar **override_cfg_file = NULL; +static gboolean playback_utc = FALSE; +static gboolean print_version = FALSE; +static gboolean show_bbox_text = FALSE; +static gboolean force_tcp = TRUE; +static gboolean print_dependencies_version = FALSE; +static gboolean quit = FALSE; +static gint return_value = 0; +static guint num_instances; +static guint num_input_files; +static GMutex fps_lock; +static gdouble fps[MAX_SOURCE_BINS]; +static gdouble fps_avg[MAX_SOURCE_BINS]; + +static Display *display = NULL; +static Window windows[MAX_INSTANCES] = { 0 }; + +static GThread *x_event_thread = NULL; +static GMutex disp_lock; + +static guint rrow, rcol, rcfg; +static gboolean rrowsel = FALSE, selecting = FALSE; +static AppConfigAnalyticsModel model_used = APP_CONFIG_ANALYTICS_MODELS_UNKNOWN; + +static struct timeval ota_request_time; +static struct timeval ota_completion_time; + +typedef struct _OTAInfo +{ + AppCtx *appCtx; + gchar *override_cfg_file; +} OTAInfo; + +/** @} imported from deepstream-app as is */ +GOptionEntry entries[] = { + {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version, + "Print DeepStreamSDK version", NULL} + , + {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text, + "Display Bounding box labels in tiled mode", NULL} + , + {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version, + "Print DeepStreamSDK and dependencies version", NULL} + , + {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files, + "Set the config file", NULL} + , + {"override-cfg-file", 'o', 0, G_OPTION_ARG_FILENAME_ARRAY, &override_cfg_file, + "Set the override config file, used for on-the-fly model update feature", + NULL} + , + {"input-file", 'i', 0, G_OPTION_ARG_FILENAME_ARRAY, &input_files, + "Set the input file", NULL} + , + {"playback-utc", 'p', 0, G_OPTION_ARG_INT, &playback_utc, + "Playback utc; default=false (base UTC from file-URL or RTCP Sender Report) =true (base UTC from file/rtsp URL)", + NULL} + , + {"pgie-model-used", 'm', 0, G_OPTION_ARG_INT, &model_used, + "PGIE Model used; {0 - Unknown [DEFAULT]}, {1: FSL}, {2: MTMC}, {3: FSL Dual Head SGIE}}", + NULL} + , + {"no-force-tcp", 0, G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &force_tcp, + "Do not force TCP for RTP transport", NULL} + , + {NULL} + , +}; + +/** + * @brief Fill NvDsVehicleObject with the NvDsClassifierMetaList + * information in NvDsObjectMeta + * NOTE: This function assumes the test-application is + * run with 3 X SGIEs sample config: + * test5_config_file_src_infer_tracker_sgie.txt + * or an equivalent config + * NOTE: If user is adding custom SGIEs, make sure to + * edit this function implementation + * @param obj_params [IN] The NvDsObjectMeta as detected and kept + * in NvDsBatchMeta->NvDsFrameMeta(List)->NvDsObjectMeta(List) + * @param obj [IN/OUT] The NvDSMeta-Schema defined Vehicle metadata + * structure + */ +static void schema_fill_sample_sgie_vehicle_metadata (NvDsObjectMeta * + obj_params, NvDsVehicleObject * obj); + +/** + * @brief Performs model update OTA operation + * Sets "model-engine-file" configuration parameter + * on infer plugin to initiate model switch OTA process + * @param ota_appCtx [IN] App context pointer + */ +void apply_ota (AppCtx * ota_appCtx); + +/** + * @brief Thread which handles the model-update OTA functionlity + * 1) Adds watch on the changes made in the provided ota-override-file, + * if changes are detected, validate the model-update change request, + * intiate model-update OTA process + * 2) Frame drops / frames without inference should NOT be detected in + * this on-the-fly model update process + * 3) In case of model update OTA fails, error message will be printed + * on the console and pipeline continues to run with older + * model configuration + * @param gpointer [IN] Pointer to OTAInfo structure + * @param gpointer [OUT] Returns NULL in case of thread exits + */ +gpointer ota_handler_thread (gpointer data); + +static void generate_ts_rfc3339(char *buf, int buf_size) { + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime(CLOCK_REALTIME, &ts); + memcpy(&tloc, (void *)(&ts.tv_sec), sizeof(time_t)); + gmtime_r(&tloc, &tm_log); + strftime(buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec / 1000000; + g_snprintf(strmsec, sizeof(strmsec), ".%.3dZ", ms); + strncat(buf, strmsec, buf_size); +} + +static GstClockTime generate_ts_rfc3339_from_ts(char *buf, int buf_size, + GstClockTime ts, gchar *src_uri, + gint stream_id) { + time_t tloc; + struct tm tm_log; + char strmsec[6]; //.nnnZ\0 + int ms; + + GstClockTime ts_generated; + + if (playback_utc || (appCtx[0]->config.multi_source_config[stream_id].type != + NV_DS_SOURCE_RTSP)) { + if (testAppCtx->streams[stream_id].meta_number == 0) { + testAppCtx->streams[stream_id].timespec_first_frame = + extract_utc_from_uri(src_uri); + memcpy( + &tloc, + (void *)(&testAppCtx->streams[stream_id].timespec_first_frame.tv_sec), + sizeof(time_t)); + ms = + testAppCtx->streams[stream_id].timespec_first_frame.tv_nsec / 1000000; + testAppCtx->streams[stream_id].gst_ts_first_frame = ts; + ts_generated = GST_TIMESPEC_TO_TIME( + testAppCtx->streams[stream_id].timespec_first_frame); + if (ts_generated == 0) { + // Expected warning for default stream + // g_print( + // "WARNING; playback mode used with URI [%s] not conforming to " + // "timestamp format;" + // " check README; using system-time\n", + // src_uri); + + clock_gettime(CLOCK_REALTIME, + &testAppCtx->streams[stream_id].timespec_first_frame); + ts_generated = GST_TIMESPEC_TO_TIME( + testAppCtx->streams[stream_id].timespec_first_frame); + } + } else { + GstClockTime ts_current = + GST_TIMESPEC_TO_TIME( + testAppCtx->streams[stream_id].timespec_first_frame) + + (ts - testAppCtx->streams[stream_id].gst_ts_first_frame); + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC(ts_current, timespec_current); + memcpy(&tloc, (void *)(×pec_current.tv_sec), sizeof(time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts_current; + } + } else { + /** ts itself is UTC Time in ns */ + struct timespec timespec_current; + GST_TIME_TO_TIMESPEC(ts, timespec_current); + memcpy(&tloc, (void *)(×pec_current.tv_sec), sizeof(time_t)); + ms = timespec_current.tv_nsec / 1000000; + ts_generated = ts; + } + gmtime_r(&tloc, &tm_log); + strftime(buf, buf_size, "%Y-%m-%dT%H:%M:%S", &tm_log); + g_snprintf(strmsec, sizeof(strmsec), ".%.3dZ", ms); + strncat(buf, strmsec, buf_size); + + return ts_generated; +} + +static gpointer meta_copy_func(gpointer data, gpointer user_data) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *)user_meta->user_meta_data; + NvDsEventMsgMeta *dstMeta = NULL; + + dstMeta = (NvDsEventMsgMeta *)g_memdup(srcMeta, sizeof(NvDsEventMsgMeta)); + + if (srcMeta->ts) dstMeta->ts = g_strdup(srcMeta->ts); + + if (srcMeta->objSignature.size > 0) { + dstMeta->objSignature.signature = (gdouble *)g_memdup( + srcMeta->objSignature.signature, srcMeta->objSignature.size); + dstMeta->objSignature.size = srcMeta->objSignature.size; + } + + if (srcMeta->objectId) { + dstMeta->objectId = g_strdup(srcMeta->objectId); + } + + if (srcMeta->sensorStr) { + dstMeta->sensorStr = g_strdup(srcMeta->sensorStr); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *srcObj = (NvDsPersonObject *)srcMeta->extMsg; + NvDsPersonObject *obj = + (NvDsPersonObject *)g_malloc0(sizeof(NvDsPersonObject)); + + obj->age = srcObj->age; + + if (srcObj->gender) obj->gender = g_strdup(srcObj->gender); + if (srcObj->cap) obj->cap = g_strdup(srcObj->cap); + if (srcObj->hair) obj->hair = g_strdup(srcObj->hair); + if (srcObj->apparel) obj->apparel = g_strdup(srcObj->apparel); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof(NvDsPersonObject); + } else if (srcMeta->objType == NVDS_OBJECT_TYPE_PRODUCT) { + NvDsProductObject *srcObj = (NvDsProductObject *)srcMeta->extMsg; + NvDsProductObject *obj = + (NvDsProductObject *)g_malloc0(sizeof(NvDsProductObject)); + if (srcObj->brand) obj->brand = g_strdup(srcObj->brand); + if (srcObj->type) obj->type = g_strdup(srcObj->type); + if (srcObj->shape) obj->shape = g_strdup(srcObj->shape); + + dstMeta->extMsg = obj; + dstMeta->extMsgSize = sizeof(NvDsProductObject); + } + } + + if (srcMeta->embedding.embedding_length > 0) { + dstMeta->embedding.embedding_length = srcMeta->embedding.embedding_length; + dstMeta->embedding.embedding_vector = + g_memdup(srcMeta->embedding.embedding_vector, + srcMeta->embedding.embedding_length * sizeof(float)); + } + + return dstMeta; +} + +static void meta_free_func(gpointer data, gpointer user_data) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)data; + NvDsEventMsgMeta *srcMeta = (NvDsEventMsgMeta *)user_meta->user_meta_data; + user_meta->user_meta_data = NULL; + + if (srcMeta->ts) { + g_free(srcMeta->ts); + } + + if (srcMeta->objSignature.size > 0) { + g_free(srcMeta->objSignature.signature); + srcMeta->objSignature.size = 0; + } + + if (srcMeta->objectId) { + g_free(srcMeta->objectId); + } + + if (srcMeta->sensorStr) { + g_free(srcMeta->sensorStr); + } + + if (srcMeta->extMsgSize > 0) { + if (srcMeta->objType == NVDS_OBJECT_TYPE_PERSON) { + NvDsPersonObject *obj = (NvDsPersonObject *)srcMeta->extMsg; + + if (obj->gender) g_free(obj->gender); + if (obj->cap) g_free(obj->cap); + if (obj->hair) g_free(obj->hair); + if (obj->apparel) g_free(obj->apparel); + } + //! Extensions for Fewshot Learning + else if (srcMeta->objType == NVDS_OBJECT_TYPE_PRODUCT) { + NvDsProductObject *obj = (NvDsProductObject *)srcMeta->extMsg; + + if (obj->brand) g_free(obj->brand); + if (obj->type) g_free(obj->type); + if (obj->shape) g_free(obj->shape); + } + + g_free(srcMeta->extMsg); + srcMeta->extMsg = NULL; + srcMeta->extMsgSize = 0; + } + + if (srcMeta->embedding.embedding_vector) { + g_free(srcMeta->embedding.embedding_vector); + } + srcMeta->embedding.embedding_length = 0; + + g_free(srcMeta); +} + +#ifdef GENERATE_DUMMY_META_EXT +static void generate_person_meta(gpointer data) { + NvDsPersonObject *obj = (NvDsPersonObject *)data; + obj->age = 45; + obj->cap = g_strdup("none-dummy-person-info"); + obj->hair = g_strdup("black"); + obj->gender = g_strdup("male"); + obj->apparel = g_strdup("formal"); +} +//! Extensions for Fewshot Learning +// Create product meta object +static void generate_product_meta(gpointer data) { + NvDsProductObject *obj = (NvDsProductObject *)data; + obj->brand = g_strdup(""); + obj->type = g_strdup(""); + obj->shape = g_strdup(""); +} +#endif + +static void generate_event_msg_meta(AppCtx *appCtx, gpointer data, + gint class_id, gboolean useTs, + GstClockTime ts, gchar *src_uri, + gint stream_id, guint sensor_id, + NvDsObjectMeta *obj_params, float scaleW, + float scaleH, NvDsFrameMeta *frame_meta, + NvDsInferTensorMeta *tensor_meta, + NvDsBatchMeta *batch_meta) { + NvDsEventMsgMeta *meta = (NvDsEventMsgMeta *)data; + GstClockTime ts_generated = 0; + + meta->objType = NVDS_OBJECT_TYPE_UNKNOWN; /**< object unknown */ + /* The sensor_id is parsed from the source group name which has the format + * [source]. */ + meta->sensorId = sensor_id; + meta->placeId = sensor_id; + meta->moduleId = sensor_id; + meta->frameId = frame_meta->frame_num; + meta->ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1); + meta->objectId = (gchar *)g_malloc0(MAX_LABEL_SIZE); + meta->bbox.left = obj_params->rect_params.left * scaleW; + meta->bbox.top = obj_params->rect_params.top * scaleH; + meta->bbox.width = obj_params->rect_params.width * scaleW; + meta->bbox.height = obj_params->rect_params.height * scaleH; + meta->confidence = obj_params->confidence; + /** tracking ID */ + meta->trackingId = obj_params->object_id; + + strncpy(meta->objectId, obj_params->obj_label, MAX_LABEL_SIZE); + + /** INFO: This API is called once for every 30 frames (now) */ + if (useTs && src_uri) { + ts_generated = generate_ts_rfc3339_from_ts(meta->ts, MAX_TIME_STAMP_LEN, ts, + src_uri, stream_id); + } else { + generate_ts_rfc3339(meta->ts, MAX_TIME_STAMP_LEN); + } + /** sensor ID when streams are added using nvmultiurisrcbin REST API */ + NvDsSensorInfo *sensorInfo = get_sensor_info(appCtx, stream_id); + if (sensorInfo) { + /** this stream was added using REST API; we have Sensor Info! */ + // g_print( + // "this stream [%d:%s] was added using REST API; we have Sensor + // Info\n", sensorInfo->source_id, sensorInfo->sensor_id); + meta->sensorStr = g_strdup(sensorInfo->sensor_id); + } + + (void)ts_generated; + /* + * This demonstrates how to attach custom objects. + * Any custom object as per requirement can be generated and attached + * like NvDsVehicleObject / NvDsPersonObject / NvDsProductObject. Then that + * object should be handled in gst-nvmsgconv component accordingly. + */ + if (model_used == APP_CONFIG_ANALYTICS_RETAIL) { + + } else if (model_used == APP_CONFIG_ANALYTICS_PEOPLE_REID) { + if (class_id == SECONDARY_GIE_MTMC_PERSON) { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_PERSON; +#ifdef GENERATE_DUMMY_META_EXT + // MTMC does not require to send dummy sgie attributes + // NvDsPersonObject *obj = + // (NvDsPersonObject *)g_malloc0(sizeof(NvDsPersonObject)); + // generate_person_meta(obj); + + // meta->extMsg = obj; + // meta->extMsgSize = sizeof(NvDsPersonObject); +#endif + } + } + // + bool embedding_on_device = TRUE; + gchar dual_head_label[MAX_CHAR_LENGTH_PER_LINE]; + if (tensor_meta) { + // Postprocess - embedding vector + if (model_used == APP_CONFIG_ANALYTICS_RETAIL) { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_PRODUCT; +#ifdef GENERATE_DUMMY_META_EXT + NvDsProductObject *obj = + (NvDsProductObject *)g_malloc0(sizeof(NvDsProductObject)); + generate_product_meta(obj); + + meta->extMsg = obj; + meta->extMsgSize = sizeof(NvDsProductObject); +#endif + NvDsInferDims embedding_dims = + tensor_meta->output_layers_info[0].inferDims; + int numElements = embedding_dims.d[0]; + meta->embedding.embedding_vector = + (float *)g_malloc0(numElements * sizeof(float)); + cudaMemcpy(meta->embedding.embedding_vector, + (float *)(tensor_meta->out_buf_ptrs_dev[0]), + numElements * sizeof(float), cudaMemcpyDeviceToHost); + meta->embedding.embedding_length = numElements; + } + else if (model_used == APP_CONFIG_ANALYTICS_PEOPLE_REID) + { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_PERSON; +#ifdef GENERATE_DUMMY_META_EXT + // MTMC does not require to send dummy sgie attributes + // NvDsPersonObject *obj = + // (NvDsPersonObject *)g_malloc0(sizeof(NvDsPersonObject)); + // generate_person_meta(obj); + + // meta->extMsg = obj; + // meta->extMsgSize = sizeof(NvDsPersonObject); +#endif + NvDsInferDims embedding_dims = + tensor_meta->output_layers_info[0].inferDims; + int numElements = embedding_dims.d[0]; + meta->embedding.embedding_vector = + (float *)g_malloc0(numElements * sizeof(float)); + cudaMemcpy(meta->embedding.embedding_vector, + (float *)(tensor_meta->out_buf_ptrs_dev[0]), + numElements * sizeof(float), cudaMemcpyDeviceToHost); + meta->embedding.embedding_length = numElements; + } + else if (model_used == APP_CONFIG_ANALYTICS_RETAIL_DUAL_HEAD_SGIE) + { + meta->type = NVDS_EVENT_MOVING; + meta->objType = NVDS_OBJECT_TYPE_PRODUCT; + + NvDsInferDims embedding_dims = + tensor_meta->output_layers_info[0].inferDims; + int numElements = embedding_dims.d[0]; + meta->embedding.embedding_vector = + (float *)g_malloc0(numElements * sizeof(float)); + cudaMemcpy(meta->embedding.embedding_vector, + (float *)(tensor_meta->out_buf_ptrs_dev[0]), + numElements * sizeof(float), cudaMemcpyDeviceToHost); + meta->embedding.embedding_length = numElements; + + NvDsInferDimsCHW dims; + + /* Access the 1-indexed output layer to get probs */ + getDimsCHWFromDims(dims, tensor_meta->output_layers_info[1].inferDims); + dual_head_classes = dims.c; + + float *outputCoverageBuffer = + (float *)tensor_meta->out_buf_ptrs_host[1]; + float maxProbability = 0; + bool attrFound = false; + NvDsInferAttribute attr; + + /* Get the output with max probability */ + for (unsigned int c = 0; c < dual_head_classes; c++) { + float probability = outputCoverageBuffer[c]; + if (probability > 0 && probability > maxProbability) { + maxProbability = probability; + attrFound = true; + attr.attributeIndex = 0; + attr.attributeValue = c; + attr.attributeConfidence = probability; + } + } +#ifdef GENERATE_DUMMY_META_EXT + NvDsProductObject *obj = + (NvDsProductObject *)g_malloc0(sizeof(NvDsProductObject)); + generate_product_meta(obj); + +#endif + /* Generate classifer metadata and attach to obj_meta */ + if (attrFound) { + NvDsClassifierMeta *classifier_meta = + nvds_acquire_classifier_meta_from_pool(batch_meta); + + classifier_meta->unique_component_id = tensor_meta->unique_id; + + /* Create NvDsLabel Info*/ + NvDsLabelInfo *label_info = + nvds_acquire_label_info_meta_from_pool(batch_meta); + label_info->result_class_id = attr.attributeValue; + label_info->result_prob = attr.attributeConfidence; + + strcpy(label_info->result_label, labels[label_info->result_class_id]); + + gchar *temp = obj_params->text_params.display_text; + obj_params->text_params.display_text = + g_strconcat(temp, " ", label_info->result_label, NULL); + g_free(temp); + + nvds_add_label_info_meta_to_classifier(classifier_meta, label_info); + nvds_add_classifier_meta_to_object(obj_params, classifier_meta); + strcpy(dual_head_label, label_info->result_label); +#ifdef GENERATE_DUMMY_META_EXT + obj->brand = g_strdup(dual_head_label); + meta->extMsg = obj; + meta->extMsgSize = sizeof(NvDsProductObject); +#endif + + } + +#ifdef GENERATE_DUMMY_META_EXT + meta->extMsg = obj; + meta->extMsgSize = sizeof(NvDsProductObject); +#endif + + } + } +} + +/** + * Callback function to be called once all inferences (Primary + Secondary) + * are done. This is opportunity to modify content of the metadata. + * e.g. Here Person is being replaced with Man/Woman and corresponding counts + * are being maintained. It should be modified according to network classes + * or can be removed altogether if not required. + */ +static void +bbox_generated_probe_after_analytics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + NvDsObjectMeta *obj_meta = NULL; + GstClockTime buffer_pts = 0; + guint32 stream_id = 0; + + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + stream_id = frame_meta->source_id; + GstClockTime buf_ntp_time = 0; + if (playback_utc == FALSE) { + /** Calculate the buffer-NTP-time + * derived from this stream's RTCP Sender Report here: + */ + StreamSourceInfo *src_stream = &testAppCtx->streams[stream_id]; + buf_ntp_time = frame_meta->ntp_timestamp; + + if (buf_ntp_time < src_stream->last_ntp_time) { + NVGSTDS_WARN_MSG_V ("Source %d: NTP timestamps are backward in time." + " Current: %lu previous: %lu", stream_id, buf_ntp_time, + src_stream->last_ntp_time); + } + src_stream->last_ntp_time = buf_ntp_time; + } + + float scaleW = 0; + float scaleH = 0; + if (!appCtx->config.streammux_config.pipeline_width || + !appCtx->config.streammux_config.pipeline_height) { + g_print("[ERROR] invalid pipeline params\n"); + + return; + } + + scaleW = (float)frame_meta->source_frame_width / + appCtx->config.streammux_config.pipeline_width; + scaleH = (float)frame_meta->source_frame_height / + appCtx->config.streammux_config.pipeline_height; + + buffer_pts = frame_meta->buf_pts; + if (playback_utc == FALSE) { + /** Use the buffer-NTP-time derived from this stream's RTCP Sender + * Report here: + */ + buffer_pts = buf_ntp_time; + } + + GList *l; + for (l = frame_meta->obj_meta_list; l != NULL; l = l->next) { + /* Now using above information we need to form a text that should + * be displayed on top of the bounding box, so lets form it here. */ + + obj_meta = (NvDsObjectMeta *)(l->data); + NvDsInferTensorMeta *tensor_meta = NULL; + //! Attaching Embedding tensor metadata + for (NvDsMetaList *l_user = obj_meta->obj_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDSINFER_TENSOR_OUTPUT_META) { + /* convert to tensor metadata */ + tensor_meta = (NvDsInferTensorMeta *)user_meta->user_meta_data; + //==Retrieve output tensor from model== + // = (float *)tensor_meta->out_buf_ptrs_host[0]; + } + } + /** Generate NvDsEventMsgMeta for every object */ + NvDsEventMsgMeta *msg_meta = + (NvDsEventMsgMeta *)g_malloc0(sizeof(NvDsEventMsgMeta)); + + generate_event_msg_meta(appCtx, msg_meta, obj_meta->class_id, TRUE, + buffer_pts, + appCtx->config.multi_source_config[stream_id].uri, + stream_id, stream_id, obj_meta, scaleW, scaleH, + frame_meta, tensor_meta, batch_meta); + /* Acquire NvDsUserMeta user meta from pool */ + NvDsUserMeta *new_user_meta = + nvds_acquire_user_meta_from_pool(batch_meta); + /* Set NvDsUserMeta below */ + new_user_meta->user_meta_data = (void *)msg_meta; + new_user_meta->base_meta.batch_meta = batch_meta; + new_user_meta->base_meta.meta_type = NVDS_EVENT_MSG_META; + new_user_meta->base_meta.copy_func = (NvDsMetaCopyFunc)meta_copy_func; + new_user_meta->base_meta.release_func = + (NvDsMetaReleaseFunc)meta_free_func; + /* We want to add NvDsUserMeta to frame level */ + nvds_add_user_meta_to_frame(frame_meta, new_user_meta); + } + testAppCtx->streams[stream_id].frameCount++; + } +} + +/** @{ imported from deepstream-app as is */ + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + cintr = TRUE; +} + +/** + * callback function to print the performance numbers of each stream. + */ +static void +perf_cb (gpointer context, NvDsAppPerfStruct * str) +{ + static guint header_print_cnt = 0; + guint i; + AppCtx *appCtx = (AppCtx *) context; + guint numf = str->num_instances; + + g_mutex_lock (&fps_lock); + guint active_src_count = 0; + for (i = 0; i < numf; i++) { + fps[i] = str->fps[i]; + if (fps[i]){ + active_src_count++; + } + fps_avg[i] = str->fps_avg[i]; + } + g_print("Active sources : %u\n", active_src_count); + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < numf; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + printf ("%s", asctime (tm)); + if (num_instances > 1) + g_print ("PERF(%d): ", appCtx->index); + else + g_print ("**PERF: "); + + for (i = 0; i < numf; i++) { + g_print ("%.2f (%.2f)\t", fps[i], fps_avg[i]); + } + g_print ("\n"); + g_mutex_unlock (&fps_lock); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (quit) { + return FALSE; + } + + if (cintr) { + cintr = FALSE; + + quit = TRUE; + g_main_loop_quit (main_loop); + + return FALSE; + } + return TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +static gboolean +kbhit (void) +{ + struct timeval tv; + fd_set rdfs; + + tv.tv_sec = 0; + tv.tv_usec = 0; + + FD_ZERO (&rdfs); + FD_SET (STDIN_FILENO, &rdfs); + + select (STDIN_FILENO + 1, &rdfs, NULL, NULL, &tv); + return FD_ISSET (STDIN_FILENO, &rdfs); +} + +/* + * Function to enable / disable the canonical mode of terminal. + * In non canonical mode input is available immediately (without the user + * having to type a line-delimiter character). + */ +static void +changemode (int dir) +{ + static struct termios oldt, newt; + + if (dir == 1) { + tcgetattr (STDIN_FILENO, &oldt); + newt = oldt; + newt.c_lflag &= ~(ICANON); + tcsetattr (STDIN_FILENO, TCSANOW, &newt); + } else + tcsetattr (STDIN_FILENO, TCSANOW, &oldt); +} + +static void +print_runtime_commands (void) +{ + g_print ("\nRuntime commands:\n" + "\th: Print this help\n" + "\tq: Quit\n\n" "\tp: Pause\n" "\tr: Resume\n\n"); + + if (appCtx[0]->config.tiled_display_config.enable) { + g_print + ("NOTE: To expand a source in the 2D tiled display and view object details," + " left-click on the source.\n" + " To go back to the tiled display, right-click anywhere on the window.\n\n"); + } +} + +/** + * Loop function to check keyboard inputs and status of each pipeline. + */ +static gboolean +event_thread_func (gpointer arg) +{ + guint i; + gboolean ret = TRUE; + + // Check if all instances have quit + for (i = 0; i < num_instances; i++) { + if (!appCtx[i]->quit) + break; + } + + if (i == num_instances) { + quit = TRUE; + g_main_loop_quit (main_loop); + return FALSE; + } + // Check for keyboard input + if (!kbhit ()) { + //continue; + return TRUE; + } + int c = fgetc (stdin); + g_print ("\n"); + + gint source_id; + GstElement *tiler = appCtx[rcfg]->pipeline.tiled_display_bin.tiler; + + if (appCtx[rcfg]->config.tiled_display_config.enable) + { + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (selecting) { + if (rrowsel == FALSE) { + if (c >= '0' && c <= '9') { + rrow = c - '0'; + g_print ("--selecting source row %d--\n", rrow); + rrowsel = TRUE; + } + } else { + if (c >= '0' && c <= '9') { + int tile_num_columns = appCtx[rcfg]->config.tiled_display_config.columns; + rcol = c - '0'; + selecting = FALSE; + rrowsel = FALSE; + source_id = tile_num_columns * rrow + rcol; + g_print ("--selecting source col %d sou=%d--\n", rcol, source_id); + if (source_id >= (gint) appCtx[rcfg]->config.num_source_sub_bins) { + source_id = -1; + } else { + appCtx[rcfg]->show_bbox_text = TRUE; + appCtx[rcfg]->active_source_index = source_id; + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + } + } + } + } + } + switch (c) { + case 'h': + print_runtime_commands (); + break; + case 'p': + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + case 'r': + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + case 'q': + quit = TRUE; + g_main_loop_quit (main_loop); + ret = FALSE; + break; + case 'c': + if (appCtx[rcfg]->config.tiled_display_config.enable && selecting == FALSE && source_id == -1) { + g_print("--selecting config file --\n"); + c = fgetc(stdin); + if (c >= '0' && c <= '9') { + rcfg = c - '0'; + if (rcfg < num_instances) { + g_print("--selecting config %d--\n", rcfg); + } else { + g_print("--selected config file %d out of bound, reenter\n", rcfg); + rcfg = 0; + } + } + } + break; + case 'z': + if (appCtx[rcfg]->config.tiled_display_config.enable && source_id == -1 && selecting == FALSE) { + g_print ("--selecting source --\n"); + selecting = TRUE; + } else { + if (!show_bbox_text) { + GstElement *nvosd = appCtx[rcfg]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + } + appCtx[rcfg]->active_source_index = -1; + selecting = FALSE; + rcfg = 0; + g_print("--tiled mode --\n"); + } + break; + default: + break; + } + return ret; +} + +static int +get_source_id_from_coordinates (float x_rel, float y_rel, AppCtx *appCtx) +{ + int tile_num_rows = appCtx->config.tiled_display_config.rows; + int tile_num_columns = appCtx->config.tiled_display_config.columns; + + int source_id = (int) (x_rel * tile_num_columns); + source_id += ((int) (y_rel * tile_num_rows)) * tile_num_columns; + + /* Don't allow clicks on empty tiles. */ + if (source_id >= (gint) appCtx->config.num_source_sub_bins) + source_id = -1; + + return source_id; +} + +/** + * Thread to monitor X window events. + */ +static gpointer +nvds_x_event_thread (gpointer data) +{ + g_mutex_lock (&disp_lock); + while (display) { + XEvent e; + guint index; + while (XPending (display)) { + XNextEvent (display, &e); + switch (e.type) { + case ButtonPress: + { + XWindowAttributes win_attr; + XButtonEvent ev = e.xbutton; + gint source_id; + GstElement *tiler; + + XGetWindowAttributes (display, ev.window, &win_attr); + + for (index = 0; index < MAX_INSTANCES; index++) + if (ev.window == windows[index]) + break; + + tiler = appCtx[index]->pipeline.tiled_display_bin.tiler; + g_object_get (G_OBJECT (tiler), "show-source", &source_id, NULL); + + if (ev.button == Button1 && source_id == -1) { + source_id = + get_source_id_from_coordinates (ev.x * 1.0 / win_attr.width, + ev.y * 1.0 / win_attr.height, appCtx[index]); + if (source_id > -1) { + g_object_set (G_OBJECT (tiler), "show-source", source_id, NULL); + appCtx[index]->active_source_index = source_id; + appCtx[index]->show_bbox_text = TRUE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", TRUE, NULL); + } + } else if (ev.button == Button3) { + g_object_set (G_OBJECT (tiler), "show-source", -1, NULL); + appCtx[index]->active_source_index = -1; + if (!show_bbox_text) { + appCtx[index]->show_bbox_text = FALSE; + GstElement *nvosd = appCtx[index]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set (G_OBJECT (nvosd), "display-text", FALSE, NULL); + } + } + } + break; + case KeyRelease: + { + KeySym p, r, q; + guint i; + p = XKeysymToKeycode (display, XK_P); + r = XKeysymToKeycode (display, XK_R); + q = XKeysymToKeycode (display, XK_Q); + if (e.xkey.keycode == p) { + for (i = 0; i < num_instances; i++) + pause_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == r) { + for (i = 0; i < num_instances; i++) + resume_pipeline (appCtx[i]); + break; + } + if (e.xkey.keycode == q) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + case ClientMessage: + { + Atom wm_delete; + for (index = 0; index < MAX_INSTANCES; index++) + if (e.xclient.window == windows[index]) + break; + + wm_delete = XInternAtom (display, "WM_DELETE_WINDOW", 1); + if (wm_delete != None && wm_delete == (Atom) e.xclient.data.l[0]) { + quit = TRUE; + g_main_loop_quit (main_loop); + } + } + break; + } + } + g_mutex_unlock (&disp_lock); + g_usleep (G_USEC_PER_SEC / 20); + g_mutex_lock (&disp_lock); + } + g_mutex_unlock (&disp_lock); + return NULL; +} + +/** + * callback function to add application specific metadata. + * Here it demonstrates how to display the URI of source in addition to + * the text generated after inference. + */ +static gboolean +overlay_graphics (AppCtx * appCtx, GstBuffer * buf, + NvDsBatchMeta * batch_meta, guint index) +{ + return TRUE; +} + +/** + * Callback function to notify the status of the model update + */ +static void +infer_model_updated_cb (GstElement * gie, gint err, const gchar * config_file) +{ + double otaTime = 0; + gettimeofday (&ota_completion_time, NULL); + + otaTime = (ota_completion_time.tv_sec - ota_request_time.tv_sec) * 1000.0; + otaTime += (ota_completion_time.tv_usec - ota_request_time.tv_usec) / 1000.0; + + const char *err_str = (err == 0 ? "ok" : "failed"); + g_print + ("\nModel Update Status: Updated model : %s, OTATime = %f ms, result: %s \n\n", + config_file, otaTime, err_str); +} + +/** + * Function to print detected Inotify handler events + * Used only for debugging purposes + */ +static void +display_inotify_event (struct inotify_event *i_event) +{ + printf (" watch decriptor =%2d; ", i_event->wd); + if (i_event->cookie > 0) + printf ("cookie =%4d; ", i_event->cookie); + + printf ("mask = "); + if (i_event->mask & IN_ACCESS) + printf ("IN_ACCESS "); + if (i_event->mask & IN_ATTRIB) + printf ("IN_ATTRIB "); + if (i_event->mask & IN_CLOSE_NOWRITE) + printf ("IN_CLOSE_NOWRITE "); + if (i_event->mask & IN_CLOSE_WRITE) + printf ("IN_CLOSE_WRITE "); + if (i_event->mask & IN_CREATE) + printf ("IN_CREATE "); + if (i_event->mask & IN_DELETE) + printf ("IN_DELETE "); + if (i_event->mask & IN_DELETE_SELF) + printf ("IN_DELETE_SELF "); + if (i_event->mask & IN_IGNORED) + printf ("IN_IGNORED "); + if (i_event->mask & IN_ISDIR) + printf ("IN_ISDIR "); + if (i_event->mask & IN_MODIFY) + printf ("IN_MODIFY "); + if (i_event->mask & IN_MOVE_SELF) + printf ("IN_MOVE_SELF "); + if (i_event->mask & IN_MOVED_FROM) + printf ("IN_MOVED_FROM "); + if (i_event->mask & IN_MOVED_TO) + printf ("IN_MOVED_TO "); + if (i_event->mask & IN_OPEN) + printf ("IN_OPEN "); + if (i_event->mask & IN_Q_OVERFLOW) + printf ("IN_Q_OVERFLOW "); + if (i_event->mask & IN_UNMOUNT) + printf ("IN_UNMOUNT "); + + if (i_event->mask & IN_CLOSE) + printf ("IN_CLOSE "); + if (i_event->mask & IN_MOVE) + printf ("IN_MOVE "); + if (i_event->mask & IN_UNMOUNT) + printf ("IN_UNMOUNT "); + if (i_event->mask & IN_IGNORED) + printf ("IN_IGNORED "); + if (i_event->mask & IN_Q_OVERFLOW) + printf ("IN_Q_OVERFLOW "); + printf ("\n"); + + if (i_event->len > 0) + printf (" name = %s mask= %x \n", i_event->name, i_event->mask); +} + +/** + * Perform model-update OTA operation + */ +void +apply_ota (AppCtx * ota_appCtx) +{ + GstElement *primary_gie = NULL; + + if (ota_appCtx->override_config.primary_gie_config.enable) { + primary_gie = + ota_appCtx->pipeline.common_elements.primary_gie_bin.primary_gie; + gchar *model_engine_file_path = + ota_appCtx->override_config.primary_gie_config.model_engine_file_path; + + gettimeofday (&ota_request_time, NULL); + if (model_engine_file_path) { + g_print ("\nNew Model Update Request %s ----> %s\n", + GST_ELEMENT_NAME (primary_gie), model_engine_file_path); + g_object_set (G_OBJECT (primary_gie), "model-engine-file", + model_engine_file_path, NULL); + } else { + g_print + ("\nInvalid New Model Update Request received. Property model-engine-path is not set\n"); + } + } +} + +/** + * Independent thread to perform model-update OTA process based on the inotify events + * It handles currently two scenarios + * 1) Local Model Update Request (e.g. Standalone Appliation) + * In this case, notifier handler watches for the ota_override_file changes + * 2) Cloud Model Update Request (e.g. EGX with Kubernetes) + * In this case, notifier handler watches for the ota_override_file changes along with + * ..data directory which gets mounted by EGX deployment in Kubernetes environment. + */ +gpointer +ota_handler_thread (gpointer data) +{ + + int length, i = 0; + char buffer[INOTIFY_EVENT_BUF_LEN]; + OTAInfo *ota = (OTAInfo *) data; + gchar *ota_ds_config_file = ota->override_cfg_file; + AppCtx *ota_appCtx = ota->appCtx; + struct stat file_stat = { 0 }; + GstElement *primary_gie = NULL; + gboolean connect_pgie_signal = FALSE; + + ota_appCtx->ota_inotify_fd = inotify_init (); + + if (ota_appCtx->ota_inotify_fd < 0) { + perror ("inotify_init"); + return NULL; + } + + char *real_path_ds_config_file = realpath (ota_ds_config_file, NULL); + g_print ("REAL PATH = %s\n", real_path_ds_config_file); + + gchar *ota_dir = g_path_get_dirname (real_path_ds_config_file); + ota_appCtx->ota_watch_desc = + inotify_add_watch (ota_appCtx->ota_inotify_fd, ota_dir, IN_ALL_EVENTS); + + int ret = lstat (ota_ds_config_file, &file_stat); + ret = ret; + + if (S_ISLNK (file_stat.st_mode)) { + printf (" Override File Provided is Soft Link\n"); + gchar *parent_ota_dir = g_strdup_printf ("%s/..", ota_dir); + ota_appCtx->ota_watch_desc = + inotify_add_watch (ota_appCtx->ota_inotify_fd, parent_ota_dir, + IN_ALL_EVENTS); + } + + while (1) { + i = 0; + length = read (ota_appCtx->ota_inotify_fd, buffer, INOTIFY_EVENT_BUF_LEN); + + if (length < 0) { + perror ("read"); + } + + if (quit == TRUE) + goto done; + + while (i < length) { + struct inotify_event *event = (struct inotify_event *) &buffer[i]; + + // Enable below function to print the inotify events, used for debugging purpose + if (0) { + display_inotify_event (event); + } + + if (connect_pgie_signal == FALSE) { + primary_gie = + ota_appCtx->pipeline.common_elements.primary_gie_bin.primary_gie; + if (primary_gie) { + g_signal_connect (G_OBJECT (primary_gie), "model-updated", + G_CALLBACK (infer_model_updated_cb), NULL); + connect_pgie_signal = TRUE; + } else { + printf + ("Gstreamer pipeline element nvinfer is yet to be created or invalid\n"); + continue; + } + } + + if (event->len) { + if (event->mask & IN_MOVED_TO) { + if (strstr ("..data", event->name)) { + memset (&ota_appCtx->override_config, 0, + sizeof (ota_appCtx->override_config)); + if (!IS_YAML(ota_ds_config_file)) { + if (!parse_config_file (&ota_appCtx->override_config, + ota_ds_config_file)) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", + ota_ds_config_file); + g_print + ("Error: ota_handler_thread: Failed to parse config file '%s'", + ota_ds_config_file); + } else { + apply_ota (ota_appCtx); + } + } else if (IS_YAML(ota_ds_config_file)) { + if (!parse_config_file_yaml (&ota_appCtx->override_config, + ota_ds_config_file)) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", + ota_ds_config_file); + g_print + ("Error: ota_handler_thread: Failed to parse config file '%s'", + ota_ds_config_file); + } else { + apply_ota (ota_appCtx); + } + } + } + } + if (event->mask & IN_CLOSE_WRITE) { + if (!(event->mask & IN_ISDIR)) { + if (strstr (ota_ds_config_file, event->name)) { + g_print ("File %s modified.\n", event->name); + + memset (&ota_appCtx->override_config, 0, + sizeof (ota_appCtx->override_config)); + if (!IS_YAML(ota_ds_config_file)) { + if (!parse_config_file (&ota_appCtx->override_config, + ota_ds_config_file)) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", + ota_ds_config_file); + g_print + ("Error: ota_handler_thread: Failed to parse config file '%s'", + ota_ds_config_file); + } else { + apply_ota (ota_appCtx); + } + } else if (IS_YAML(ota_ds_config_file)) { + if (!parse_config_file_yaml (&ota_appCtx->override_config, + ota_ds_config_file)) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", + ota_ds_config_file); + g_print + ("Error: ota_handler_thread: Failed to parse config file '%s'", + ota_ds_config_file); + } else { + apply_ota (ota_appCtx); + } + } + } + } + } + } + i += INOTIFY_EVENT_SIZE + event->len; + } + } +done: + inotify_rm_watch (ota_appCtx->ota_inotify_fd, ota_appCtx->ota_watch_desc); + close (ota_appCtx->ota_inotify_fd); + + free (real_path_ds_config_file); + g_free (ota_dir); + + g_free (ota); + return NULL; +} + +/** + * Parse labels from label file and fill them in the global variable: `labels` + */ +void get_labels_from_file(gchar *classifier_label_file) { + // Use file pointer + FILE *file = fopen(classifier_label_file, "r"); + + // Return if file is not found + if (file == NULL) { + perror("Error opening file."); + return; + } + + guint line_count = 0; + while (fgets(labels[line_count], MAX_CHAR_LENGTH_PER_LINE, file) != NULL) { + // Remove newline character from the end of the line + labels[line_count][strcspn(labels[line_count], "\n")] = '\0'; + line_count++; + + if (line_count == dual_head_classes) + break; + else if (line_count > dual_head_classes) { + g_print( + "[WARNING] You have more labels than classes. Labels after line %d " + "will not be " + "used.\n", + dual_head_classes); + break; + } + } + + if (line_count < dual_head_classes) + g_print( + "[WARNING] You have more classes than labels. Empty strings will be " + "used in " + "metadata field when a class without label is predicted.\n"); + + // Close the file + fclose(file); +} + +/** @} imported from deepstream-app as is */ + +int +main (int argc, char *argv[]) +{ + testAppCtx = (TestAppCtx *) g_malloc0 (sizeof (TestAppCtx)); + GOptionContext *ctx = NULL; + GOptionGroup *group = NULL; + GError *error = NULL; + guint i; + OTAInfo *otaInfo = NULL; + + ctx = g_option_context_new ("Nvidia DeepStream MDX Perception App"); + group = g_option_group_new ("abc", NULL, NULL, NULL, NULL); + g_option_group_add_entries (group, entries); + + g_option_context_set_main_group (ctx, group); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL); + + if (!g_option_context_parse (ctx, &argc, &argv, &error)) { + NVGSTDS_ERR_MSG_V ("%s", error->message); + g_print ("%s",g_option_context_get_help (ctx, TRUE, NULL)); + return -1; + } + + if (print_version) { + g_print ("deepstream-MDXPerception-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (print_dependencies_version) { + g_print ("deepstream-MDXPerception-app version %d.%d.%d\n", + NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO); + return 0; + } + + if (cfg_files) { + num_instances = g_strv_length (cfg_files); + } + if (input_files) { + num_input_files = g_strv_length (input_files); + } + + if (!cfg_files || num_instances == 0) { + NVGSTDS_ERR_MSG_V ("Specify config file with -c option"); + return_value = -1; + goto done; + } + + for (i = 0; i < num_instances; i++) { + appCtx[i] = (AppCtx *) g_malloc0 (sizeof (AppCtx)); + appCtx[i]->person_class_id = -1; + appCtx[i]->car_class_id = -1; + appCtx[i]->index = i; + appCtx[i]->active_source_index = -1; + if (show_bbox_text) { + appCtx[i]->show_bbox_text = TRUE; + } + + if (input_files && input_files[i]) { + appCtx[i]->config.multi_source_config[0].uri = + g_strdup_printf ("file://%s", input_files[i]); + g_free (input_files[i]); + } + + if(IS_YAML(cfg_files[i])) { + if (!parse_config_file_yaml (&appCtx[i]->config, cfg_files[i])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[i]); + appCtx[i]->return_value = -1; + goto done; + } + } else { + if (!parse_config_file (&appCtx[i]->config, cfg_files[i])) { + NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[i]); + appCtx[i]->return_value = -1; + goto done; + } + } + + if (override_cfg_file && override_cfg_file[i]) { + if (!g_file_test (override_cfg_file[i], + (GFileTest)(G_FILE_TEST_IS_REGULAR | G_FILE_TEST_IS_SYMLINK))) + { + g_print ("Override file %s does not exist, quitting...\n", + override_cfg_file[i]); + appCtx[i]->return_value = -1; + goto done; + } + otaInfo = (OTAInfo *) g_malloc0 (sizeof (OTAInfo)); + otaInfo->appCtx = appCtx[i]; + otaInfo->override_cfg_file = override_cfg_file[i]; + appCtx[i]->ota_handler_thread = g_thread_new ("ota-handler-thread", + ota_handler_thread, otaInfo); + } + } + + for (i = 0; i < num_instances; i++) { + for (guint j = 0; j < appCtx[i]->config.num_source_sub_bins; j++) { + /** Force the source (applicable only if RTSP) + * to use TCP for RTP/RTCP channels. + * forcing TCP to avoid problems with UDP port usage from within docker- + * container. + * The UDP RTCP channel when run within docker had issues receiving + * RTCP Sender Reports from server + */ + if (force_tcp) + appCtx[i]->config.multi_source_config[j].select_rtp_protocol = 0x04; + } + if (!create_pipeline (appCtx[i], bbox_generated_probe_after_analytics, + NULL, perf_cb, overlay_graphics)) { + NVGSTDS_ERR_MSG_V ("Failed to create pipeline"); + return_value = -1; + goto done; + } + /** Now add probe to RTPSession plugin src pad */ + for (guint j = 0; j < appCtx[i]->pipeline.multi_src_bin.num_bins; j++) { + testAppCtx->streams[j].id = j; + } + /** In test5 app, as we could have several sources connected + * for a typical IoT use-case, raising the nvstreammux's + * buffer-pool-size to 16 */ + g_object_set (appCtx[i]->pipeline.multi_src_bin.streammux, + "buffer-pool-size", STREAMMUX_BUFFER_POOL_SIZE, NULL); + } + + if (model_used == APP_CONFIG_ANALYTICS_RETAIL_DUAL_HEAD_SGIE) { + NvDsGieConfig *classifier_config = + &appCtx[0]->config.secondary_gie_sub_bin_config[0]; + gboolean classifier_enabled = classifier_config->enable; + + if (!classifier_enabled) { + perror("Dual Head Classifier is disabled."); + return 1; + } + + gchar *classifier_label_file = classifier_config->label_file_path; + get_labels_from_file(classifier_label_file); + } + + main_loop = g_main_loop_new (NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, NULL); + + g_mutex_init (&disp_lock); + display = XOpenDisplay (NULL); + for (i = 0; i < num_instances; i++) { + guint j; + + if (!show_bbox_text) { + GstElement *nvosd = appCtx[i]->pipeline.instance_bins[0].osd_bin.nvosd; + g_object_set(G_OBJECT(nvosd), "display-text", FALSE, NULL); + } + + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) { + NVGSTDS_ERR_MSG_V ("Failed to set pipeline to PAUSED"); + return_value = -1; + goto done; + } + + for (j = 0; j < appCtx[i]->config.num_sink_sub_bins; j++) { + XTextProperty xproperty; + gchar *title; + guint width, height; + XSizeHints hints = {0}; + + if (!GST_IS_VIDEO_OVERLAY (appCtx[i]->pipeline.instance_bins[0].sink_bin. + sub_bins[j].sink)) { + continue; + } + + if (!display) { + NVGSTDS_ERR_MSG_V ("Could not open X Display"); + return_value = -1; + goto done; + } + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width) + width = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width; + else + width = appCtx[i]->config.tiled_display_config.width; + + if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height) + height = + appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height; + else + height = appCtx[i]->config.tiled_display_config.height; + + width = (width) ? width : DEFAULT_X_WINDOW_WIDTH; + height = (height) ? height : DEFAULT_X_WINDOW_HEIGHT; + + hints.flags = PPosition | PSize; + hints.x = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_x; + hints.y = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_y; + hints.width = width; + hints.height = height; + + windows[i] = + XCreateSimpleWindow (display, RootWindow (display, + DefaultScreen (display)), hints.x, hints.y, width, height, 2, + 0x00000000, 0x00000000); + + XSetNormalHints(display, windows[i], &hints); + + if (num_instances > 1) + title = g_strdup_printf (APP_TITLE "-%d", i); + else + title = g_strdup (APP_TITLE); + if (XStringListToTextProperty ((char **) &title, 1, &xproperty) != 0) { + XSetWMName (display, windows[i], &xproperty); + XFree (xproperty.value); + } + + XSetWindowAttributes attr = { 0 }; + if ((appCtx[i]->config.tiled_display_config.enable && + appCtx[i]->config.tiled_display_config.rows * + appCtx[i]->config.tiled_display_config.columns == 1) || + (appCtx[i]->config.tiled_display_config.enable == 0)) { + attr.event_mask = KeyRelease; + } else if (appCtx[i]->config.tiled_display_config.enable) { + attr.event_mask = ButtonPress | KeyRelease; + } + XChangeWindowAttributes (display, windows[i], CWEventMask, &attr); + + Atom wmDeleteMessage = XInternAtom (display, "WM_DELETE_WINDOW", False); + if (wmDeleteMessage != None) { + XSetWMProtocols (display, windows[i], &wmDeleteMessage, 1); + } + XMapRaised (display, windows[i]); + XSync (display, 1); //discard the events for now + gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (appCtx + [i]->pipeline.instance_bins[0].sink_bin.sub_bins[j].sink), + (gulong) windows[i]); + gst_video_overlay_expose (GST_VIDEO_OVERLAY (appCtx[i]->pipeline. + instance_bins[0].sink_bin.sub_bins[j].sink)); + if (!x_event_thread) + x_event_thread = g_thread_new ("nvds-window-event-thread", + nvds_x_event_thread, NULL); + } + } + + /* Dont try to set playing state if error is observed */ + if (return_value != -1) { + for (i = 0; i < num_instances; i++) { + if (gst_element_set_state (appCtx[i]->pipeline.pipeline, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { + + g_print ("\ncan't set pipeline to playing state.\n"); + return_value = -1; + goto done; + } + } + } + + print_runtime_commands (); + + changemode (1); + + g_timeout_add (40, event_thread_func, NULL); + g_main_loop_run (main_loop); + + changemode (0); + +done: + + g_print ("Quitting\n"); + for (i = 0; i < num_instances; i++) { + if (appCtx[i] == NULL) + continue; + + if (appCtx[i]->return_value == -1) + return_value = -1; + + destroy_pipeline (appCtx[i]); + + if (appCtx[i]->ota_handler_thread && override_cfg_file[i]) { + inotify_rm_watch (appCtx[i]->ota_inotify_fd, appCtx[i]->ota_watch_desc); + g_thread_join (appCtx[i]->ota_handler_thread); + } + + g_mutex_lock (&disp_lock); + if (windows[i]) + XDestroyWindow (display, windows[i]); + windows[i] = 0; + g_mutex_unlock (&disp_lock); + + g_free (appCtx[i]); + } + + g_mutex_lock (&disp_lock); + if (display) + XCloseDisplay (display); + display = NULL; + g_mutex_unlock (&disp_lock); + g_mutex_clear (&disp_lock); + + if (main_loop) { + g_main_loop_unref (main_loop); + } + + if (ctx) { + g_option_context_free (ctx); + } + + if (return_value == 0) { + g_print ("App run successful\n"); + } else { + g_print ("App run failed\n"); + } + + gst_deinit (); + + return return_value; + + g_free (testAppCtx); + + return 0; +} + +static gchar * +get_first_result_label (NvDsClassifierMeta * classifierMeta) +{ + GList *n; + for (n = classifierMeta->label_info_list; n != NULL; n = n->next) { + NvDsLabelInfo *labelInfo = (NvDsLabelInfo *) (n->data); + if (labelInfo->result_label[0] != '\0') { + return g_strdup (labelInfo->result_label); + } + } + return NULL; +} + +static void +schema_fill_sample_sgie_vehicle_metadata (NvDsObjectMeta * obj_params, + NvDsVehicleObject * obj) +{ + if (!obj_params || !obj) { + return; + } + + /** The JSON obj->classification, say type, color, or make + * according to the schema shall have null (unknown) + * classifications (if the corresponding sgie failed to provide a label) + */ + obj->type = NULL; + obj->make = NULL; + obj->model = NULL; + obj->color = NULL; + obj->license = NULL; + obj->region = NULL; + + GList *l; + for (l = obj_params->classifier_meta_list; l != NULL; l = l->next) { + NvDsClassifierMeta *classifierMeta = (NvDsClassifierMeta *) (l->data); + switch (classifierMeta->unique_component_id) { + case SECONDARY_GIE_VEHICLE_TYPE_UNIQUE_ID: + obj->type = get_first_result_label (classifierMeta); + break; + case SECONDARY_GIE_VEHICLE_COLOR_UNIQUE_ID: + obj->color = get_first_result_label (classifierMeta); + break; + case SECONDARY_GIE_VEHICLE_MAKE_UNIQUE_ID: + obj->make = get_first_result_label (classifierMeta); + break; + default: + break; + } + } +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.h b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.h new file mode 100644 index 00000000..82684669 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __DEEPSTREAM_FEWSHOT_LEARNING_APP_H__ +#define __DEEPSTREAM_FEWSHOT_LEARNING_APP_H__ + +#include +#include "deepstream_config.h" + +/* From deepstream_utc.c */ +#include +#include +#include +#include + +#define _DEFAULT_SOURCE +#define _XOPEN_SOURCE + +typedef struct +{ + gint anomaly_count; + gint meta_number; + struct timespec timespec_first_frame; + GstClockTime gst_ts_first_frame; + GMutex lock_stream_rtcp_sr; + guint32 id; + gint frameCount; + GstClockTime last_ntp_time; +} StreamSourceInfo; + +typedef struct +{ + StreamSourceInfo streams[MAX_SOURCE_BINS]; +} TestAppCtx; + +/** URI sample: + * HWY_20_AND_BRYANT__WB__4_11_2018_4_59_59_485_AM_UTC-07_00.mp4 + * + * Specification format: + * __M_DD_YYYY_H_MIN_SEC_MSEC_AM/PM_UTC.mp4 + * a) Offset is: + * <+/-HOURS_MIN> + * b) In ..AM/PM_TIMEZONE.mp4 + * TIMEZONE will always be UTC + * + * The __M_DD_YYYY_H_MIN_SEC_MSEC_AM/PM time + * shall behind UTC + * + */ +#define URI_UTC_START_DELIM "__" +#define URI_UTC_END_DELIM "_UTC" +#define MAX_UTC_STRING_LEN (256) +#define LENGTH__AMPM_UTC (3) //"_AM" +#define LENGTH_UTC_END_DELIM (4) + +static gboolean extract_ms_from_utc(gchar *utc, guint32 *ms) { + /** find _UTC delim */ + gchar *utc_delim = (gchar *)strstr(utc, URI_UTC_END_DELIM); + gint32 utc_string_length; + if (!utc_delim) { + return FALSE; + } + + /** find the immediately preceeding '_' */ + utc_string_length = + (guint32)(((size_t)utc_delim) - ((size_t)utc)) - LENGTH__AMPM_UTC - 1; + gint32 i_ms = 0; /**< index into utc at which ms is */ + for (i_ms = utc_string_length; ((i_ms >= 0) && (utc[i_ms] != '_')); i_ms--) + ; + if (i_ms < 0) { + return FALSE; + } + + guint32 input_items = sscanf(&utc[i_ms], "_%u_", ms); + if (input_items != 1) { + return FALSE; + } + + /** also remove the ms part from utc string */ + strncpy(&utc[i_ms], (utc_delim - LENGTH__AMPM_UTC), + MAX_UTC_STRING_LEN - utc_string_length); + + return TRUE; +} + +/** + * @brief Extracts and returns the in nanoseconds + */ +static gboolean extract_offset_from_utc(gchar *utc, gint64 *offset_nsec) { + /** find _UTC delim */ + gchar *utc_offset = (gchar *)strstr(utc, URI_UTC_END_DELIM); + if (!utc_offset) { + return FALSE; + } + gint32 hours; + guint32 minutes; + utc_offset += LENGTH_UTC_END_DELIM; + sscanf(utc_offset, "%d_%u", &hours, &minutes); + *offset_nsec = ((ABS(hours) * 60 + minutes) * 60) * GST_SECOND; + if (hours > 0) { + /** if positive, offset shall be subtracted from + * __M_DD_YYYY_H_MIN_SEC_MSEC_AM/PM to arrive at UTC + * Otherwise added + */ + *offset_nsec = (*offset_nsec) * -1; + } + return TRUE; +} + +struct timespec extract_utc_from_uri(gchar *uri) { + gchar utc_string[MAX_UTC_STRING_LEN]; + struct tm utc_tmbroken = {0}; + struct timespec utc_timespec = {0}; + gchar *utc_iter = uri; + gchar *utc = NULL; + guint32 ms = 0; + gint64 time_ns = 0; + /** Find the beginning of UTC time field in URI + * The URI delimiter + */ + do { + utc = utc_iter; + utc_iter = (gchar *)strstr(utc, URI_UTC_START_DELIM); + if (utc_iter) { + /** skip the starting delim */ + utc_iter += strlen(URI_UTC_START_DELIM); + } + } while (utc_iter); + + if (utc == uri) { + /** Invalid URI */ + return utc_timespec; + } + + /** extract ms and remove ms_ */ + g_strlcpy(utc_string, utc, MAX_UTC_STRING_LEN); + + gboolean ok = extract_ms_from_utc(utc_string, &ms); + if (!ok) { + /** Invalid URI */ + return utc_timespec; + } + + /** First generate the time tm structure from provided string; + * Note: Assuming UTC always */ + gchar *first_char_not_processed = + strptime((const char *)utc_string, "%m_%d_%Y_%I_%M_%S_%p", &utc_tmbroken); + if (!first_char_not_processed || + strncmp(first_char_not_processed, URI_UTC_END_DELIM, + LENGTH_UTC_END_DELIM) != 0) { + /** first_char_not_processed should be URI_UTC_END_DELIM + * Otherwise, it is an error condition */ + return utc_timespec; + } + + ok = extract_offset_from_utc(utc_string, &time_ns); + if (!ok) { + /** Invalid URI */ + return utc_timespec; + } + + /** mktime(): Now convert the broken down tm struct to time_t, calendar time + * representation*/ + utc_timespec.tv_sec = timegm(&utc_tmbroken); + utc_timespec.tv_nsec = ms * GST_MSECOND; + + /** final time is */ + time_ns = (gint64)(GST_TIMESPEC_TO_TIME(utc_timespec)) + time_ns; + + GST_TIME_TO_TIMESPEC(time_ns, utc_timespec); + + return utc_timespec; +} + +#endif /**< __DEEPSTREAM_TEST5_APP_H__ */ diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/mdx_perception_pipeline.png b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/mdx_perception_pipeline.png new file mode 100644 index 00000000..85c33589 Binary files /dev/null and b/src/apps/tao_apps/apps/tao_others/deepstream-mdx-perception-app/mdx_perception_pipeline.png differ diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/Makefile new file mode 100644 index 00000000..e512f9a8 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/Makefile @@ -0,0 +1,64 @@ +################################################################################ +# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= deepstream-nvocdr-app + +CXX=g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= deepstream_nvocdr_app.cpp \ + ../common/ds_yml_parse.cpp + +OBJS:= $(SRCS:.cpp=.o) + +PKGS:= gstreamer-1.0 + +CFLAGS:= $(shell pkg-config --cflags $(PKGS)) +CFLAGS+= -I../../../../../../includes/ \ + -I/usr/local/cuda/include \ + -I../common + +LIBS:= $(shell pkg-config --libs $(PKGS)) +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \ + -lm -lstdc++ -lnvds_yml_parser -lgstrtspserver-1.0\ + -L/usr/local/cuda/lib64/ -lcudart -lcuda -lyaml-cpp \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + make -C nvocdr + make -C nvocdr/deepstream + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + cp nvocdr/libnvocdr.so $(LIB_INSTALL_DIR) + cp nvocdr/deepstream/libnvocdr_impl.so $(LIB_INSTALL_DIR) + +clean: + rm -rf *.o $(OBJS) $(APP) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/README.md b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/README.md new file mode 100644 index 00000000..5f6164ac --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/README.md @@ -0,0 +1,89 @@ +## Description +The nvOCDR deepstream sample application for optical character detection and recognition. + +The TAO pretrained models used in the sample application are: +* [OCDNet deployable_onnx_v2.4](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocdnet) +* [OCRNet deployable_v2.1.1](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/ocrnet) + +## Prerequisition +* DeepStream SDK 6.2 GA and above + +#### **Set up the development environment**: +$ git lfs pull + +$ sudo apt update && sudo apt install -y libopencv-dev + +#### **Prepare the nvocdr library**: +Refer to [NVIDIA-Optical-Character-Detection-and-Recognition-Solution](https://github.com/NVIDIA-AI-IOT/NVIDIA-Optical-Character-Detection-and-Recognition-Solution/tree/main) + +1.prepare the libnvocdr.so/libnvocdr_impl.so lib + +```shell +# git clone https://github.com/NVIDIA-AI-IOT/NVIDIA-Optical-Character-Detection-and-Recognition-Solution.git nvocdr +$ git submodule update --init --recursive +$ export CUDA_VER= + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 +$ make -C nvocdr +$ make -C nvocdr/deepstream +$ cp nvocdr/libnvocdr.so /opt/nvidia/deepstream/deepstream/lib/ +$ cp nvocdr/deepstream/libnvocdr_impl.so /opt/nvidia/deepstream/deepstream/lib/ +$ sudo ldconfig +``` + +#### **Get the TensorRT OSS plugin library (Optional)**: + +**Notes: If you're using DeepStream 6.4 and above, you can skip this step.** + +1.To avoid affecting the results of other apps, please replace the TensorRT plugin with the original one after running this app. + +2.Please replace the 'x' of libnvinfer_plugin.so.8.x.x in the shell command line with the actual value in your environment. + +For X86 platform +```shell +$wget https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/secure/8.6.0/local_repos/nv-tensorrt-local-repo-ubuntu2004-8.6.0-cuda-11.8_1.0-1_amd64.deb +$dpkg-deb -xv nv-tensorrt-local-repo-ubuntu2004-8.6.0-cuda-11.8_1.0-1_amd64.deb debs +$cd debs/var/nv-tensorrt-local-repo-ubuntu2004-8.6.0-cuda-11.8 +$dpkg-deb -xv libnvinfer-plugin8_8.6.0.12-1+cuda11.8_amd64.deb deb_file +$cp deb_file/usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8.6.0 /usr/lib/x86_64-linux-gnu/libnvinfer_plugin.so.8.x.x +``` + +For Jetson platform +- Get TensorRT OSS repository +```shell +git clone -b release/8.6 https://github.com/NVIDIA/TensorRT.git +cd TensorRT +git submodule update --init --recursive +``` + +- Compile TensorRT `libnvinfer_plugin.so`: +```shell +mkdir build && cd build +cmake .. -DTRT_LIB_DIR=/usr/lib/aarch64-linux-gnu/ +make nvinfer_plugin -j4 +``` + +- Copy the `libnvinfer_plugin.so` to the system library path +```shell +cp libnvinfer_plugin.so.8.6.x /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8.x.x +``` + +## Build And Run +The application can be build and run seperately. + +Prepare OCDNet/OCRNet engine file +```bash +#Generate OCDNet engine with dynmaic batch size and max batch size is 4: +/usr/src/tensorrt/bin/trtexec --onnx=../../../models/nvocdr/ocdnet.onnx --minShapes=input:1x3x736x1280 --optShapes=input:1x3x736x1280 --maxShapes=input:1x3x736x1280 --fp16 --saveEngine=../../../models/nvocdr/ocdnet.fp16.engine + +#Generate OCRNet engine with dynamic batch size and max batch size is 32: +/usr/src/tensorrt/bin/trtexec --onnx=../../../models/nvocdr/ocrnet.onnx --minShapes=input:1x1x64x200 --optShapes=input:32x1x64x200 --maxShapes=input:32x1x64x200 --fp16 --saveEngine=../../../models/nvocdr/ocrnet.fp16.engine +``` + +```bash +# Go to the deepstream-nvocdr-app directory +make + +# Set the 'source-list' as the path of your sources in the nvocdr_app_config.yml +./deepstream-nvocdr-app ../../../configs/app/nvocdr_app_config.yml +``` \ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.cpp new file mode 100644 index 00000000..54328721 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.cpp @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include "nvds_yml_parser.h" +#include "ds_yml_parse.h" +#include +#include +#include "cuda_runtime_api.h" + +GstElement *pipeline = NULL; + +static void +signal_catch_callback(int signum) +{ + g_print("User Interrupted..\n"); + if(pipeline != NULL) { + gst_element_send_event(pipeline, gst_event_new_eos()); + g_print("Send EOS to pipline!\n"); + } +} + +static void +signal_catch_setup() +{ + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_handler = signal_catch_callback; + sigaction(SIGINT, &action, NULL); +} + +typedef struct _DsSourceBin +{ + GstElement *source_bin; + GstElement *uri_decode_bin; + GstElement *vidconv; + GstElement *nvvidconv; + GstElement *capsfilt; + GstElement *capsraw; + gint index; +}DsSourceBinStruct; + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad to videoconvert if no hardware decoder is used */ + if (bin_struct->vidconv) { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->vidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to" + " converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + if (!gst_element_link_many (bin_struct->vidconv, bin_struct->capsraw, + bin_struct->nvvidconv, NULL)) { + g_printerr ("Failed to link videoconvert to nvvideoconvert\n"); + } + } else { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->nvvidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to " + "converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + } + if (gst_caps_features_contains (features, "memory:NVMM")) { + g_print ("###Decodebin pick nvidia decoder plugin.\n"); + } else { + /* Get the source bin ghost pad */ + g_print ("###Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) user_data; + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + if (g_strstr_len (name, -1, "pngdec") == name) { + bin_struct->vidconv = gst_element_factory_make ("videoconvert", + "source_vidconv"); + bin_struct->capsraw = gst_element_factory_make ("capsfilter", + "raw_caps"); + GstCaps *caps = gst_caps_new_simple ("video/x-raw", "format", + G_TYPE_STRING, "I420", NULL); + g_object_set (G_OBJECT (bin_struct->capsraw), "caps", caps, NULL); + gst_bin_add_many (GST_BIN (bin_struct->source_bin), bin_struct->vidconv, + bin_struct->capsraw, NULL); + } else { + bin_struct->vidconv = NULL; + } +} + +static bool +create_source_bin (DsSourceBinStruct *ds_source_struct, gchar * uri) +{ + gchar bin_name[16] = { }; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + ds_source_struct->nvvidconv = NULL; + ds_source_struct->capsfilt = NULL; + ds_source_struct->source_bin = NULL; + ds_source_struct->uri_decode_bin = NULL; + + g_snprintf (bin_name, 15, "source-bin-%02d", ds_source_struct->index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + ds_source_struct->source_bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + ds_source_struct->uri_decode_bin = gst_element_factory_make ("uridecodebin", + "uri-decode-bin"); + ds_source_struct->nvvidconv = gst_element_factory_make ("nvvideoconvert", + "source_nvvidconv"); + ds_source_struct->capsfilt = gst_element_factory_make ("capsfilter", + "source_capset"); + + if (!ds_source_struct->source_bin || !ds_source_struct->uri_decode_bin || + !ds_source_struct->nvvidconv + || !ds_source_struct->capsfilt) { + g_printerr ("One element in source bin could not be created.\n"); + return false; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (ds_source_struct->uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), ds_source_struct); + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), ds_source_struct); + + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (ds_source_struct->capsfilt), "caps", caps, NULL); + + gst_bin_add_many (GST_BIN (ds_source_struct->source_bin), + ds_source_struct->uri_decode_bin, ds_source_struct->nvvidconv, + ds_source_struct->capsfilt, NULL); + + if (!gst_element_link (ds_source_struct->nvvidconv, + ds_source_struct->capsfilt)) { + g_printerr ("Could not link vidconv and capsfilter\n"); + return false; + } + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + GstPad *gstpad = gst_element_get_static_pad (ds_source_struct->capsfilt, + "src"); + if (!gstpad) { + g_printerr ("Could not find srcpad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + return false; + } + if(!gst_element_add_pad (ds_source_struct->source_bin, + gst_ghost_pad_new ("src", gstpad))) { + g_printerr ("Could not add ghost pad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + } + gst_object_unref (gstpad); + + return true; +} + +int main (int argc, char *argv[]) { + GMainLoop *loop = NULL; + GstElement *streammux = NULL, *sink = NULL, + *nvvidconv = NULL, *nvosd = NULL, *nvvidconv1 = NULL, + *outenc = NULL, *capfilt = NULL, *nvtile = NULL, + *nvocdr_template = NULL, *mux = NULL, *encparse = NULL; + DsSourceBinStruct source_struct[128]; + + GstBus *bus = NULL; + guint bus_watch_id; + GstCaps *caps = NULL; + + static guint src_cnt = 0; + guint tiler_rows, tiler_columns; + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + bool isImage=false; + bool isStreaming=false; + GList* g_list = NULL; + GList* iterator = NULL; + bool isH264 = true; + int enc_type = ENCODER_TYPE_HW; + gchar *filepath = NULL; + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + /* setup singal handler */ + signal_catch_setup(); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("nvocdr-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One main element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN(pipeline), streammux); + + if (NVDS_YAML_PARSER_SUCCESS != nvds_parse_source_list(&g_list, argv[1], "source-list")) { + g_printerr ("No source is found. Exiting.\n"); + return -1; + } + + for (iterator = g_list, src_cnt=0; iterator; iterator = iterator->next,src_cnt++) { + /* Source element for reading from the file */ + source_struct[src_cnt].index = src_cnt; + if (g_strrstr ((gchar *)iterator->data, ".jpg") || + g_strrstr ((gchar *)iterator->data, ".jpeg") || + g_strrstr ((gchar *)iterator->data, ".png")) + isImage = true; + else + isImage = false; + + if (g_strrstr ((gchar *)iterator->data, "rtsp://") || + g_strrstr ((gchar *)iterator->data, "v4l2://") || + g_strrstr ((gchar *)iterator->data, "http://") || + g_strrstr ((gchar *)iterator->data, "rtmp://")) { + isStreaming = true; + } else { + isStreaming = false; + } + + if (!create_source_bin (&(source_struct[src_cnt]), (gchar *)iterator->data)) + { + g_printerr ("Source bin could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_struct[src_cnt].source_bin); + + g_snprintf (pad_name_sink, 64, "sink_%d", src_cnt); + sinkpad = gst_element_request_pad_simple (streammux, pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_struct[src_cnt].source_bin, + pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + } + + nvocdr_template = gst_element_factory_make ("nvdsvideotemplate", + "nvocdr_template"); + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvid-converter"); + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + nvvidconv1 = gst_element_factory_make ("nvvideoconvert", "nvvid-converter1"); + capfilt = gst_element_factory_make ("capsfilter", "nvvideo-caps"); + nvtile = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + guint output_type = 0; + + output_type = ds_parse_group_type(argv[1], "output"); + if(!output_type){ + g_printerr ("No output setting. Exiting.\n"); + return -1; + } + + if (output_type == 1) { + GString * filename = NULL; + filename = ds_parse_file_name(argv[1], "output"); + + if (isImage) { + outenc = gst_element_factory_make ("jpegenc", "jpegenc"); + caps = + gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, + "I420", NULL); + g_object_set (G_OBJECT (capfilt), "caps", caps, NULL); + filepath = g_strconcat(filename->str,".jpg",NULL); + } else { + mux = gst_element_factory_make ("qtmux", "mp4-mux"); + isH264 = !(ds_parse_enc_codec(argv[1], "output")); + enc_type = ds_parse_enc_type(argv[1], "output"); + + create_video_encoder(isH264, enc_type, &capfilt, &outenc, &encparse, NULL); + filepath = g_strconcat(filename->str,".mp4",NULL); + if(enc_type == ENCODER_TYPE_HW) + ds_parse_enc_config (outenc, argv[1], "output"); + } + sink = gst_element_factory_make ("filesink", "nvvideo-renderer"); + } + else if (output_type == 2) + sink = gst_element_factory_make ("fakesink", "fake-renderer"); + else if (output_type == 3) { + if(prop.integrated) + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); + else +#ifdef __aarch64__ + sink = gst_element_factory_make ("nv3dsink", "nvvideo-renderer"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!nvvidconv || !nvocdr_template || !nvosd || !sink || !capfilt) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + nvds_parse_streammux(streammux, argv[1], "streammux"); + + if (isStreaming) + g_object_set (G_OBJECT (streammux), "live-source", true, NULL); + g_object_set (G_OBJECT (streammux), "batch-size", src_cnt, NULL); + + tiler_rows = (guint) sqrt (src_cnt); + tiler_columns = (guint) ceil (1.0 * src_cnt / tiler_rows); + g_object_set (G_OBJECT (nvtile), "rows", tiler_rows, "columns", + tiler_columns, "width", 1280, "height", 720, NULL); + + ds_parse_ocdr_videotemplate_config(nvocdr_template, argv[1], "video-template"); + + /* we add a bus message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many (GST_BIN (pipeline), + nvocdr_template, nvvidconv, nvosd, nvtile, sink, NULL); + + if (!gst_element_link_many (streammux, nvocdr_template, nvtile, nvvidconv, nvosd, NULL)) { + g_printerr ("Inferring and tracking elements link failure.\n"); + return -1; + } + + g_object_set (G_OBJECT (sink), "sync", 0, "async", false,NULL); + + if (output_type == 1) { + g_object_set (G_OBJECT (sink), "location", filepath, NULL); + g_object_set (G_OBJECT (sink), "enable-last-sample", false,NULL); + if (!isImage) { + gst_bin_add_many (GST_BIN (pipeline), nvvidconv1, outenc, capfilt, + encparse, mux, NULL); + if (!gst_element_link_many (nvosd, nvvidconv1, capfilt, + outenc, encparse, mux, sink, NULL)) { + g_printerr ("OSD and sink elements link failure.\n"); + return -1; + } + } else { + gst_bin_add_many (GST_BIN (pipeline), nvvidconv1, outenc, capfilt, + NULL); + if (!gst_element_link_many (nvosd, nvvidconv1, capfilt, + outenc, sink, NULL)) { + g_printerr ("OSD and sink elements link failure.\n"); + return -1; + } + } + g_free(filepath); + } else if (output_type == 2) { + if (!gst_element_link (nvosd, sink)) { + g_printerr ("OSD and sink elements link failure.\n"); + return -1; + } + } else if (output_type == 3) { + if (!gst_element_link_many (nvosd, sink, NULL)) { + g_printerr ("OSD and sink elements link failure.\n"); + return -1; + } + } + + /* Set the pipeline to "playing" state */ + g_print (" Now playing! \n"); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + + return 0; +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/nvocdr b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/nvocdr new file mode 160000 index 00000000..b5f3d248 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-nvocdr-app/nvocdr @@ -0,0 +1 @@ +Subproject commit b5f3d248c3a2e43284c373f42cf621dc9e43c9bc diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/Makefile new file mode 100644 index 00000000..0ef5bfa4 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/Makefile @@ -0,0 +1,94 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX=g++ -std=c++14 + +APP:= deepstream-pose-classification-app + + +CUDA_HOME:= /usr/local/cuda-$(CUDA_VER) +DEEPSTREAM_HOME:= /opt/nvidia/deepstream/deepstream + +LIB_INSTALL_DIR?=$(DEEPSTREAM_HOME)/lib/ +APP_INSTALL_DIR?=$(DEEPSTREAM_HOME)/bin/ + +SRCS:= deepstream_pose_classification_app.cpp ../common/ds_yml_parse.cpp + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 + +OBJS:= $(patsubst %.c,%.o, $(patsubst %.cpp,%.o, $(SRCS))) + +CFLAGS+= \ + -I$(CUDA_HOME)/include \ + -I$(DEEPSTREAM_HOME)/sources/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/apps-common/includes \ + -I$(DEEPSTREAM_HOME)/sources/apps/sample_apps/deepstream-app \ + -I../common + +LIBS+= \ + -L$(CUDA_HOME)/lib64 -lcudart -lcuda -lnvds_yml_parser -lyaml-cpp\ + -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvds_utils -lgstapp-1.0 -lgstrtspserver-1.0 \ + -lpthread -lm -ldl -Wl,-rpath,$(LIB_INSTALL_DIR) + +CFLAGS+= $(shell pkg-config --cflags $(PKGS)) + +LIBS+= $(shell pkg-config --libs $(PKGS)) + +all: $(APP) + make -C nvdsinfer_custom_impl_BodyPose3DNet + make -C nvdspreprocess_lib + make -C infer_pose_classification_parser + make -C postprocesslib_impl + +debug: CXXFLAGS += -DDEBUG -ggdb +debug: CCFLAGS += -DDEBUG -ggdb +debug: CFLAGS += -DDEBUG -ggdb +debug: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cpp $(INCS) Makefile + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CXX) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) + make clean -C nvdsinfer_custom_impl_BodyPose3DNet + make clean -C nvdspreprocess_lib + make clean -C infer_pose_classification_parser + make clean -C postprocesslib_impl + + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/README.md b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/README.md new file mode 100644 index 00000000..e4699468 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/README.md @@ -0,0 +1,37 @@ +# deepstream-pose-classification + +## Introduction +The project contains pose classification application built using Deepstream SDK. + +## Prerequisites: +DeepStream SDK not less than 6.2 installed which is available at http://developer.nvidia.com/deepstream-sdk + +Please follow instructions in the `/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-app/README` on how to install the prequisites for building Deepstream SDK apps. + +## Installation +1. Install libeigen3-dev +```bash + $ sudo apt install libeigen3-dev + $ cd /usr/include + $ sudo ln -sf eigen3/Eigen Eigen +``` +2. prepare the models + [PeopleNet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet) + [BodyPose3DNet](https://ngc.nvidia.com/models/nvstaging:tao:bodypose3dnet) + [poseclassificationnet](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/poseclassificationnet) + [tracker model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/reidentificationnet) +## Build the applications +```bash +make +``` + +## Run the applications +### `deepstream-pose-classification-app` +The command line options of this application are listed below: +```bash +$ ./deepstream-pose-classification-app ../../../configs/app/deepstream_pose_classification_config.yaml +``` + +## Tips + +1. if FPS is getting smaller and smaller when there are many objects in the sources, please increase scaling-buf-pool-size and tensor-buf-pool-size in config_preprocess_bodypose_classification.txt. diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.cpp new file mode 100644 index 00000000..6bf4b24d --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.cpp @@ -0,0 +1,833 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include +#include +#include + +#include "cuda_runtime_api.h" +#include "gstnvdsmeta.h" +#include "nvdsmeta_schema.h" +#include "deepstream_common.h" +#include "deepstream_perf.h" +#include "nvds_yml_parser.h" +#include "ds_yml_parse.h" + +#define MAX_DISPLAY_LEN 64 + +// Default camera attributes +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Padding due to AR SDK model requires bigger bboxes*/ +#define PAD_DIM 128 + +#define PGIE_CLASS_ID_PERSON 0 + +/* Check for parsing error. */ +#define RETURN_ON_PARSER_ERROR(parse_expr) \ + if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ + g_printerr("Error in parsing configuration file.\n"); \ + return -1; \ + } + +//---Global variables derived from program arguments--- +static guint _cintr = FALSE; +static gboolean _quit = FALSE; +int _image_width = MUXER_OUTPUT_WIDTH; +int _image_height = MUXER_OUTPUT_HEIGHT; +int _pad_dim = PAD_DIM;// A scaled version of PAD_DIM + +static GstElement *pipeline = NULL; + +gint frame_number = 0; + +#define ACQUIRE_DISP_META(dmeta) \ + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_labels == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) \ + { \ + dmeta = nvds_acquire_display_meta_from_pool(bmeta);\ + nvds_add_display_meta_to_frame(frame_meta, dmeta);\ + }\ + +static float _sgie_classifier_threshold = FLT_MIN; + + +/* pgie_src_pad_buffer_probe will extract metadata received from pgie + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +pgie_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + gchar *msg = NULL; + GstBuffer *buf = (GstBuffer *)info->data; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + float sizex = obj_meta->rect_params.width * .5f; + float sizey = obj_meta->rect_params.height * .5f; + float centrx = obj_meta->rect_params.left + sizex; + float centry = obj_meta->rect_params.top + sizey; + sizex *= (1.25f); + sizey *= (1.25f); + if (sizex < sizey) + sizex = sizey; + else + sizey = sizex; + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + + sizex= obj_meta->rect_params.width * .5f, sizey = obj_meta->rect_params.height * .5f; + centrx = obj_meta->rect_params.left + sizex, centry = obj_meta->rect_params.top + sizey; + // Make sure box has same aspect ratio as 3D Body Pose model's input dimensions + // (e.g 192x256 -> 0.75 aspect ratio) by enlarging in the appropriate dimension. + float xScale = (float)192.0 / (float)sizex, yScale = (float)256.0 / (float)sizey; + if (xScale < yScale) { // expand on height + sizey = (float)256.0/ xScale; + } + else { // expand on width + sizex = (float)192.0 / yScale; + } + + obj_meta->rect_params.width = roundf(2.f *sizex); + obj_meta->rect_params.height = roundf(2.f *sizey); + obj_meta->rect_params.left = roundf (centrx - obj_meta->rect_params.width/2.f); + obj_meta->rect_params.top = roundf (centry - obj_meta->rect_params.height/2.f); + if (obj_meta->rect_params.left < 0.0) { + obj_meta->rect_params.left = 0.0; + } + if (obj_meta->rect_params.top < 0.0) { + obj_meta->rect_params.top = 0.0; + } + if (obj_meta->rect_params.left + obj_meta->rect_params.width > muxer_output_width_pad -1){ + obj_meta->rect_params.width = muxer_output_width_pad - 1 - obj_meta->rect_params.left; + } + if (obj_meta->rect_params.top + obj_meta->rect_params.height > muxer_output_height_pad -1){ + obj_meta->rect_params.height = muxer_output_height_pad - 1 - obj_meta->rect_params.top; + } + + } + } + return GST_PAD_PROBE_OK; +} + +/* osd_sink_pad_buffer_probe will extract metadata received from OSD + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) +{ + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) + { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + + /* Parameters to draw text onto the On-Screen-Display */ + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Frame Number %d", frame_number); + offset = snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, " "); + + txt_params->x_offset = 50; + txt_params->y_offset = 100; + + char font_name[] = "Mono"; + txt_params->font_params.font_name = font_name; + txt_params->font_params.font_size = 15; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + frame_number++; + return GST_PAD_PROBE_OK; +} + +typedef struct _DsSourceBin +{ + GstElement *source_bin; + GstElement *uri_decode_bin; + GstElement *vidconv; + GstElement *nvvidconv; + GstElement *capsfilt; + GstElement *capsraw; + gint index; +}DsSourceBinStruct; + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) +{ + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR:{ + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad to videoconvert if no hardware decoder is used */ + if (bin_struct->vidconv) { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->vidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to" + " converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + if (!gst_element_link_many (bin_struct->vidconv, bin_struct->capsraw, + bin_struct->nvvidconv, NULL)) { + g_printerr ("Failed to link videoconvert to nvvideoconvert\n"); + } + } else { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->nvvidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to " + "converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + } + if (gst_caps_features_contains (features, "memory:NVMM")) { + g_print ("###Decodebin pick nvidia decoder plugin.\n"); + } else { + /* Get the source bin ghost pad */ + g_print ("###Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) user_data; + g_print ("Decodebin child added: %s\n", name); + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + if (g_strstr_len (name, -1, "pngdec") == name) { + bin_struct->vidconv = gst_element_factory_make ("videoconvert", + "source_vidconv"); + bin_struct->capsraw = gst_element_factory_make ("capsfilter", + "raw_caps"); + GstCaps *caps = gst_caps_new_simple ("video/x-raw", "format", + G_TYPE_STRING, "I420", NULL); + g_object_set (G_OBJECT (bin_struct->capsraw), "caps", caps, NULL); + gst_bin_add_many (GST_BIN (bin_struct->source_bin), bin_struct->vidconv, + bin_struct->capsraw, NULL); + } else { + bin_struct->vidconv = NULL; + } +} + +static bool +create_source_bin (DsSourceBinStruct *ds_source_struct, gchar * uri) +{ + gchar bin_name[16] = { }; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + ds_source_struct->nvvidconv = NULL; + ds_source_struct->capsfilt = NULL; + ds_source_struct->source_bin = NULL; + ds_source_struct->uri_decode_bin = NULL; + + g_snprintf (bin_name, 15, "source-bin-%02d", ds_source_struct->index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + ds_source_struct->source_bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + ds_source_struct->uri_decode_bin = gst_element_factory_make ("uridecodebin", + "uri-decode-bin"); + ds_source_struct->nvvidconv = gst_element_factory_make ("nvvideoconvert", + "source_nvvidconv"); + ds_source_struct->capsfilt = gst_element_factory_make ("capsfilter", + "source_capset"); + + if (!ds_source_struct->source_bin || !ds_source_struct->uri_decode_bin || + !ds_source_struct->nvvidconv + || !ds_source_struct->capsfilt) { + g_printerr ("One element in source bin could not be created.\n"); + return false; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (ds_source_struct->uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), ds_source_struct); + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), ds_source_struct); + + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (ds_source_struct->capsfilt), "caps", caps, NULL); + + gst_bin_add_many (GST_BIN (ds_source_struct->source_bin), + ds_source_struct->uri_decode_bin, ds_source_struct->nvvidconv, + ds_source_struct->capsfilt, NULL); + + if (!gst_element_link (ds_source_struct->nvvidconv, + ds_source_struct->capsfilt)) { + g_printerr ("Could not link vidconv and capsfilter\n"); + return false; + } + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + GstPad *gstpad = gst_element_get_static_pad (ds_source_struct->capsfilt, + "src"); + if (!gstpad) { + g_printerr ("Could not find srcpad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + return false; + } + if(!gst_element_add_pad (ds_source_struct->source_bin, + gst_ghost_pad_new ("src", gstpad))) { + g_printerr ("Could not add ghost pad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + } + gst_object_unref (gstpad); + + return true; +} + +/** + * Function to handle program interrupt signal. + * It installs default handler after handling the interrupt. + */ +static void +_intr_handler (int signum) +{ + struct sigaction action; + + NVGSTDS_ERR_MSG_V ("User Interrupted.. \n"); + + memset (&action, 0, sizeof (action)); + action.sa_handler = SIG_DFL; + + sigaction (SIGINT, &action, NULL); + + _cintr = TRUE; +} + +/* + * Function to install custom handler for program interrupt signal. + */ +static void +_intr_setup (void) +{ + struct sigaction action; + + memset (&action, 0, sizeof (action)); + action.sa_handler = _intr_handler; + + sigaction (SIGINT, &action, NULL); +} + +/** + * Loop function to check the status of interrupts. + * It comes out of loop if application got interrupted. + */ +static gboolean +check_for_interrupt (gpointer data) +{ + if (_quit) { + return FALSE; + } + + if (_cintr) { + _cintr = FALSE; + + _quit = TRUE; + GMainLoop *loop = (GMainLoop *) data; + g_main_loop_quit (loop); + + return FALSE; + } + return TRUE; +} + +int main(int argc, char *argv[]) +{ + guint num_sources = 0; + + GMainLoop *loop = NULL; + GstCaps *caps = NULL; + GstElement *streammux = NULL, *pgie = NULL, *sgie = NULL, *postprocess = NULL, *preprocess1 = NULL, *sgie1 = NULL;; + // Padding the image and removing the padding + GstElement *nvvideoconvert_enlarge = NULL, *nvvideoconvert_reduce = NULL, + *capsFilter_enlarge = NULL, *capsFilter_reduce = NULL; + GstElement *nvvidconv = NULL, *nvtile = NULL, *nvosd = NULL, *tracker = NULL, *nvdslogger = NULL; + GstElement *sink = NULL; + DsSourceBinStruct source_struct[128]; + GstBus *bus = NULL; + guint bus_watch_id; + + gboolean useDisplay = FALSE; + gboolean useFakeSink = FALSE; + gboolean useFileSink = FALSE; + guint tiler_rows, tiler_columns; + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + bool isStreaming=false; + GList* g_list = NULL; + GList* iterator = NULL; + bool isH264 = true; + gchar *filepath = NULL; + + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + /* Standard GStreamer initialization */ + // signal(SIGINT, sigintHandler); + gst_init(&argc, &argv); + loop = g_main_loop_new(NULL, FALSE); + + _intr_setup (); + g_timeout_add (400, check_for_interrupt, NULL); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new("deepstream_pose_classfication_app"); + if (!pipeline) { + g_printerr ("Pipeline could not be created. Exiting.\n"); + return -1; + } + + /* we add a message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "streammux-pgie"); + if (!streammux) { + g_printerr ("PGIE streammux could not be created. Exiting.\n"); + return -1; + } + gst_bin_add(GST_BIN(pipeline), streammux); + + parse_streammux_width_height_yaml(&_image_width, &_image_height, argv[1]); + g_print("width %d hight %d\n", _image_width, _image_height); + + _pad_dim = PAD_DIM * _image_width / MUXER_OUTPUT_WIDTH; + //---Set properties of streammux--- + + if (NVDS_YAML_PARSER_SUCCESS != nvds_parse_source_list(&g_list, argv[1], "source-list")) { + g_printerr ("No source is found. Exiting.\n"); + return -1; + } + + for (iterator = g_list, num_sources=0; iterator; iterator = iterator->next,num_sources++) { + /* Source element for reading from the file */ + source_struct[num_sources].index = num_sources; + + if (g_strrstr ((gchar *)iterator->data, "rtsp://") || + g_strrstr ((gchar *)iterator->data, "v4l2://") || + g_strrstr ((gchar *)iterator->data, "http://") || + g_strrstr ((gchar *)iterator->data, "rtmp://")) { + isStreaming = true; + } else { + isStreaming = false; + } + + g_print("video %s\n", (gchar *)iterator->data); + + if (!create_source_bin (&(source_struct[num_sources]), (gchar *)iterator->data)) + { + g_printerr ("Source bin could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_struct[num_sources].source_bin); + + g_snprintf (pad_name_sink, 64, "sink_%d", num_sources); + sinkpad = gst_element_request_pad_simple (streammux, pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_struct[num_sources].source_bin, + pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + GstPadLinkReturn ret = gst_pad_link (srcpad, sinkpad); + if ( ret != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting. %d\n",ret); + return -1; + } + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + } + + nvds_parse_streammux(streammux, argv[1], "streammux"); + + if (isStreaming) + g_object_set (G_OBJECT (streammux), "live-source", true, NULL); + g_object_set (G_OBJECT (streammux), "batch-size", num_sources, NULL); + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER; + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&pgie_type, argv[1], "primary-gie")); + if (pgie_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + pgie = gst_element_factory_make("nvinferserver", "primary-nvinference-engine"); + } else { + pgie = gst_element_factory_make("nvinfer", "primary-nvinference-engine"); + } + if (!pgie) { + g_printerr ("PGIE element could not be created. Exiting.\n"); + return -1; + } + nvds_parse_gie (pgie, argv[1], "primary-gie"); + + /* Override the batch-size set in the config file with the number of sources. */ + guint pgie_batch_size = 0; + g_object_get(G_OBJECT(pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, num_sources); + + g_object_set(G_OBJECT(pgie), "batch-size", num_sources, NULL); + } + + //---Set pgie properties--- + + /* We need to have a tracker to track the identified objects */ + tracker = gst_element_factory_make ("nvtracker", "tracker"); + if (!tracker) { + g_printerr ("Nvtracker could not be created. Exiting.\n"); + return -1; + } + nvds_parse_tracker(tracker, argv[1], "tracker"); + + nvdslogger = gst_element_factory_make ("nvdslogger", "nvdslogger"); + if (!nvdslogger) { + g_printerr ("Nvdslogger could not be created. Exiting.\n"); + return -1; + } + g_object_set (G_OBJECT(nvdslogger), "fps-measurement-interval-sec", + 1, NULL); + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* pgie_src_pad = gst_element_get_static_pad(tracker, "src"); + if (!pgie_src_pad) + g_printerr ("Unable to get src pad for pgie\n"); + else + gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + pgie_src_pad_buffer_probe, NULL, NULL); + gst_object_unref (pgie_src_pad); + + /* 3d bodypose secondary gie */ + NvDsGieType sgie0_type = NVDS_GIE_PLUGIN_INFER; + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&sgie0_type, argv[1], "secondary-gie0")); + if (sgie0_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + sgie = gst_element_factory_make("nvinferserver", "secondary-nvinference-engine"); + } else { + sgie = gst_element_factory_make("nvinfer", "secondary-nvinference-engine"); + } + if (!sgie) { + g_printerr ("Secondary nvinfer could not be created. Exiting.\n"); + return -1; + } + //---Set sgie properties--- + /* Configure the nvinfer element using the nvinfer config file. */ + nvds_parse_gie (sgie, argv[1], "secondary-gie0"); + + /* Override the batch-size set in the config file with the number of sources. */ + guint sgie_batch_size = 0; + g_object_get(G_OBJECT(sgie), "batch-size", &sgie_batch_size, NULL); + if (sgie_batch_size < num_sources) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + sgie_batch_size, num_sources); + + g_object_set(G_OBJECT(sgie), "batch-size", num_sources, NULL); + } + //---Set sgie properties--- + + /* postprocess for 3d bodypose secondary gie */ + postprocess = gst_element_factory_make("nvdspostprocess", "postprocess-plugin"); + nvds_parse_postprocess(postprocess, argv[1], "secondary-postprocess0"); + + /* preprocess + bodypose classification */ + preprocess1 = gst_element_factory_make("nvdspreprocess", "preprocess-plugin"); + nvds_parse_preprocess(preprocess1, argv[1], "secondary-preprocess1"); + NvDsGieType sgie1_type = NVDS_GIE_PLUGIN_INFER; + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&sgie1_type, argv[1], "secondary-gie1")); + if (sgie1_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + sgie1 = gst_element_factory_make("nvinferserver", "bodypose-classification-nvinference-engine"); + } else { + sgie1 = gst_element_factory_make("nvinfer", "bodypose-classification-nvinference-engine"); + } + if (!sgie1) { + g_printerr ("sgie1 could not be created. Exiting.\n"); + return -1; + } + nvds_parse_gie (sgie1, argv[1], "secondary-gie1"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvideo-converter"); + if (!nvvidconv) { + g_printerr ("nvvidconv could not be created. Exiting.\n"); + return -1; + } + + //---Manipulate image size so that PGIE bbox is large enough--- + // Enlarge image so that PeopleNet detected bbox is larger which would fully cover the + // detected object in the original sized image. + nvvideoconvert_enlarge = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_enlarge"); + if (!nvvideoconvert_enlarge) { + g_printerr ("nvvideoconvert_enlarge could not be created. Exiting.\n"); + return -1; + } + capsFilter_enlarge = gst_element_factory_make("capsfilter", "capsFilter_enlarge"); + if (!capsFilter_enlarge) { + g_printerr ("capsFilter_enlarge could not be created. Exiting.\n"); + return -1; + } + + // Reduce the previously enlarged image frame so that the final output video retains the + // same dimension as the pipeline's input video dimension. + nvvideoconvert_reduce = gst_element_factory_make("nvvideoconvert", "nvvideoconvert_reduce"); + if (!nvvideoconvert_reduce) { + g_printerr ("nvvideoconvert_reduce could not be created. Exiting.\n"); + return -1; + } + capsFilter_reduce = gst_element_factory_make("capsfilter", "capsFilter_reduce"); + if (!capsFilter_reduce) { + g_printerr ("capsFilter_reduce could not be created. Exiting.\n"); + return -1; + } + + gchar *string1 = NULL; + asprintf (&string1, "%d:%d:%d:%d", _pad_dim, _pad_dim, _image_width, _image_height); + + // "dest-crop" - input size < output size + g_object_set(G_OBJECT(nvvideoconvert_enlarge), "dest-crop", string1,"interpolation-method",1 ,NULL); + // "src-crop" - input size > output size + g_object_set(G_OBJECT(nvvideoconvert_reduce), "src-crop", string1,"interpolation-method",1 ,NULL); + free(string1); + + /* Padding due to AR SDK model requires bigger bboxes*/ + const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + muxer_output_width_pad, muxer_output_height_pad); + GstCaps *caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_enlarge),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + + asprintf (&string1, "video/x-raw(memory:NVMM),width=%d,height=%d", + _image_width, _image_height); + caps1 = gst_caps_from_string (string1); + g_object_set(G_OBJECT(capsFilter_reduce),"caps", caps1, NULL); + free(string1); + gst_caps_unref(caps1); + //---Manipulate image size so that PGIE bbox is large enough--- + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); + if (!nvosd) { + g_printerr ("Nvdsosd could not be created. Exiting.\n"); + return -1; + } + nvtile = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + tiler_rows = (guint) sqrt (num_sources); + tiler_columns = (guint) ceil (1.0 * num_sources / tiler_rows); + g_object_set (G_OBJECT (nvtile), "rows", tiler_rows, "columns", + tiler_columns, "width", MUXER_OUTPUT_WIDTH, "height", MUXER_OUTPUT_HEIGHT, NULL); + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + GstPad* osd_sink_pad = gst_element_get_static_pad(nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, NULL, NULL); + gst_object_unref(osd_sink_pad); + + /* Set output file location */ + int sink_type = 0; + parse_sink_type_yaml(&sink_type, argv[1]); + int enc_type = 0; + parse_sink_enc_type_yaml(&enc_type, argv[1]); + g_print("sink_type:%d, enc_type:%d\n", sink_type, enc_type); + + if(sink_type == 1) { + sink = gst_element_factory_make("nvvideoencfilesinkbin", "nv-filesink"); + if (!sink) { + g_printerr ("Filesink could not be created. Exiting.\n"); + return -1; + } + g_object_set(G_OBJECT(sink), "output-file", "out.mp4", NULL); + g_object_set(G_OBJECT(sink), "bitrate", 4000000, NULL); + //g_object_set(G_OBJECT(sink), "profile", 3, NULL); + g_object_set(G_OBJECT(sink), "codec", 1, NULL);//hevc + // g_object_set(G_OBJECT(sink), "control-rate", 0, NULL);//hevc + g_object_set(G_OBJECT(sink), "enc-type", enc_type, NULL); + } else if(sink_type == 2) { + sink = gst_element_factory_make("nvrtspoutsinkbin", "nv-rtspsink"); + if (!sink) { + g_printerr ("Filesink could not be created. Exiting.\n"); + return -1; + } + g_object_set(G_OBJECT(sink), "enc-type", enc_type, NULL); + } else if(sink_type == 3) { + if (prop.integrated) { + sink = gst_element_factory_make("nv3dsink", "nv-sink"); + } else { +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nv-sink"); +#else + sink = gst_element_factory_make("nveglglessink", "nv-sink"); +#endif + } + } else { + sink = gst_element_factory_make("fakesink", "nv-fakesink"); + } + + /* Add all elements to the pipeline */ + // streammux has been added into pipeline already. + gst_bin_add_many(GST_BIN(pipeline), + nvvideoconvert_enlarge, capsFilter_enlarge, + pgie, tracker, sgie, postprocess, preprocess1, sgie1, nvtile, + nvvidconv, nvosd, sink, nvdslogger, + nvvideoconvert_reduce, capsFilter_reduce, NULL); + + // Link elements + if (!gst_element_link_many(streammux, + nvvideoconvert_enlarge, capsFilter_enlarge, pgie, tracker, sgie, postprocess, preprocess1, sgie1, + nvdslogger, nvvideoconvert_reduce, capsFilter_reduce, nvtile, nvvidconv, nvosd, sink, NULL)) { + g_printerr ("Elements could not be linked. Exiting.\n"); + return -1; + } + + /* Set the pipeline to "playing" state */ + g_print("Now playing!\n"); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + GST_DEBUG_BIN_TO_DOT_FILE((GstBin*)pipeline, GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(loop); + + /* Out of the main loop, clean up nicely */ + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline, GST_STATE_NULL); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + + return 0; + +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/Makefile new file mode 100644 index 00000000..813d423d --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/Makefile @@ -0,0 +1,36 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +CC:= g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes + +LIBS:= -lnvinfer +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= infer_pose_classification_parser.cpp +TARGET_LIB:= libnvdsinfer_pose_classfication_parser.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/infer_pose_classification_parser.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/infer_pose_classification_parser.cpp new file mode 100644 index 00000000..6478ac61 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/infer_pose_classification_parser.cpp @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer.h" +#include +#include +using namespace std; +using std::string; +using std::vector; + +const char classes_str[6][32] = { + "sitting_down", "getting_up", "sitting", "standing", "walking", "jumping" +}; + +extern "C" +{ + +/* convert poseclassifiction's ouptut logits to probalilities */ +std::vector +logits_to_probabilities(float *array, int arrayLen) +{ + std::vector ret; + float sum = 0; + for (int i = 0; i < arrayLen; i++) { + sum += exp(array[i]); + } + for (int i = 0; i < arrayLen; i++) { + float probability = exp(array[i]); + probability /= sum; + ret.push_back(probability); + } + return ret; +} + +bool NvDsParseCustomPoseClassification(std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, float classifierThreshold, + std::vector &attrList, std::string &attrString) +{ + float maxProbability = 0; + bool attrFound = false; + NvDsInferAttribute attr; + + NvDsInferDimsCHW dims; + getDimsCHWFromDims (dims, outputLayersInfo[0].inferDims); + unsigned int numClasses = dims.c; + float *outputCoverageBuffer = (float *) outputLayersInfo[0].buffer; + /* Iterate through all the probabilities that the object belongs to + * each class. Find the maximum probability and the corresponding class + * which meets the minimum threshold. */ + std::vector probabilities; + probabilities = logits_to_probabilities(outputCoverageBuffer, numClasses); + for (unsigned int c = 0; c < numClasses; c++) { + float probability = probabilities[c]; + if (probability > classifierThreshold && probability > maxProbability) { + //printf("c:%d, probability:%f\n", c, probability); + maxProbability = probability; + attrFound = true; + attr.attributeIndex = 0; + attr.attributeValue = c; + attr.attributeConfidence = probability; + attrString = classes_str[attr.attributeValue]; + } + } + + if (attrFound) { + attr.attributeLabel = strdup(attrString.c_str()); + attrList.push_back(attr); + } + + return true; +} + +}//end of extern "C" diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/Makefile new file mode 100644 index 00000000..134a0307 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/Makefile @@ -0,0 +1,41 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CC:= g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes -I/usr/local/cuda-$(CUDA_VER)/include + + +LIBS:= -lnvinfer -lnvinfer_plugin +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvdsinitinputlayers_BodyPose3DNet.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_BodyPose3DNet.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp new file mode 100644 index 00000000..ef19daad --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/nvdsinitinputlayers_BodyPose3DNet.cpp @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "nvdsinfer_custom_impl.h" +#include +/* Assumes only one input layer "im_info" needs to be initialized */ +bool NvDsInferInitializeInputLayers (std::vector const &inputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + unsigned int maxBatchSize) +{ + float scale_normalized_mean_limb_lengths[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; + float mean_limb_lengths[] = { + 246.3427, 246.3427, 492.6854, 402.4380, 487.0321, 128.6856, 391.6295, + 281.9928, 249.9478, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 169.1832, 402.2611, 488.1824, 128.6848, 407.5836, + 281.9897, 249.9489, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 168.6137, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000}; + + //k_inv would change for camera parameters + float k_inv[] = {0.00124876620338, 0, -0.119881555525, + 0, 0.00124876620338, -0.159842074033, + 0, 0, 1}; + + float t_form_inv[] = {1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0}; + for (auto v : inputLayersInfo){ + if (!strcmp(v.layerName, "scale_normalized_mean_limb_lengths")){ + memcpy(v.buffer,scale_normalized_mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "mean_limb_lengths")){ + memcpy(v.buffer,mean_limb_lengths,sizeof(float)*36); + } + if (!strcmp(v.layerName, "k_inv")){ + memcpy(v.buffer,k_inv,sizeof(float)*9); + } + if (!strcmp(v.layerName, "t_form_inv")){ + memcpy(v.buffer,t_form_inv,sizeof(float)*9); + } + } + + return true; +} + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/Makefile new file mode 100644 index 00000000..dcca11b0 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/Makefile @@ -0,0 +1,66 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX:= g++ +NVCC:=/usr/local/cuda-$(CUDA_VER)/bin/nvcc + +SRCS:= nvdspreprocess_lib.cpp + +INCS:= $(wildcard *.h) +LIB:=libcustom2d_preprocess.so + +CFLAGS+= -fPIC -DHAVE_CONFIG_H -std=c++17 -Wall -Werror \ + -I /usr/local/cuda-$(CUDA_VER)/include \ + -I /opt/nvidia/deepstream/deepstream/sources/gst-plugins/include \ + -I /opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdspreprocess/include \ + -I /opt/nvidia/deepstream/deepstream/sources/includes + +GST_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/gst-plugins/ +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +LIBS := -shared -Wl,-no-undefined \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -ldl \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lnvdsgst_meta -lnvds_meta -lnvbufsurface -lnvbufsurftransform\ + -lcuda -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.cpp=.o) +OBJS:= $(OBJS:.cu=.o) + +PKGS:= gstreamer-1.0 gstreamer-base-1.0 gstreamer-video-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + @echo $(CFLAGS) + $(NVCC) -c -o $@ --compiler-options '-fPIC' $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) + +install: $(LIB) + cp -rv $(LIB) $(GST_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.cpp new file mode 100644 index 00000000..0a56051c --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.cpp @@ -0,0 +1,303 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" + +#include "nvdspreprocess_lib.h" +#include "nvdsmeta_schema.h" +#include +#include + +using std::vector; + +#define _MIN_FRAME_ 3 +#define _MAX_FRAME_ 300 +#define _MAX_OBJECT_NUM_ 20 +#define _TIME_OUT_ 2 +#define _KEYPOINTS_NUM 34 +#define FREE(p) (free(p), p=NULL) + +/*wrap keypoints*/ +struct SObjectContex +{ + uint64_t object_id; + float *x, *y, *z; + long tv_sec; + SObjectContex() { + object_id = UNTRACKED_OBJECT_ID; + x = y = z = NULL; + tv_sec = 0; + }; + ~SObjectContex() { + FREE(x); + FREE(y); + FREE(z); + }; +}; + +struct CustomCtx +{ + std::mutex mtx; + /* vector for obejct context*/ + vector multi_objects; + int one_channel_element_num; + int one_channel_bytes; + int move_element_num; + int move_element_bytes; + ~CustomCtx() + { + int size = multi_objects.size(); + printf("objects size:%d\n", size); + for (int i = 0; i < size; i++) + { + delete multi_objects[i]; + multi_objects[i] = NULL; + } + multi_objects.clear(); + }; +}; + +/* find Object by object_id */ +SObjectContex* findObjectCtx(CustomCtx *ctx, guint64 object_id) +{ + std::unique_lock lck(ctx->mtx); + SObjectContex* pSObjectCtx = NULL; + for (vector::iterator itor = ctx->multi_objects.begin(); + itor != ctx->multi_objects.end(); itor++) { + if( (*itor)->object_id == object_id) { + pSObjectCtx = (*itor); + } + } + return pSObjectCtx; +} + +/* find unused object by object_id */ +SObjectContex* +findUnusedObjectCtx(CustomCtx *ctx, guint64 object_id) +{ + std::unique_lock lck(ctx->mtx); + SObjectContex* pSObjectCtx = NULL; + for (vector::iterator itor = ctx->multi_objects.begin(); + itor != ctx->multi_objects.end(); itor++) { + if( (*itor)->object_id == UNTRACKED_OBJECT_ID) { + pSObjectCtx = (*itor); + } + } + return pSObjectCtx; +} + +/* extend object group */ +SObjectContex* +CreateObjectCtx(CustomCtx *ctx) +{ + std::unique_lock lck(ctx->mtx); + SObjectContex* pSObjectCtx = new SObjectContex; + if(pSObjectCtx) + { + pSObjectCtx->object_id = UNTRACKED_OBJECT_ID; + pSObjectCtx->x = (float*)calloc(ctx->one_channel_element_num, sizeof(float)); + pSObjectCtx->y = (float*)calloc(ctx->one_channel_element_num, sizeof(float)); + pSObjectCtx->z = (float*)calloc(ctx->one_channel_element_num, sizeof(float)); + ctx->multi_objects.push_back(pSObjectCtx); + } + return pSObjectCtx; +} + +/* extend object group */ +void +ResetObjectCtx(CustomCtx *ctx, SObjectContex *pSObjectCtx) +{ + if(pSObjectCtx) + { + printf("ResetObjectCtx, object_id:%ld\n", pSObjectCtx->object_id); + pSObjectCtx->object_id = UNTRACKED_OBJECT_ID; + memset(pSObjectCtx->x, 0, ctx->one_channel_bytes); + memset(pSObjectCtx->y, 0, ctx->one_channel_bytes); + memset(pSObjectCtx->z, 0, ctx->one_channel_bytes); + pSObjectCtx->tv_sec = 0; + } +} + +void LoopObjectCtx(CustomCtx *ctx) +{ + std::unique_lock lck(ctx->mtx); + struct timeval tv; + SObjectContex* pSObjectCtx = NULL; + for (vector::iterator itor = ctx->multi_objects.begin(); + itor != ctx->multi_objects.end(); itor++) { + pSObjectCtx = (*itor); + gettimeofday (&tv, NULL); + if(pSObjectCtx->object_id != UNTRACKED_OBJECT_ID && + (tv.tv_sec - pSObjectCtx->tv_sec) > _TIME_OUT_){ + ResetObjectCtx(ctx, pSObjectCtx); + } + } +} + +/* save 34 keypoints to local */ +void +sveKeypoints(CustomCtx *ctx, void *user_meta_data, SObjectContex *pSObjectCtx) +{ + std::unique_lock lck(ctx->mtx); + if(pSObjectCtx) + { + NvDsJoints *ds_joints = (NvDsJoints *) user_meta_data; + //move from tail to head + memmove(pSObjectCtx->x, pSObjectCtx->x + _KEYPOINTS_NUM, ctx->move_element_bytes); + memmove(pSObjectCtx->y, pSObjectCtx->y + _KEYPOINTS_NUM, ctx->move_element_bytes); + memmove(pSObjectCtx->z, pSObjectCtx->z + _KEYPOINTS_NUM, ctx->move_element_bytes); + + //save keypoints + for(int i = 0; i < ds_joints->num_joints; i++){ + *(pSObjectCtx->x + ctx->move_element_num + i) = ds_joints->joints[i].x; + *(pSObjectCtx->y + ctx->move_element_num + i) = ds_joints->joints[i].y; + *(pSObjectCtx->z + ctx->move_element_num + i) = ds_joints->joints[i].z; + } + + //update time + struct timeval tv; + gettimeofday (&tv, NULL); + pSObjectCtx->tv_sec = tv.tv_sec; + } +} + +NvDsPreProcessStatus +CustomTensorPreparation(CustomCtx *ctx, NvDsPreProcessBatch *batch, NvDsPreProcessCustomBuf *&buf, + CustomTensorParams &tensorParam, NvDsPreProcessAcquirer *acquirer) +{ + NvDsPreProcessStatus status = NVDSPREPROCESS_TENSOR_NOT_READY; + + /** acquire a buffer from tensor pool */ + buf = acquirer->acquire(); + float * pDst = (float*)buf->memory_ptr; + int units = batch->units.size(); + for(int i = 0; i < units; i++) + { + guint64 object_id = batch->units[i].roi_meta.object_meta->object_id; + GstBuffer *inbuf = (GstBuffer *)batch->inbuf; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_user = NULL; + SObjectContex* pSObjectCtx = NULL; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(inbuf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) + { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) + { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)l_obj->data; + if(obj_meta->object_id != object_id) + continue; + + for (l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) + { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type == NVDS_OBJ_META) + { + /* find by objectid */ + pSObjectCtx = findObjectCtx(ctx, obj_meta->object_id); + if(!pSObjectCtx) + { + /* can't find objectid, find one whose objectid is -1 */ + pSObjectCtx = findUnusedObjectCtx(ctx, obj_meta->object_id); + if(pSObjectCtx) + { + /*can find one whose objectid is not -1, copy keypoints*/ + pSObjectCtx->object_id = obj_meta->object_id; + sveKeypoints(ctx, user_meta->user_meta_data, pSObjectCtx); + } else { + /* if no, extent Skeypoints, then copy keypoints*/ + pSObjectCtx = CreateObjectCtx(ctx); + printf("extendObjectCtx pSObjectCtx:%p\n", pSObjectCtx); + if(pSObjectCtx) + { + pSObjectCtx->object_id = obj_meta->object_id; + sveKeypoints(ctx, user_meta->user_meta_data, pSObjectCtx); + } + } + } else { + /* can find, copy keypoints */ + sveKeypoints(ctx, user_meta->user_meta_data, pSObjectCtx); + } + } + } + } + } + + /* copy to buffer, 3 X 300 X 34 X 1 (C T V M) */ + if(pSObjectCtx) + { + cudaMemcpy(pDst, pSObjectCtx->x, ctx->one_channel_bytes, cudaMemcpyHostToDevice); + pDst = pDst + ctx->one_channel_element_num; + cudaMemcpy(pDst, pSObjectCtx->y, ctx->one_channel_bytes, cudaMemcpyHostToDevice); + pDst = pDst + ctx->one_channel_element_num; + cudaMemcpy(pDst, pSObjectCtx->z, ctx->one_channel_bytes, cudaMemcpyHostToDevice); + pDst = pDst + ctx->one_channel_element_num; + } + } + + //reset object context if timeout + LoopObjectCtx(ctx); + status = NVDSPREPROCESS_SUCCESS; + return status; +} + +NvDsPreProcessStatus +CustomTransformation(NvBufSurface *in_surf, NvBufSurface *out_surf, CustomTransformParams ¶ms) +{ + /* do nothing, bodypose data is in object's metadata, here we can't access object */ + return NVDSPREPROCESS_SUCCESS; +} + +CustomCtx *initLib(CustomInitParams initparams) +{ + CustomCtx *ctx = new CustomCtx; + std::string sframeSeqLen = initparams.user_configs[NVDSPREPROCESS_USER_CONFIGS_FRAMES_SEQUENCE_LENGHTH]; + int len = atoi(sframeSeqLen.c_str()); + printf("frameSeqLen:%d\n", len); + if(len < _MIN_FRAME_ || len > _MAX_FRAME_) + { + printf("frameSeqLen iilegal, use default vaule 300\n"); + len = _MAX_FRAME_; + } + ctx->one_channel_element_num = len*_KEYPOINTS_NUM; + ctx->one_channel_bytes = ctx->one_channel_element_num*sizeof(float); + ctx->move_element_num = (len-1)*_KEYPOINTS_NUM; + ctx->move_element_bytes = ctx->move_element_num*sizeof(float); + + /* initial vector for multi_keypoints*/ + for(int i = 0; i < _MAX_OBJECT_NUM_; i++){ + CreateObjectCtx(ctx); + } + + return ctx; +} + +void deInitLib(CustomCtx *ctx) +{ + delete ctx; +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.h b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.h new file mode 100644 index 00000000..4214db99 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/nvdspreprocess_lib.h @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +/** + * @file nvdspreprocess_lib.h + * NVIDIA DeepStream Preprocess lib specifications + * + * @b Description: This file defines common elements used in the API + * exposed by the Gst-nvdspreprocess plugin. + */ + +/** + * @defgroup gstreamer_nvdspreprocess_api NvDsPreProcess Plugin + * Defines an API for the GStreamer NvDsPreProcess custom lib. + * @ingroup custom_gstreamer + * @{ + */ + +#ifndef __NVDSPREPROCESS_LIB__ +#define __NVDSPREPROCESS_LIB__ + +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" +#include "nvdspreprocess_interface.h" + +/** Maximum file path length */ +#define _PATH_MAX 4096 + +/** actual frames sequence length */ +#define NVDSPREPROCESS_USER_CONFIGS_FRAMES_SEQUENCE_LENGHTH "frames-sequence-length" + +/** + * Custom transformation function for group + */ +extern "C" +NvDsPreProcessStatus CustomTransformation(NvBufSurface *in_surf, + NvBufSurface *out_surf, + CustomTransformParams ¶ms); + +/** + * Custom Asynchronus group transformation function + */ +extern "C" +NvDsPreProcessStatus CustomAsyncTransformation(NvBufSurface *in_surf, + NvBufSurface *out_surf, + CustomTransformParams ¶ms); + +/** + * Custom tensor preparation function for NCHW/NHWC network order + */ +extern "C" +NvDsPreProcessStatus CustomTensorPreparation(CustomCtx *ctx, NvDsPreProcessBatch *batch, NvDsPreProcessCustomBuf *&buf, + CustomTensorParams &tensorParam, NvDsPreProcessAcquirer *acquirer); + +/** + * custom library initialization function + */ +extern "C" +CustomCtx *initLib(CustomInitParams initparams); + +/** + * custom library deinitialization function + */ +extern "C" +void deInitLib(CustomCtx *ctx); + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/Makefile new file mode 100644 index 00000000..32f246f4 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/Makefile @@ -0,0 +1,75 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX:= g++ -std=c++17 + +SRCS:= postprocesslib_impl.cpp post_processor.cpp post_processor_bodypose.cpp + +INCS:= $(wildcard *.h) +LIB:=libpostprocess_impl.so + +CFLAGS+= -fPIC \ + -I /usr/local/cuda-$(CUDA_VER)/include \ + -I ../../../includes -I../includes -I../../eigen/ -I /opt/nvidia/deepstream/deepstream/sources/includes -I./ \ + -I /opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdspostprocess/includes + +GST_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/gst-plugins/ +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + + + +LIBS += -shared -Wl,-no-undefined -L/usr/lib/x86_64-linux-gnu/ -lyaml-cpp \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -ldl -lpthread \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lnvdsgst_meta -lnvds_meta \ + -lnvbufsurface -lnvbufsurftransform -lnvdsbufferpool -lnvds_inferutils\ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.cpp=.o) +PKGS:= gstreamer-1.0 gstreamer-base-1.0 gstreamer-video-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.cpp $(INCS) Makefile + @echo $(CFLAGS) + $(CXX) -c -o $@ $(CFLAGS) $< + +$(LIB): $(OBJS) $(DEP) Makefile + @echo $(CFLAGS) + $(CXX) -o $@ $(OBJS) $(LIBS) + +$(DEP): $(DEP_FILES) + $(MAKE) -C postprocesslib_impl/ + +install: $(LIB) + cp -rv $(LIB) $(GST_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.cpp new file mode 100644 index 00000000..523ec481 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.cpp @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "post_processor.h" + +using namespace std; + +/* Parse the labels file and extract the class label strings. For format of + * the labels file, please refer to the custom models section in the + * DeepStreamSDK documentation. + */ +NvDsPostProcessStatus +ModelPostProcessor::parseLabelsFile(const std::string& labelsFilePath) +{ + std::ifstream labels_file(labelsFilePath); + std::string delim{';'}; + if (!labels_file.is_open()) + { + printError("Could not open labels file:%s", safeStr(labelsFilePath)); + return NVDSPOSTPROCESS_CONFIG_FAILED; + } + while (labels_file.good() && !labels_file.eof()) + { + std::string line, word; + std::vector l; + size_t pos = 0, oldpos = 0; + + std::getline(labels_file, line, '\n'); + if (line.empty()) + continue; + + while ((pos = line.find(delim, oldpos)) != std::string::npos) + { + word = line.substr(oldpos, pos - oldpos); + l.push_back(word); + oldpos = pos + delim.length(); + } + l.push_back(line.substr(oldpos)); + m_Labels.push_back(l); + } + + if (labels_file.bad()) + { + printError("Failed to parse labels file:%s, iostate:%d", + safeStr(labelsFilePath), (int)labels_file.rdstate()); + return NVDSPOSTPROCESS_CONFIG_FAILED; + } + return NVDSPOSTPROCESS_SUCCESS; +} + + +NvDsPostProcessStatus +ModelPostProcessor::initResource(NvDsPostProcessContextInitParams& initParams) +{ + + if (!string_empty(initParams.labelsFilePath)) + { + if (NVDSPOSTPROCESS_SUCCESS != parseLabelsFile(initParams.labelsFilePath)){ + printError("parse label file:%s failed", initParams.labelsFilePath); + } + } + return NVDSPOSTPROCESS_SUCCESS; +} + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.h b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.h new file mode 100644 index 00000000..11a55804 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor.h @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __POST_PROCESSOR_HPP__ +#define __POST_PROCESSOR_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" +#include "gst-nvquery.h" +#include "gstnvdsmeta.h" +#include "gst-nvevent.h" +#include "nvdsinfer_dbscan.h" +#include "post_processor_struct.h" + + +#ifndef PP_DISABLE_CLASS_COPY +#define PP_DISABLE_CLASS_COPY(NoCopyClass) \ + NoCopyClass(const NoCopyClass&) = delete; \ + void operator=(const NoCopyClass&) = delete +#endif + + +class ModelPostProcessor +{ + +protected: + ModelPostProcessor(NvDsPostProcessNetworkType type, int id, int gpuId) + : m_NetworkType(type), m_UniqueID(id), m_GpuID(gpuId){} + +public: + virtual ~ModelPostProcessor() = default; + + virtual NvDsPostProcessStatus + initResource(NvDsPostProcessContextInitParams& initParams); + const std::vector>& getLabels() const + { + return m_Labels; + } + void freeBatchOutput(NvDsPostProcessBatchOutput& batchOutput); + void setNetworkInfo (NvDsInferNetworkInfo networkInfo){ + m_NetworkInfo = networkInfo; + } + + virtual NvDsPostProcessStatus parseEachFrame( + const std::vector &outputLayers, + NvDsPostProcessFrameOutput& result) = 0; + + + virtual void + attachMetadata (NvBufSurface *surf, gint batch_idx, + NvDsBatchMeta *batch_meta, + NvDsFrameMeta *frame_meta, + NvDsObjectMeta *object_meta, + NvDsObjectMeta *parent_obj_meta, + NvDsPostProcessFrameOutput & detection_output, + NvDsPostProcessDetectionParams *all_params, + std::set & filterOutClassIds, + int32_t unique_id, + gboolean output_instance_mask, + gboolean process_full_frame, + float segmentationThreshold, + gboolean maintain_aspect_ratio, + NvDsRoiMeta *roi_meta, + gboolean symmetric_padding) = 0; + + virtual void + prcoessMetadata (NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) = 0; + + virtual void releaseFrameOutput(NvDsPostProcessFrameOutput& frameOutput) = 0; + +protected: + NvDsPostProcessStatus parseLabelsFile(const std::string &path); + +private: + PP_DISABLE_CLASS_COPY(ModelPostProcessor); + +protected: + /* Processor type */ + NvDsPostProcessNetworkType m_NetworkType = NvDsPostProcessNetworkType_Other; + + int m_UniqueID = 0; + uint32_t m_GpuID = 0; + + /* Network input information. */ + NvDsInferNetworkInfo m_NetworkInfo = {0}; + std::vector m_AllLayerInfo; + std::vector m_OutputLayerInfo; + + /* Holds the string labels for classes. */ + std::vector> m_Labels; + +}; + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.cpp new file mode 100644 index 00000000..7857e8a3 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.cpp @@ -0,0 +1,761 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "post_processor_bodypose.h" +#include +#include + +using namespace std; +#define MAX_TIME_STAMP_LEN 32 + +Eigen::Matrix3f _K;// Camera intrinsic matrix +//===Global variables=== +std::unordered_map> g_filter_pose25d; +OneEuroFilter m_filterRootDepth; // Root node in pose25d. +static Eigen::Matrix3f m_K_inv_transpose; +const float m_scale_ll[] = { + 0.5000, 0.5000, 1.0000, 0.8175, 0.9889, 0.2610, 0.7942, 0.5724, 0.5078, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3433, 0.8171, + 0.9912, 0.2610, 0.8259, 0.5724, 0.5078, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.3422, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000}; +const float m_mean_ll[] = { + 246.3427f, 246.3427f, 492.6854f, 402.4380f, 487.0321f, 128.6856f, 391.6295f, + 281.9928f, 249.9478f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 169.1832f, 402.2611f, 488.1824f, 128.6848f, 407.5836f, + 281.9897f, 249.9489f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f, 0.0000f, 168.6137f, 0.0000f, 0.0000f, 0.0000f, 0.0000f, + 0.0000f}; + +// Default camera attributes +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 +#define FOCAL_LENGTH 800.79041f +double _focal_length_dbl = FOCAL_LENGTH; +float _focal_length = (float)_focal_length_dbl; +int _image_width = MUXER_OUTPUT_WIDTH; +int _image_height = MUXER_OUTPUT_HEIGHT; +static float _sgie_classifier_threshold = FLT_MIN; +/* Padding due to AR SDK model requires bigger bboxes*/ +#define PAD_DIM 128 +int _pad_dim = PAD_DIM;// A scaled version of PAD_DIM + +#define ACQUIRE_DISP_META(dmeta) \ + if (dmeta->num_circles == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_labels == MAX_ELEMENTS_IN_DISPLAY_META || \ + dmeta->num_lines == MAX_ELEMENTS_IN_DISPLAY_META) \ + { \ + dmeta = nvds_acquire_display_meta_from_pool(bmeta);\ + nvds_add_display_meta_to_frame(frame_meta, dmeta);\ + }\ + +#define GET_LINE(lparams) \ + ACQUIRE_DISP_META(dmeta)\ + lparams = &dmeta->line_params[dmeta->num_lines];\ + dmeta->num_lines++;\ + +static +void generate_ts_rfc3339 (char *buf, int buf_size) +{ + time_t tloc; + struct tm tm_log; + struct timespec ts; + char strmsec[6]; //.nnnZ\0 + + clock_gettime(CLOCK_REALTIME, &ts); + memcpy(&tloc, (void *)(&ts.tv_sec), sizeof(time_t)); + gmtime_r(&tloc, &tm_log); + strftime(buf, buf_size,"%Y-%m-%dT%H:%M:%S", &tm_log); + int ms = ts.tv_nsec/1000000; + g_snprintf(strmsec, sizeof(strmsec),".%.3dZ", ms); + strncat(buf, strmsec, buf_size); +} + +static +gpointer copy_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsJoints *src_joints = (NvDsJoints *) user_meta->user_meta_data; + NvDsJoints *dst_joints = NULL; + + dst_joints = (NvDsJoints *)g_memdup2 ((gpointer)src_joints, sizeof(NvDsJoints)); + dst_joints->num_joints = src_joints->num_joints; + dst_joints->pose_type = src_joints->pose_type; + dst_joints->joints = (NvDsJoint *)g_memdup2 ((gpointer)src_joints->joints, + sizeof(NvDsJoint)*src_joints->num_joints); + return dst_joints; +} + +static void +release_bodypose_meta (gpointer data, gpointer user_data) +{ + NvDsUserMeta *user_meta = (NvDsUserMeta *) data; + NvDsJoints *src_joints = (NvDsJoints *) user_meta->user_meta_data; + g_free (src_joints->joints); + g_free (user_meta->user_meta_data); + user_meta->user_meta_data = NULL; +} + +NvDsPostProcessStatus +BodyPoseModelPostProcessor::initResource(NvDsPostProcessContextInitParams& initParams) +{ + ModelPostProcessor::initResource(initParams); + m_ClassificationThreshold = initParams.classifierThreshold; + _K.row(0) << _focal_length, 0, _image_width / 2.f; + _K.row(1) << 0, _focal_length, _image_height / 2.f; + _K.row(2) << 0, 0, 1.f; + + _pad_dim = PAD_DIM * _image_width / MUXER_OUTPUT_WIDTH; + return NVDSPOSTPROCESS_SUCCESS; +} + +NvDsPostProcessStatus +BodyPoseModelPostProcessor::parseEachFrame( + const std::vector& outputLayers, + NvDsPostProcessFrameOutput& result) +{ + result.outputType = NvDsPostProcessNetworkType_BodyPose; + fillBodyPoseOutput(outputLayers, result.bodyPoseOutput); + return NVDSPOSTPROCESS_SUCCESS; +} + +NvDsPostProcessStatus +BodyPoseModelPostProcessor::fillBodyPoseOutput( + const std::vector& outputLayers, + NvDsPostProcessBodyPoseOutput& output) +{ + movenetposeFromTensorMeta(outputLayers, output); + return NVDSPOSTPROCESS_SUCCESS; +} + +typedef struct NvAR_Point3f { + float x, y, z; +} NvAR_Point3f; + +/* Given 2D and ZRel, we need to find the depth of the root to reconstruct the scale normalized 3D Pose. + While there exists many 3D poses that can have the same 2D projection, given the 2.5D pose and intrinsic camera parameters, + there exists a unique 3D pose that satisfies (Xˆn − Xˆm)**2 + (Yˆn − Yˆm)**2 + (Zˆn − Zˆm)**2 = C**2. + Refer Section 3.3 of https://arxiv.org/pdf/1804.09534.pdf for more details. +*/ +std::vector calculateZRoots(const std::vector& X0, const std::vector& X1, + const std::vector& Y0, const std::vector& Y1, + const std::vector& Zrel0, + const std::vector& Zrel1, const std::vector& C) { + std::vector zRoots(X0.size()); + for (int i = 0; i < X0.size(); i++) { + double x0 = (double)X0[i], x1 = (double)X1[i], y0 = (double)Y0[i], y1 = (double)Y1[i], + z0 = (double)Zrel0[i], z1 = (double)Zrel1[i]; + double a = ((x1 - x0) * (x1 - x0)) + ((y1 - y0) * (y1 - y0)); + double b = 2 * (z1 * ((x1 * x1) + (y1 * y1) - x1 * x0 - y1 * y0) + + z0 * ((x0 * x0) + (y0 * y0) - x1 * x0 - y1 * y0)); + double c = ((x1 * z1 - x0 * z0) * (x1 * z1 - x0 * z0)) + + ((y1 * z1 - y0 * z0) * (y1 * z1 - y0 * z0)) + + ((z1 - z0) * (z1 - z0)) - (C[i] * C[i]); + double d = (b * b) - (4 * a * c); + + // make sure the solutions are valid + a = fmax(DBL_EPSILON, a); + d = fmax(DBL_EPSILON, d); + zRoots[i] = (float) ((-b + sqrt(d)) / (2 * a + 1e-8)); + } + return zRoots; +} + +float median(std::vector& v) { + size_t n = v.size() / 2; + nth_element(v.begin(), v.begin() + n, v.end()); + return v[n]; +} + +/* Given 2D keypoints and the relative depth of each keypoint w.r.t the root, we find the depth of the root + to reconstruct the scale normalized 3D pose. +*/ +std::vector liftKeypoints25DTo3D(const float* p2d, + const float* pZRel, + const int numKeypoints, + const Eigen::Matrix3f& KInv, + const float limbLengths[]) { + + const int ROOT = 0; + + // Contains the relative depth values of each keypoints + std::vector zRel(numKeypoints, 0.f); + + // Matrix containing the 2D keypoints. + Eigen::MatrixXf XY1 = Eigen::MatrixXf(numKeypoints, 3); + + // Mean distance between a specific pair and its parent. + std::vector C; + + // Indices representing keypoints and its parents for limb lengths > 0. + // In our dataset, we only have limb length information for few keypoints. + std::vector idx0 = { 0, 3, 6, 8, 5, 2, 2, 21, 23, 21, 7, 4, 1, 1, 20, 22, 20 }; + std::vector idx1 = { 3, 6, 0, 5, 2, 0, 21, 23, 25, 6, 4, 1, 0, 20, 22, 24, 6 }; + + std::vector X0(idx0.size(), 0.f), Y0(idx0.size(), 0.f), X1(idx0.size(), 0.f), Y1(idx0.size(), 0.f), + zRel0(idx0.size(), 0.f), zRel1(idx0.size(), 0.f); + + for (int i = 0; i < numKeypoints; i++) { + zRel[i] = pZRel[i]; + + XY1.row(i) << p2d[i * 2], p2d[(i * 2) + 1], 1.f; + + if (limbLengths[i] > 0.f) C.push_back(limbLengths[i]); + } + + // Set relative depth of root to be 0 as the relative depth is measure w.r.t the root. + zRel[ROOT] = 0.f; + +/* redundant logic + for (int i = 0; i < XY1.rows(); i++) { + float x = XY1(i, 0); + float y = XY1(i, 1); + float z = XY1(i, 2); + XY1.row(i) << x, y, z; + } +*/ + XY1 = XY1 * KInv; + + for (int i = 0; i < idx0.size(); i++) { + X0[i] = XY1(idx0[i], 0); + Y0[i] = XY1(idx0[i], 1); + X1[i] = XY1(idx1[i], 0); + Y1[i] = XY1(idx1[i], 1); + zRel0[i] = zRel[idx0[i]]; + zRel1[i] = zRel[idx1[i]]; + } + + std::vector zRoots = calculateZRoots(X0, X1, Y0, Y1, zRel0, zRel1, C); + + float zRootsMedian = median(zRoots); + + zRootsMedian = m_filterRootDepth.filter(zRootsMedian); + + std::vector p3d(numKeypoints, { 0.f, 0.f, 0.f }); + + for (int i = 0; i < numKeypoints; i++) { + p3d[i].x = XY1(i, 0) * zRel[i]; + p3d[i].y = XY1(i, 1) * zRel[i]; + p3d[i].z = XY1(i, 2) * zRel[i]; + } + + return p3d; +} + +void osd_upper_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 24; + const int idx_joints[] = { 0, 1, 2, 3, 6, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}; + const int num_bones = 25; + const int idx_bones[] = { 21, 6, 20, 6, 21, 23, 20, 22, 24, 22, 23, 25, + 27, 25, 31, 25, 33, 25, 29, 25, 24, 30, 24, 26, + 24, 32, 24, 28, 2, 21, 1, 20, 3, 6, 6, 15, + 15, 16, 15, 17, 19, 17, 18, 16, 0, 1, 0, 2, + 0, 3}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void osd_lower_body(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3 * _image_width / MUXER_OUTPUT_WIDTH;//6;//3; + const int keypoint_line_width = 2 * _image_width / MUXER_OUTPUT_WIDTH;//4;//2; + + const int num_joints = 10; + const int idx_joints[] = { 4, 5, 7, 8, 9, 10, 11, 12, 13, 14}; + const int num_bones = 10; + const int idx_bones[] = { 2, 5, 5, 8, 1, 4, 4, 7, 7, 13, + 8, 14, 8, 10, 7, 9, 11, 9, 12, 10}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = idx_joints[ii]; + + if (keypoints_confidence[i] < _sgie_classifier_threshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{0.96, 0.26, 0.21, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < _sgie_classifier_threshold) || + (keypoints_confidence[i1] < _sgie_classifier_threshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + +void parse_25dpose_from_tensor_meta(NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + // const int pelvis = 0; + // const int left_hip = 1; + // const int right_hip = 2; + // const int torso = 3; + // const int left_knee = 4; + // const int right_knee = 5; + // const int neck = 6; + // const int left_ankle = 7; + // const int right_ankle = 8; + // const int left_big_toe = 9; + // const int right_big_toe = 10; + // const int left_small_toe = 11; + // const int right_small_toe = 12; + // const int left_heel = 13; + // const int right_heel = 14; + // const int nose = 15; + // const int left_eye = 16; + // const int right_eye = 17; + // const int left_ear = 18; + // const int right_ear = 19; + // const int left_shoulder = 20; + // const int right_shoulder = 21; + // const int left_elbow = 22; + // const int right_elbow = 23; + // const int left_wrist = 24; + // const int right_wrist = 25; + // const int left_pinky_knuckle = 26; + // const int right_pinky_knuckle = 27; + // const int left_middle_tip = 28; + // const int right_middle_tip = 29; + // const int left_index_knuckle = 30; + // const int right_index_knuckle = 31; + // const int left_thumb_tip = 32; + // const int right_thumb_tip = 33; + + const int numKeyPoints = 34; + float keypoints[2 * numKeyPoints]; + float keypointsZRel[numKeyPoints]; + float keypoints_confidence[numKeyPoints]; + + m_K_inv_transpose = _K.inverse().eval(); + m_K_inv_transpose = m_K_inv_transpose.transpose().eval(); + + NvDsBatchMeta *bmeta = frame_meta->base_meta.batch_meta; + NvDsDisplayMeta *dmeta = nvds_acquire_display_meta_from_pool(bmeta); + nvds_add_display_meta_to_frame(frame_meta, dmeta); + + for (unsigned int m=0; m < tensor_meta->num_output_layers;m++){ + NvDsInferLayerInfo *info = &tensor_meta->output_layers_info[m]; + + if (!strcmp(info->layerName, "pose25d")) { + float *data = (float *)tensor_meta->out_buf_ptrs_host[m]; + // for (int j =0 ; j < 34; j++) { + // printf ("a=%f b=%f c=%f d=%f\n",data[j*4],data[j*4+1],data[j*4+2], data[j*4+3]); + // } + + // Initialize + if (g_filter_pose25d.find(obj_meta->object_id) == g_filter_pose25d.end()) { + const float m_oneEuroSampleRate = 30.0f; + // const float m_oneEuroMinCutoffFreq = 0.1f; + // const float m_oneEuroCutoffSlope = 0.05f; + const float m_oneEuroDerivCutoffFreq = 1.0f;// Hz + + //std::vector filter_vec; + std::vector filter_vec; + + for (int j=0; j < numKeyPoints*3; j++) { + //TODO:Pending delete especially when object goes out of view, or ID switch + //will cause memleak, cleanup required wrap into class + // filter_vec.push_back(SF1eFilterCreate(30, 1.0, 0.0, 1.0)); + + // filters for x and y + // for (auto& fil : m_filterKeypoints2D) fil.reset(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.1f, 0.05, m_oneEuroDerivCutoffFreq)); + + // filters for z (depth) + // for (auto& fil : m_filterKeypointsRelDepth) fil.reset(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq); + filter_vec.push_back(OneEuroFilter(m_oneEuroSampleRate, 0.5f, 0.05, m_oneEuroDerivCutoffFreq)); + } + g_filter_pose25d[obj_meta->object_id] = filter_vec; + + // Filters depth of root keypoint + m_filterRootDepth.reset(m_oneEuroSampleRate, 0.1f, 0.05f, m_oneEuroDerivCutoffFreq); + } + + int batchSize_offset = 0; + + //std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + std::vector &filt_val = g_filter_pose25d[obj_meta->object_id]; + + // x,y,z,c + for (int i = 0; i < numKeyPoints; i++) { + int index = batchSize_offset + i * 4; + + // Update with filtered results + keypoints[2 * i ] = filt_val[3 * i ].filter(data[index ] * + (obj_meta->rect_params.width / 192.0) + obj_meta->rect_params.left); + keypoints[2 * i + 1] = filt_val[3 * i + 1].filter(data[index + 1] * + (obj_meta->rect_params.height / 256.0) + obj_meta->rect_params.top); + keypointsZRel[i] = filt_val[3 * i + 2].filter(data[index + 2]); + + keypoints_confidence[i] = data[index + 3]; + } + + // Since we have cropped and resized the image buffer provided to the SDK from the app, + // we scale and offset the points back to the original resolution + float scaleOffsetXY[] = {1.0f, 0.0f, 1.0f, 0.0f}; + + // Render upper body + if (1) { + osd_upper_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + // Render lower body + if (1) { + osd_lower_body(frame_meta, bmeta, dmeta, numKeyPoints, keypoints, keypoints_confidence); + } + + // SGIE operates on an enlarged/padded image buffer. + // const int muxer_output_width_pad = _pad_dim * 2 + _image_width; + // const int muxer_output_height_pad = _pad_dim * 2 + _image_height; + // Before outputting result, the image frame with overlay is cropped by removing _pad_dim. + // The final pose estimation result should counter the padding before deriving 3D keypoints. + for (int i = 0; i < numKeyPoints; i++) { + keypoints[2 * i ]-= _pad_dim; + keypoints[2 * i + 1]-= _pad_dim; + } + + // Recover pose 3D + std::vector p3dLifted; + p3dLifted = liftKeypoints25DTo3D(keypoints, keypointsZRel, numKeyPoints, m_K_inv_transpose, m_scale_ll); + // float scale = recoverScale(p3dLifted, keypoints_confidence, m_mean_ll); + // printf("scale = %f\n", scale); + // for (auto i = 0; i < p3dLifted.size(); i++) { + // p3dLifted[i].x *= scale; + // p3dLifted[i].y *= scale; + // p3dLifted[i].z *= scale; + // } + + NvDsUserMeta *user_meta = nvds_acquire_user_meta_from_pool (bmeta); + NvDsJoints *ds_joints = (NvDsJoints *)g_malloc(sizeof(NvDsJoints)); + ds_joints->num_joints = numKeyPoints; + ds_joints->pose_type = 1;// datapose3D + ds_joints->joints = (NvDsJoint *)g_malloc (numKeyPoints * sizeof(NvDsJoint)); + + //attach 3D to the user meta data + for (int i = 0; i < numKeyPoints; i++) { + ds_joints->joints[i].confidence = keypoints_confidence[i]; + ds_joints->joints[i].x = p3dLifted[i].x; + ds_joints->joints[i].y = p3dLifted[i].y; + ds_joints->joints[i].z = p3dLifted[i].z; + + //g_print("%d point [%f, %f, %f]\n", i, p3dLifted[i].x, p3dLifted[i].y, p3dLifted[i].z); + } + user_meta->user_meta_data = ds_joints; + user_meta->base_meta.meta_type = (NvDsMetaType) NVDS_OBJ_META; + user_meta->base_meta.release_func = release_bodypose_meta; + user_meta->base_meta.copy_func = copy_bodypose_meta; + nvds_add_user_meta_to_obj (obj_meta, user_meta); + } + } +} + +void +BodyPoseModelPostProcessor::attachMetadata (NvBufSurface *surf, gint batch_idx, + NvDsBatchMeta *batch_meta, + NvDsFrameMeta *frame_meta, + NvDsObjectMeta *obj_meta, + NvDsObjectMeta *parent_obj_meta, + NvDsPostProcessFrameOutput & detection_output, + NvDsPostProcessDetectionParams *all_params, + std::set & filterOutClassIds, + int32_t unique_id, + gboolean output_instance_mask, + gboolean process_full_frame, + float segmentationThreshold, + gboolean maintain_aspect_ratio, + NvDsRoiMeta *roi_meta, + gboolean symmetric_padding) +{ + +} + +void +BodyPoseModelPostProcessor::prcoessMetadata (NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) +{ + parse_25dpose_from_tensor_meta(tensor_meta, frame_meta, obj_meta); +} + +void +BodyPoseModelPostProcessor::releaseFrameOutput(NvDsPostProcessFrameOutput& frameOutput) +{ + switch (frameOutput.outputType) + { + case NvDsPostProcessNetworkType_BodyPose: + //Release if meta not attached + //delete[] frameOutput.segmentationOutput.class_map; + break; + default: + break; + } +} + +float BodyPoseModelPostProcessor::median(std::vector& v) { + size_t n = v.size() / 2; + nth_element(v.begin(), v.begin() + n, v.end()); + return v[n]; +} + +void BodyPoseModelPostProcessor::osdBody(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]) +{ + const int keypoint_radius = 3;//6;//3; + const int keypoint_line_width = 2;//4;//2; + + const int num_joints = 17; + const int num_bones = 18; + const int idx_bones[] = { 0,1, 0,2, 1,3, 2,4, + 0,5, 0,6, 5,6, 5,7, + 7,9, 6,8, 8,10, 11,12, + 5,11, 11,13, 13,15, 6,12, + 12,14, 14,16}; + const NvOSD_ColorParams bone_colors[] = { + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}, + NvOSD_ColorParams{0, 0, 1.0, 1}, + NvOSD_ColorParams{1.0, 0, 0, 1}, + NvOSD_ColorParams{0, 1.0, 0, 1}}; + + for (int ii = 0; ii < num_joints; ii++) { + int i = ii;// idx_joints[ii]; + + + if (keypoints_confidence[i] < m_ClassificationThreshold) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_CircleParams &cparams = dmeta->circle_params[dmeta->num_circles]; + cparams.xc = keypoints[2 * i ]; + cparams.yc = keypoints[2 * i + 1]; + cparams.radius = keypoint_radius; + cparams.circle_color = NvOSD_ColorParams{1.0, 0, 0, 1}; + cparams.has_bg_color = 1; + cparams.bg_color = NvOSD_ColorParams{1.0, 0, 0, 1}; + dmeta->num_circles++; + } + + for (int i = 0; i < num_bones; i++) { + int i0 = idx_bones[2 * i ]; + int i1 = idx_bones[2 * i + 1]; + + if ((keypoints_confidence[i0] < m_ClassificationThreshold) || + (keypoints_confidence[i1] < m_ClassificationThreshold)) + continue; + + ACQUIRE_DISP_META(dmeta); + NvOSD_LineParams *lparams = &dmeta->line_params[dmeta->num_lines]; + lparams->x1 = keypoints[2 * i0]; + lparams->y1 = keypoints[2 * i0 + 1]; + lparams->x2 = keypoints[2 * i1]; + lparams->y2 = keypoints[2 * i1 + 1]; + lparams->line_width = keypoint_line_width; + lparams->line_color = bone_colors[i]; + dmeta->num_lines++; + } + + return; +} + + +void BodyPoseModelPostProcessor::movenetposeFromTensorMeta( + const std::vector& outputLayers, + NvDsPostProcessBodyPoseOutput& output) +{ + // const int pelvis = 0; + // const int left_hip = 1; + // const int right_hip = 2; + // const int torso = 3; + // const int left_knee = 4; + // const int right_knee = 5; + // const int neck = 6; + // const int left_ankle = 7; + // const int right_ankle = 8; + // const int left_big_toe = 9; + // const int right_big_toe = 10; + // const int left_small_toe = 11; + // const int right_small_toe = 12; + // const int left_heel = 13; + // const int right_heel = 14; + // const int nose = 15; + // const int left_eye = 16; + // const int right_eye = 17; + // const int left_ear = 18; + // const int right_ear = 19; + // const int left_shoulder = 20; + // const int right_shoulder = 21; + // const int left_elbow = 22; + // const int right_elbow = 23; + // const int left_wrist = 24; + // const int right_wrist = 25; + // const int left_pinky_knuckle = 26; + // const int right_pinky_knuckle = 27; + // const int left_middle_tip = 28; + // const int right_middle_tip = 29; + // const int left_index_knuckle = 30; + // const int right_index_knuckle = 31; + // const int left_thumb_tip = 32; + // const int right_thumb_tip = 33; + + unsigned int numAttributes = outputLayers.size(); + for (unsigned int m=0; m < numAttributes;m++){ + const NvDsInferLayerInfo *info = &outputLayers[m]; + if (!strcmp(info->layerName, "output_0")) { + output.data = (float *)info->buffer; + + } + } +} + + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.h b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.h new file mode 100644 index 00000000..91972871 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_bodypose.h @@ -0,0 +1,175 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __POST_PROCESSOR_BODYPOSE_HPP__ +#define __POST_PROCESSOR_BODYPOSE_HPP__ + +#include "post_processor.h" +#include "nvdsmeta_schema.h" + +#include + +class OneEuroFilter { +public: + /// Default constructor + OneEuroFilter() { + reset(30.0f /* Hz */, 0.1f /* Hz */, 0.09f /* ??? */, 0.5f /* Hz */); + } + /// Constructor + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + OneEuroFilter(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(dataUpdateRate, minCutoffFreq, cutoffSlope, derivCutoffFreq); + } + /// Reset all parameters of the filter. + /// @param dataUpdateRate the sampling rate, i.e. the number of samples per unit of time. + /// @param minCutoffFreq the lowest bandwidth filter applied. + /// @param cutoffSlope the rate at which the filter adapts: higher levels reduce lag. + /// @param derivCutoffFreq the bandwidth of the filter applied to smooth the derivative, default 1 Hz. + void reset(float dataUpdateRate, float minCutoffFreq, float cutoffSlope, float derivCutoffFreq) { + reset(); _rate = dataUpdateRate; _minCutoff = minCutoffFreq; _beta = cutoffSlope; _dCutoff = derivCutoffFreq; + } + /// Reset only the initial condition of the filter, leaving parameters the same. + void reset() { _firstTime = true; _xFilt.reset(); _dxFilt.reset(); } + /// Apply the one euro filter to the given input. + /// @param x the unfiltered input value. + /// @return the filtered output value. + float filter(float x) + { + float dx, edx, cutoff; + if (_firstTime) { + _firstTime = false; + dx = 0; + } else { + dx = (x - _xFilt.hatXPrev()) * _rate; + } + edx = _dxFilt.filter(dx, alpha(_rate, _dCutoff)); + cutoff = _minCutoff + _beta * fabsf(edx); + return _xFilt.filter(x, alpha(_rate, cutoff)); + } + + +private: + class LowPassFilter { + public: + LowPassFilter() { reset(); } + void reset() { _firstTime = true; } + float hatXPrev() const { return _hatXPrev; } + float filter(float x, float alpha){ + if (_firstTime) { + _firstTime = false; + _hatXPrev = x; + } + float hatX = alpha * x + (1.f - alpha) * _hatXPrev; + _hatXPrev = hatX; + return hatX; + + } + private: + float _hatXPrev; + bool _firstTime; + }; + inline float alpha(float rate, float cutoff) { + const float kOneOverTwoPi = 0.15915494309189533577f; // 1 / (2 * pi) + // The paper has 4 divisions, but we only use one + // float tau = kOneOverTwoPi / cutoff, te = 1.f / rate; + // return 1.f / (1.f + tau / te); + return cutoff / (rate * kOneOverTwoPi + cutoff); +} + bool _firstTime; + float _rate, _minCutoff, _dCutoff, _beta; + LowPassFilter _xFilt, _dxFilt; +}; + +constexpr int NVDS_OBJECT_TYPE_PERSON_EXT_POSE = 0x103; + +typedef struct NvDsPersonPoseExt { + gint num_poses; + NvDsJoints *poses; +}NvDsPersonPoseExt; + + +class BodyPoseModelPostProcessor : public ModelPostProcessor{ + +public: + BodyPoseModelPostProcessor(int id, int gpuId = 0) + : ModelPostProcessor (NvDsPostProcessNetworkType_BodyPose, id, gpuId) {} + + ~BodyPoseModelPostProcessor() override = default; + + NvDsPostProcessStatus + initResource(NvDsPostProcessContextInitParams& initParams) override; + + NvDsPostProcessStatus parseEachFrame(const std::vector & + outputLayers, + NvDsPostProcessFrameOutput &result) override; + + void + attachMetadata (NvBufSurface *surf, gint batch_idx, + NvDsBatchMeta *batch_meta, + NvDsFrameMeta *frame_meta, + NvDsObjectMeta *object_meta, + NvDsObjectMeta *parent_obj_meta, + NvDsPostProcessFrameOutput & detection_output, + NvDsPostProcessDetectionParams *all_params, + std::set & filterOutClassIds, + int32_t unique_id, + gboolean output_instance_mask, + gboolean process_full_frame, + float segmentationThreshold, + gboolean maintain_aspect_ratio, + NvDsRoiMeta *roi_meta, + gboolean symmetric_padding) + override; + + void + prcoessMetadata (NvDsInferTensorMeta *tensor_meta, + NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta) override; + + void releaseFrameOutput(NvDsPostProcessFrameOutput& frameOutput) override; +private: + NvDsPostProcessStatus fillBodyPoseOutput( + const std::vector& outputLayers, + NvDsPostProcessBodyPoseOutput& output); + void osdBody(NvDsFrameMeta* frame_meta, + NvDsBatchMeta *bmeta, + NvDsDisplayMeta *dmeta, + const int numKeyPoints, + const float keypoints[], + const float keypoints_confidence[]); + + void movenetposeFromTensorMeta( + const std::vector& outputLayers, + NvDsPostProcessBodyPoseOutput& output); + + float median(std::vector& v); + + float m_ClassificationThreshold = 0.50; + std::unordered_map> m_filter_pose; + OneEuroFilter m_filterRootDepth; // Root node in pose25d. + fpos_t m_fp_25_pos; +}; + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_struct.h b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_struct.h new file mode 100644 index 00000000..eabbf57f --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/post_processor_struct.h @@ -0,0 +1,624 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __POST_PROCESS_STRUCT_HPP__ +#define __POST_PROCESS_STRUCT_HPP__ + +#include +#include +#include +#include "gstnvdsinfer.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +#define _PATH_MAX 4096 +#define _MAX_STR_LENGTH 1024 +/** + * Sets values on a @ref NvDsInferDimsCHW structure from a @ref NvDsInferDims + * structure. + */ +#define getDimsCHWFromDims(dimsCHW,dims) \ + do { \ + (dimsCHW).c = (dims).d[0]; \ + (dimsCHW).h = (dims).d[1]; \ + (dimsCHW).w = (dims).d[2]; \ + } while (0) + +#define getDimsHWCFromDims(dimsCHW,dims) \ + do { \ + (dimsCHW).h = (dims).d[0]; \ + (dimsCHW).w = (dims).d[1]; \ + (dimsCHW).c = (dims).d[2]; \ + } while (0) + +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +inline const char *safeStr(const std::string &str) +{ + return str.c_str(); +} + +inline bool string_empty(const char* str) { + return !str || strlen(str) == 0; +} + +constexpr float DEFAULT_PRE_CLUSTER_THRESHOLD = 0.2; +constexpr float DEFAULT_POST_CLUSTER_THRESHOLD = 0.0; +constexpr float DEFAULT_EPS = 0.0; +constexpr int DEFAULT_GROUP_THRESHOLD = 0; +constexpr int DEFAULT_MIN_BOXES = 0; +constexpr float DEFAULT_DBSCAN_MIN_SCORE = 0; +constexpr float DEFAULT_NMS_IOU_THRESHOLD = 0.3; +constexpr int DEFAULT_TOP_K = -1; +constexpr bool ATHR_ENABLED = true; +constexpr float ATHR_THRESHOLD = 60.0; +constexpr int PROCESS_MODEL_FULL_FRAME = 1; +constexpr int PROCESS_MODEL_OBJECTS = 2; + +/** + * Enum for the status codes returned by NvDsPostProcessAlgorithm. + */ +typedef enum { + /** operation succeeded. */ + NVDSPOSTPROCESS_SUCCESS = 0, + /** Failed to configure the instance possibly due to an + * erroneous initialization property. */ + NVDSPOSTPROCESS_CONFIG_FAILED, + /** Custom Library interface implementation failed. */ + NVDSPOSTPROCESS_CUSTOM_LIB_FAILED, + /** Invalid parameters were supplied. */ + NVDSPOSTPROCESS_INVALID_PARAMS, + /** Output parsing failed. */ + NVDSPOSTPROCESS_OUTPUT_PARSING_FAILED, + /** CUDA error was encountered. */ + NVDSPOSTPROCESS_CUDA_ERROR, + /** Resource error was encountered. */ + NVDSPOSTPROCESS_RESOURCE_ERROR, + /** Unknown error was encountered. */ + NVDSPOSTPROCESS_UNKNOWN_ERROR +} NvDsPostProcessStatus; + + +typedef enum { + NVDSPOSTPROCESS_LOG_ERROR = 0, + NVDSPOSTPROCESS_LOG_WARNING, + NVDSPOSTPROCESS_LOG_INFO, + NVDSPOSTPROCESS_LOG_DEBUG, +} NvDsPostProcesLogLevel; + +#define printMsg(level, tag_str, fmt, ...) \ + do { \ + char* baseName = strrchr((char*)__FILE__, '/'); \ + baseName = (baseName) ? (baseName + 1) : (char*)__FILE__; \ + char logMsgBuffer[5 * _MAX_STR_LENGTH + 1]; \ + snprintf(logMsgBuffer, 5 * _MAX_STR_LENGTH, \ + tag_str " NvDsPostProcess::%s() <%s:%d> : " fmt, \ + __func__, baseName, __LINE__, ##__VA_ARGS__); \ + fprintf(stderr, "%s\n", logMsgBuffer); \ + } while (0) + +#define printError(fmt, ...) \ + do { \ + printMsg (NVDSINFER_LOG_ERROR, "Error in", fmt, ##__VA_ARGS__); \ + } while (0) + +#define printWarning(fmt, ...) \ + do { \ + printMsg (NVDSINFER_LOG_WARNING, "Warning from", fmt, ##__VA_ARGS__); \ + } while (0) + +#define printInfo(fmt, ...) \ + do { \ + printMsg (NVDSINFER_LOG_INFO, "Info from", fmt, ##__VA_ARGS__); \ + } while (0) + +#define printDebug(fmt, ...) \ + do { \ + printMsg (NVDSINFER_LOG_DEBUG, "DEBUG", fmt, ##__VA_ARGS__); \ + } while (0) + +typedef struct +{ + unsigned int roiTopOffset; + unsigned int roiBottomOffset; + unsigned int detectionMinWidth; + unsigned int detectionMinHeight; + unsigned int detectionMaxWidth; + unsigned int detectionMaxHeight; +} NvDsPostProcessDetectionFilterParams; + +/** + * Holds the bounding box coloring information for one class; + */ +typedef struct +{ + int have_border_color; + NvOSD_ColorParams border_color; + + int have_bg_color; + NvOSD_ColorParams bg_color; +} NvDsPostProcessColorParams; + + +/** + * Enum for clustering mode for detectors + */ +typedef enum +{ + NVDSPOSTPROCESS_CLUSTER_GROUP_RECTANGLES = 0, + NVDSPOSTPROCESS_CLUSTER_DBSCAN, + NVDSPOSTPROCESS_CLUSTER_NMS, + NVDSPOSTPROCESS_CLUSTER_DBSCAN_NMS_HYBRID, + NVDSPOSTPROCESS_CLUSTER_NONE +} NvDsPostProcessClusterMode; + +/** + * Defines UFF layer orders. + */ +typedef enum { + NvDsPostProcessTensorOrder_kNCHW, + NvDsPostProcessTensorOrder_kNHWC, + NvDsPostProcessTensorOrder_kNC, +} NvDsPostProcessTensorOrder; + + +/** + * Defines network types. + */ +typedef enum +{ + /** Specifies a detector. Detectors find objects and their coordinates, + and their classes in an input frame. */ + NvDsPostProcessNetworkType_Detector, + /** Specifies a classifier. Classifiers classify an entire frame into + one of several classes. */ + NvDsPostProcessNetworkType_Classifier, + /** Specifies a segmentation network. A segmentation network classifies + each pixel into one of several classes. */ + NvDsPostProcessNetworkType_Segmentation, + /** Specifies a instance segmentation network. A instance segmentation + network detects objects, bounding box and mask for objects, and + their classes in an input frame */ + NvDsPostProcessNetworkType_InstanceSegmentation, + /** Bodypose 3D */ + NvDsPostProcessNetworkType_BodyPose, + /** Specifies other. Output layers of an "other" network are not parsed by + NvDsPostProcessContext. This is useful for networks that produce custom output. + Output can be parsed by the NvDsPostProcessContext client or can be combined + with the Gst-nvinfer feature to flow output tensors as metadata. */ + NvDsPostProcessNetworkType_Other = 100 +} NvDsPostProcessNetworkType; + +/** + * Holds detection and bounding box grouping parameters. + */ +typedef struct +{ + /** Holds the bounding box detection threshold to be applied prior + * to clustering operation. */ + float preClusterThreshold; + + /** Hold the bounding box detection threshold to be applied post + * clustering operation. */ + float postClusterThreshold; + + /** Holds the epsilon to control merging of overlapping boxes. Refer to OpenCV + * groupRectangles and DBSCAN documentation for more information on epsilon. */ + float eps; + /** Holds the minimum number of boxes in a cluster to be considered + an object during grouping using DBSCAN. */ + int minBoxes; + /** Holds the minimum number boxes in a cluster to be considered + an object during grouping using OpenCV groupRectangles. */ + int groupThreshold; + /** Minimum score in a cluster for the cluster to be considered an object + during grouping. Different clustering may cause the algorithm + to use different scores. */ + float minScore; + /** IOU threshold to be used with NMS mode of clustering. */ + float nmsIOUThreshold; + /** Number of objects with objects to be filtered in the decensding order + * of probability */ + int topK; + + unsigned int roiTopOffset; + unsigned int roiBottomOffset; + unsigned int detectionMinWidth; + unsigned int detectionMinHeight; + unsigned int detectionMaxWidth; + unsigned int detectionMaxHeight; + NvDsPostProcessColorParams color_params; + +} NvDsPostProcessDetectionParams; + + + +/** + * Holds the initialization parameters required for the NvDsPostProcessContext interface. + */ +typedef struct _NvDsPostProcessContextInitParams +{ + /** Holds a unique identifier for the instance. This can be used + to identify the instance that is generating log and error messages. */ + unsigned int uniqueID; + + /** Holds the maximum number of frames to be inferred together in a batch. + The number of input frames in a batch must be + less than or equal to this. */ + unsigned int maxBatchSize; + + /** Holds the pathname of the labels file containing strings for the class + labels. The labels file is optional. The file format is described in the + custom models section of the DeepStream SDK documentation. */ + char labelsFilePath[_PATH_MAX]; + + + /** Holds the network type. */ + NvDsPostProcessNetworkType networkType; + + /** Holds the number of classes detected by a detector network. */ + unsigned int numDetectedClasses; + + /** Holds per-class detection parameters. The array's size must be equal + to @a numDetectedClasses. */ + NvDsPostProcessDetectionParams *perClassDetectionParams; + + /** Holds the minimum confidence threshold for the classifier to consider + a label valid. */ + float classifierThreshold; + + float segmentationThreshold; + + /** Holds a pointer to an array of pointers to output layer names. */ + char ** outputLayerNames; + /** Holds the number of output layer names. */ + unsigned int numOutputLayers; + + /** Holds the ID of the GPU which is to run the inference. */ + unsigned int gpuID; + + /** Inference input dimensions for runtime engine */ + NvDsInferDimsCHW inferInputDims; + + /** Holds the type of clustering mode */ + NvDsPostProcessClusterMode clusterMode; + + /** Holds the name of the bounding box and instance mask parse function + in the custom library. */ + char customBBoxInstanceMaskParseFuncName[_MAX_STR_LENGTH]; + + /** Holds the name of the custom bounding box function + in the custom library. */ + char customBBoxParseFuncName[_MAX_STR_LENGTH]; + /** Name of the custom classifier attribute parsing function in the custom + * library. */ + char customClassifierParseFuncName[_MAX_STR_LENGTH]; + + /** Holds output order for segmentation network */ + NvDsPostProcessTensorOrder segmentationOutputOrder; + + char *classifier_type; + /** Holds boolean value to show whether preprocessor support is there. */ + gboolean preprocessor_support = FALSE; +} NvDsPostProcessContextInitParams; + +/** + * Holds information about one parsed object from a detector's output. + */ +typedef struct +{ + /** Holds the ID of the class to which the object belongs. */ + unsigned int classId; + + /** Holds the horizontal offset of the bounding box shape for the object. */ + float left; + /** Holds the vertical offset of the object's bounding box. */ + float top; + /** Holds the width of the object's bounding box. */ + float width; + /** Holds the height of the object's bounding box. */ + float height; + + /** Holds the object detection confidence level; must in the range + [0.0,1.0]. */ + float detectionConfidence; +} NvDsPostProcessObjectDetectionInfo; + +/** + * Holds information about one classified attribute. + */ +typedef struct +{ + /** Holds the index of the attribute's label. This index corresponds to + the order of output layers specified in the @a outputCoverageLayerNames + vector during initialization. */ + unsigned int attributeIndex; + /** Holds the the attribute's output value. */ + unsigned int attributeValue; + /** Holds the attribute's confidence level. */ + float attributeConfidence; + /** Holds a pointer to a string containing the attribute's label. + Memory for the string must not be freed. Custom parsing functions must + allocate strings on heap using strdup or equivalent. */ + char *attributeLabel; +} NvDsPostProcessAttribute; + + +/** + * A typedef defined to maintain backward compatibility. + */ +typedef NvDsPostProcessObjectDetectionInfo NvDsPostProcessParseObjectInfo; + +/** + * Holds information about one parsed object and instance mask from a detector's output. + */ +typedef struct +{ + /** Holds the ID of the class to which the object belongs. */ + unsigned int classId; + + /** Holds the horizontal offset of the bounding box shape for the object. */ + float left; + /** Holds the vertical offset of the object's bounding box. */ + float top; + /** Holds the width of the object's bounding box. */ + float width; + /** Holds the height of the object's bounding box. */ + float height; + + /** Holds the object detection confidence level; must in the range + [0.0,1.0]. */ + float detectionConfidence; + + /** Holds object segment mask */ + float *mask; + /** Holds width of mask */ + unsigned int mask_width; + /** Holds height of mask */ + unsigned int mask_height; + /** Holds size of mask in bytes*/ + unsigned int mask_size; +} NvDsPostProcessInstanceMaskInfo; + +/** + * Holds information about one detected object. + */ +typedef struct +{ + /** Holds the object's offset from the left boundary of the frame. */ + float left; + /** Holds the object's offset from the top boundary of the frame. */ + float top; + /** Holds the object's width. */ + float width; + /** Holds the object's height. */ + float height; + /** Holds the index for the object's class. */ + int classIndex; + /** Holds a pointer to a string containing a label for the object. */ + char *label; + /* confidence score of the detected object. */ + float confidence; + /* Instance mask information for the object. */ + float *mask; + /** Holds width of mask */ + unsigned int mask_width; + /** Holds height of mask */ + unsigned int mask_height; + /** Holds size of mask in bytes*/ + unsigned int mask_size; +} NvDsPostProcessObject; + +/** + * Holds information on all objects detected by a detector network in one + * frame. + */ +typedef struct +{ + /** Holds a pointer to an array of objects. */ + NvDsPostProcessObject *objects; + /** Holds the number of objects in @a objects. */ + unsigned int numObjects; +} NvDsPostProcessDetectionOutput; + +/** + * Holds information on all attributes classifed by a classifier network for + * one frame. + */ +typedef struct +{ + /** Holds a pointer to an array of attributes. There may be more than + one attribute, depending on the number of output coverage layers + (multi-label classifiers). */ + NvDsPostProcessAttribute *attributes; + /** Holds the size of the @a attributes array. */ + unsigned int numAttributes; + /** Holds a pointer to a string containing a label for the + classified output. */ + char *label; +} NvDsPostProcessClassificationOutput; + +/** + * Holds information parsed from segmentation network output for one frame. + */ +typedef struct +{ + /** Holds the width of the output. Same as network width. */ + unsigned int width; + /** Holds the height of the output. Same as network height. */ + unsigned int height; + /** Holds the number of classes supported by the network. */ + unsigned int classes; + /** Holds a pointer to an array for the 2D pixel class map. + The output for pixel (x,y) is at index (y*width+x). */ + int *class_map; + /** Holds a pointer to an array containing raw probabilities. + The probability for class @a c and pixel (x,y) is at index + (c*width*height + y*width+x). */ + float *class_probability_map; +} NvDsPostProcessSegmentationOutput; + +typedef struct NvDsPoint3f { + float x; + float y; + float z; +} NvDsPoint3f; + +/** + * Holds information parsed from bodypose network output for one frame. + */ +typedef struct +{ + /** Holds the width of the output. Same as network width. */ + unsigned int width; + /** Holds the height of the output. Same as network height. */ + unsigned int height; + + unsigned int num_key_points; + float *data; +} NvDsPostProcessBodyPoseOutput; + + +/** + * Holds the information inferred by the network on one frame. + */ +typedef struct +{ + /** Holds an output type indicating the valid member in the union + of @a detectionOutput, @a classificationOutput, and @a segmentationOutput. + This is basically the network type. */ + NvDsPostProcessNetworkType outputType; + /** Holds a union of supported outputs. The valid member is determined by + @a outputType. */ + union + { + /** Holds detector output. Valid when @a outputType is + @ref NvDsPostProcessNetworkType_Detector. */ + NvDsPostProcessDetectionOutput detectionOutput; + /** Holds classifier output. Valid when @a outputType is + @ref NvDsPostProcessNetworkType_Classifier. */ + NvDsPostProcessClassificationOutput classificationOutput; + /** Holds classifier output. Valid when @a outputType is + @ref NvDsPostProcessNetworkType_Classifier. */ + NvDsPostProcessSegmentationOutput segmentationOutput; + /** Holds classifier output. Valid when @a outputType is + @ref NvDsPostProcessNetworkType_Classifier. */ + NvDsPostProcessBodyPoseOutput bodyPoseOutput; + }; +} NvDsPostProcessFrameOutput; + +/** + * Holds the output for all of the frames in a batch (an array of frame), + * and related buffer information. + */ +typedef struct +{ + /** Holds a pointer to an array of outputs for each frame in the batch. */ + NvDsPostProcessFrameOutput *frames; + /** Holds the number of elements in @a frames. */ + unsigned int numFrames; + + /** Holds a pointer to an array of pointers to output device buffers + for this batch. The array elements are set by */ + void **outputDeviceBuffers; + /** Holds the number of elements in @a *outputDeviceBuffers. */ + unsigned int numOutputDeviceBuffers; + + /** Holds a pointer to an array of pointers to host buffers for this batch. + The array elements are set by */ + void **hostBuffers; + /** Holds the number of elements in hostBuffers. */ + unsigned int numHostBuffers; + + /** Holds a private context pointer for the set of output buffers. */ + void* priv; +} NvDsPostProcessBatchOutput; + +#ifdef __cplusplus +} +#endif + +/** + * Holds the detection parameters required for parsing objects. + */ +typedef struct +{ + /** Holds the number of classes requested to be parsed, starting with + class ID 0. Parsing functions may only output objects with + class ID less than this value. */ + unsigned int numClassesConfigured; + /** Holds a per-class vector of detection confidence thresholds + to be applied prior to the clustering operation. + Parsing functions may only output an object with detection confidence + greater than or equal to the vector element indexed by the object's + class ID. */ + std::vector perClassPreclusterThreshold; + /* Per-class threshold to be applied after the clustering operation. */ + std::vector perClassPostclusterThreshold; + +} NvDsPostProcessParseDetectionParams; + + +/** Holds the cached information of an object. */ +struct NvDsPostProcessObjectInfo { + /** Vector of cached classification attributes. */ + std::vector attributes; + /** Cached string label. */ + std::string label; + + NvDsPostProcessObjectInfo(const NvDsPostProcessObjectInfo&) = delete; + NvDsPostProcessObjectInfo() = default; + ~NvDsPostProcessObjectInfo(){ + for (auto &attr : attributes) { + if (attr.attributeLabel) + free (attr.attributeLabel); + } + } +}; + + +/** + * Holds the inference information/history for one object based on it's + * tracking id. + */ +typedef struct _NvDsPostProcessObjectHistory +{ + /** Boolean indicating if the object is already being inferred on. */ + int under_inference; + /** Bounding box co-ordinates of the object when it was last inferred on. */ + NvOSD_RectParams last_inferred_coords; + /** Number of the frame in the stream when the object was last inferred on. */ + unsigned long last_inferred_frame_num; + /** Number of the frame in the stream when the object was last accessed. This + * is useful for clearing stale enteries in map of the object histories and + * keeping the size of the map in check. */ + unsigned long last_accessed_frame_num; + /** Cached object information. */ + NvDsPostProcessObjectInfo cached_info; +} NvDsPostProcessObjectHistory; + + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.cpp b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.cpp new file mode 100644 index 00000000..bf0de8d6 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.cpp @@ -0,0 +1,831 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "postprocesslib_impl.h" +#include + +using namespace std; + +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + +/* This quark is required to identify NvDsMeta when iterating through + * the buffer metadatas */ +static GQuark _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING); + +extern "C" IDSPostProcessLibrary *CreateCustomAlgoCtx(DSPostProcess_CreateParams *params) +{ + return new PostProcessAlgorithm(params); +} + + +/*Separate a config file entry with delimiters + *to be able to parse it.*/ +std::vector +PostProcessAlgorithm::SplitString (std::string input) { + std::stringstream longStr(input); + std::string item; + std::vector ret; + while (std::getline (longStr, item, ';')){ + ret.push_back(item); + } + return ret; +} + +std::set +PostProcessAlgorithm::SplitStringInt (std::string input) { + + std::stringstream longStr(input); + std::string item; + std::set ret; + while (std::getline (longStr, item, ';')){ + ret.insert(stoi(item)); + } + return ret; +} + +/* Get the absolute path of a file mentioned in the config given a + * file path absolute/relative to the config file. */ + +bool +PostProcessAlgorithm::GetAbsFilePath ( + const gchar * cfg_file_path, const gchar * file_path, + char *abs_path_str) +{ + gchar abs_cfg_path[PATH_MAX + 1]; + gchar abs_real_file_path[PATH_MAX + 1]; + gchar *abs_file_path; + gchar *delim; + + /* Absolute path. No need to resolve further. */ + if (file_path[0] == '/') { + /* Check if the file exists, return error if not. */ + if (!realpath (file_path, abs_real_file_path)) { + /* Ignore error if file does not exist and use the unresolved path. */ + if (errno != ENOENT) + return FALSE; + } + g_strlcpy (abs_path_str, abs_real_file_path, _PATH_MAX); + return TRUE; + } + + /* Get the absolute path of the config file. */ + if (!realpath (cfg_file_path, abs_cfg_path)) { + return FALSE; + } + + /* Remove the file name from the absolute path to get the directory of the + * config file. */ + delim = g_strrstr (abs_cfg_path, "/"); + *(delim + 1) = '\0'; + + /* Get the absolute file path from the config file's directory path and + * relative file path. */ + abs_file_path = g_strconcat (abs_cfg_path, file_path, nullptr); + + /* Resolve the path.*/ + if (realpath (abs_file_path, abs_real_file_path) == nullptr) { + /* Ignore error if file does not exist and use the unresolved path. */ + if (errno == ENOENT) + g_strlcpy (abs_real_file_path, abs_file_path, _PATH_MAX); + else { + g_free (abs_file_path); + return FALSE; + } + } + + g_free (abs_file_path); + + g_strlcpy (abs_path_str, abs_real_file_path, _PATH_MAX); + return TRUE; +} + +/* Parse the labels file and extract the class label strings. For format of + * the labels file, please refer to the custom models section in the + * DeepStreamSDK documentation. + */ +bool PostProcessAlgorithm::ParseLabelsFile( + const std::string& labelsFilePath) { + std::ifstream labels_file(labelsFilePath, std::ios_base::in); + std::string delim{';'}; + if (!labels_file) { + printError("Could not open labels file:%s", safeStr(labelsFilePath)); + return false; + } + while (labels_file.good() && !labels_file.eof()) { + std::string line, word; + std::vector l; + size_t pos = 0, oldpos = 0; + + std::getline(labels_file, line, '\n'); + if (line.empty()) + continue; + + while ((pos = line.find(delim, oldpos)) != std::string::npos) { + word = line.substr(oldpos, pos - oldpos); + l.push_back(word); + oldpos = pos + delim.length(); + } + l.push_back(line.substr(oldpos)); + m_Labels.push_back(l); + } + + if (labels_file.bad()) { + printError("Failed to parse labels file:%s, iostate:%d", + safeStr(labelsFilePath), (int)labels_file.rdstate()); + return false; + } + return true; +} + + +bool PostProcessAlgorithm::SetConfigFile (const gchar *cfg_file_path){ + bool ret = true; + NvDsPostProcessStatus status = NVDSPOSTPROCESS_SUCCESS; + + if (!cfg_file_path || !std::filesystem::exists(cfg_file_path)) + { + printError("Config File input not provided or doesn't exist"); + return false; + } + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + if (!(configyml.size() > 0)) { + printError("Unable to parse config file '%s' ", cfg_file_path); + return false; + } + + m_processLock.lock(); + // Parse the config file here + if(configyml["property"]) { + for(YAML::const_iterator itr = configyml["property"].begin(); + itr != configyml["property"].end(); ++itr){ + std::string paramKey = itr->first.as(); + if (paramKey == "gpu-id"){ + m_gpuId = itr->second.as(); + m_initParams.gpuID = m_gpuId; + } + else if (paramKey == "preprocessor-support"){ + m_preprocessor_support = itr->second.as(); + m_initParams.preprocessor_support = m_preprocessor_support; + } + else if (paramKey == "network-type"){ + switch (itr->second.as()) { + case NvDsPostProcessNetworkType_Detector: + case NvDsPostProcessNetworkType_Classifier: + case NvDsPostProcessNetworkType_Segmentation: + case NvDsPostProcessNetworkType_InstanceSegmentation: + case NvDsPostProcessNetworkType_BodyPose: + case NvDsPostProcessNetworkType_Other: + m_networkType = static_cast(itr->second.as()); + break; + default: + g_printerr ("Error. Invalid value for 'network-type':'%d'\n", + itr->second.as()); + return false; + break; + } + m_initParams.networkType = m_networkType; + } + else if (paramKey == "process-mode"){ + m_processMode = itr->second.as(); + } + else if (paramKey == "num-detected-classes"){ + m_numDetectedClasses = itr->second.as(); + m_initParams.numDetectedClasses = m_numDetectedClasses; + } + else if (paramKey == "gie-unique-id"){ + m_gieUniqueId = itr->second.as(); + m_initParams.uniqueID = m_gieUniqueId; + } + else if (paramKey == "labelfile-path"){ + ret = ParseLabelsFile(itr->second.as()); + if (ret ==false){ + return ret; + } + std::strncpy(m_initParams.labelsFilePath, + (itr->second.as()).c_str(), sizeof(m_initParams.labelsFilePath)-1); + } + else if (paramKey == "cluster-mode"){ + switch (itr->second.as()) + { + case 0: + m_clusterMode = NVDSPOSTPROCESS_CLUSTER_GROUP_RECTANGLES; + break; + case 1: + m_clusterMode = NVDSPOSTPROCESS_CLUSTER_DBSCAN; + break; + case 2: + m_clusterMode = NVDSPOSTPROCESS_CLUSTER_NMS; + break; + case 3: + m_clusterMode = NVDSPOSTPROCESS_CLUSTER_DBSCAN_NMS_HYBRID; + break; + case 4: + m_clusterMode = NVDSPOSTPROCESS_CLUSTER_NONE; + break; + default: + g_printerr ("Error. Invalid value for 'cluster-mode':'%d'\n", + itr->second.as()); + return false; + break; + } + m_initParams.clusterMode = m_clusterMode; + } + else if (paramKey == "release-tensor-meta"){ + m_releaseTensorMeta = itr->second.as(); + } + else if (paramKey == "output-instance-mask"){ + m_outputInstanceMask = itr->second.as(); + } + else if (paramKey == "is-classifier"){ + m_isClassifier = itr->second.as(); + } + else if (paramKey == "classifier-threshold"){ + m_classifierThreshold = itr->second.as(); + m_initParams.classifierThreshold = m_classifierThreshold; + } + else if (paramKey == "classifier-type"){ + m_classifierType = itr->second.as(); + g_free (m_initParams.classifier_type); + m_initParams.classifier_type = g_strdup(m_classifierType.c_str()); + } + else if (paramKey == "segmentation-threshold"){ + m_segmentationThreshold = itr->second.as(); + m_initParams.segmentationThreshold = m_segmentationThreshold; + } + else if (paramKey == "segmentation-output-order"){ + m_initParams.segmentationOutputOrder = + static_cast(itr->second.as()); + } + else if (paramKey == "parse-classifier-func-name") { + std::string temp = itr->second.as(); + std::strncpy (m_initParams.customClassifierParseFuncName, temp.c_str(), + sizeof(m_initParams.customClassifierParseFuncName)-1); + } + else if (paramKey == "parse-bbox-func-name") { + std::string temp = itr->second.as(); + std::strncpy (m_initParams.customBBoxParseFuncName, temp.c_str(), + sizeof(m_initParams.customBBoxParseFuncName)-1); + } else if (paramKey == "parse-bbox-instance-mask-func-name") { + std::string temp = itr->second.as(); + std::strncpy (m_initParams.customBBoxInstanceMaskParseFuncName, temp.c_str(), + sizeof(m_initParams.customBBoxInstanceMaskParseFuncName)-1); + } + else if (paramKey == "output-blob-names"){ + m_outputBlobNames = SplitString (itr->second.as()); + gchar **values; + int len = (int) m_outputBlobNames.size(); + if (m_initParams.outputLayerNames){ + for (guint i=0; i < m_initParams.numOutputLayers; i++){ + g_free(m_initParams.outputLayerNames[i]); + m_initParams.outputLayerNames[i] = NULL; + } + g_free (m_initParams.outputLayerNames); + } + values = g_new (gchar *, len + 1); + for (int i = 0; i < len; i++) { + int size = 64; + char* str2 = (char*) g_malloc0(sizeof(char) * size); + std::strncpy (str2, m_outputBlobNames[i].c_str(), size-1); + values[i] = str2; + } + values[len] = NULL; + m_initParams.outputLayerNames = values; + m_initParams.numOutputLayers = len; + } + else if (paramKey == "operate-on-class-ids"){ + m_operateOnClassIds = SplitStringInt (itr->second.as()); + } + else if (paramKey == "filter-out-class-ids"){ + m_filterOutClassIds = SplitStringInt (itr->second.as()); + } + else { + printWarning ("Unknown parameter %s ",paramKey.c_str()); + } + } + } + else { + printError("property group not present in config file '%s' ", cfg_file_path); + m_processLock.unlock(); + return false; + } + + if (m_initParams.networkType == NvDsPostProcessNetworkType_Detector || + m_initParams.networkType == NvDsPostProcessNetworkType_InstanceSegmentation){ + NvDsPostProcessDetectionParams detection_params{DEFAULT_PRE_CLUSTER_THRESHOLD, + DEFAULT_POST_CLUSTER_THRESHOLD, DEFAULT_EPS, + DEFAULT_GROUP_THRESHOLD, DEFAULT_MIN_BOXES, + DEFAULT_DBSCAN_MIN_SCORE, DEFAULT_NMS_IOU_THRESHOLD, DEFAULT_TOP_K, + 0, 0, 0, 0, 0, 0, + {TRUE, (NvOSD_ColorParams){1.0,0.0,0.0,1.0}, + FALSE,(NvOSD_ColorParams){1.0,0.0,0.0,1.0}}}; + detection_params.color_params.have_border_color = TRUE; + detection_params.color_params.border_color = (NvOSD_ColorParams) {1, 0, 0, 1}; + detection_params.color_params.have_bg_color = FALSE; + + /* Parse the parameters for "all" classes if the group has been specified. + * Detection/Segmentation */ + if (configyml["class-attrs-all"]) { + ret = ParseConfAttr (configyml["class-attrs-all"], -1, detection_params); + if (ret ==false){ + printError("Parsing 'class-attrs-all' group failed"); + return ret; + } + } + + /* Initialize the per-class vector with the same default/parsed values for + * all classes. */ + if (m_initParams.perClassDetectionParams){ + delete [] m_initParams.perClassDetectionParams; + } + m_initParams.perClassDetectionParams = + new NvDsPostProcessDetectionParams[m_initParams.numDetectedClasses]; + + for (uint32_t icnt = 0; icnt < m_initParams.numDetectedClasses; icnt++){ + m_initParams.perClassDetectionParams[icnt] = detection_params; + } + + for(YAML::const_iterator itr = configyml.begin(); itr != configyml.end(); ++itr) { + std::string paramKey = itr->first.as(); + std::string class_str = "class-attrs-"; + if ((paramKey != "class-attrs-all") && + (paramKey.size() >= class_str.size())) { + if (class_str.compare(0,class_str.size(),paramKey.c_str(), + class_str.size()) == 0) { + std::string num_str = paramKey.substr(class_str.size()); + gint64 class_index = stoi(num_str); + m_initParams.perClassDetectionParams[class_index] = detection_params; + ret = ParseConfAttr (configyml[paramKey], class_index, + m_initParams.perClassDetectionParams[class_index]); + if (ret ==false){ + printError("Parsing '%s' group failed",paramKey.c_str()); + return ret; + } + } + } + } + status = preparePostProcess(); + if (status != NVDSPOSTPROCESS_SUCCESS){ + return false; + } + } + else if (m_initParams.networkType == NvDsPostProcessNetworkType_Classifier || + m_initParams.networkType == NvDsPostProcessNetworkType_Segmentation || + m_initParams.networkType == NvDsPostProcessNetworkType_BodyPose){ + + status = preparePostProcess(); + if (status != NVDSPOSTPROCESS_SUCCESS){ + return false; + } + } + else { + printError("Parsing for network type %d is not supported", + m_initParams.networkType); + return false; + } + + m_processLock.unlock(); + return true; +} + +NvDsPostProcessStatus +PostProcessAlgorithm::preparePostProcess(){ + NvDsPostProcessStatus ret = NVDSPOSTPROCESS_CONFIG_FAILED; + + m_Postprocessor.reset(); + switch (m_initParams.networkType){ + case NvDsPostProcessNetworkType_BodyPose: + m_Postprocessor = std::make_unique(m_gieUniqueId, m_gpuId); + ret = m_Postprocessor->initResource(m_initParams); + break; + //FIXME: + case NvDsPostProcessNetworkType_Other: + printWarning(" Failed to validate the network type not supported, %d",m_initParams.networkType); + return ret; + default: + printError(" Failed to validate the network type, unknown network %d",m_initParams.networkType); + return ret; + } + if (ret != NVDSPOSTPROCESS_SUCCESS){ + m_Postprocessor.reset(); + } + return ret; +} + +bool PostProcessAlgorithm::ParseConfAttr (YAML::Node node, gint64 class_index, + NvDsPostProcessDetectionParams& params) +{ + bool ret = true; + + for(YAML::const_iterator itr = node.begin(); itr != node.end(); ++itr) { + + std::string paramKey = itr->first.as(); + if (paramKey == "detected-min-w"){ + params.detectionMinWidth = itr->second.as(); + } + else if (paramKey == "detected-min-h"){ + params.detectionMinHeight = itr->second.as(); + } + else if (paramKey == "detected-max-w"){ + params.detectionMaxWidth = itr->second.as(); + } + else if (paramKey == "detected-max-h"){ + params.detectionMaxHeight = itr->second.as(); + } + else if (paramKey == "minBoxes"){ + params.minBoxes = itr->second.as(); + } + else if (paramKey == "pre-cluster-threshold"){ + params.preClusterThreshold = itr->second.as(); + } + else if (paramKey == "post-cluster-threshold"){ + params.postClusterThreshold = itr->second.as(); + } + else if (paramKey == "eps"){ + params.eps = itr->second.as(); + } + else if (paramKey == "group-threshold"){ + params.groupThreshold = itr->second.as(); + } + else if (paramKey == "min-score"){ + params.minScore = itr->second.as(); + } + else if (paramKey == "dbscan-min-score"){ + params.minScore = itr->second.as(); + } + else if (paramKey == "nms-iou-threshold"){ + params.nmsIOUThreshold = itr->second.as(); + } + else if (paramKey == "topk"){ + params.topK = itr->second.as(); + } + else if (paramKey == "roi-top-offset"){ + params.roiTopOffset = itr->second.as(); + } + else if (paramKey == "roi-bottom-offset"){ + params.roiBottomOffset = itr->second.as(); + } + else if (paramKey == "border-color") { + std::string values = itr->second.as(); + std::vector vec = SplitString(values); + if (vec.size() != 4){ + g_printerr + ("Error: in border-color, Number of Color params should be exactly 4 " + "floats {r, g, b, a} between 0 and 1"); + ret = false; + goto done; + } + params.color_params.border_color.red = std::stod(vec[0]); + params.color_params.border_color.green = std::stod(vec[1]); + params.color_params.border_color.blue = std::stod(vec[2]); + params.color_params.border_color.alpha = std::stod(vec[3]); + } + else if (paramKey == "bg-color") { + std::string values = itr->second.as(); + std::vector vec = SplitString(values); + + if (vec.size() != 4) { + g_printerr + ("Error: Group bg-color, Number of Color params should be exactly 4 " + "floats {r, g, b, a} between 0 and 1"); + ret = false; + goto done; + } + params.color_params.bg_color.red = std::stod(vec[0]); + params.color_params.bg_color.green = std::stod(vec[1]); + params.color_params.bg_color.blue = std::stod(vec[2]); + params.color_params.bg_color.alpha = std::stod(vec[3]); + params.color_params.have_bg_color = TRUE; + } + else { + printWarning ("Unknown parameter '%s' ",paramKey.c_str()); + } + } + m_detectorClassAttr[class_index] = params; +done: + return ret; +} + + +bool PostProcessAlgorithm::HandleEvent (GstEvent *event) +{ + switch (GST_EVENT_TYPE(event)) + { + case GST_EVENT_EOS: + m_processLock.lock(); + m_stop = TRUE; + m_processCV.notify_all(); + m_processLock.unlock(); + while (outputthread_stopped == FALSE) + { + //g_print ("waiting for processq to be empty, buffers in processq = %ld\n", m_processQ.size()); + g_usleep (1000); + } + break; + default: + break; + } + if ((GstNvEventType)GST_EVENT_TYPE(event) == GST_NVEVENT_STREAM_EOS) + { + gst_nvevent_parse_stream_eos (event, &source_id); + } + if ((GstNvEventType)GST_EVENT_TYPE(event) == GST_NVEVENT_PAD_ADDED) + { + gst_nvevent_parse_pad_added (event, &source_id); + } + if ((GstNvEventType)GST_EVENT_TYPE(event) == GST_NVEVENT_PAD_DELETED) + { + gst_nvevent_parse_pad_deleted (event, &source_id); + } + return true; +} + + +/* Deinitialize the Custom Lib context */ +PostProcessAlgorithm::~PostProcessAlgorithm() +{ + std::unique_lock lk(m_processLock); + m_processCV.wait(lk, [&]{return m_processQ.empty();}); + m_stop = TRUE; + m_processCV.notify_all(); + lk.unlock(); + + /* Wait for OutputThread to complete */ + if (m_outputThread) { + m_outputThread->join(); + } + + if (m_initParams.perClassDetectionParams){ + delete[] m_initParams.perClassDetectionParams; + } + if (m_initParams.outputLayerNames){ + for (uint32_t i = 0; i < m_initParams.numOutputLayers; i++){ + g_free(m_initParams.outputLayerNames[i]); + m_initParams.outputLayerNames[i] = NULL; + } + g_free (m_initParams.outputLayerNames); + } +} + +// Returns NvDsBatchMeta if present in the gstreamer buffer else NULL +NvDsBatchMeta *PostProcessAlgorithm::GetNVDS_BatchMeta (GstBuffer *buffer) +{ + gpointer state = NULL; + GstMeta *gst_meta = NULL; + NvDsBatchMeta *batch_meta = NULL; + + while ((gst_meta = gst_buffer_iterate_meta(buffer, &state))) { + if (!gst_meta_api_type_has_tag (gst_meta->info->api, _dsmeta_quark)) { + continue; + } + NvDsMeta *dsmeta = (NvDsMeta *) gst_meta; + + if (dsmeta->meta_type == NVDS_BATCH_GST_META) { + if (batch_meta != NULL) { + GST_WARNING("Multiple NvDsBatchMeta found on buffer %p", buffer); + } + batch_meta = (NvDsBatchMeta *) dsmeta->meta_data; + } + } + return batch_meta; +} + + +/* Process Buffer */ +BufferResult PostProcessAlgorithm::ProcessBuffer (GstBuffer *inbuf) +{ + GstMapInfo in_map_info = GST_MAP_INFO_INIT; + + GST_DEBUG_OBJECT (m_element, "PostProcessLib: ---> Inside %s frame_num = %d\n", __func__, m_frameNum++); + + /* Map the buffer contents and get the pointer to NvBufSurface. */ + if (!gst_buffer_map (inbuf, &in_map_info, GST_MAP_READ)) { + GST_ELEMENT_ERROR (m_element, STREAM, FAILED, + ("%s:gst buffer map to get pointer to NvBufSurface failed", __func__), (NULL)); + return BufferResult::Buffer_Error; + } + gst_buffer_unmap(inbuf, &in_map_info); + + // Push buffer to process thread for further processing + PacketInfo packetInfo; + packetInfo.inbuf = inbuf; + packetInfo.frame_num = m_frameNum; + + // Add custom preprocessing logic if required, here + // Pass the buffer to output_loop for further processing and pushing to next component + + // Enable for dumping the input frame, for debugging purpose + m_processLock.lock(); + m_processQ.push(packetInfo); + m_processCV.notify_all(); + m_processLock.unlock(); + + return BufferResult::Buffer_Async; +} + + + +/* Output Processing Thread */ +void PostProcessAlgorithm::OutputThread(void) +{ + GstFlowReturn flow_ret; + GstBuffer *outBuffer = NULL; + std::unique_lock lk(m_processLock); + NvDsBatchMeta *batch_meta = NULL; + int32_t frame_cnt = 0; + /* Run till signalled to stop. */ + while (1) { + + /* Wait if processing queue is empty. */ + if (m_processQ.empty()) { + if (m_stop == TRUE) { + break; + } + m_processCV.wait(lk); + continue; + } + + PacketInfo packetInfo = m_processQ.front(); + m_processQ.pop(); + + m_processCV.notify_all(); + lk.unlock(); + + // Add post process algorithm logic here + // Once buffer processing is done, push the buffer to the downstream + // by using gst_pad_push function + NvBufSurface *in_surf = getNvBufSurface (packetInfo.inbuf); + batch_meta = GetNVDS_BatchMeta (packetInfo.inbuf); + outBuffer = packetInfo.inbuf; + nvds_set_input_system_timestamp (outBuffer, GST_ELEMENT_NAME(m_element)); + if(m_preprocessor_support) + { + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + /* Iterate user metadata in frames to search PGIE's tensor metadata */ + for (NvDsMetaList * l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *roi_user_meta = (NvDsUserMeta *) l_user->data; + if (roi_user_meta->base_meta.meta_type != NVDS_ROI_META) + continue; + /* convert to roi metadata */ + NvDsRoiMeta *roi_meta = + (NvDsRoiMeta *) roi_user_meta->user_meta_data; + for (NvDsUserMetaList * r_user = roi_meta->roi_user_meta_list; + r_user != NULL; r_user = r_user->next){ + NvDsUserMeta *tensor_user_meta = (NvDsUserMeta *) r_user->data; + if (tensor_user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + /* convert to tensor metadata */ + NvDsInferTensorMeta *meta = + (NvDsInferTensorMeta *) tensor_user_meta->user_meta_data; + /* PGIE and operate on meta->unique_id data only */ + if (meta->unique_id == m_gieUniqueId){ + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + info->buffer = meta->out_buf_ptrs_host[i]; + } + std::vector < NvDsInferLayerInfo > + outputLayersInfo (meta->output_layers_info, + meta->output_layers_info + meta->num_output_layers); + NvDsPostProcessFrameOutput output; + memset (&output, 0, sizeof(output)); + if (m_Postprocessor){ + m_Postprocessor->setNetworkInfo(meta->network_info); + m_Postprocessor->parseEachFrame(outputLayersInfo, output); + m_Postprocessor->attachMetadata (in_surf, frame_meta->batch_id, + batch_meta, frame_meta, NULL, NULL, + output, + m_initParams.perClassDetectionParams, + m_filterOutClassIds, + m_gieUniqueId, + m_outputInstanceMask, + m_processMode, m_segmentationThreshold, + meta->maintain_aspect_ratio, roi_meta, + meta->symmetric_padding); + m_Postprocessor->releaseFrameOutput (output); + } + else { + GST_WARNING_OBJECT(m_element, "Post Processor not initialized for network"); + } + } + } + } + } + } + else + { + /* Iterate each frame metadata in batch */ + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + + if (m_processMode == PROCESS_MODEL_FULL_FRAME){ + /* Iterate user metadata in frames to search PGIE's tensor metadata */ + for (NvDsMetaList * l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + + /* convert to tensor metadata */ + NvDsInferTensorMeta *meta = + (NvDsInferTensorMeta *) user_meta->user_meta_data; + //PGIE and operate on meta->unique_id data only + if (meta->unique_id == m_gieUniqueId){ + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + info->buffer = meta->out_buf_ptrs_host[i]; + } + /* Parse output tensor and fill detection results into objectList. */ + std::vector < NvDsInferLayerInfo > + outputLayersInfo (meta->output_layers_info, + meta->output_layers_info + meta->num_output_layers); + NvDsPostProcessFrameOutput output; + memset (&output, 0, sizeof(output)); + if (m_Postprocessor){ + m_Postprocessor->setNetworkInfo(meta->network_info); + m_Postprocessor->parseEachFrame(outputLayersInfo, output); + m_Postprocessor->attachMetadata (in_surf, frame_meta->batch_id, + batch_meta, frame_meta, NULL, NULL, + output, + m_initParams.perClassDetectionParams, + m_filterOutClassIds, + m_gieUniqueId, + m_outputInstanceMask, + m_processMode, m_segmentationThreshold, + meta->maintain_aspect_ratio, NULL, + meta->symmetric_padding); + m_Postprocessor->releaseFrameOutput (output); + } + else { + GST_WARNING_OBJECT(m_element, "Post Processor not initialized for network"); + } + } + } + } + else if (m_processMode == PROCESS_MODEL_OBJECTS){ + /* Iterate object metadata in frame */ + for (NvDsMetaList * l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + NvDsObjectMeta *obj_meta = (NvDsObjectMeta *) l_obj->data; + + /* Iterate user metadata in object to search SGIE's tensor data */ + for (NvDsMetaList * l_user = obj_meta->obj_user_meta_list; l_user != NULL; + l_user = l_user->next) { + + NvDsUserMeta *user_meta = (NvDsUserMeta *) l_user->data; + if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + + /* convert to tensor metadata */ + NvDsInferTensorMeta *meta = + (NvDsInferTensorMeta *) user_meta->user_meta_data; + + if (meta->unique_id == m_gieUniqueId){ + m_Postprocessor->setNetworkInfo(meta->network_info); + m_Postprocessor->prcoessMetadata(meta, frame_meta, obj_meta); + } + } + } + } + } + } + + nvds_set_output_system_timestamp (outBuffer, GST_ELEMENT_NAME(m_element)); + flow_ret = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (m_element), outBuffer); + GST_DEBUG_OBJECT (m_element, + "CustomLib: %s in_surf=%p, Pushing Frame %d to downstream... Frame %d flow_ret = %d"\ + " TS=%" GST_TIME_FORMAT " \n", + __func__, in_surf, packetInfo.frame_num, frame_cnt++, + flow_ret, + GST_TIME_ARGS(GST_BUFFER_PTS(outBuffer))); + + lk.lock(); + continue; + } + outputthread_stopped = true; + lk.unlock(); + return; +} + + diff --git a/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.h b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.h new file mode 100644 index 00000000..c332224c --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream-pose-classification/postprocesslib_impl/postprocesslib_impl.h @@ -0,0 +1,191 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __POSTPROCESSLIB_HPP__ +#define __POSTPROCESSLIB_HPP__ + +#include +#include +#include "post_processor.h" +#include "post_processor_bodypose.h" +#include "nvdspostprocesslib_base.hpp" + +#define FORMAT_NV12 "NV12" +#define FORMAT_RGBA "RGBA" +#define FORMAT_I420 "I420" +#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM" + +/* Strcture used to share between the threads */ +struct PacketInfo { + GstBuffer *inbuf; + guint frame_num; +}; + +class PostProcessAlgorithm : public DSPostProcessLibraryBase +{ +public: + + explicit PostProcessAlgorithm(DSPostProcess_CreateParams *createParams) { + m_vectorProperty.clear(); + outputthread_stopped = false; + + if (createParams){ + m_element = createParams->m_element; + m_gpuId = createParams->m_gpuId; + m_cudaStream = createParams->m_cudaStream; + m_preprocessor_support = createParams->m_preprocessor_support; + } + else { + m_element = NULL; + m_gpuId = 0; + m_cudaStream = 0; + m_preprocessor_support = FALSE; + } + m_initParams.preprocessor_support = FALSE; + m_outputThread = new std::thread(&PostProcessAlgorithm::OutputThread, this); + m_initParams.uniqueID = 0; + m_initParams.maxBatchSize = 1; + + std::memset(m_initParams.labelsFilePath,0, sizeof(m_initParams.labelsFilePath)); + m_initParams.networkType = NvDsPostProcessNetworkType_Other; + std::memset(m_initParams.customClassifierParseFuncName, 0, + sizeof(m_initParams.customClassifierParseFuncName)-1); + std::memset(m_initParams.customBBoxParseFuncName, 0, + sizeof(m_initParams.customBBoxParseFuncName)-1); + std::memset(m_initParams.customBBoxInstanceMaskParseFuncName, 0, + sizeof(m_initParams.customBBoxInstanceMaskParseFuncName)-1); + + + /** Holds the number of classes detected by a detector network. */ + m_initParams.numDetectedClasses = 0; + + /** Holds per-class detection parameters. The array's size must be equal + to @a numDetectedClasses. */ + m_initParams.perClassDetectionParams = NULL; + + /** Holds the minimum confidence threshold for the classifier to consider + a label valid. */ + m_initParams.classifierThreshold = 0.5; + + m_initParams.segmentationThreshold = 0.5; + + /** Holds a pointer to an array of pointers to output layer names. */ + m_initParams.outputLayerNames = NULL; + /** Holds the number of output layer names. */ + m_initParams.numOutputLayers = 0; + + /** Holds the ID of the GPU which is to run the inference. */ + m_initParams.gpuID = 0; + + m_initParams.inferInputDims = {0,0,0}; + + /** Holds the type of clustering mode */ + m_initParams.clusterMode = NVDSPOSTPROCESS_CLUSTER_NMS; + + m_initParams.classifier_type = NULL; + } + + /* Pass GST events to the library */ + virtual bool HandleEvent(GstEvent *event); + + /* Read Config file */ + virtual bool SetConfigFile (const gchar *config_file); + + /* Process Incoming Buffer */ + virtual BufferResult ProcessBuffer(GstBuffer *inbuf); + + std::vector SplitString (std::string input); + + std::set SplitStringInt (std::string input); + + bool GetAbsFilePath (const gchar * cfg_file_path, const gchar * file_path, + char *abs_path_str); + + gboolean hw_caps; + + /* Deinit members */ + ~PostProcessAlgorithm(); + +private: + /* Helper Function to Extract Batch Meta from buffer */ + NvDsBatchMeta * GetNVDS_BatchMeta (GstBuffer *buffer); + + /* Output Processing Thread, push buffer to downstream */ + void OutputThread(void); + + /* Helper function to Dump NvBufSurface RAW content */ + void DumpNvBufSurface (NvBufSurface *in_surface, NvDsBatchMeta *batch_meta); + + bool ParseLabelsFile(const std::string& labelsFilePath); + bool ParseConfAttr (YAML::Node node, gint64 class_id, NvDsPostProcessDetectionParams& params); + NvDsPostProcessStatus preparePostProcess(); + + NvDsPostProcessNetworkType m_networkType; + NvDsPostProcessClusterMode m_clusterMode; + gfloat m_classifierThreshold; + gfloat m_segmentationThreshold; + gint m_numDetectedClasses; + gint m_processMode = 1; + guint m_gieUniqueId = 0; + gboolean m_isClassifier = 0; + gboolean m_releaseTensorMeta = FALSE; + gboolean m_outputInstanceMask = FALSE; + gboolean m_preprocessor_support = FALSE; + std::string m_classifierType; + std::set m_filterOutClassIds; + std::set m_operateOnClassIds; + std::vector m_outputBlobNames; + + std::vector m_InstanceMaskList; + std::unordered_map m_detectorClassAttr; + /* Vector of NvDsPostProcessInstanceMaskInfo vectors for each class. */ + std::vector> m_PerClassInstanceMaskList; + std::vector> m_Labels; + + std::unique_ptr m_Postprocessor; + NvDsPostProcessContextInitParams m_initParams; +public: + guint source_id = 0; + guint m_frameNum = 0; + bool outputthread_stopped = false; + + /* Output Thread Pointer */ + std::thread *m_outputThread = NULL; + + /* Queue and Lock Management */ + std::queue m_processQ; + std::mutex m_processLock; + std::condition_variable m_processCV; + /* Aysnc Stop Handling */ + gboolean m_stop = FALSE; + + /* Vector Containing Key:Value Pair of Custom Lib Properties */ + std::vector m_vectorProperty; + +}; + +extern "C" IDSPostProcessLibrary *CreateCustomAlgoCtx(DSPostProcess_CreateParams *params); +// Create Custom Algorithm / Library Context + + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/Makefile new file mode 100644 index 00000000..74e44951 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/Makefile @@ -0,0 +1,8 @@ +all: + @make -C nvdspreprocess_lib + +install: + @make install -C nvdspreprocess_lib + +clean: + @make clean -C nvdspreprocess_lib diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/README.md b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/README.md new file mode 100644 index 00000000..2c85b41a --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/README.md @@ -0,0 +1,537 @@ +# NV12/RGBA-To-Tensor-Preprocess + +## Prerequisites + +Please follow instructions in the apps/sample_apps/deepstream-app/README on how +to install the prerequisites for the Deepstream SDK, the DeepStream SDK itself, +and the apps. + +You must have the following development packages installed + GStreamer-1.0 + GStreamer-1.0 Base Plugins + GStreamer-1.0 gstrtspserver + X11 client-side library + +To install these packages, execute the following command: + +```bash +sudo apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ +libgstrtspserver-1.0-dev libx11-dev +``` + +## Purpose + +This sample demonstrates how to replace the Deepstream native NV12/RGBA->RGB(tensor) scaling, padding and normalization functionality with [roiconvert](https://github.com/NVIDIA-AI-IOT/Lidar_AI_Solution/tree/master/libraries/roiconvert). + +## Features + +1. The input to nvdspreprocess can be NV12 (Block Linear / Pitch) or RGBA. +2. Support Dgpu and Jetson. +3. Support normalization formula R/G/B_output = (R/G/B - offset_R/G/B) * scale_R/G/B. +4. Support processing full frame/ROI/Objects. +5. Support affine transformation. + +## To Compile + +```bash +$ cd deepstream_custom_preprocessing_app +$ git submodule update --init --recursive +$ cd nvdspreprocess_lib +$ export CUDA_VER= + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 +$ make +``` + +NOTE: To improve performance on specific GPUs, please add "-gencode=arch=compute_xx,code=sm_xx" in Makefile. Computing capability can be found in this link https://developer.nvidia.com/zh-cn/cuda-gpus#compute. + +## Usage + +### Processing Full Frame(nvdspreprocess + pgie) +Sample: + +```bash +gst-launch-1.0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 \ +nvstreammux name=mux batch-size=2 width=1920 height=1080 ! nvdspreprocess config-file=config_preprocess_frame.txt ! \ +nvinfer input-tensor-meta=true config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.yml ! \ +nvmultistreamtiler ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdsosd ! nvvideoconvert ! nvv4l2h264enc ! h264parse ! qtmux ! filesink location=out.mp4 +``` + +### Processing ROIs(nvdspreprocess + pgie) +Sample: + +```bash +gst-launch-1.0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 \ +nvstreammux name=mux batch-size=2 width=1920 height=1080 ! nvdspreprocess config-file=config_preprocess_roi.txt ! \ +nvinfer input-tensor-meta=true config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.yml ! \ +nvmultistreamtiler ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdsosd ! nvvideoconvert ! nvv4l2h264enc ! h264parse ! qtmux ! filesink location=out.mp4 +``` + +### Processing Objects(pgie + nvdspreprocess + sgie) +Sample: + +```bash +gst-launch-1.0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 \ +nvstreammux name=mux batch-size=2 width=1920 height=1080 ! \ +nvinfer config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.yml ! \ +nvdspreprocess config-file=config_preprocess_sgie.txt ! \ +nvinfer unique-id=4 input-tensor-meta=true config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_secondary_vehicletypes.yml ! \ +nvmultistreamtiler ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdsosd ! nvvideoconvert ! nvv4l2h264enc ! h264parse ! qtmux ! filesink location=out.mp4 +``` + +### Processing RGBA Input To Nvdspreprocess +Sample: + +```bash +gst-launch-1.0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.jpg ! jpegparse ! jpegdec ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.jpg ! jpegparse ! jpegdec ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! mux.sink_1 \ +nvstreammux name=mux batch-size=2 width=1920 height=1080 ! nvdspreprocess config-file=config_preprocess_frame.txt ! \ +nvinfer input-tensor-meta=true batch-size=2 config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.txt ! nvmultistreamtiler ! nvdsosd ! nveglglessink +``` + +### Processing Affine Transformation Tensors +If you want to generate affine transformation tensors, please use get_affine_matrx.py to generate affine matrix first. Taking scaling 1920x1080 to 960x544 and rotating 45 degrees for example, run the followming code. +```bash +$ python3 get_affine_matrx.py #please modify the code if using other values. +``` +Then fill scale-type and affine-matrix in the configuration file of nvdspreprocess as shown below. +```bash +scale-type=2 +affine-matrix=0.35355339;-0.35355339;331.50757595;0.35355339;0.35355339;-258.33008589 +``` + +## Configurations In [user-configs] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigurationsMeaningExample
pixel-normalization-factor works as "scale" in the formula R/G/B_Output = (R/G/B - offset_R/G/B) * scale_R/G/Bpixel-normalization-factor=0.003921568;0.003921568;0.003921568
The default value is 1;1;1.
offsetsworks as "offset" in the formula R/G/B_output = (R/G/B - offset_R/G/B) * scale_R/G/B.offsets=10;10;10
The default value is 0;0;0.
scaling-filterScaling Interpolation method. 0=Nearest 1=Bilinear 2=Default(Nearest)scaling-filter=0
The default is 0.
scale-typeScaling type. 0: "maintain-aspect-ratio=0", 1: "maintain-aspect-ratio=1 and center" 2: matrix.scale-type:1
The default is 1.
affine-matrixif scale-type is set to 2. affine-matrix is needed.affine-matrix=1.0;0.0;0.0;0.0;1.0;0.0
The default value is 1.0;0.0;0.0;0.0;1.0;0.0.
+## Performance +The below is a time consumption comparison using the DeepStream native library and the high-performance roiconvert library. + +Prerequisites: +|Items | values | +| ---------------- | ---------------- | +|Device | RTX 6000 & AGX Orin 64G | +|DeepStream Version | 8.0 | +|Input Format | NV12 | +|batch-num range | 40-45 | +|batch-size |16(fullframe) 64(ROI) | +|Power Mode |MAX perf and power | + + +### Processing Full Frame +```bash +#Pipeline for pitch layout +gst-launch-1.0 -v \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_2 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_3 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_4 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_5 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_6 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_7 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_8 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_9 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_10 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_11 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_12 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_13 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_14 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_15 \ +nvstreammux name=mux batch-size=16 width=1920 height=1080 ! nvdspreprocess config-file=config_preprocess_frame.txt ! fakesink + +# Pipeline for block linear +gst-launch-1.0 -v \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_2 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_3 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_4 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_5 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_6 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_7 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_8 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_9 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_10 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_11 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_12 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_13 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_14 \ +filesrc location=/opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_15 \ +nvstreammux name=mux batch-size=16 width=1280 height=720 ! nvdspreprocess config-file=config_preprocess_frame.txt ! fakesink +``` +Perf On Jetson Orin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
InputFunctionnativeroiconvert
pitchLayout(Nearest) + + + + +
CustomAsyncTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
5.540
8.024
13.564(ms)
+ 		+ + + + +
0
9.174
9.174(ms)
+
pitchLayout(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
6.316
8.219
14.535(ms)
+ 		+ + + + +
0
12.923
12.923(ms)
+
blockLinear(Nearest) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
6.085
7.862
13.947(ms)
+ 		+ + + + +
0
8.709
8.709(ms)
+
blockLinear(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
6.031
7.728
13.759(ms)
+ 		+ + + + +
0
13.551
13.551(ms)
+
+ +Perf On Dgpu RTX A6000 + + + + + + + + + + + + + + + + + + + +
InputFunctionnativeroiconvert
pitchLayout(Nearest) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
1.641
3.749
5.391(ms)
+ 		+ + + + +
0
4.643
4.643(ms)
+
pitchLayout(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors with batch-size=16
+ 		+ + + + +
1.588
5.815
7.403(ms)
+ 		+ + + + +
0
4.597
4.597(ms)
+
+ +### Processing Objects +```bash +#Pipeline for pitch layout +gst-launch-1.0 -v filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_0 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_1 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_2 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_3 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_4 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_5 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_6 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_7 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_8 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_9 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_10 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_11 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_12 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_13 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_14 filesrc location=$object_file ! qtdemux ! h264parse ! nvv4l2decoder ! mux.sink_15 nvstreammux name=mux batch-size=16 width=1920 height=1080 ! nvinfer config-file-path=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_infer_primary.yml ! nvdspreprocess config-file=config_preprocess_sgie.txt ! fakesink +``` +Perf on Jetson Orin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
InputFunctionNativeroiconvert
pitchLayout(Nearest) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
9.723
69.228
78.951(ms)
+ 		+ + + + +
0
17.596
17.596(ms)
+
pitchLayout(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
15.929
86.215
102.144(ms)
+ 		+ + + + +
0
21.906
21.906(ms)
+
blockLinear(Nearest) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
7.347
11.238
18.585(ms)
+ 		+ + + + +
0
10.439
10.439(ms)
+
blockLinear(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
12.820
12.675
25.495(ms)
+ 		+ + + + +
0
12.502
12.502(ms)
+
+ +Perf on Dgpu RTX A6000 + + + + + + + + + + + + + + + + + + + +
InputFunctionNativeroiconvert
pitchLayout(Nearest) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
37.743
21.874
59.617(ms)
+ 		+ + + + +
0
7.837
7.837(ms)
+
pitchLayout(Bilinear) + + + + +
CustomTransformation
CustomTensorPreparation
Time to generate 6 batches of tensors for objects
+ 		+ + + + +
20.757
15.944
36.701(ms)
+ 		+ + + + +
0
17.554
17.554(ms)
+
\ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_frame.txt b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_frame.txt new file mode 100644 index 00000000..98ad9390 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_frame.txt @@ -0,0 +1,80 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 + # list of component gie-id for which tensor is prepared +target-unique-ids=1 + # 0=NCHW, 1=NHWC, 2=CUSTOM +network-input-order=0 + # 0=process on objects 1=process on frames +process-on-frame=1 + #uniquely identify the metadata generated by this element +unique-id=5 + # gpu-id to be used +gpu-id=0 + # if enabled maintain the aspect ratio while scaling +maintain-aspect-ratio=1 + # if enabled pad symmetrically with maintain-aspect-ratio enabled +symmetric-padding=1 + # processig width/height at which image scaled +processing-width=960 +processing-height=544 + # max buffer in scaling buffer pool +scaling-buf-pool-size=6 + # max buffer in tensor buffer pool +tensor-buf-pool-size=6 + # tensor shape based on network-input-order +network-input-shape= 16;3;544;960 + # 0=RGB, 1=BGR, 2=GRAY +network-color-format=0 + # 0=FP32, 1=UINT8, 2=INT8, 3=UINT32, 4=INT32, 5=FP16 +tensor-data-type=0 + # tensor name same as input layer name +tensor-name=input_1:0 + # 0=NVBUF_MEM_DEFAULT 1=NVBUF_MEM_CUDA_PINNED 2=NVBUF_MEM_CUDA_DEVICE 3=NVBUF_MEM_CUDA_UNIFIED +scaling-pool-memory-type=0 + # 0=NvBufSurfTransformCompute_Default 1=NvBufSurfTransformCompute_GPU 2=NvBufSurfTransformCompute_VIC +scaling-pool-compute-hw=1 + # Scaling Interpolation method + # 0=NvBufSurfTransformInter_Nearest 1=NvBufSurfTransformInter_Bilinear 6=NvBufSurfTransformInter_Default +scaling-filter=0 + # custom library .so path having custom functionality +custom-lib-path=./nvdspreprocess_lib/libcustom2d_preprocess.so + # custom tensor preparation function name having predefined input/outputs + # check the default custom library nvdspreprocess_lib for more info +custom-tensor-preparation-function=CustomTensorPreparation + +[user-configs] + # Below parameters get used when using default custom library nvdspreprocess_lib + # network scaling factor +pixel-normalization-factor=0.003921568;0.003921568;0.003921568 + # array of offsets for each channel +#offsets= + # Scaling Interpolation method + # 0=Nearest 1=Bilinear 2=Default(Nearest) +scaling-filter=0 + # scale type + # 0: "maintain-aspect-ratio=0", 1: "maintain-aspect-ratio=1 and center" 2: matrix +scale-type=1 + # When scale-type is set to 2, a 3x2 user-defined affine matrix +affine-matrix=1.0;0.0;0.0;0.0;1.0;0.0 + +[group-0] +src-ids=0;1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20;21;22;23 +custom-input-transformation-function=CustomAsyncTransformation +process-on-roi=0 +roi-params-src-0=300;200;700;800;1300;300;600;700 +roi-params-src-1=860;300;900;500;50;300;500;700 diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_roi.txt b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_roi.txt new file mode 100644 index 00000000..193341a9 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_roi.txt @@ -0,0 +1,80 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 + # list of component gie-id for which tensor is prepared +target-unique-ids=1 + # 0=NCHW, 1=NHWC, 2=CUSTOM +network-input-order=0 + # 0=process on objects 1=process on frames +process-on-frame=1 + #uniquely identify the metadata generated by this element +unique-id=5 + # gpu-id to be used +gpu-id=0 + # if enabled maintain the aspect ratio while scaling +maintain-aspect-ratio=1 + # if enabled pad symmetrically with maintain-aspect-ratio enabled +symmetric-padding=1 + # processig width/height at which image scaled +processing-width=960 +processing-height=544 + # max buffer in scaling buffer pool +scaling-buf-pool-size=6 + # max buffer in tensor buffer pool +tensor-buf-pool-size=6 + # tensor shape based on network-input-order +network-input-shape= 16;3;544;960 + # 0=RGB, 1=BGR, 2=GRAY +network-color-format=0 + # 0=FP32, 1=UINT8, 2=INT8, 3=UINT32, 4=INT32, 5=FP16 +tensor-data-type=0 + # tensor name same as input layer name +tensor-name=input_1:0 + # 0=NVBUF_MEM_DEFAULT 1=NVBUF_MEM_CUDA_PINNED 2=NVBUF_MEM_CUDA_DEVICE 3=NVBUF_MEM_CUDA_UNIFIED +scaling-pool-memory-type=0 + # 0=NvBufSurfTransformCompute_Default 1=NvBufSurfTransformCompute_GPU 2=NvBufSurfTransformCompute_VIC +scaling-pool-compute-hw=1 + # Scaling Interpolation method + # 0=NvBufSurfTransformInter_Nearest 1=NvBufSurfTransformInter_Bilinear 6=NvBufSurfTransformInter_Default +scaling-filter=0 + # custom library .so path having custom functionality +custom-lib-path=./nvdspreprocess_lib/libcustom2d_preprocess.so + # custom tensor preparation function name having predefined input/outputs + # check the default custom library nvdspreprocess_lib for more info +custom-tensor-preparation-function=CustomTensorPreparation + +[user-configs] + # Below parameters get used when using default custom library nvdspreprocess_lib + # network scaling factor +pixel-normalization-factor=0.003921568;0.003921568;0.003921568 + # array of offsets for each channel +#offsets= + # Scaling Interpolation method + # 0=Nearest 1=Bilinear 2=Default(Nearest) +scaling-filter=0 + # scale type + # 0: "maintain-aspect-ratio=0", 1: "maintain-aspect-ratio=1 and center" 2: matrix +scale-type=1 + # When scale-type is set to 2, a 3x2 user-defined affine matrix +affine-matrix=1.0;0.0;0.0;0.0;1.0;0.0 + +[group-0] +src-ids=0;1 +custom-input-transformation-function=CustomAsyncTransformation +process-on-roi=1 +roi-params-src-0=300;200;700;800;1300;300;600;700 +roi-params-src-1=860;300;900;500;50;300;500;700 diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_sgie.txt b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_sgie.txt new file mode 100644 index 00000000..8ea5aaf6 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/config_preprocess_sgie.txt @@ -0,0 +1,85 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 + # list of component gie-id for which tensor is prepared +target-unique-ids=3;4;5 + # preprocess on metadata generated by this unique gie-id +operate-on-gie-id=1 + # 0=NCHW, 1=NHWC, 2=CUSTOM +network-input-order=0 + # 0=process on objects 1=process on frames +process-on-frame=0 + #uniquely identify the metadata generated by this element +unique-id=5 + # gpu-id to be used +gpu-id=0 + # processig width/height at which image scaled +processing-width=224 +processing-height=224 + # max buffer in scaling buffer pool +scaling-buf-pool-size=6 + # max buffer in tensor buffer pool +tensor-buf-pool-size=6 + # tensor shape based on network-input-order +network-input-shape= 64;3;224;224 + # 0=RGB, 1=BGR, 2=GRAY +network-color-format=1 + # 0=FP32, 1=UINT8, 2=INT8, 3=UINT32, 4=INT32, 5=FP16 +tensor-data-type=0 + # tensor name same as input layer name +tensor-name=input_1:0 + # 0=NVBUF_MEM_DEFAULT 1=NVBUF_MEM_CUDA_PINNED 2=NVBUF_MEM_CUDA_DEVICE 3=NVBUF_MEM_CUDA_UNIFIED +scaling-pool-memory-type=0 + # 0=NvBufSurfTransformCompute_Default 1=NvBufSurfTransformCompute_GPU 2=NvBufSurfTransformCompute_VIC +scaling-pool-compute-hw=1 + # Scaling Interpolation method + # 0=NvBufSurfTransformInter_Nearest 1=NvBufSurfTransformInter_Bilinear 6=NvBufSurfTransformInter_Default +scaling-filter=0 + # custom library .so path having custom functionality +custom-lib-path=./nvdspreprocess_lib/libcustom2d_preprocess.so + # custom tensor preparation function name having predefined input/outputs + # check the default custom library nvdspreprocess_lib for more info +custom-tensor-preparation-function=CustomTensorPreparation + +[user-configs] + # Below parameters get used when using default custom library nvdspreprocess_lib + # network scaling factor +pixel-normalization-factor=1;1;1 + # array of offsets for each channel +#offsets= + # Scaling Interpolation method + # 0=Nearest 1=Bilinear 2=Default(Nearest) +scaling-filter=0 + # scale type + # 0: "maintain-aspect-ratio=0", 1: "maintain-aspect-ratio=1 and center" 2: matrix +scale-type=0 + # When scale-type is set to 2, a 3x2 user-defined affine matrix +affine-matrix=1.0;0.0;0.0;0.0;1.0;0.0 + + +[group-0] +src-ids=0;1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16;17;18;19;20;21;22;23 +#0:Car 1:Bicycle 2:Person 3:Roadsign +operate-on-class-ids=0 +custom-input-transformation-function=CustomAsyncTransformation +process-on-all-objects=1 +roi-params-src-0=300;200;700;800;1300;300;600;700 +roi-params-src-1=860;300;900;500;50;300;500;700 +input-object-min-width=20 +input-object-min-height=20 +input-object-max-width=2000 +input-object-max-height=2000 diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/get_affine_matrx.py b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/get_affine_matrx.py new file mode 100644 index 00000000..5f595197 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/get_affine_matrx.py @@ -0,0 +1,44 @@ +import numpy as np +import math +np.set_printoptions(suppress=True) + + +Input_Width = 1920 +Input_Height = 1080 +Output_Width = 960 +Output_Height = 544 +alpha = np.pi/4 + + +T = np.array([ + [1, 0, -Input_Width * 0.5], + [0, 1, -Input_Height * 0.5], + [0, 0, 1] +]) + +R = np.array([ + [math.cos(alpha), -math.sin(alpha), 0], + [math.sin(alpha), math.cos(alpha), 0], + [0, 0, 1] +]) + +w_scale = Output_Width/Input_Width +h_scale = Output_Height/Input_Height +sx = sy = min(w_scale, h_scale) + +S = np.array([ + [sx, 0, 0], + [0, sy, 0], + [0, 0, 1] +]) + +invT = np.array([ + [1, 0, Output_Width/2], + [0, 1, Output_Height/2], + [0, 0, 1] +]) + +M = invT @ S @ R @ T + +print(M) + \ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/modules/Lidar_AI_Solution b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/modules/Lidar_AI_Solution new file mode 160000 index 00000000..6dfdcd21 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/modules/Lidar_AI_Solution @@ -0,0 +1 @@ +Subproject commit 6dfdcd219a33dba66c0b00bd59d0cc532c0b919a diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/Makefile new file mode 100644 index 00000000..9eeb20d4 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/Makefile @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +CXX:= g++ +NVCC:=/usr/local/cuda-$(CUDA_VER)/bin/nvcc + +# CUDA code generation flags +GPU_ARCHS = 75 80 86 89 90 100 120 +SMS = $(GPU_ARCHS) +ifeq ($(GENCODES),) + $(foreach sm,$(SMS),$(eval GENCODES += -gencode arch=compute_$(sm),code=sm_$(sm))) + GENCODES += -gencode arch=compute_90,code=compute_90 +endif + +GENCODE_FLAGS ?= $(GENCODES) + +KERNEL_PATH := ../modules/Lidar_AI_Solution/libraries/roiconvert/roi_conversion + +SRCS:= nvdspreprocess_lib.cpp \ + $(KERNEL_PATH)/roi_conversion.cu + +INCS:= $(wildcard *.h) +LIB:=libcustom2d_preprocess.so + +CFLAGS += -fPIC -DHAVE_CONFIG_H -std=c++17 -Wall -Werror \ + -I/usr/local/cuda-$(CUDA_VER)/include \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I/opt/nvidia/deepstream/deepstream/sources/gst-plugins/gst-nvdspreprocess/include \ + -I$(KERNEL_PATH) + +GST_INSTALL_DIR ?= /opt/nvidia/deepstream/deepstream/lib/gst-plugins/ +LIB_INSTALL_DIR ?= /opt/nvidia/deepstream/deepstream/lib/ + +LIBS := -shared -Wl,-no-undefined \ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart -ldl \ + -L$(LIB_INSTALL_DIR) -lnvdsgst_helper -lnvdsgst_meta -lnvds_meta \ + -lnvbufsurface -lnvbufsurftransform -lcuda \ + -Wl,-rpath,$(LIB_INSTALL_DIR) + +OBJS:= $(SRCS:.cpp=.o) +OBJS:= $(notdir $(OBJS:.cu=.o)) + +PKGS:= gstreamer-1.0 + +CFLAGS+=$(shell pkg-config --cflags $(PKGS)) +LIBS+=$(shell pkg-config --libs $(PKGS)) + +all: $(LIB) + +%.o: %.cpp $(INCS) + $(CXX) -c -o $@ $(CFLAGS) $< + +%.o: $(KERNEL_PATH)/%.cu $(KERNEL_PATH)/roi_conversion.hpp + $(NVCC) -c -O3 -o $@ $(GENCODE_FLAGS) -I$(KERNEL_PATH) --compiler-options '-fPIC' $< + +$(LIB): $(OBJS) $(DEP) + $(CXX) -o $@ $(OBJS) $(LIBS) + +install: $(LIB) + cp -rv $(LIB) $(GST_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.cpp b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.cpp new file mode 100644 index 00000000..fdccce28 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.cpp @@ -0,0 +1,493 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include +#include +#include +#include + +#include "nvdspreprocess_lib.h" +#include "roi_conversion.hpp" + +#if defined(__aarch64__) +#include "cudaEGL.h" +#include +#include +#endif + +#define NVTX_DEEPBLUE_COLOR 0xFF667EBE + +/** pixel-normalization-factor config parameter */ +#define NVDSPREPROCESS_USER_CONFIGS_PIXEL_NORMALIZATION_FACTOR "pixel-normalization-factor" + +/** offsets config parameter */ +#define NVDSPREPROCESS_USER_CONFIGS_OFFSETS "offsets" + +#define NVDSPREPROCESS_USER_CONFIGS_SCALING_FILTER "scaling-filter" + +#define NVDSPREPROCESS_USER_CONFIGS_SCALE_TYPE "scale-type" + +#define NVDSPREPROCESS_USER_CONFIGS_AFFINE_MATRIX "affine-matrix" + +#define MAX_CACHED_EGL_FRAME_SIZE 10000 + +#define checkRuntime(call) check_runtime(call, #call, __LINE__, __FILE__) +bool __inline__ check_runtime(cudaError_t e, const char *call, int line, const char *file) { + if (e != cudaSuccess) { + std::fprintf(stderr, "CUDA Runtime error %s # %s, code = %s [ %d ] in file %s:%d\n", call, + cudaGetErrorString(e), cudaGetErrorName(e), e, file, line); + return false; + } + return true; +} + +struct CustomCtx { + std::shared_ptr conv_handle; + /** Custom mean subtraction and normalization parameters */ + std::array scales{1.0f, 1.0f, 1.0f}; + std::array offsets{0.0f, 0.0f, 0.0f}; + std::array affine_matrix{1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f}; + enum ScaleType { + FitXY = 0, + FitCenter = 1, + Matrix = 2, + }; + ScaleType scale_type = FitCenter; + + /** interpolation filter for transformation */ + roiconv::Interpolation interpolation = roiconv::Interpolation::Nearest; + NvDsPreProcessNetworkSize network_size; + cudaStream_t stream; + nvtxDomainHandle_t nvtx_domain; + +#if defined(__aarch64__) + using GraphRes = + std::tuple; + using EglFrameCacheMap = std::unordered_map; + EglFrameCacheMap egl_frame_cache; +#endif +}; + +static roiconv::Interpolation to_roiconv_interpolation(const std::string &input) { + int val = std::stoi(input); + roiconv::Interpolation ret = roiconv::Interpolation::NoneEnum; + printf("%d is set for scaling-filter, ", (int)ret); + switch (val) { + case 0: + case 2: + ret = roiconv::Interpolation::Nearest; + printf("using Nearest\n"); + break; + case 1: + ret = roiconv::Interpolation::Bilinear; + printf("using Bilinear\n"); + break; + default: + printf("unsupported value, use default(Nearest)\n"); + ret = roiconv::Interpolation::Nearest; + break; + } + return ret; +} + +// Currently only supports NVBUF_COLOR_FORMAT_NV12 input +// NV12BlockLinear = 1, // Y, UV stride = width +// NV12PitchLinear = 2, // Y, UV stride = width +static roiconv::InputFormat to_roiconv_input_format(NvBufSurfaceParams *surface_params) { + switch (surface_params->colorFormat) { + case NVBUF_COLOR_FORMAT_NV12: + if (surface_params->layout == NVBUF_LAYOUT_BLOCK_LINEAR) { + return roiconv::InputFormat::NV12BlockLinear; + } else if (surface_params->layout == NVBUF_LAYOUT_PITCH) { + return roiconv::InputFormat::NV12PitchLinear; + } + case NVBUF_COLOR_FORMAT_RGBA: + return roiconv::InputFormat::RGBA; + case NVBUF_COLOR_FORMAT_YUV422: + // return roiconv::InputFormat::YUV422Packed_YUYV; + case NVBUF_COLOR_FORMAT_YUV420: + // return roiconv::InputFormat::YUVI420Separated; + case NVBUF_COLOR_FORMAT_RGB: + // return roiconv::InputFormat::RGB; + default: + printf("unsupport input surface format %d \n", surface_params->colorFormat); + return roiconv::InputFormat::NoneEnum; + } +} + +static roiconv::OutputFormat to_roiconv_output_format(NvDsPreProcessNetworkInputOrder order, + NvDsPreProcessFormat format) { + switch (format) { + case NvDsPreProcessFormat_RGB: + if (order == NvDsPreProcessNetworkInputOrder_kNCHW) { + return roiconv::OutputFormat::CHW_RGB; + } else if (order == NvDsPreProcessNetworkInputOrder_kNHWC) { + return roiconv::OutputFormat::HWC_RGB; + } + case NvDsPreProcessFormat_BGR: + if (order == NvDsPreProcessNetworkInputOrder_kNCHW) { + return roiconv::OutputFormat::CHW_BGR; + } else if (order == NvDsPreProcessNetworkInputOrder_kNHWC) { + return roiconv::OutputFormat::HWC_BGR; + } + case NvDsPreProcessFormat_GRAY: + return roiconv::OutputFormat::Gray; + default: + printf("unknow output format %d \n", format); + return roiconv::OutputFormat::NoneEnum; + } +} + +static roiconv::OutputDType to_roiconv_output_dtype(NvDsDataType data_type) { + switch (data_type) { + case NvDsDataType_UINT8: + case NvDsDataType_INT8: + return roiconv::OutputDType::Uint8; + case NvDsDataType_FP32: + return roiconv::OutputDType::Float32; + case NvDsDataType_FP16: + return roiconv::OutputDType::Float16; + default: + printf("unknow output dtype %d \n", data_type); + return roiconv::OutputDType::NoneEnum; + } +} + +template +static std::array split_string(const std::string &input, char delimiter = ';') { + std::array array; + std::istringstream iss(input); + std::string token; + size_t index = 0; + + while (std::getline(iss, token, delimiter)) { + // Check if we are within bounds + if (index < N) { + // Convert token to float and assign to array + array[index] = std::stof(token); + index++; + } else { + // Stop if we exceed the size of the array + break; + } + } + + return array; +} + +#if defined(__aarch64__) +using EglFrameCacheIter = CustomCtx::EglFrameCacheMap::iterator; +static void release_egl_frame_cache_entry(EglFrameCacheIter it) { + if (std::get<2>(it->second)) { + // y plane + checkRuntime(cudaDestroySurfaceObject(std::get<2>(it->second))); + } + if (std::get<3>(it->second)) { + // uv plane + checkRuntime(cudaDestroySurfaceObject(std::get<3>(it->second))); + } + cuGraphicsUnregisterResource(std::get<0>(it->second)); +} + +static std::pair cache_egl_frame(CustomCtx *ctx, NvBufSurface *surface, + int batch_index) { + const NvBufSurfaceParams *surface_params = &surface->surfaceList[batch_index]; + CUresult ret; + cudaSurfaceObject_t y_plane = 0; + cudaSurfaceObject_t uv_plane = 0; + + NvBufSurfaceMapEglImage(surface, batch_index); + + // EGLImage is in GPU memory address + EGLImageKHR eglimage = surface_params->mappedAddr.eglImage; + CUgraphicsResource resource; + EglFrameCacheIter null_iter; + // Register EGLImage to graphics resource + ret = cuGraphicsEGLRegisterImage(&resource, eglimage, CU_GRAPHICS_MAP_RESOURCE_FLAGS_READ_ONLY); + if (ret != CUDA_SUCCESS) { + printf("cuGraphicsEGLRegisterImage failed\n"); + return std::make_pair(null_iter, false); + } + // Map to CUeglFrame, for direct CUDA use + CUeglFrame eglFrame; + ret = cuGraphicsResourceGetMappedEglFrame(&eglFrame, resource, 0, 0); + if (ret != CUDA_SUCCESS) { + printf("cuGraphicsResourceGetMappedEglFrame failed\n"); + return std::make_pair(null_iter, false); + } + + if (surface_params->layout == NVBUF_LAYOUT_BLOCK_LINEAR) { + void *ptr0 = eglFrame.frame.pArray[0]; + void *ptr1 = eglFrame.frame.pArray[1]; + // Create the surface objects for Y + cudaResourceDesc resDesc; + + memset(&resDesc, 0, sizeof(cudaResourceDesc)); + resDesc.resType = cudaResourceTypeArray; + resDesc.res.array.array = (cudaArray_t)ptr0; + checkRuntime(cudaCreateSurfaceObject(&y_plane, &resDesc)); + + // Create the surface objects for UV + resDesc.res.array.array = (cudaArray_t)ptr1; + checkRuntime(cudaCreateSurfaceObject(&uv_plane, &resDesc)); + } + + ///\ If the saved egl mapped entries exceed the configured value, erase + /// the first entry + if (ctx->egl_frame_cache.size() > MAX_CACHED_EGL_FRAME_SIZE) { + printf("egl_frame_cache size exceeds limit: %lu / %d\n", ctx->egl_frame_cache.size(), + MAX_CACHED_EGL_FRAME_SIZE); + // Erase first entry in the map + // This is not a good strategy, as the first entry in the unorder_map may have just been + // inserted. + auto it = ctx->egl_frame_cache.begin(); + release_egl_frame_cache_entry(it); + ctx->egl_frame_cache.erase(it); + } + auto item = std::make_tuple(resource, eglFrame, y_plane, uv_plane); + return ctx->egl_frame_cache.insert(std::make_pair(surface_params, item)); +} +#endif + +// get y / uv plane address +static std::tuple get_planes_addr(CustomCtx *ctx, NvBufSurface *surface, + int batch_index) { + const NvBufSurfaceParams *surface_params = &surface->surfaceList[batch_index]; + if (surface_params->colorFormat != NVBUF_COLOR_FORMAT_NV12 && + surface_params->colorFormat != NVBUF_COLOR_FORMAT_RGBA) { + printf("unsupported colorformat %d \n", surface_params->colorFormat); + return std::make_tuple(nullptr, nullptr); + } + +#if defined(__aarch64__) + if(surface->memType == NVBUF_MEM_SURFACE_ARRAY) { + auto it = ctx->egl_frame_cache.find(surface_params); + ///\ Get the cuda mem pointers by EGL mapping + if (it == ctx->egl_frame_cache.end()) { + auto ret = cache_egl_frame(ctx, surface, batch_index); + if (ret.second) { + it = ret.first; + } else { + printf("cache egl frame failed \n"); + return std::make_tuple(nullptr, nullptr); + } + } + + if (surface_params->layout == NVBUF_LAYOUT_PITCH) { + void *luma = std::get<1>(it->second).frame.pPitch[0]; + void *chroma = std::get<1>(it->second).frame.pPitch[1]; + return std::make_tuple(luma, chroma); + } else if (surface_params->layout == NVBUF_LAYOUT_BLOCK_LINEAR) { + void *luma = reinterpret_cast(std::get<2>(it->second)); + void *chroma = reinterpret_cast(std::get<3>(it->second)); + return std::make_tuple(luma, chroma); + } else { + printf("unsupported layout %d \n", surface_params->layout); + return std::make_tuple(nullptr, nullptr); + } + } +#endif + auto input_height = surface->surfaceList[0].height; + auto input_stride = surface->surfaceList[0].pitch; + void *luma = surface->surfaceList[batch_index].dataPtr; + void *chroma = static_cast(surface->surfaceList[batch_index].dataPtr) + + input_stride * input_height; + return std::make_tuple(luma, chroma); +} + +CustomCtx *initLib(CustomInitParams initparams) { + auto ctx = std::make_unique(); + + ctx->conv_handle = roiconv::create(); + auto it = initparams.user_configs.find(NVDSPREPROCESS_USER_CONFIGS_PIXEL_NORMALIZATION_FACTOR); + if (it != initparams.user_configs.end()) { + ctx->scales = split_string<3>(it->second); + } + + it = initparams.user_configs.find(NVDSPREPROCESS_USER_CONFIGS_OFFSETS); + if (it != initparams.user_configs.end()) { + ctx->offsets = split_string<3>(it->second); + printf("Using offsets: %f, %f, %f\n", ctx->offsets[0], ctx->offsets[1], ctx->offsets[2]); + //convert "scale*(x-mean)" to "roi*alpha + beta" + for(int i = 0; i < 3; i++){ + ctx->offsets[i] *= -ctx->scales[i]; + } + } + + it = initparams.user_configs.find(NVDSPREPROCESS_USER_CONFIGS_SCALING_FILTER); + if (it != initparams.user_configs.end()) { + ctx->interpolation = to_roiconv_interpolation(it->second); + } + + it = initparams.user_configs.find(NVDSPREPROCESS_USER_CONFIGS_SCALE_TYPE); + if (it != initparams.user_configs.end()) { + ctx->scale_type = static_cast(stoi(it->second)); + printf("%d is set for scale_type, ", ctx->scale_type); + if(ctx->scale_type == CustomCtx::ScaleType::FitXY) { + printf("using FitXY\n"); + } else if(ctx->scale_type == CustomCtx::ScaleType::FitCenter) { + printf("using FitCenter\n"); + } else if (ctx->scale_type == CustomCtx::ScaleType::Matrix) { + printf("using Matrix\n"); + it = initparams.user_configs.find(NVDSPREPROCESS_USER_CONFIGS_AFFINE_MATRIX); + if (it != initparams.user_configs.end()) { + ctx->affine_matrix = split_string<6>(it->second); + } + } else { + printf("unsupported scale_type, use default(FitXY)\n"); + } + } + + printf("Using scales: %.5f, %.5f, %.5f\n", ctx->scales[0], ctx->scales[1], ctx->scales[2]); + printf("affine_matrix: "); + for (int i = 0; i < 6; i++) { + printf("%.3f ", ctx->affine_matrix[i]); + } + printf("\n"); + + // network initialization + const auto &tensor_params = initparams.tensor_params; + if (tensor_params.network_input_order == NvDsPreProcessNetworkInputOrder_kNCHW) { + ctx->network_size.channels = tensor_params.network_input_shape[1]; + ctx->network_size.height = tensor_params.network_input_shape[2]; + ctx->network_size.width = tensor_params.network_input_shape[3]; + } else if (tensor_params.network_input_order == NvDsPreProcessNetworkInputOrder_kNHWC) { + ctx->network_size.height = tensor_params.network_input_shape[1]; + ctx->network_size.width = tensor_params.network_input_shape[2]; + ctx->network_size.channels = tensor_params.network_input_shape[3]; + } else { + printf("network-input-order = %d not supported\n", tensor_params.network_input_order); + return nullptr; + } + switch (tensor_params.network_color_format) { + case NvDsPreProcessFormat_RGB: + case NvDsPreProcessFormat_BGR: + if (ctx->network_size.channels != 3) { + printf("RGB/BGR input format specified but network input channels is not 3\n"); + return nullptr; + } + break; + case NvDsPreProcessFormat_GRAY: + if (ctx->network_size.channels != 1) { + printf("GRAY input format specified but network input channels is not 1.\n"); + return nullptr; + } + break; + case NvDsPreProcessFormat_Tensor: + default: + printf("Unknown input format\n"); + return nullptr; + } + + checkRuntime(cudaStreamCreateWithFlags(&ctx->stream, cudaStreamNonBlocking)); + return ctx.release(); +} + +void deInitLib(CustomCtx *ctx) { +#if defined(__aarch64__) + auto it = ctx->egl_frame_cache.begin(); + while (it != ctx->egl_frame_cache.end()) { + release_egl_frame_cache_entry(it); + ++it; + } + ctx->egl_frame_cache.clear(); +#endif + + ctx->conv_handle.reset(); + checkRuntime(cudaStreamDestroy(ctx->stream)); + delete ctx; +} + +NvDsPreProcessStatus CustomTransformation(NvBufSurface *in_surf, NvBufSurface *out_surf, + CustomTransformParams ¶ms) { + return NVDSPREPROCESS_SUCCESS; +} + +NvDsPreProcessStatus CustomAsyncTransformation(NvBufSurface *in_surf, NvBufSurface *out_surf, + CustomTransformParams ¶ms) { + return NVDSPREPROCESS_SUCCESS; +} + +NvDsPreProcessStatus CustomTensorPreparation(CustomCtx *ctx, NvDsPreProcessBatch *batch, + NvDsPreProcessCustomBuf *&buf, + CustomTensorParams &tensorParam, + NvDsPreProcessAcquirer *acquirer) { + /** acquire a buffer from tensor pool */ + buf = acquirer->acquire(); + void *dst = buf->memory_ptr; + GstBuffer *inbuf = (GstBuffer *)batch->inbuf; + GstMapInfo inmap = GST_MAP_INFO_INIT; + if (!gst_buffer_map(inbuf, &inmap, GST_MAP_READ)) { + GST_ERROR("input buffer mapinfo failed"); + return NVDSPREPROCESS_CUSTOM_TENSOR_FAILED; + } + NvBufSurface *surface = (NvBufSurface *)inmap.data; + gst_buffer_unmap(inbuf, &inmap); + + nvtxEventAttributes_t eventAttrib = {0}; + eventAttrib.version = NVTX_VERSION; + eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + eventAttrib.colorType = NVTX_COLOR_ARGB; + eventAttrib.color = NVTX_DEEPBLUE_COLOR; + eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; + std::string nvtx_str = " tensorPrepare " + std::to_string(batch->inbuf_batch_num); + eventAttrib.message.ascii = nvtx_str.c_str(); + nvtxDomainRangePushEx(ctx->nvtx_domain, &eventAttrib); + + uint8_t fill_color[3] = {0, 0, 0}; + tensorParam.params.network_input_shape[0] = (int)batch->units.size(); + auto layer_bytes = tensorParam.params.buffer_size / tensorParam.params.network_input_shape[0]; + auto count = 0; + for (const auto &it : batch->units) { + roiconv::Task task; + task.x0 = it.roi_meta.roi.left; + task.y0 = it.roi_meta.roi.top; + task.x1 = task.x0 + it.roi_meta.roi.width; + task.y1 = task.y0 + it.roi_meta.roi.height; + + auto planes = get_planes_addr(ctx, surface, it.batch_index); + task.input_planes[0] = std::get<0>(planes); + task.input_planes[1] = std::get<1>(planes); + task.input_planes[2] = nullptr; + task.input_width = surface->surfaceList[0].width; + task.input_height = surface->surfaceList[0].height; + task.input_stride = surface->surfaceList[0].pitch; + + task.output_width = ctx->network_size.width; + task.output_height = ctx->network_size.height; + task.output = (uint8_t *)dst + layer_bytes * count++; + if (ctx->scale_type == CustomCtx::ScaleType::FitXY) { + task.resize_affine(); + } else if (ctx->scale_type == CustomCtx::ScaleType::FitCenter) { + task.center_resize_affine(); + } else if (ctx->scale_type == CustomCtx::ScaleType::Matrix) { + memcpy(task.affine_matrix, &ctx->affine_matrix[0], sizeof(ctx->affine_matrix)); + } + memcpy(task.alpha, &ctx->scales[0], sizeof(ctx->scales)); + memcpy(task.beta, &ctx->offsets[0], sizeof(ctx->offsets)); + memcpy(task.fillcolor, fill_color, 3); + ctx->conv_handle->add(task); + } + + auto input_format = to_roiconv_input_format(&surface->surfaceList[0]); + auto output_dtype = to_roiconv_output_dtype(tensorParam.params.data_type); + auto output_format = to_roiconv_output_format(tensorParam.params.network_input_order, + tensorParam.params.network_color_format); + + ctx->conv_handle->run(input_format, output_dtype, output_format, ctx->interpolation, ctx->stream, + false); + checkRuntime(cudaStreamSynchronize(ctx->stream)); + nvtxDomainRangePop(ctx->nvtx_domain); + + return NVDSPREPROCESS_SUCCESS; +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.h b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.h new file mode 100644 index 00000000..eca32410 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_custom_preprocessing_app/nvdspreprocess_lib/nvdspreprocess_lib.h @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +/** + * @file nvdspreprocess_lib.h + * NVIDIA DeepStream Preprocess lib specifications + * + * @b Description: This file defines common elements used in the API + * exposed by the Gst-nvdspreprocess plugin. + */ + +/** + * @defgroup gstreamer_nvdspreprocess_api NvDsPreProcess Plugin + * Defines an API for the GStreamer NvDsPreProcess custom lib. + * @ingroup custom_gstreamer + * @{ + */ + +#ifndef __NVDSPREPROCESS_LIB__ +#define __NVDSPREPROCESS_LIB__ + +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" +#include "nvdspreprocess_interface.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * custom library initialization function + */ +CustomCtx *initLib(CustomInitParams initparams); + +/** + * custom library deinitialization function + */ +void deInitLib(CustomCtx *ctx); + +/** + * Custom transformation function for group + */ +NvDsPreProcessStatus CustomTransformation(NvBufSurface *in_surf, NvBufSurface *out_surf, + CustomTransformParams ¶ms); + +/** + * Custom Asynchronus group transformation function + */ +NvDsPreProcessStatus CustomAsyncTransformation(NvBufSurface *in_surf, NvBufSurface *out_surf, + CustomTransformParams ¶ms); + +/** + * Custom tensor preparation function for NCHW/NHWC network order + */ +NvDsPreProcessStatus CustomTensorPreparation(CustomCtx *ctx, NvDsPreProcessBatch *batch, + NvDsPreProcessCustomBuf *&buf, + CustomTensorParams &tensorParam, + NvDsPreProcessAcquirer *acquirer); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/Makefile new file mode 100644 index 00000000..a82ce0aa --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/Makefile @@ -0,0 +1,64 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= deepstream-lpr-app + +CXX=g++ + + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ +APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/bin/ + +SRCS:= deepstream_lpr_app.cpp \ + deepstream_nvdsanalytics_meta.cpp \ + ../common/ds_yml_parse.cpp + +OBJS:= $(SRCS:.cpp=.o) + +PKGS:= gstreamer-1.0 +CFLAGS:=$(shell pkg-config --cflags $(PKGS)) +CFLAGS+=-Wall -Werror -Wno-unused-function \ + -I/opt/nvidia/deepstream/deepstream/sources/includes \ + -I/usr/local/cuda/include \ + -I../common/ + +LIBS:= $(shell pkg-config --libs $(PKGS)) +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lm -lstdc++\ + -lnvds_yml_parser -lyaml-cpp -lgstrtspserver-1.0 -Wl,-rpath,$(LIB_INSTALL_DIR) \ + -L/usr/local/cuda/lib64/ -lcudart + +all: $(APP) + +%.o: %.cpp + $(CXX) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) + @$(CC) -o $(APP) $(OBJS) $(LIBS) + @make -C nvinfer_custom_lpr_parser + +install: $(APP) + cp -rv $(APP) $(APP_INSTALL_DIR) + @make install -C nvinfer_custom_lpr_parser + +clean: + rm -rf $(OBJS) $(APP) + @make -C nvinfer_custom_lpr_parser clean \ No newline at end of file diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/README.md b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/README.md new file mode 100644 index 00000000..e7a9dad2 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/README.md @@ -0,0 +1,129 @@ +# Sample For Car License Recognization + - [Description](#description) + - [Performance](#performance) + - [Prerequisition](#prerequisition) + - [Download](#download) + - [Prepare Triton Server](#prepare-triton-server) + - [Build and Run](#build-and-run) + - [Notice](#notice) + +--- + +## Description +This sample is to show how to use graded models for detection and classification with DeepStream SDK version not less than 5.0.1. The models in this sample are all TAO3.0 models. + +`PGIE(car detection) -> SGIE(car license plate detection) -> SGIE(car license plate recognization)` + +![LPR/LPD application](lpr.png) + +This pipeline is based on three TAO models below + +* Car detection model https://ngc.nvidia.com/catalog/models/nvidia:tao:trafficcamnet +* LPD (car license plate detection) model https://ngc.nvidia.com/catalog/models/nvidia:tao:lpdnet +* LPR (car license plate recognization/text extraction) model https://ngc.nvidia.com/catalog/models/nvidia:tao:lprnet + +More details for TAO3.0 LPD and LPR models and TAO training, please refer to [TAO document](https://docs.nvidia.com/tao/tao-toolkit/text/overview.html). + +## Performance +Below table shows the end-to-end performance of processing 1080p videos with this sample application. + +| Device | Number of streams | Batch Size | Total FPS | +|-----------| ----------------- | -----------|-----------| +|Jetson Nano| 1 | 1 | 9.2 | +|Jetson NX | 3 | 3 | 80.31 | +|Jetson Xavier | 5 | 5 | 146.43 | +|Jetson Orin| 5 | 5 | 341.65 | +|T4 | 14 | 14 | 447.15 | + +## Prerequisition + +* [DeepStream SDK 6.0 or above](https://developer.nvidia.com/deepstream-getting-started) + + Make sure deepstream-test1 sample can run successful to verify your DeepStream installation + +* [tao-converter](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/resources/tao-converter/version) + + Download x86 or Jetson tao-converter which is compatible to your platform from the links in https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/resources/tao-converter/version. +* [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server) + + The LPR sample application can work as Triton client on x86 platforms. + +## Download + +1. Download Project with SSH or HTTPS + +```shell + # SSH + git clone git@github.com:NVIDIA/deepstream.git + # or HTTPS + git clone https://github.com/NVIDIA/deepstream.git +``` + +2. Prepare Models + +All models can be downloaded with the following commands: + +```shell + cd deepstream/src/apps/tao_apps/ + ./download_models.sh +``` + +## Prepare Triton Server +From DeepStream 6.1, LPR sample application supports three inferencing modes: +* gst-nvinfer inferencing based on TensorRT +* gst-nvinferserver inferencing as Triton CAPI client(only for x86) +* gst-nvinferserver inferencing as Triton gRPC client(only for x86) + +The following instructions are only needed for the LPR sample application working with gst-nvinferserver inferencing on x86 platforms as the Triton client. For LPR sample application works with nvinfer mode, please go to [Build and Run](#build-and-run) part directly. + +The Triton Inference Server libraries are required to be installed if the DeepStream LPR sample application should work as the Triton client, the Triton client [document](https://github.com/triton-inference-server/client) instructs how to install the necessary libraries. A easier way is to run DeepStream application in the [DeepStream Triton container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream). + +* Setting up Triton Inference Server for native cAPI inferencing, please refer to [triton_server.md](../../../triton_server.md). + +* Setting up Triton Inference Server for gRPC inferencing, please refer to [triton_server_grpc.md](../../../triton_server_grpc.md). + +## Build and Run + +```shell + # Build + cd apps/tao_others/deepstream_lpr_app + make +``` + +A sample of US car plate recognition: + +```shell + cp dict_us.txt dict.txt + # nvinfer is used by default, please modify the configuration file + # for nvinferserver capi/nvinferserver grpc + ./deepstream-lpr-app ../../../configs/app/lpr_app_us_config.yml +``` + +A sample of Chinese car plate recognition: + +```shell + cp dict_ch.txt dict.txt + # nvinfer is used by default, please modify the configuration file + # for nvinferserver capi/nvinferserver grpc + ./deepstream-lpr-app ../../../configs/app/lpr_app_ch_config.yml +``` + +## Notice +1. This sample application only support mp4 files which contain H264 videos as input files. +2. For Chinese plate recognition, please make sure the OS supports Chinese language. +3. The second argument of deepstream-lpr-app should be 2(fakesink) for performance test. +4. The trafficcamnet and LPD models are all INT8 models, the LPR model is FP16 model. +5. There is a bug for Triton gprc mode: the first two character can't be recognized. +6. For some yolo models, some layers of the models should use FP32 precision. This is a network characteristics that the accuracy drops rapidly when maximum layers are run in INT8 precision. Please refer the [layer-device-precision](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvinfer.html) for more details. +7. For Chinese plate recognition, please make sure the Chinese language support is in the OS. `Take Ubuntu as an example :` + + - Install Chinese Language package . + ```bash + sudo apt-get install language-pack-zh-hans + ``` + + - Set the Chinese language enviroment + ```bash + export LANG=zh_CN.UTF-8 + export LANGUAGE="zh_CN:zh:en_US:en" + ``` diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.cpp b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.cpp new file mode 100644 index 00000000..c6e5764f --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.cpp @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ds_yml_parse.h" +#include "gst-nvmessage.h" +#include "gstnvdsmeta.h" +#include "nvds_yml_parser.h" +#include "nvdsmeta.h" + +#define MAX_DISPLAY_LEN 64 + +#define MEASURE_ENABLE 1 + +#define PGIE_CLASS_ID_VEHICLE 0 +#define PGIE_CLASS_ID_PERSON 2 + +#define SGIE_CLASS_ID_LPD 0 + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 4000000 + +/* Check for parsing error. */ +#define RETURN_ON_PARSER_ERROR(parse_expr) \ + if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ + g_printerr("Error in parsing configuration file.\n"); \ + return -1; \ + } + +gint frame_number = 0; +gint total_plate_number = 0; +gchar pgie_classes_str[4][32] = {"Vehicle", "TwoWheeler", "Person", "Roadsign"}; + +extern "C" void parse_nvdsanalytics_meta_data(NvDsBatchMeta *batch_meta); + +#define PRIMARY_DETECTOR_UID 1 +#define SECONDARY_DETECTOR_UID 2 +#define SECONDARY_CLASSIFIER_UID 3 + +typedef struct _perf_measure { + GstClockTime pre_time; + GstClockTime total_time; + guint count; +} perf_measure; + +/* osd_sink_pad_buffer_probe will extract metadata received on OSD sink pad + * and update params for drawing rectangle, object information etc. */ +static GstPadProbeReturn +osd_sink_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { + GstBuffer *buf = (GstBuffer *)info->data; + NvDsObjectMeta *obj_meta = NULL; + guint vehicle_count = 0; + guint person_count = 0; + guint lp_count = 0; + guint label_i = 0; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsMetaList *l_class = NULL; + NvDsMetaList *l_label = NULL; + NvDsDisplayMeta *display_meta = NULL; + NvDsClassifierMeta *class_meta = NULL; + NvDsLabelInfo *label_info = NULL; + GstClockTime now; + perf_measure *perf = (perf_measure *)(u_data); + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + now = g_get_monotonic_time(); + + if (perf->pre_time == GST_CLOCK_TIME_NONE) { + perf->pre_time = now; + perf->total_time = GST_CLOCK_TIME_NONE; + } else { + if (perf->total_time == GST_CLOCK_TIME_NONE) { + perf->total_time = (now - perf->pre_time); + } else { + perf->total_time += (now - perf->pre_time); + } + perf->pre_time = now; + perf->count++; + } + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + if (!frame_meta) + continue; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + + if (!obj_meta) + continue; + + /* Check that the object has been detected by the primary detector + * and that the class id is that of vehicles/persons. */ + if (obj_meta->unique_component_id == PRIMARY_DETECTOR_UID) { + if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) + vehicle_count++; + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) + person_count++; + } + + if (obj_meta->unique_component_id == SECONDARY_DETECTOR_UID) { + if (obj_meta->class_id == SGIE_CLASS_ID_LPD) { + lp_count++; + /* Print this info only when operating in secondary model. */ + if (obj_meta->parent) + g_print("License plate found for parent object %p (type=%s)\n", + obj_meta->parent, + pgie_classes_str[obj_meta->parent->class_id]); + + obj_meta->text_params.set_bg_clr = 1; + obj_meta->text_params.text_bg_clr.red = 0.0; + obj_meta->text_params.text_bg_clr.green = 0.0; + obj_meta->text_params.text_bg_clr.blue = 0.0; + obj_meta->text_params.text_bg_clr.alpha = 0.0; + + obj_meta->text_params.font_params.font_color.red = 1.0; + obj_meta->text_params.font_params.font_color.green = 1.0; + obj_meta->text_params.font_params.font_color.blue = 0.0; + obj_meta->text_params.font_params.font_color.alpha = 1.0; + obj_meta->text_params.font_params.font_size = 12; + } + } + + for (l_class = obj_meta->classifier_meta_list; l_class != NULL; + l_class = l_class->next) { + class_meta = (NvDsClassifierMeta *)(l_class->data); + if (!class_meta) + continue; + if (class_meta->unique_component_id == SECONDARY_CLASSIFIER_UID) { + for (label_i = 0, l_label = class_meta->label_info_list; + label_i < class_meta->num_labels && l_label; + label_i++, l_label = l_label->next) { + label_info = (NvDsLabelInfo *)(l_label->data); + if (label_info) { + if (label_info->label_id == 0 && + label_info->result_class_id == 1) { + g_print("Plate License %s\n", label_info->result_label); + } + } + } + } + } + } + + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (char *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", + person_count); + offset += snprintf(txt_params->display_text + offset, MAX_DISPLAY_LEN, + "Vehicle = %d ", vehicle_count); + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + /* Font , font-color and font-size */ + char font_n[6]; + snprintf(font_n, 6, "Serif"); + txt_params->font_params.font_name = font_n; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + + g_print("Frame Number = %d Vehicle Count = %d Person Count = %d" + " License Plate Count = %d\n", + frame_number, vehicle_count, person_count, lp_count); + frame_number++; + total_plate_number += lp_count; + return GST_PAD_PROBE_OK; +} + +static gboolean bus_call(GstBus *bus, GstMessage *msg, gpointer data) { + GMainLoop *loop = (GMainLoop *)data; + switch (GST_MESSAGE_TYPE(msg)) { + case GST_MESSAGE_EOS: + g_print("End of stream\n"); + g_main_loop_quit(loop); + break; + case GST_MESSAGE_ERROR: { + gchar *debug; + GError *error; + gst_message_parse_error(msg, &error, &debug); + g_printerr("ERROR from element %s: %s\n", GST_OBJECT_NAME(msg->src), + error->message); + if (debug) + g_printerr("Error details: %s\n", debug); + g_free(debug); + g_error_free(error); + g_main_loop_quit(loop); + break; + } + default: + break; + } + return TRUE; +} + +static void cb_new_pad(GstElement *element, GstPad *pad, GstElement *data) { + GstCaps *new_pad_caps = NULL; + GstStructure *new_pad_struct = NULL; + const gchar *new_pad_type = NULL; + GstPadLinkReturn ret; + + GstPad *sink_pad = gst_element_get_static_pad(data, "sink"); + if (gst_pad_is_linked(sink_pad)) { + g_print("h264parser already linked. Ignoring.\n"); + goto exit; + } + + new_pad_caps = gst_pad_get_current_caps(pad); + new_pad_struct = gst_caps_get_structure(new_pad_caps, 0); + new_pad_type = gst_structure_get_name(new_pad_struct); + g_print("qtdemux pad %s\n", new_pad_type); + + if (g_str_has_prefix(new_pad_type, "video/x-h264")) { + ret = gst_pad_link(pad, sink_pad); + if (GST_PAD_LINK_FAILED(ret)) + g_print("fail to link parser and mp4 demux.\n"); + } else { + g_print("%s output, not 264 stream\n", new_pad_type); + } + +exit: + gst_object_unref(sink_pad); +} + +/* nvdsanalytics_src_pad_buffer_probe will extract metadata received on + * nvdsanalytics src pad and extract nvanalytics metadata etc. */ +static GstPadProbeReturn +nvdsanalytics_src_pad_buffer_probe(GstPad *pad, GstPadProbeInfo *info, + gpointer u_data) { + GstBuffer *buf = (GstBuffer *)info->data; + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + parse_nvdsanalytics_meta_data(batch_meta); + + return GST_PAD_PROBE_OK; +} + +static inline const char *infer_plugin(NvDsGieType type) { + switch (type) { + case NVDS_GIE_PLUGIN_INFER: + return "nvinfer"; + case NVDS_GIE_PLUGIN_INFER_SERVER: + return "nvinferserver"; + default: + return "unknown"; + } +} + +int main(int argc, char *argv[]) { + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *streammux = NULL, *sink = NULL, + *primary_detector = NULL, *secondary_detector = NULL, + *nvvidconv = NULL, *nvosd = NULL, *nvvidconv1 = NULL, + *outenc = NULL, *capfilt = NULL, *mux = NULL, + *secondary_classifier = NULL, *nvtile = NULL, *encparse = NULL; + GstElement *tracker = NULL, *nvdsanalytics = NULL; + GstElement *queue1 = NULL, *queue2 = NULL, *queue3 = NULL, *queue4 = NULL, + *queue5 = NULL, *queue6 = NULL, *queue7 = NULL, *queue8 = NULL, + *queue9 = NULL, *queue10 = NULL; + GstElement *h264parser[128], *source[128], *decoder[128], *mp4demux[128], + *parsequeue[128]; + GstBus *bus = NULL; + guint bus_watch_id; + GstPad *osd_sink_pad = NULL; + // int i; + static guint src_cnt = 0; + guint tiler_rows, tiler_columns; + perf_measure perf_measure; + + gchar ele_name[64]; + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + bool isH264 = true; + int enc_type = ENCODER_TYPE_HW; + GList *g_list = NULL; + GList *iterator = NULL; + + NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER; + NvDsGieType sgie0_type = NVDS_GIE_PLUGIN_INFER; + NvDsGieType sgie1_type = NVDS_GIE_PLUGIN_INFER; + + if (argc != 2) { + g_printerr("Usage: %s \n", argv[0]); + return -1; + } + + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + // For Chinese language supporting + setlocale(LC_CTYPE, ""); + /* Standard GStreamer initialization */ + gst_init(&argc, &argv); + loop = g_main_loop_new(NULL, FALSE); + + perf_measure.pre_time = GST_CLOCK_TIME_NONE; + perf_measure.total_time = GST_CLOCK_TIME_NONE; + perf_measure.count = 0; + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new("pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr("One element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add(GST_BIN(pipeline), streammux); + + RETURN_ON_PARSER_ERROR( + nvds_parse_source_list(&g_list, argv[1], "source-list")); + RETURN_ON_PARSER_ERROR( + nvds_parse_gie_type(&pgie_type, argv[1], "primary-gie")); + RETURN_ON_PARSER_ERROR( + nvds_parse_gie_type(&sgie0_type, argv[1], "secondary-gie0")); + RETURN_ON_PARSER_ERROR( + nvds_parse_gie_type(&sgie1_type, argv[1], "secondary-gie1")); + + /* Multiple source files */ + for (iterator = g_list, src_cnt = 0; iterator; + iterator = iterator->next, src_cnt++) { + /* Only h264 element stream with mp4 container is supported. */ + g_snprintf(ele_name, 64, "file_src_%d", src_cnt); + + /* Source element for reading from the file */ + source[src_cnt] = gst_element_factory_make("filesrc", ele_name); + + g_snprintf(ele_name, 64, "mp4demux_%d", src_cnt); + mp4demux[src_cnt] = gst_element_factory_make("qtdemux", ele_name); + + g_snprintf(ele_name, 64, "h264parser_%d", src_cnt); + h264parser[src_cnt] = gst_element_factory_make("h264parse", ele_name); + + g_snprintf(ele_name, 64, "parsequeue_%d", src_cnt); + parsequeue[src_cnt] = gst_element_factory_make("queue", ele_name); + + /* Use nvdec_h264 for hardware accelerated decode on GPU */ + g_snprintf(ele_name, 64, "decoder_%d", src_cnt); + decoder[src_cnt] = gst_element_factory_make("nvv4l2decoder", ele_name); + + if (!source[src_cnt] || !h264parser[src_cnt] || !decoder[src_cnt] || + !mp4demux[src_cnt]) { + g_printerr("One element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add_many(GST_BIN(pipeline), source[src_cnt], mp4demux[src_cnt], + h264parser[src_cnt], parsequeue[src_cnt], decoder[src_cnt], + NULL); + + g_snprintf(pad_name_sink, 64, "sink_%d", src_cnt); + sinkpad = gst_element_request_pad_simple(streammux, pad_name_sink); + g_print("Request %s pad from streammux\n", pad_name_sink); + if (!sinkpad) { + g_printerr("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad(decoder[src_cnt], pad_name_src); + if (!srcpad) { + g_printerr("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link(srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + + if (!gst_element_link_pads(source[src_cnt], "src", mp4demux[src_cnt], + "sink")) { + g_printerr("Elements could not be linked: 0. Exiting.\n"); + return -1; + } + + g_signal_connect(mp4demux[src_cnt], "pad-added", G_CALLBACK(cb_new_pad), + h264parser[src_cnt]); + + if (!gst_element_link_many(h264parser[src_cnt], parsequeue[src_cnt], + decoder[src_cnt], NULL)) { + g_printerr("Elements could not be linked: 1. Exiting.\n"); + } + + /* we set the input filename to the source element */ + g_object_set(G_OBJECT(source[src_cnt]), "location", (gchar *)iterator->data, + NULL); + + gst_object_unref(sinkpad); + gst_object_unref(srcpad); + } + g_list_free(g_list); + + /* Create three nvinfer instances for two detectors and one classifier*/ + primary_detector = gst_element_factory_make(infer_plugin(pgie_type), + "primary-infer-engine1"); + + secondary_detector = gst_element_factory_make(infer_plugin(sgie0_type), + "secondary-infer-engine1"); + + secondary_classifier = gst_element_factory_make(infer_plugin(sgie1_type), + "secondary-infer-engine2"); + + /* Use convertor to convert from NV12 to RGBA as required by nvosd */ + nvvidconv = gst_element_factory_make("nvvideoconvert", "nvvid-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + nvosd = gst_element_factory_make("nvdsosd", "nv-onscreendisplay"); + + nvvidconv1 = gst_element_factory_make("nvvideoconvert", "nvvid-converter1"); + + capfilt = gst_element_factory_make("capsfilter", "nvvideo-caps"); + + nvtile = gst_element_factory_make("nvmultistreamtiler", "nvtiler"); + + tracker = gst_element_factory_make("nvtracker", "nvtracker"); + + /* Use nvdsanalytics to perform analytics on object */ + nvdsanalytics = gst_element_factory_make("nvdsanalytics", "nvdsanalytics"); + + queue1 = gst_element_factory_make("queue", "queue1"); + queue2 = gst_element_factory_make("queue", "queue2"); + queue3 = gst_element_factory_make("queue", "queue3"); + queue4 = gst_element_factory_make("queue", "queue4"); + queue5 = gst_element_factory_make("queue", "queue5"); + queue6 = gst_element_factory_make("queue", "queue6"); + queue7 = gst_element_factory_make("queue", "queue7"); + queue8 = gst_element_factory_make("queue", "queue8"); + queue9 = gst_element_factory_make("queue", "queue9"); + queue10 = gst_element_factory_make("queue", "queue10"); + + /* set properties for nvdsanalytics */ + ds_parse_nvdsanalytics(nvdsanalytics, argv[1], "analytics"); + + guint output_type = 2; + output_type = ds_parse_group_type(argv[1], "output"); + if (output_type == 1) { + sink = gst_element_factory_make("filesink", "nvvideo-renderer"); + } else if (output_type == 2) { + sink = gst_element_factory_make("fakesink", "fake-renderer"); + } else if (output_type == 3) { + if (prop.integrated) + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + else +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + sink = gst_element_factory_make("nveglglessink", "nvvideo-renderer"); +#endif + } + + if (!primary_detector || !secondary_detector || !nvvidconv || !nvosd || + !sink) { + g_printerr("One element could not be created. Exiting.\n"); + return -1; + } + + g_object_set(G_OBJECT(streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", src_cnt, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + tiler_rows = (guint)sqrt(src_cnt); + tiler_columns = (guint)ceil(1.0 * src_cnt / tiler_rows); + g_object_set(G_OBJECT(nvtile), "rows", tiler_rows, "columns", tiler_columns, + "width", 1280, "height", 720, NULL); + + /* Set the config files for the two detectors and one classifier. The PGIE + * detects the cars. The first SGIE detects car plates from the cars and the + * second SGIE classifies the caracters in the car plate to identify the car + * plate string. */ + nvds_parse_gie(primary_detector, argv[1], "primary-gie"); + nvds_parse_gie(secondary_detector, argv[1], "secondary-gie0"); + nvds_parse_gie(secondary_classifier, argv[1], "secondary-gie1"); + nvds_parse_tracker(tracker, argv[1], "tracker"); + + /* we add a bus message handler */ + bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline)); + bus_watch_id = gst_bus_add_watch(bus, bus_call, loop); + gst_object_unref(bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + gst_bin_add_many(GST_BIN(pipeline), primary_detector, secondary_detector, + tracker, nvdsanalytics, queue1, queue2, queue3, queue4, + queue5, queue6, queue7, queue8, secondary_classifier, + nvvidconv, nvosd, nvtile, sink, NULL); + + if (!gst_element_link_many(streammux, queue1, primary_detector, queue2, + tracker, queue3, nvdsanalytics, queue4, + secondary_detector, queue5, secondary_classifier, + queue6, nvtile, queue7, nvvidconv, queue8, nvosd, + NULL)) { + g_printerr("Inferring and tracking elements link failure.\n"); + return -1; + } + + if (output_type == 1) { + isH264 = !(ds_parse_enc_codec(argv[1], "output")); + enc_type = ds_parse_enc_type(argv[1], "output"); + create_video_encoder(isH264, enc_type, &capfilt, &outenc, &encparse, NULL); + if (!capfilt || !outenc || !encparse) { + g_printerr("enc element could not be created. Exiting.\n"); + return -1; + } + gchar *filepath = NULL; + mux = gst_element_factory_make("qtmux", "mp4-mux"); + + GString *output_file = ds_parse_file_name(argv[1], "output"); + filepath = g_strconcat(output_file->str, ".mp4", NULL); + ds_parse_enc_config(outenc, argv[1], "output"); + + g_object_set(G_OBJECT(sink), "async", FALSE, NULL); + g_object_set(G_OBJECT(sink), "sync", TRUE, NULL); + g_object_set(G_OBJECT(sink), "location", filepath, NULL); + gst_bin_add_many(GST_BIN(pipeline), queue9, nvvidconv1, capfilt, queue10, + outenc, encparse, mux, sink, NULL); + + if (!gst_element_link_many(nvosd, queue9, nvvidconv1, capfilt, queue10, + outenc, encparse, mux, sink, NULL)) { + g_printerr("OSD and sink elements link failure.\n"); + return -1; + } + } else if (output_type == 2) { + g_object_set(G_OBJECT(sink), "sync", 0, "async", false, NULL); + if (!gst_element_link(nvosd, sink)) { + g_printerr("OSD and sink elements link failure.\n"); + return -1; + } + } else if (output_type == 3) { + gst_bin_add(GST_BIN(pipeline), queue9); + if (!gst_element_link_many(nvosd, queue9, sink, NULL)) { + g_printerr("OSD and sink elements link failure.\n"); + return -1; + } + } + + /* Lets add probe to get informed of the meta data generated, we add probe to + * the sink pad of the osd element, since by that time, the buffer would have + * had got all the metadata. */ + osd_sink_pad = gst_element_get_static_pad(nvosd, "sink"); + if (!osd_sink_pad) + g_print("Unable to get sink pad\n"); + else + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + osd_sink_pad_buffer_probe, &perf_measure, NULL); + gst_object_unref(osd_sink_pad); + + osd_sink_pad = gst_element_get_static_pad(nvdsanalytics, "src"); + if (!osd_sink_pad) + g_print("Unable to get src pad\n"); + else + gst_pad_add_probe(osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, + nvdsanalytics_src_pad_buffer_probe, NULL, NULL); + gst_object_unref(osd_sink_pad); + + /* Set the pipeline to "playing" state */ + g_print("Now playing: %s\n", argv[1]); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print("Running...\n"); + g_main_loop_run(loop); + + /* Out of the main loop, clean up nicely */ + g_print("Returned, stopping playback\n"); + gst_element_set_state(pipeline, GST_STATE_NULL); + + g_print("Average fps %f\n", ((perf_measure.count - 1) * src_cnt * 1000000.0) / + perf_measure.total_time); + g_print("Totally %d plates are inferred\n", total_plate_number); + g_print("Deleting pipeline\n"); + gst_object_unref(GST_OBJECT(pipeline)); + g_source_remove(bus_watch_id); + g_main_loop_unref(loop); + return 0; +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_nvdsanalytics_meta.cpp b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_nvdsanalytics_meta.cpp new file mode 100644 index 00000000..68d5a4be --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/deepstream_nvdsanalytics_meta.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include + +#include "gstnvdsmeta.h" +#include "nvds_analytics_meta.h" + +/* parse_nvdsanalytics_meta_data + * and extract nvanalytics metadata etc. */ +extern "C" void parse_nvdsanalytics_meta_data(NvDsBatchMeta *batch_meta) { + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + + // NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + std::stringstream out_string; + /* Iterate user metadata in frames to search analytics metadata */ + for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type != NVDS_USER_FRAME_META_NVDSANALYTICS) + continue; + + /* convert to metadata */ + NvDsAnalyticsFrameMeta *meta = + (NvDsAnalyticsFrameMeta *)user_meta->user_meta_data; + /* Get the labels from nvdsanalytics config file */ + for (std::pair status : meta->objInROIcnt) { + out_string << " Objs in ROI "; + out_string << status.first; + out_string << " = "; + out_string << status.second; + out_string << "\n"; + } + for (std::pair status : meta->objLCCumCnt) { + out_string << " LineCrossing Cumulative "; + out_string << status.first; + out_string << " = "; + out_string << status.second; + out_string << "\n"; + } + for (std::pair status : meta->objLCCurrCnt) { + out_string << " LineCrossing Current Frame "; + out_string << status.first; + out_string << " = "; + out_string << status.second; + out_string << "\n"; + } + for (std::pair status : meta->ocStatus) { + out_string << " Overcrowding status "; + out_string << status.first; + out_string << " = "; + out_string << status.second; + out_string << "\n"; + } + } + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + + // Access attached user meta for each object + for (NvDsMetaList *l_user_meta = obj_meta->obj_user_meta_list; + l_user_meta != NULL; l_user_meta = l_user_meta->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)(l_user_meta->data); + if (user_meta->base_meta.meta_type == + NVDS_USER_OBJ_META_NVDSANALYTICS) { + NvDsAnalyticsObjInfo *user_meta_data = + (NvDsAnalyticsObjInfo *)user_meta->user_meta_data; + if (user_meta_data->dirStatus.length()) { + out_string << " object " << obj_meta->object_id << " is moving in " + << user_meta_data->dirStatus; + } + } + } + } + + if (out_string.str().size()) { + g_print("Frame Number = %d of Stream = %d, %s\n", frame_meta->frame_num, + frame_meta->pad_index, out_string.str().c_str()); + } + } +} diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_ch.txt b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_ch.txt new file mode 100644 index 00000000..6595afdd --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_ch.txt @@ -0,0 +1,67 @@ +皖 +沪 +津 +渝 +冀 +晋 +蒙 +辽 +吉 +黑 +苏 +浙 +京 +闽 +赣 +鲁 +豫 +鄂 +湘 +粤 +桂 +琼 +川 +贵 +云 +藏 +陕 +甘 +青 +宁 +新 +警 +学 +A +B +C +D +E +F +G +H +J +K +L +M +N +P +Q +R +S +T +U +V +W +X +Y +Z +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_us.txt b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_us.txt new file mode 100644 index 00000000..ce7f2546 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/dict_us.txt @@ -0,0 +1,35 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F +G +H +I +J +K +L +M +N +P +Q +R +S +T +U +V +W +X +Y +Z diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/lpr.png b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/lpr.png new file mode 100644 index 00000000..f8b905ae Binary files /dev/null and b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/lpr.png differ diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/Makefile b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/Makefile new file mode 100644 index 00000000..706c2d90 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/Makefile @@ -0,0 +1,45 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +CC:= g++ + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CFLAGS:= -Wall -Werror -std=c++11 -shared -fPIC + +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes + +LIBS:= -lnvinfer +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +SRCFILES:= nvinfer_custom_lpr_parser.cpp +TARGET_LIB:= libnvdsinfer_custom_impl_lpr.so + +all: $(TARGET_LIB) + +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS) $(LFLAGS) + +install: $(TARGET_LIB) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) diff --git a/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp new file mode 100644 index 00000000..ecc3e43b --- /dev/null +++ b/src/apps/tao_apps/apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/nvinfer_custom_lpr_parser.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvdsinfer.h" + +static bool dict_ready = false; +std::vector dict_table; + +extern "C" bool NvDsInferParseCustomNVPlate( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, float classifierThreshold, + std::vector &attrList, std::string &attrString) { + int *outputStrBuffer = NULL; + float *outputConfBuffer = NULL; + NvDsInferAttribute LPR_attr; + + int seq_len = 0; + + // Get list + std::vector str_idxes; + int prev = 100; + + // For confidence + double bank_softmax_max[16] = {0.0}; + unsigned int valid_bank_count = 0; + bool do_softmax = false; + std::ifstream fdict; + + setlocale(LC_CTYPE, ""); + + if (!dict_ready) { + fdict.open("dict.txt"); + if (!fdict.is_open()) { + printf("open dictionary file failed. %s \n", strerror(errno)); + return false; + } + while (!fdict.eof()) { + std::string strLineAnsi; + if (getline(fdict, strLineAnsi)) { + dict_table.push_back(strLineAnsi); + } + } + dict_ready = true; + fdict.close(); + } + + int layer_size = outputLayersInfo.size(); + + LPR_attr.attributeConfidence = 1.0; + + seq_len = networkInfo.width / 4; + + for (int li = 0; li < layer_size; li++) { + if (!outputLayersInfo[li].isInput) { + if (outputLayersInfo[li].dataType == 0) { + if (!outputConfBuffer) + outputConfBuffer = static_cast(outputLayersInfo[li].buffer); + } else if (outputLayersInfo[li].dataType == 3) { + if (!outputStrBuffer) + outputStrBuffer = static_cast(outputLayersInfo[li].buffer); + } + } + } + + for (int seq_id = 0; seq_id < seq_len; seq_id++) { + do_softmax = false; + + int curr_data = outputStrBuffer[seq_id]; + if (curr_data < 0 || curr_data > static_cast(dict_table.size())) { + continue; + } + if (seq_id == 0) { + prev = curr_data; + str_idxes.push_back(curr_data); + if (curr_data != static_cast(dict_table.size())) + do_softmax = true; + } else { + if (curr_data != prev) { + str_idxes.push_back(curr_data); + if (static_cast(curr_data) != dict_table.size()) + do_softmax = true; + } + prev = curr_data; + } + + // Do softmax + if (do_softmax) { + do_softmax = false; + bank_softmax_max[valid_bank_count] = outputConfBuffer[seq_id]; + valid_bank_count++; + } + } + + attrString = ""; + for (unsigned int id = 0; id < str_idxes.size(); id++) { + if (static_cast(str_idxes[id]) != dict_table.size()) { + attrString += dict_table[str_idxes[id]]; + } + } + + // Ignore the short string, it may be wrong plate string + if (valid_bank_count >= 3) { + LPR_attr.attributeIndex = 0; + LPR_attr.attributeValue = 1; + LPR_attr.attributeLabel = strdup(attrString.c_str()); + for (unsigned int count = 0; count < valid_bank_count; count++) { + LPR_attr.attributeConfidence *= bank_softmax_max[count]; + } + attrList.push_back(LPR_attr); + } + + return true; +} diff --git a/src/apps/tao_apps/apps/tao_segmentation/Makefile b/src/apps/tao_apps/apps/tao_segmentation/Makefile new file mode 100644 index 00000000..c9073558 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_segmentation/Makefile @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +APP:= ds-tao-segmentation + +CC:=g++ + +VERBOSE?=0 +ifeq ($(VERBOSE), 1) +AT= +else +AT=@ +endif + +DS_VER = $(shell deepstream-app -v | awk '$$1~/DeepStreamSDK/ {print substr($$2,1,3)}' ) + +DS_SRC_PATH := /opt/nvidia/deepstream/deepstream-$(DS_VER) + +# Change to your deepstream SDK includes +CFLAGS+= -I$(DS_SRC_PATH)/sources/includes -I /usr/local/cuda/include +LIB_INSTALL_DIR?=$(DS_SRC_PATH)/lib/ +APP_INSTALL_DIR?=$(DS_SRC_PATH)/bin/ + +SRCS:= $(wildcard *.c) + +INCS:= $(wildcard *.h) + +PKGS:= gstreamer-1.0 + +OBJS:= $(SRCS:.c=.o) + +CFLAGS+= `pkg-config --cflags $(PKGS)` +CFLAGS+= -std=c++14 + +LIBS:= `pkg-config --libs $(PKGS)` + +LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper\ + -L/usr/local/cuda-$(CUDA_VER)/lib64/ -lcudart \ + -lnvds_yml_parser -lyaml-cpp -Wl,-rpath,$(LIB_INSTALL_DIR) + +all: $(APP) + +%.o: %.c $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +$(APP): $(OBJS) Makefile + $(CC) -o $(APP) $(OBJS) $(LIBS) + +install: $(APP) + @mkdir -p $(APP_INSTALL_DIR) + cp -rv $(APP) $(APP_INSTALL_DIR) + +clean: + rm -rf $(OBJS) $(APP) diff --git a/src/apps/tao_apps/apps/tao_segmentation/deepstream_seg_app.c b/src/apps/tao_apps/apps/tao_segmentation/deepstream_seg_app.c new file mode 100644 index 00000000..ff8a4be7 --- /dev/null +++ b/src/apps/tao_apps/apps/tao_segmentation/deepstream_seg_app.c @@ -0,0 +1,1090 @@ +/* + * Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "gstnvdsmeta.h" +#include "gstnvdsinfer.h" +#include "nvds_yml_parser.h" +#include "cuda_runtime_api.h" + + +/* The muxer output resolution must be set if the input streams will be of + * different resolution. The muxer will scale all the input frames to this + * resolution. */ +#define MUXER_OUTPUT_WIDTH 1280 +#define MUXER_OUTPUT_HEIGHT 720 + +/* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set + * based on the fastest source's framerate. */ +#define MUXER_BATCH_TIMEOUT_USEC 40000 + +#define TILED_OUTPUT_WIDTH 1280 +#define TILED_OUTPUT_HEIGHT 720 + +/*segvisual's dimention must be greater than or equal to model's. + read model's dimention from cfg, or use this default value*/ +#define SEG_OUTPUT_WIDTH 1920 +#define SEG_OUTPUT_HEIGHT 1080 + +/* NVIDIA Decoder source pad memory feature. This feature signifies that source + * pads having this capability will push GstBuffers containing cuda buffers. */ +#define GST_CAPS_FEATURES_NVMM "memory:NVMM" + +#define MAX_SOURCE_BINS 1024 + +static guint fileLoop; +static guint networkType; +static guint numDetectedClasses; + +typedef struct +{ + gdouble fps[MAX_SOURCE_BINS]; + gdouble fps_avg[MAX_SOURCE_BINS]; + guint num_instances; +}PerfStruct; + +typedef struct +{ + guint buffer_cnt; + guint total_buffer_cnt; + struct timeval total_fps_time; + struct timeval start_fps_time; + struct timeval last_fps_time; + struct timeval last_sample_fps_time; +}InstancePerfStruct; + +typedef struct +{ + guint num_instances; + GMutex struct_lock; + GstPad *sink_bin_pad; + InstancePerfStruct instance_str[MAX_SOURCE_BINS]; +} PerfStructInt; + +typedef struct _DsSourceBin +{ + GstElement *source_bin; + GstElement *uri_decode_bin; + GstElement *vidconv; + GstElement *nvvidconv; + GstElement *capsfilt; + GstElement *capsraw; + gint index; + gboolean is_imagedec; + gboolean is_streaming; + guint64 accumulated_base; + guint64 prev_accumulated_base; +}DsSourceBinStruct; + +typedef struct +{ + guint file_loop; + guint network_type; + guint num_detected_classes; + std::string config_path; + guint seg_gpu_id; + guint seg_width; + guint seg_height; + gboolean seg_background; + float seg_alpha; +} YamlParasStruct; + +static const char* dgpus_unsupport_hw_enc[] = { + "NVIDIA A100", + "NVIDIA A30", + "NVIDIA H100", // NVIDIA H100 SXM, NVIDIA H100 PCIe, NVIDIA H100 NVL + "NVIDIA T500", + "GeForce MX570 A", + "DGX A100" +}; + +/* Separate a config file entry with delimiters + * into strings. */ +static std::vector +split_string (std::string input) { + std::vector positions; + for (unsigned int i = 0; i < input.size(); i++) { + if (input[i] == ';') + positions.push_back(i); + } + std::vector ret; + int prev = 0; + for (auto &j: positions) { + std::string temp = input.substr(prev, j - prev); + ret.push_back(temp); + prev = j + 1; + } + ret.push_back(input.substr(prev, input.size() - prev)); + return ret; +} + +static void +parse_tests_yaml (YamlParasStruct *yaml_paras, const gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + std::string paramKey = ""; + + for(YAML::const_iterator itr = configyml["tests"].begin(); + itr != configyml["tests"].end(); ++itr) + { + paramKey = itr->first.as(); + if (paramKey == "file-loop") { + yaml_paras->file_loop = itr->second.as(); + } + } + + for(YAML::const_iterator itr = configyml["primary-gie"].begin(); + itr != configyml["primary-gie"].end(); ++itr) + { + paramKey = itr->first.as(); + if (paramKey == "config-file-path") { + yaml_paras->config_path = itr->second.as(); + } + } + + for(YAML::const_iterator itr = configyml["property"].begin(); + itr != configyml["property"].end(); ++itr) + { + paramKey = itr->first.as(); + if (paramKey == "network-type") { + yaml_paras->network_type = itr->second.as(); + } + if (paramKey == "num-detected-classes") { + yaml_paras->num_detected_classes = itr->second.as(); + } + } +} + +static void +parse_filesink_yaml (gint *enc_type, gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + for(YAML::const_iterator itr = configyml["filesink"].begin(); + itr != configyml["filesink"].end(); ++itr) + { + std::string paramKey = itr->first.as(); + if (paramKey == "enc-type") { + int value = itr->second.as(); + if(value == 0 || value == 1){ + *enc_type = value; + } + } else { + *enc_type = 0; + } + } + g_print("enc_type:%d\n", *enc_type); +} + +static void +parse_segvisual_yaml (YamlParasStruct *yaml_paras, const gchar *cfg_file_path) +{ + YAML::Node configyml = YAML::LoadFile(cfg_file_path); + + std::string paramKey = ""; + + for(YAML::const_iterator itr = configyml["segvisual"].begin(); + itr != configyml["segvisual"].end(); ++itr) + { + paramKey = itr->first.as(); + if (paramKey == "gpu-id") { + yaml_paras->seg_gpu_id = itr->second.as(); + } + + if (paramKey == "width") { + yaml_paras->seg_width = itr->second.as(); + } + + if (paramKey == "height") { + yaml_paras->seg_height = itr->second.as(); + } + + if (paramKey == "orig_background") { + yaml_paras->seg_background = itr->second.as(); + } + if (paramKey == "alpha") { + yaml_paras->seg_alpha = itr->second.as(); + } + } + +} + +/* + * Function to seek the source stream to start. + * It is required to play the stream in loop. + */ +static gboolean +seek_decode (gpointer data) +{ + DsSourceBinStruct *bin = (DsSourceBinStruct *) data; + gboolean ret = TRUE; + + gst_element_set_state (bin->source_bin, GST_STATE_PAUSED); + + ret = gst_element_seek (bin->source_bin, 1.0, GST_FORMAT_TIME, + (GstSeekFlags) (GST_SEEK_FLAG_KEY_UNIT | GST_SEEK_FLAG_FLUSH), + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); + + if (!ret) + GST_WARNING ("Error in seeking pipeline"); + + gst_element_set_state (bin->source_bin, GST_STATE_PLAYING); + + return FALSE; +} + +/** + * Probe function to drop certain events to support custom + * logic of looping of each source stream. + */ +static GstPadProbeReturn +restart_stream_buf_prob (GstPad * pad, GstPadProbeInfo * info, + gpointer u_data) +{ + GstEvent *event = GST_EVENT (info->data); + DsSourceBinStruct *bin = (DsSourceBinStruct *) u_data; + + if ((info->type & GST_PAD_PROBE_TYPE_BUFFER)) { + GST_BUFFER_PTS(GST_BUFFER(info->data)) += bin->prev_accumulated_base; + } + if ((info->type & GST_PAD_PROBE_TYPE_EVENT_BOTH)) { + if (GST_EVENT_TYPE (event) == GST_EVENT_EOS) { + g_timeout_add (1, seek_decode, bin); + } + + if (GST_EVENT_TYPE (event) == GST_EVENT_SEGMENT) { + GstSegment *segment; + + gst_event_parse_segment (event, (const GstSegment **) &segment); + segment->base = bin->accumulated_base; + bin->prev_accumulated_base = bin->accumulated_base; + bin->accumulated_base += segment->stop; + } + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + /* QOS events from downstream sink elements cause decoder to drop + * frames after looping the file since the timestamps reset to 0. + * We should drop the QOS events since we have custom logic for + * looping individual sources. */ + case GST_EVENT_QOS: + case GST_EVENT_SEGMENT: + case GST_EVENT_FLUSH_START: + case GST_EVENT_FLUSH_STOP: + return GST_PAD_PROBE_DROP; + default: + break; + } + } + return GST_PAD_PROBE_OK; +} + +static gboolean +perf_measurement_callback (gpointer data) +{ + PerfStructInt *str = (PerfStructInt *) data; + guint buffer_cnt[MAX_SOURCE_BINS]; + PerfStruct perf_struct; + struct timeval current_fps_time; + guint i; + static guint header_print_cnt = 0; + if (header_print_cnt % 20 == 0) { + g_print ("\n**PERF: "); + for (i = 0; i < str->num_instances; i++) { + g_print ("FPS %d (Avg)\t", i); + } + g_print ("\n"); + header_print_cnt = 0; + } + header_print_cnt++; + + time_t t = time (NULL); + struct tm *tm = localtime (&t); + g_print ("%s", asctime (tm)); + + g_mutex_lock (&str->struct_lock); + + for (i = 0; i < str->num_instances; i++) { + buffer_cnt[i] = + str->instance_str[i].buffer_cnt; + str->instance_str[i].buffer_cnt = 0; + } + + perf_struct.num_instances = str->num_instances; + gettimeofday (¤t_fps_time, NULL); + + g_print ("**PERF: "); + for (i = 0; i < str->num_instances; i++) { + InstancePerfStruct *str1 = &str->instance_str[i]; + gdouble time1 = + (str1->total_fps_time.tv_sec + + str1->total_fps_time.tv_usec / 1000000.0) + + (current_fps_time.tv_sec + current_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + + gdouble time2; + + if (str1->last_sample_fps_time.tv_sec == 0 && + str1->last_sample_fps_time.tv_usec == 0) { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->start_fps_time.tv_sec + + str1->start_fps_time.tv_usec / 1000000.0); + } else { + time2 = + (str1->last_fps_time.tv_sec + + str1->last_fps_time.tv_usec / 1000000.0) - + (str1->last_sample_fps_time.tv_sec + + str1->last_sample_fps_time.tv_usec / 1000000.0); + } + str1->total_buffer_cnt += buffer_cnt[i]; + perf_struct.fps[i] = buffer_cnt[i] / time2; + if (isnan (perf_struct.fps[i])) + perf_struct.fps[i] = 0; + + perf_struct.fps_avg[i] = str1->total_buffer_cnt / time1; + if (isnan (perf_struct.fps_avg[i])) + perf_struct.fps_avg[i] = 0; + + str1->last_sample_fps_time = str1->last_fps_time; + + g_print ("%.2f(%.2f)\t", perf_struct.fps[i], perf_struct.fps_avg[i]); + } + + g_print("\n"); + + g_mutex_unlock (&str->struct_lock); + + return TRUE; +} + +/** + * Buffer probe function on element. + */ +static GstPadProbeReturn +buf_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) +{ + PerfStructInt *str = (PerfStructInt *) u_data; + NvDsBatchMeta *batch_meta = + gst_buffer_get_nvds_batch_meta (GST_BUFFER (info->data)); + + if (!batch_meta) + return GST_PAD_PROBE_OK; + + g_mutex_lock (&str->struct_lock); + for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) l_frame->data; + InstancePerfStruct *str1 = &str->instance_str[frame_meta->pad_index]; + gettimeofday (&str1->last_fps_time, NULL); + if (str1->start_fps_time.tv_sec == 0 && str1->start_fps_time.tv_usec == 0) { + str1->start_fps_time = str1->last_fps_time; + } else { + str1->buffer_cnt++; + } + } + g_mutex_unlock (&str->struct_lock); + return GST_PAD_PROBE_OK; +} + +static void +cb_newpad (GstElement * decodebin, GstPad * decoder_src_pad, gpointer data) +{ + g_print ("In cb_newpad\n"); + GstCaps *caps = gst_pad_get_current_caps (decoder_src_pad); + const GstStructure *str = gst_caps_get_structure (caps, 0); + const gchar *name = gst_structure_get_name (str); + DsSourceBinStruct *bin_struct = (DsSourceBinStruct *) data; + GstCapsFeatures *features = gst_caps_get_features (caps, 0); + + /* Need to check if the pad created by the decodebin is for video and not + * audio. */ + if (!strncmp (name, "video", 5)) { + /* Link the decodebin pad to videoconvert if no hardware decoder is used */ + if (bin_struct->vidconv) { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->vidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + if (!gst_element_link_many (bin_struct->vidconv, bin_struct->capsraw, + bin_struct->nvvidconv, NULL)) { + g_printerr ("Failed to link videoconvert to nvvideoconvert\n"); + } + } else { + GstPad *conv_sink_pad = gst_element_get_static_pad (bin_struct->nvvidconv, + "sink"); + if (gst_pad_link (decoder_src_pad, conv_sink_pad)) { + g_printerr ("Failed to link decoderbin src pad to converter sink pad\n"); + } + g_object_unref(conv_sink_pad); + } + if (gst_caps_features_contains (features, GST_CAPS_FEATURES_NVMM)) { + g_print ("###Decodebin pick nvidia decoder plugin.\n"); + } else { + /* Get the source bin ghost pad */ + g_print ("###Decodebin did not pick nvidia decoder plugin.\n"); + } + } +} + +static void +decodebin_child_added (GstChildProxy * child_proxy, GObject * object, + gchar * name, gpointer user_data) +{ + g_print ("Decodebin child added: %s\n", name); + DsSourceBinStruct *src_bin_st = (DsSourceBinStruct *) user_data; + if (g_strrstr (name, "decodebin") == name) { + g_signal_connect (G_OBJECT (object), "child-added", + G_CALLBACK (decodebin_child_added), user_data); + } + + if (g_strstr_len (name, -1, "nvv4l2decoder") == name) { + if (fileLoop && !src_bin_st->is_streaming) { + g_print ("loop model: %s\n", name); + GstPad *gstpad = gst_element_get_static_pad (GST_ELEMENT(object), "sink"); + gst_pad_add_probe(gstpad, (GstPadProbeType) (GST_PAD_PROBE_TYPE_EVENT_BOTH | + GST_PAD_PROBE_TYPE_EVENT_FLUSH | GST_PAD_PROBE_TYPE_BUFFER), restart_stream_buf_prob, user_data, NULL); + gst_object_unref (gstpad); + } + } +} + +static bool +create_source_bin (DsSourceBinStruct *ds_source_struct, gchar * uri) +{ + gchar bin_name[16] = { }; + GstCaps *caps = NULL; + GstCapsFeatures *feature = NULL; + + ds_source_struct->nvvidconv = NULL; + ds_source_struct->capsfilt = NULL; + ds_source_struct->source_bin = NULL; + ds_source_struct->uri_decode_bin = NULL; + + g_snprintf (bin_name, 15, "source-bin-%02d", ds_source_struct->index); + /* Create a source GstBin to abstract this bin's content from the rest of the + * pipeline */ + ds_source_struct->source_bin = gst_bin_new (bin_name); + + /* Source element for reading from the uri. + * We will use decodebin and let it figure out the container format of the + * stream and the codec and plug the appropriate demux and decode plugins. */ + ds_source_struct->uri_decode_bin = gst_element_factory_make ("uridecodebin", + "uri-decode-bin"); + ds_source_struct->nvvidconv = gst_element_factory_make ("nvvideoconvert", + "source_nvvidconv"); + ds_source_struct->capsfilt = gst_element_factory_make ("capsfilter", + "source_capset"); + + if (!ds_source_struct->source_bin || !ds_source_struct->uri_decode_bin + || !ds_source_struct->nvvidconv + || !ds_source_struct->capsfilt) { + g_printerr ("One element in source bin could not be created.\n"); + return false; + } + + /* We set the input uri to the source element */ + g_object_set (G_OBJECT (ds_source_struct->uri_decode_bin), "uri", uri, NULL); + + /* Connect to the "pad-added" signal of the decodebin which generates a + * callback once a new pad for raw data has beed created by the decodebin */ + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "pad-added", + G_CALLBACK (cb_newpad), ds_source_struct); + g_signal_connect (G_OBJECT (ds_source_struct->uri_decode_bin), "child-added", + G_CALLBACK (decodebin_child_added), ds_source_struct); + + caps = gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "NV12", + NULL); + feature = gst_caps_features_new ("memory:NVMM", NULL); + gst_caps_set_features (caps, 0, feature); + g_object_set (G_OBJECT (ds_source_struct->capsfilt), "caps", caps, NULL); + + gst_bin_add_many (GST_BIN (ds_source_struct->source_bin), + ds_source_struct->uri_decode_bin, ds_source_struct->nvvidconv, + ds_source_struct->capsfilt, NULL); + + if (!gst_element_link (ds_source_struct->nvvidconv, + ds_source_struct->capsfilt)) { + g_printerr ("Could not link vidconv and capsfilter\n"); + return false; + } + + /* We need to create a ghost pad for the source bin which will act as a proxy + * for the video decoder src pad. The ghost pad will not have a target right + * now. Once the decode bin creates the video decoder and generates the + * cb_newpad callback, we will set the ghost pad target to the video decoder + * src pad. */ + GstPad *gstpad = gst_element_get_static_pad (ds_source_struct->capsfilt, + "src"); + if (!gstpad) { + g_printerr ("Could not find srcpad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + return false; + } + if(!gst_element_add_pad (ds_source_struct->source_bin, + gst_ghost_pad_new("src", gstpad))) { + g_printerr ("Could not add ghost pad in '%s'", + GST_ELEMENT_NAME(ds_source_struct->capsfilt)); + } + gst_object_unref (gstpad); + + return true; +} + +static gboolean +bus_call (GstBus * bus, GstMessage * msg, gpointer data) { + GMainLoop *loop = (GMainLoop *) data; + switch (GST_MESSAGE_TYPE (msg)) { + case GST_MESSAGE_EOS: + g_print ("End of stream\n"); + g_main_loop_quit (loop); + break; + case GST_MESSAGE_ERROR: { + gchar *debug; + GError *error; + gst_message_parse_error (msg, &error, &debug); + g_printerr ("ERROR from element %s: %s\n", + GST_OBJECT_NAME (msg->src), error->message); + if (debug) + g_printerr ("Error details: %s\n", debug); + g_free (debug); + g_error_free (error); + g_main_loop_quit (loop); + break; + } + default: + break; + } + return TRUE; +} + +static bool +is_enc_hw_support() { + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + bool enc_hw_support = TRUE; + if (prop.integrated) { + char device_name[50]; + FILE* ptr = fopen("/proc/device-tree/model", "r"); + + if (ptr) { + while (fgets(device_name, 50, ptr) != NULL) { + if (strstr(device_name,"Orin") && (strstr(device_name,"Nano"))) + enc_hw_support = FALSE; + } + fclose(ptr); + } + } else { + for (int i = 0; i < sizeof(dgpus_unsupport_hw_enc)/sizeof(dgpus_unsupport_hw_enc[0]); i++) { + if (!strncasecmp(prop.name, dgpus_unsupport_hw_enc[i], strlen(dgpus_unsupport_hw_enc[i]))) { + enc_hw_support = FALSE; + break; + } + } + } + return enc_hw_support; +} + +/* Check for parsing error. */ +#define RETURN_ON_PARSER_ERROR(parse_expr) \ + if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ + g_printerr("Error in parsing configuration file.\n"); \ + return -1; \ + } + +static void printUsage(const char* cmd) { + g_printerr ("\tUsage: %s -c pgie_config_file -i [-b BATCH]" + " [-d]\n\tOR\n\t %s yml_config_file\n", cmd, cmd); + g_printerr ("-h: \n\tprint help info \n"); + g_printerr ("-c: \n\tpgie config file, e.g. pgie_frcnn_tao_config.txt \n"); + g_printerr ("-i: \n\tH264 or JPEG input file \n"); + g_printerr ("-b: \n\tbatch size, this will override the value of \"batch-size\" in pgie config file \n"); + g_printerr ("-d: \n\tenable display, otherwise dump to output H264 or JPEG file \n"); + g_printerr ("-f: \n\tuse fake_sink to test the performace\n"); + g_printerr ("-l: \n\tloop mode for the pipeline\n"); + g_printerr ("-o: \n\tOriginal background On\n"); + g_printerr ("-a: \n\tAlpha value with original background setting\n"); + g_printerr ("-w: \n\tThe model output width\n"); + g_printerr ("-e: \n\tThe model output height\n"); + g_printerr ("yml_config_file: \n\tYAML config file, e.g. seg_app_unet.yml \n"); +} +int +main (int argc, char *argv[]) { + GMainLoop *loop = NULL; + GstElement *pipeline = NULL, *source_bin = NULL, + *streammux = NULL, *sink = NULL, + *pgie = NULL, *nvvidconv = NULL, *segvisual = NULL, + *parser1 = NULL, *nvvidconv1 = NULL, *enc = NULL, + *tiler = NULL, *mux = NULL; + + DsSourceBinStruct source_struct[128]; + + GstPad *sinkpad, *srcpad; + gchar pad_name_sink[16] = "sink_0"; + gchar pad_name_src[16] = "src"; + + GstBus *bus = NULL; + guint bus_watch_id; + + gboolean isImage = FALSE; + gboolean useDisplay = FALSE; + gboolean useFakeSink = FALSE; + gboolean original_background = FALSE; + float alpha=1.0f; + guint tiler_rows, tiler_cols; + guint batchSize = 0; + guint pgie_batch_size; + guint c; + const char* optStr = "a:b:c:w:e:dohfli:"; + std::string pgie_config; + gboolean isYAML = FALSE; + GList* g_list = NULL; + GList* iterator = NULL; + static guint src_cnt = 0; + PerfStructInt str; + YamlParasStruct yaml_paras; + fileLoop = 0; + int enc_type = 0; + networkType = 2; + numDetectedClasses = 1; + guint model_width = 0; + guint model_height = 0; + NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER; + int current_device = -1; + cudaGetDevice(¤t_device); + struct cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, current_device); + + if (argc==2 && (g_str_has_suffix(argv[1], ".yml") || + g_str_has_suffix(argv[1], ".yaml"))) { + isYAML = TRUE; + if (NVDS_YAML_PARSER_SUCCESS != nvds_parse_source_list(&g_list, argv[1], "source-list")) { + g_printerr ("No source is found. Exiting.\n"); + return -1; + } + + RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&pgie_type, argv[1], + "primary-gie")); + if(pgie_type == NVDS_GIE_PLUGIN_INFER){ + parse_tests_yaml(&yaml_paras, argv[1]); + fileLoop = yaml_paras.file_loop; + yaml_paras.config_path.erase(0,2); + yaml_paras.config_path = "configs" + yaml_paras.config_path; + parse_tests_yaml(&yaml_paras, yaml_paras.config_path.c_str()); + networkType = yaml_paras.network_type; + numDetectedClasses = yaml_paras.num_detected_classes; + } + parse_tests_yaml(&yaml_paras, argv[1]); + + fileLoop = yaml_paras.file_loop; + yaml_paras.config_path.erase(0,2); + yaml_paras.config_path = "configs" + yaml_paras.config_path; + networkType = yaml_paras.network_type; + numDetectedClasses = yaml_paras.num_detected_classes; + } else { + while ((c = getopt(argc, argv, optStr)) != -1) { + switch (c) { + case 'a': + alpha = std::atof(optarg); + break; + case 'b': + batchSize = std::atoi(optarg); + batchSize = batchSize == 0 ? 1:batchSize; + break; + case 'c': + { + pgie_config.assign(optarg); + GKeyFile *key_file = g_key_file_new (); + GError *error = NULL; + g_key_file_load_from_file (key_file, pgie_config.c_str(), G_KEY_FILE_NONE, &error); + gboolean has_key; + has_key = g_key_file_has_key(key_file, "property", "network-type", &error); + if (has_key) { + networkType = g_key_file_get_integer (key_file, "property", "network-type", &error); + } + has_key = g_key_file_has_key(key_file, "property", "num-detected-classes", &error); + if (has_key) { + numDetectedClasses = g_key_file_get_integer (key_file, "property", "num-detected-classes", &error); + } + g_key_file_free(key_file); + } + break; + case 'd': + useDisplay = TRUE; + break; + case 'f': + useFakeSink = TRUE; + break; + case 'i': + g_list = g_list_append(g_list, optarg); + break; + case 'l': + fileLoop = 1; + break; + case 'o': + original_background = TRUE; + break; + case 'w': + model_width = std::atoi(optarg); + break; + case 'e': + model_height = std::atoi(optarg); + break; + case 'h': + default: + printUsage(argv[0]); + return -1; + } + } + } + + /* Check input arguments */ + if (argc == 1) { + printUsage(argv[0]); + return -1; + } + + if(useDisplay) useFakeSink = FALSE; + + /* Standard GStreamer initialization */ + gst_init (&argc, &argv); + loop = g_main_loop_new (NULL, FALSE); + + /* Create gstreamer elements */ + /* Create Pipeline element that will form a connection of other elements */ + pipeline = gst_pipeline_new ("ds-custom-pipeline"); + + /* Create nvstreammux instance to form batches from one or more sources. */ + streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); + + if (!pipeline || !streammux) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), streammux); + + for (iterator = g_list, src_cnt=0; iterator; iterator = iterator->next,src_cnt++) { + /* Source element for reading from the file */ + source_struct[src_cnt].index = src_cnt; + + if (g_strrstr ((gchar *)iterator->data, ".jpg") || g_strrstr ((gchar *)iterator->data, ".jpeg") + || g_strrstr ((gchar *)iterator->data, ".png")) + isImage = TRUE; + else + isImage = FALSE; + if (g_strrstr ((gchar *)iterator->data, "rtsp://") || g_strrstr ((gchar *)iterator->data, "v4l2://") + || g_strrstr ((gchar *)iterator->data, "http://") || g_strrstr ((gchar *)iterator->data, "rtmp://")) { + source_struct[src_cnt].is_streaming = TRUE; + } else { + source_struct[src_cnt].is_streaming = FALSE; + } + if (!create_source_bin (&(source_struct[src_cnt]), (gchar *)iterator->data)) + { + g_printerr ("Source bin could not be created. Exiting.\n"); + return -1; + } + + gst_bin_add (GST_BIN (pipeline), source_struct[src_cnt].source_bin); + + g_snprintf (pad_name_sink, 64, "sink_%d", src_cnt); + sinkpad = gst_element_get_request_pad (streammux, pad_name_sink); + g_print("Request %s pad from streammux\n",pad_name_sink); + if (!sinkpad) { + g_printerr ("Streammux request sink pad failed. Exiting.\n"); + return -1; + } + + srcpad = gst_element_get_static_pad (source_struct[src_cnt].source_bin, + pad_name_src); + if (!srcpad) { + g_printerr ("Decoder request src pad failed. Exiting.\n"); + return -1; + } + + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { + g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); + return -1; + } + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + } + + str.num_instances = src_cnt; + g_timeout_add (5000, perf_measurement_callback, &str); + + const char* batch_size = std::getenv("BATCH_SIZE"); + if(batch_size != NULL ) { + batchSize = std::stoi(batch_size); + g_printerr("batch size is %d \n", batchSize); + } + + /* Use nvinfer to run inferencing on decoder's output, + * behaviour of inferencing is set through config file */ + if (pgie_type == NVDS_GIE_PLUGIN_INFER_SERVER) { + pgie = gst_element_factory_make ("nvinferserver", "primary-nvinference-engine"); + } else { + pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine"); + } + + /* Use convertor to convert from NV12 to RGBA as required by segvisual */ + //nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter"); + + /* Create OSD to draw on the converted RGBA buffer */ + if(!model_width) + model_width = SEG_OUTPUT_WIDTH; + if(!model_height) + model_height = SEG_OUTPUT_HEIGHT; + if(isYAML) { + parse_segvisual_yaml(&yaml_paras, argv[1]); + if (!yaml_paras.seg_width || !yaml_paras.seg_height) { + g_printerr ("segvisual resolution should not be zero. Exiting.\n"); + return -1; + } + model_height = yaml_paras.seg_height; + model_width = yaml_paras.seg_width; + original_background = yaml_paras.seg_background; + alpha = yaml_paras.seg_alpha; + } + segvisual = gst_element_factory_make ("nvsegvisual", "nv-segvisual"); + g_object_set (G_OBJECT (segvisual), "original-background", original_background, NULL); + g_object_set (G_OBJECT (segvisual), "alpha", alpha, NULL); + + tiler = gst_element_factory_make ("nvmultistreamtiler", "nvtiler"); + + /* Finally render the osd output */ +#ifdef PLATFORM_TEGRA + transform = gst_element_factory_make ("nvegltransform", "nvegl-transform"); +#endif + if(isYAML) { + GstElement *eglsink = gst_element_factory_make ("nveglglessink", "test-egl-sink"); + GstElement *filesink = gst_element_factory_make ("filesink", "test-file-sink"); + GstElement *fakesink = gst_element_factory_make("fakesink", "test-fake-sink"); + if(NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "eglsink")){ + useDisplay = TRUE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_file_sink(filesink, argv[1], "filesink")){ + useDisplay = FALSE; + } else if (NVDS_YAML_PARSER_DISABLED != nvds_parse_egl_sink(eglsink, argv[1], "fakesink")){ + useDisplay = FALSE; + useFakeSink = TRUE; + } else { + g_printerr ("No sink is configured. Exiting.\n"); + return -1; + } + g_object_unref(eglsink); + g_object_unref(filesink); + g_object_unref(fakesink); + } + + const char* use_display = std::getenv("USE_DISPLAY"); + if(use_display != NULL && std::stoi(use_display) == 1) { + useDisplay = true; + } + + if(useDisplay == FALSE) { + if(isImage == FALSE){ + parser1 = gst_element_factory_make ("h264parse", "h264-parser1"); + if (isYAML) { + parse_filesink_yaml(&enc_type, argv[1]); + } else { + // 0: HW 1: SW + enc_type = is_enc_hw_support() ? 0 : 1; + } + if(enc_type == 0){ + enc = gst_element_factory_make ("nvv4l2h264enc", "h264-enc"); + } else { + enc = gst_element_factory_make ("x264enc", "h264-enc"); + } + if(!useFakeSink) { + mux = gst_element_factory_make ("qtmux", "mp4-mux"); + if (!mux) { + g_printerr ("Failed to create mp4-mux"); + return -1; + } + gst_bin_add (GST_BIN (pipeline), mux); + } + } else { + parser1 = gst_element_factory_make ("jpegparse", "jpeg-parser1"); + enc = gst_element_factory_make ("jpegenc", "jpeg-enc"); + } + nvvidconv1 = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter1"); + if(!useFakeSink) { + sink = gst_element_factory_make ("filesink", "file-sink"); + } else { + sink = gst_element_factory_make("fakesink", "file-sink"); + } + if (!pgie + || !tiler || !nvvidconv1 || !segvisual || !enc || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + + //save the file to local dir + if(isImage == FALSE) + g_object_set (G_OBJECT (sink), "location", "./out.mp4", NULL); + else + g_object_set (G_OBJECT (sink), "location", "./out.jpg", NULL); + } else { + if(prop.integrated) + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); + else +#ifdef __aarch64__ + sink = gst_element_factory_make("nv3dsink", "nv3d-sink"); +#else + sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer"); +#endif + if (!pgie + || !tiler || !segvisual || !sink) { + g_printerr ("One element could not be created. Exiting.\n"); + return -1; + } + } + + if(isYAML) { + nvds_parse_streammux(streammux, argv[1], "streammux"); + if(!batchSize) { + g_object_get(G_OBJECT (streammux), "batch-size", &batchSize, NULL); + } + } + + if(!batchSize) { batchSize = src_cnt; } + + g_print ("batchSize %d...\n", batchSize); + + if(source_struct[0].is_streaming == TRUE) + g_object_set (G_OBJECT (streammux), "live-source", true, NULL); + + g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", + MUXER_OUTPUT_HEIGHT, "batch-size", batchSize, + "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); + + g_object_set (G_OBJECT (segvisual), "width", model_width, "height", + model_height, NULL); + + /* Set all the necessary properties of the nvinfer element, + * the necessary ones are : */ + if(isYAML) { + nvds_parse_gie (pgie, argv[1], "primary-gie"); + } else { + g_object_set (G_OBJECT (pgie), + "config-file-path", pgie_config.c_str(), NULL); + } + + /* Override the batch-size set in the config file with the number of sources. */ + g_object_get (G_OBJECT (pgie), "batch-size", &pgie_batch_size, NULL); + if (pgie_batch_size != batchSize) { + g_printerr + ("WARNING: Overriding infer-config batch-size (%d) with number of sources (%d)\n", + pgie_batch_size, batchSize); + g_object_set (G_OBJECT (pgie), "batch-size", batchSize, NULL); + } + + tiler_rows = (guint) sqrt (batchSize); + tiler_cols = (guint) ceil (1.0 * batchSize / tiler_rows); + /* we set the tiler properties here */ + g_object_set (G_OBJECT (tiler), "rows", tiler_rows, "columns", tiler_cols, + "width", TILED_OUTPUT_WIDTH, "height", TILED_OUTPUT_HEIGHT, NULL); + g_object_set (G_OBJECT (segvisual), "batch-size", batchSize, NULL); + + /* we add a message handler */ + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); + gst_object_unref (bus); + + /* Set up the pipeline */ + /* we add all elements into the pipeline */ + if(useDisplay == FALSE) { + gst_bin_add_many (GST_BIN (pipeline), pgie, tiler, + segvisual, nvvidconv1, enc, parser1, sink, NULL); + } else { + gst_bin_add_many (GST_BIN (pipeline), pgie, + tiler, segvisual, sink, NULL); + } + + /* We link the elements together */ + /* uridocoderbin -> + * nvinfer -> nvvideoconvert -> segvisual -> video-renderer */ + if (useDisplay == FALSE) { + if (isImage == FALSE && !useFakeSink) { + if (!gst_element_link_many (streammux, pgie, + segvisual, tiler, nvvidconv1, enc, parser1, mux, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } else { + if (!gst_element_link_many (streammux, pgie, + segvisual, tiler, nvvidconv1, enc, parser1, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + } else { + if (!gst_element_link_many (streammux, pgie, + tiler, segvisual, sink, NULL)) { + g_printerr ("Elements could not be linked: 2. Exiting.\n"); + return -1; + } + } + + /*Performance measurement video fps*/ + GstPad *streammux_src_pad = gst_element_get_static_pad (streammux, "src"); + if (!streammux_src_pad) + g_print ("Unable to get streammux src pad\n"); + else + gst_pad_add_probe(streammux_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + buf_probe, &str, NULL); + gst_object_unref (streammux_src_pad); + + /* Set the pipeline to "playing" state */ + g_print ("Now playing: %s\n", pgie_config.c_str()); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + /* Wait till pipeline encounters an error or EOS */ + g_print ("Running...\n"); + g_main_loop_run (loop); + + /* Out of the main loop, clean up nicely */ + g_print ("Returned, stopping playback\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + g_print ("Deleting pipeline\n"); + gst_object_unref (GST_OBJECT (pipeline)); + g_source_remove (bus_watch_id); + g_main_loop_unref (loop); + return 0; +} diff --git a/src/apps/tao_apps/build_triton_engine.sh b/src/apps/tao_apps/build_triton_engine.sh new file mode 100755 index 00000000..d213c0b8 --- /dev/null +++ b/src/apps/tao_apps/build_triton_engine.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +IS_JETSON_PLATFORM=`uname -i | grep aarch64` + +export PATH=$PATH:/usr/src/tensorrt/bin + +#detection +#for instance segmentation +#for mask2former +echo "Building Model mask2former..." +mkdir -p models/mask2former/1 +trtexec --onnx=./models/mask2former/mask2former.onnx --fp16 --saveEngine=./models/mask2former/1/mask2former.onnx_b1_gpu0_fp16.engine \ + --minShapes=inputs:1x3x800x800 --optShapes=inputs:1x3x800x800 --maxShapes=inputs:1x3x800x800 --precisionConstraints=obey \ + --layerPrecisions=/post_processor/Div:fp32,/post_processor/ReduceSum:fp32,/post_processor/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Where:fp32,sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.0/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.2/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.3/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.4/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.5/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.6/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.7/norm/LayerNormalization:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/Add:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_2:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Div:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_6:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_9:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Cast:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Cast_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Div_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_3:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_7:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Where:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_8:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_5:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Softmax:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_4:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Gemm:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/Add_1:fp32,/sem_seg_head/predictor/transformer_cross_attention_layers.8/norm/LayerNormalization:fp32 & + + +#for segmentation +#citysemsegformer +echo "Building Model citysemsegformer..." +mkdir -p models/citysemsegformer/1 && \ +trtexec --onnx=./models/citysemsegformer/citysemsegformer.onnx --fp16 \ + --saveEngine=./models/citysemsegformer/1/citysemsegformer.onnx_b1_gpu0_fp16.engine \ + --minShapes="input":1x3x1024x1820 --optShapes="input":1x3x1024x1820 --maxShapes="input":1x3x1024x1820& + +#peoplenet_transformer +echo "Building Model peoplenet_transformer" +mkdir -p models/peoplenet_transformer/1 +trtexec --onnx=./models/peoplenet_transformer/resnet50_peoplenet_transformer_op17.onnx --fp16 \ + --saveEngine=./models/peoplenet_transformer/1/resnet50_peoplenet_transformer_op17.onnx_b1_gpu0_fp16.engine \ + --minShapes="inputs":1x3x544x960 --optShapes="inputs":1x3x544x960 --maxShapes="inputs":1x3x544x960& + +#reidentificationnet +echo "Building Model reidentificationnet" +mkdir -p models/reidentificationnet/1 +trtexec --minShapes=input:1x3x256x128 --optShapes=input:8x3x256x128 --maxShapes=input:16x3x256x128 \ + --fp16 --saveEngine=models/reidentificationnet/1/resnet50_market1501_aicity156.onnx_b16_gpu0_fp16.engine \ + --onnx=models/reidentificationnet/resnet50_market1501_aicity156.onnx & + +#retail_object_detection_binary_dino +echo "Building Model retail_object_detection_binary_dino" +mkdir -p models/retail_object_detection_binary_dino/1 +trtexec --onnx=models/retail_object_detection_binary_dino/retail_object_detection_dino_binary.onnx \ + --saveEngine=models/retail_object_detection_binary_dino/1/retail_object_detection_dino_binary.onnx_b1_gpu0_fp32.engine \ + --minShapes=inputs:1x3x540x960 --optShapes=inputs:1x3x540x960 --maxShapes=inputs:1x3x540x960 \ + --sparsity=enable& + +#retail_object_recognition +echo "Building Model retail_object_recognition" +mkdir -p models/retail_object_recognition/1 +trtexec --onnx=models/retail_object_recognition/retail_object_recognition.onnx \ + --saveEngine=models/retail_object_recognition/1/retail_object_recognition.onnx_b16_gpu0_fp16.engine \ + --minShapes=input:1x3x224x224 --optShapes=input:16x3x224x224 --maxShapes=input:16x3x224x224 \ + --fp16 --sparsity=enable& + +#peoplenet +echo "Building Model peoplenet..." +mkdir -p models/peoplenet/1 +trtexec --onnx=./models/peoplenet/resnet34_peoplenet_int8.onnx --fp16 \ + --saveEngine=./models/peoplenet/1/resnet34_peoplenet_int8.onnx_b2_gpu0_fp16.engine \ +--minShapes="input_1:0":1x3x544x960 --optShapes="input_1:0":2x3x544x960 --maxShapes="input_1:0":2x3x544x960& + +#poseclassificationnet +echo "Building Model poseclassificationnet..." +mkdir -p models/poseclassificationnet/1 +trtexec --onnx=./models/poseclassificationnet/st-gcn_3dbp_nvidia.onnx --fp16 \ +--saveEngine=./models/poseclassificationnet/1/st-gcn_3dbp_nvidia.onnx_b4_gpu0_fp16.engine \ +--minShapes="input":1x3x300x34x1 --optShapes="input":4x3x300x34x1 --maxShapes="input":4x3x300x34x1& + +#bodypose3dnet +echo "Building Model bodypose3dnet..." +mkdir -p models/bodypose3dnet/1 +trtexec --onnx=models/bodypose3dnet/bodypose3dnet_accuracy.onnx --fp16 \ + --saveEngine=models/bodypose3dnet/1/bodypose3dnet_accuracy.onnx_b8_gpu0_fp16.engine \ + --minShapes="input0":1x3x256x192,k_inv:1x3x3,t_form_inv:1x3x3,scale_normalized_mean_limb_lengths:1x36,mean_limb_lengths:1x36 \ + --optShapes="input0":8x3x256x192,k_inv:8x3x3,t_form_inv:8x3x3,scale_normalized_mean_limb_lengths:8x36,mean_limb_lengths:8x36 \ + --maxShapes="input0":8x3x256x192,k_inv:8x3x3,t_form_inv:8x3x3,scale_normalized_mean_limb_lengths:8x36,mean_limb_lengths:8x36 + +#LPD/LPR +echo "Building Models for LPD/LPR..." +mkdir -p models/trafficcamnet/1 +trtexec --onnx=models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx --fp16 \ + --saveEngine=models/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":1x3x544x960 --maxShapes="input_1:0":1x3x544x960 + +mkdir -p models/LPD_us/1 +trtexec --onnx=models/LPD_us/LPDNet_usa_pruned_tao5.onnx --fp16 \ + --saveEngine=models/LPD_us/1/LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x480x640 \ + --optShapes="input_1:0":16x3x480x640 --maxShapes="input_1:0":16x3x480x640 + +mkdir -p models/LPD_ch/1 +trtexec --onnx=models/LPD_ch/LPDNet_CCPD_pruned_tao5.onnx --fp16 \ + --saveEngine=models/LPD_ch/1/LPDNet_CCPD_pruned_tao5.onnx_b16_gpu0_fp16.engine --minShapes="input_1:0":1x3x1168x720 \ + --optShapes="input_1:0":16x3x1168x720 --maxShapes="input_1:0":16x3x1168x720 + +mkdir -p models/LPR_us/1 + trtexec --onnx=models/LPR_us/us_lprnet_baseline18_deployable.onnx --fp16 \ + --saveEngine=models/LPR_us/1/us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine --minShapes="image_input":1x3x48x96 \ + --optShapes="image_input":8x3x48x96 --maxShapes="image_input":16x3x48x96 + +mkdir -p models/LPR_ch/1 + trtexec --onnx=models/LPR_ch/ch_lprnet_baseline18_deployable.onnx --fp16 \ + --saveEngine=models/LPR_ch/1/ch_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine --minShapes="image_input":1x3x48x96 \ + --optShapes="image_input":8x3x48x96 --maxShapes="image_input":16x3x48x96 diff --git a/src/apps/tao_apps/configs/app/classifier_app_config.yml b/src/apps/tao_apps/configs/app/classifier_app_config.yml new file mode 100644 index 00000000..609d5661 --- /dev/null +++ b/src/apps/tao_apps/configs/app/classifier_app_config.yml @@ -0,0 +1,46 @@ +################################################################################ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 + +streammux: + batch-size: 1 + width: 600 + height: 800 + batched-push-timeout: 40000 + +primary-gie: + plugin-type: 0 + #0:nvinfer, 1:nvinfeserver + config-file-path: ../nvinfer/multi_task_tao/pgie_multi_task_tao_config.yml + #config-file-path: ../triton/multi_task_tao/pgie_multi_task_tao_config.yml + #config-file-path: ../triton-grpc/multi_task_tao/pgie_multi_task_tao_config.yml + +eglsink: + enable: 0 + +filesink: + enable: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + diff --git a/src/apps/tao_apps/configs/app/config_nvdsanalytics.txt b/src/apps/tao_apps/configs/app/config_nvdsanalytics.txt new file mode 100644 index 00000000..fc2d4d13 --- /dev/null +++ b/src/apps/tao_apps/configs/app/config_nvdsanalytics.txt @@ -0,0 +1,71 @@ +# Copyright (c) 2020-2025 NVIDIA Corporation. All rights reserved. +# +# NVIDIA Corporation and its licensors retain all intellectual property +# and proprietary rights in and to this software, related documentation +# and any modifications thereto. Any use, reproduction, disclosure or +# distribution of this software and related documentation without an express +# license agreement from NVIDIA Corporation is strictly prohibited. + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +#Width height used for configuration to which below configs are configured +config-width=1920 +config-height=1080 +#osd-mode 0: Dont display any lines, rois and text +# 1: Display only lines, rois and static text i.e. labels +# 2: Display all info from 1 plus information about counts +osd-mode=2 +#Set OSD font size that has to be displayed +display-font-size=12 + +## Per stream configuration +[roi-filtering-stream-0] +#enable or disable following feature +enable=1 +#ROI to filter select objects, and remove from meta data +roi-RF=1090;100;1640;100;1640;700;1090;700 + +#remove objects in the ROI +inverse-roi=0 +class-id=0 + +[overcrowding-stream-0] +enable=0 +roi-OC=295;643;579;634;642;913;56;828 +#no of objects that will trigger OC +object-threshold=3 +class-id=0 + +[line-crossing-stream-0] +enable=1 +#Label;direction;lc +line-crossing-EntryTop=1300;150;1300;400;1090;300;1640;300 +line-crossing-EntryLeft=900;450;1200;450;1090;680;1090;120 +line-crossing-EntryBottom=1300;900;1300;600;1640;700;1090;700 +line-crossing-ExitTop=1300;400;1300;150;1090;300;1640;300 +line-crossing-ExitLeft=1200;450;900;450;1090;680;1090;120 +line-crossing-ExitBottom=1300;600;1300;900;1640;700;1090;700 +class-id=0 +#extended when 0- only counts crossing on the configured Line +# 1- assumes extended Line crossing counts all the crossing +extended=0 +#LC modes supported: +#loose : counts all crossing without strong adherence to direction +#balanced: Strict direction adherence expected compared to mode=loose +#strict : Strict direction adherence expected compared to mode=balanced +mode=strict + +[direction-detection-stream-0] +enable=1 +#Label;direction; +direction-EntryBottom=1320;900;1320;600 +direction-EntryTop=1320;150;1320;400 +direction-EntryLeft=900;470;1200;470 +direction-ExitBottom=1320;600;1320;900 +direction-ExitTop=1320;400;1320;150 +direction-ExitLeft=1200;470;900;470 +class-id=0 +mode=strict diff --git a/src/apps/tao_apps/configs/app/deepstream_pose_classification_config.yaml b/src/apps/tao_apps/configs/app/deepstream_pose_classification_config.yaml new file mode 100644 index 00000000..e8ece932 --- /dev/null +++ b/src/apps/tao_apps/configs/app/deepstream_pose_classification_config.yaml @@ -0,0 +1,61 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_walk.mov;file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_walk.mov + +streammux: + width: 1280 + height: 720 + batched-push-timeout: 40000 + +tracker: + enable: 1 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + +primary-gie: + plugin-type: 0 + config-file-path: ../nvinfer/peoplenet_tao/config_infer_primary_peoplenet.txt + #config-file-path: ../triton/peoplenet_tao/config_infer_primary_peoplenet.yml + #config-file-path: ../triton-grpc/peoplenet_tao/config_infer_primary_peoplenet.yml + +secondary-gie0: + plugin-type: 0 + config-file-path: ../nvinfer/bodypose3d_tao/config_infer_secondary_bodypose3dnet.txt + #config-file-path: ../triton/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml + #config-file-path: ../triton-grpc/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml + +secondary-postprocess0: + config-file-path: ../nvinfer/bodypose3d_tao/config_bodypose_post_process_sgie.yml + lib-name: ../../apps/tao_others/deepstream-pose-classification/postprocesslib_impl/libpostprocess_impl.so + +secondary-preprocess1: + config-file-path: ../nvinfer/bodypose_classification_tao/config_preprocess_bodypose_classification.txt + +secondary-gie1: + plugin-type: 0 + config-file-path: ../nvinfer/bodypose_classification_tao/config_infer_third_bodypose_classification.txt + #config-file-path: ../triton/bodypose_classification_tao/config_infer_third_bodypose_classification.yml + #config-file-path: ../triton-grpc/bodypose_classification_tao/config_infer_third_bodypose_classification.yml + +sink: + #0 fakesink + #1 filesink generate the out.mp4 file in the current directory + #2 rtspsink publish at rtsp://localhost:8554/ds-test + #3 displaysink + sink-type: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + + + diff --git a/src/apps/tao_apps/configs/app/det_app_config.yml b/src/apps/tao_apps/configs/app/det_app_config.yml new file mode 100644 index 00000000..43bc227e --- /dev/null +++ b/src/apps/tao_apps/configs/app/det_app_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 + +streammux: + batch-size: 1 + width: 1920 + height: 1080 + batched-push-timeout: 4000000 + +primary-gie: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + + #peoplenet-transformer + config-file-path: ../nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + #config-file-path: ../triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + #config-file-path: ../triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + +eglsink: + enable: 0 + +filesink: + enable: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + +fakesink: + enable: 0 + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/configs/app/ins_seg_app.yml b/src/apps/tao_apps/configs/app/ins_seg_app.yml new file mode 100644 index 00000000..53f2906b --- /dev/null +++ b/src/apps/tao_apps/configs/app/ins_seg_app.yml @@ -0,0 +1,50 @@ +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 + +streammux: + batch-size: 1 + width: 1280 + height: 720 + batched-push-timeout: 40000 + +primary-gie: + plugin-type: 0 + + #mask2former + config-file-path: ../nvinfer/mask2former_tao/pgie_mask2former_tao_config.yml + #config-file-path: ../triton/mask2former_tao/pgie_mask2former_tao_config.yml + #config-file-path: ../triton-grpc/mask2former_tao/pgie_mask2former_tao_config.yml + +eglsink: + enable: 0 + +filesink: + enable: 1 + +fakesink: + enable: 0 + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/configs/app/lpr_app_ch_config.yml b/src/apps/tao_apps/configs/app/lpr_app_ch_config.yml new file mode 100644 index 00000000..6b055caa --- /dev/null +++ b/src/apps/tao_apps/configs/app/lpr_app_ch_config.yml @@ -0,0 +1,80 @@ +################################################################################ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: /root/deepstream_tao_apps/apps/tao_others/deepstream_lpr_app/cn.mp4 + +#If there is ROI +analytics: + enable: 1 + config-file: config_nvdsanalytics.txt + +output: + ## 1:file ouput 2:fake output 3:eglsink output + type: 1 + ## 0: H264 encoder 1:H265 encoder + enc: 0 + ## encoder type 0=Hardware 1=Software + enc-type: 0 + bitrate: 4000000 + ##The file name without suffix + filename: test + +primary-gie: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For car detection + config-file-path: ../nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.txt + # config-file-path: ../triton/trafficcamnet_tao/pgie_trafficcamnet_config.txt + # config-file-path: ../triton-grpc/trafficcamnet_tao/pgie_trafficcamnet_config.txt + unique-id: 1 + +secondary-gie0: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For China mainland car plate + config-file-path: ../nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt + # config-file-path: ../triton/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt + # config-file-path: ../triton-grpc/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt + unique-id: 2 + process-mode: 2 + +secondary-gie1: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For China mainland car plate recognization + config-file-path: ../nvinfer/lpr_ch_tao/sgie_lpr_ch_config.txt + # config-file-path: ../triton/lpr_ch_tao/sgie_lpr_ch_config.txt + # config-file-path: ../triton-grpc/lpr_ch_tao/sgie_lpr_ch_config.txt + unique-id: 3 + process-mode: 2 + +tracker: + enable: 1 + tracker-width: 640 + tracker-height: 384 + gpu-id: 0 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + enable-batch-process: 1 + + diff --git a/src/apps/tao_apps/configs/app/lpr_app_us_config.yml b/src/apps/tao_apps/configs/app/lpr_app_us_config.yml new file mode 100644 index 00000000..a0b39ee6 --- /dev/null +++ b/src/apps/tao_apps/configs/app/lpr_app_us_config.yml @@ -0,0 +1,76 @@ +################################################################################ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: /root/deepstream_tao_apps/apps/tao_others/deepstream_lpr_app/us.mp4 + +analytics: + enable: 1 + config-file: config_nvdsanalytics.txt + +output: + ## 1:file ouput 2:fake output 3:eglsink output + type: 1 + ## 0: H264 encoder 1:H265 encoder + enc: 0 + ## encoder type 0=Hardware 1=Software + enc-type: 0 + bitrate: 4000000 + ##The file name without suffix + filename: test + +primary-gie: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For car detection + config-file-path: ../nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.txt + # config-file-path: ../triton/trafficcamnet_tao/pgie_trafficcamnet_config.txt + # config-file-path: ../triton-grpc/trafficcamnet_tao/pgie_trafficcamnet_config.txt + unique-id: 1 + +secondary-gie0: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For lpd_DetectNet2 + config-file-path: ../nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.txt + # config-file-path: ../triton/LPD_us_tao/sgie_lpd_DetectNet2_us.txt + # config-file-path: ../triton-grpc/LPD_us_tao/sgie_lpd_DetectNet2_us.txt + unique-id: 2 + process-mode: 2 + +secondary-gie1: + #0:nvinfer, 1:nvinfeserver + plugin-type: 0 + ##For US car plate recognization + config-file-path: ../nvinfer/lpr_us_tao/sgie_lpr_us_config.txt + # config-file-path: ../triton/lpr_us_tao/sgie_lpr_us_config.txt + # config-file-path: ../triton-grpc/lpr_us_tao/sgie_lpr_us_config.txt + unique-id: 3 + process-mode: 2 + +tracker: + tracker-width: 640 + tracker-height: 384 + gpu-id: 0 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + diff --git a/src/apps/tao_apps/configs/app/nvocdr_app_config.yml b/src/apps/tao_apps/configs/app/nvocdr_app_config.yml new file mode 100644 index 00000000..62938161 --- /dev/null +++ b/src/apps/tao_apps/configs/app/nvocdr_app_config.yml @@ -0,0 +1,32 @@ +source-list: + #list: file:///workspace/nvocdr/img_0.jpg;file:///workspace/nvocdr/img_1.jpg + list: file:///workspace/nvocdr/pexels-george-morina-5266796.mp4 + +output: + ## 1:file ouput 2:fake output 3:eglsink output + type: 1 + ## 0: H264 encoder 1:H265 encoder + codec: 0 + #encoder type 0=Hardware 1=Software + enc-type: 0 + bitrate: 2000000 + ##The file name without suffix + filename: test + +streammux: + width: 1280 + height: 720 + batched-push-timeout: 40000 + +video-template: + customlib-name: /opt/nvidia/deepstream/deepstream/lib/libnvocdr_impl.so + customlib-props: + - ocdnet-engine-path:../../../models/nvocdr/ocdnet.fp16.engine + - ocdnet-input-shape:3,736,1280 + - ocdnet-binarize-threshold:0.1 + - ocdnet-polygon-threshold:0.3 + - ocdnet-max-candidate:200 + - ocrnet-engine-path:../../../models/nvocdr/ocrnet.fp16.engine + - ocrnet-dict-path:../../../models/nvocdr/character_list + - ocrnet-input-shape:1,64,200 + - ocrnet-decode:Attention diff --git a/src/apps/tao_apps/configs/app/peoplenet_reidentification.txt b/src/apps/tao_apps/configs/app/peoplenet_reidentification.txt new file mode 100644 index 00000000..f8a36443 --- /dev/null +++ b/src/apps/tao_apps/configs/app/peoplenet_reidentification.txt @@ -0,0 +1,156 @@ +################################################################################ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +#(0)=nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1)=nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2)=nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3)=nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4)=nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=2 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 +# smart record specific fields, valid only for source type=4 +# 0 = disable, 1 = through cloud events, 2 = through cloud + local events +#smart-record=1 +# 0 = mp4, 1 = mkv +#smart-rec-container=0 +#smart-rec-file-prefix +#smart-rec-dir-path +# smart record cache size in seconds +#smart-rec-cache +# default duration of recording in seconds. +#smart-rec-default-duration +# duration of recording in seconds. +# this will override default value. +#smart-rec-duration +# seconds before the current time to start recording. +#smart-rec-start-time +# value in seconds to dump video stream. +#smart-rec-interval + +[sink0] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File +type=1 +sync=1 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 + +[sink1] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +## only SW mpeg4 is supported right now. +codec=1 +sync=0 +bitrate=2000000 +output-file=out.mp4 +source-id=0 + + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=15 +text-color=1;1;1;1 +text-bg-color=0.3;0.3;0.3;1 +font=Arial +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +## batch-size must be identical to the number of input sources. +batch-size=1 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 +## If set to TRUE, system timestamp will be attached as ntp timestamp +## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached +# attach-sys-ts-as-ntp=1 + +[primary-gie] +enable=1 +gie-unique-id=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;1;1;1 +bbox-border-color3=0;1;0;1 +config-file=../nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt + +[tracker] +enable=1 +gpu-id=0 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=544 +tracker-height=960 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 + +[secondary-gie] +enable=1 +gie-unique-id=2 +# operate-on-gie-id has to be specified in the main configuration file. +operate-on-gie-id=1 +config-file=../nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.txt + +[tests] +file-loop=0 diff --git a/src/apps/tao_apps/configs/app/peoplenet_reidentification.yml b/src/apps/tao_apps/configs/app/peoplenet_reidentification.yml new file mode 100644 index 00000000..cdb5b6df --- /dev/null +++ b/src/apps/tao_apps/configs/app/peoplenet_reidentification.yml @@ -0,0 +1,137 @@ +################################################################################ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + +tiled-display: + enable: 1 + rows: 1 + columns: 1 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File + type: 1 + sync: 1 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 3=mpeg4 + ## only SW mpeg4 is supported right now. + codec: 1 + sync: 0 + bitrate: 2000000 + output-file: out.mp4 + source-id: 0 + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Arial + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + ## batch-size must be identical to the number of input sources. + batch-size: 1 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + ## If set to TRUE, system timestamp will be attached as ntp timestamp + ## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached + # attach-sys-ts-as-ntp: 1 + +primary-gie: + enable: 1 + plugin-type: 0 + gie-unique-id: 1 + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;1;1;1 + bbox-border-color3: 0;1;0;1 + config-file: ../nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + #config-file: ../triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + #config-file: ../triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml + +tracker: + enable: 1 + gpu-id: 0 + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 544 + tracker-height: 960 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 + +secondary-gie0: + enable: 1 + plugin-type: 0 + gie-unique-id: 2 + # operate-on-gie-id has to be specified in the main configuration file. + operate-on-gie-id: 1 + config-file: ../nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml + #config-file: ../triton/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml + #config-file: ../triton-grpc/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/configs/app/retail_object_detection_recognition.txt b/src/apps/tao_apps/configs/app/retail_object_detection_recognition.txt new file mode 100644 index 00000000..5e958a81 --- /dev/null +++ b/src/apps/tao_apps/configs/app/retail_object_detection_recognition.txt @@ -0,0 +1,181 @@ +################################################################################ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 +#(0)=nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1)=nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2)=nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3)=nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4)=nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI 4=RTSP +type=2 +uri=file://../../sample/streams/retail_object_h264.mp4 +num-sources=1 +gpu-id=0 +nvbuf-memory-type=0 +# smart record specific fields, valid only for source type=4 +# 0 = disable, 1 = through cloud events, 2 = through cloud + local events +#smart-record=1 +# 0 = mp4, 1 = mkv +#smart-rec-container=0 +#smart-rec-file-prefix +#smart-rec-dir-path +# smart record cache size in seconds +#smart-rec-cache +# default duration of recording in seconds. +#smart-rec-default-duration +# duration of recording in seconds. +# this will override default value. +#smart-rec-duration +# seconds before the current time to start recording. +#smart-rec-start-time +# value in seconds to dump video stream. +#smart-rec-interval + +[sink0] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File +type=1 +sync=0 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 + +[sink1] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +## only SW mpeg4 is supported right now. +codec=1 +sync=0 +bitrate=2000000 +output-file=out.mp4 +source-id=0 + + +[sink2] +# set to 1 to enable sending metadata to kafka +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvoverlaysink 6=MsgConvBroker +type=6 +msg-conv-config=/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/configs/dstest5_msgconv_sample_config.txt +#(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload +#(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal +#(2): PAYLOAD_DEEPSTREAM_PROTOBUF - Deepstream schema payload protobuf +#(256): PAYLOAD_RESERVED - Reserved type +#(257): PAYLOAD_CUSTOM - Custom schema payload +msg-conv-payload-type=1 +#(0): Create payload using NvdsEventMsgMeta +#(1): New Api to create payload using NvDsFrameMeta +msg-conv-msg2p-new-api=0 +#Frame interval at which payload is generated +msg-conv-frame-interval=1 +msg-broker-proto-lib=/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so +#Provide your msg-broker-conn-str here +msg-broker-conn-str=127.0.0.1;9092;test +topic=test + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=15 +text-color=1;1;1;1 +text-bg-color=0.3;0.3;0.3;1 +font=Arial +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +## batch-size must be identical to the number of input sources. +batch-size=1 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 +## If set to TRUE, system timestamp will be attached as ntp timestamp +## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached +# attach-sys-ts-as-ntp=1 + +[primary-gie] +enable=1 +gie-unique-id=1 +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;1;1;1 +bbox-border-color3=0;1;0;1 +config-file=../nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml +#config-file=../triton/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml +#config-file=../triton-grpc/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml + +[tracker] +enable=1 +gpu-id=0 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 + +[secondary-gie0] +enable=1 +gie-unique-id=2 +# operate-on-gie-id has to be specified in the main configuration file. +operate-on-gie-id=1 +config-file=../nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml +labelfile-path=../../models/retail_object_recognition/retail_object_recognition_labels.txt + +[tests] +file-loop=0 diff --git a/src/apps/tao_apps/configs/app/retail_object_detection_recognition.yml b/src/apps/tao_apps/configs/app/retail_object_detection_recognition.yml new file mode 100644 index 00000000..54a4fdd2 --- /dev/null +++ b/src/apps/tao_apps/configs/app/retail_object_detection_recognition.yml @@ -0,0 +1,161 @@ +################################################################################ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 5 + +tiled-display: + enable: 1 + rows: 1 + columns: 1 + width: 1280 + height: 720 + gpu-id: 0 + #(0): nvbuf-mem-default - Default memory allocated, specific to particular platform + #(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla + #(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla + #(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla + #(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson + nvbuf-memory-type: 0 + +source: + csv-file-path: sources_retail_object.csv + +sink0: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File + type: 1 + sync: 0 + source-id: 0 + gpu-id: 0 + nvbuf-memory-type: 0 + +sink1: + enable: 1 + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvdrmvideosink 6=MsgConvBroker + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 3=mpeg4 + ## only SW mpeg4 is supported right now. + codec: 1 + sync: 0 + bitrate: 2000000 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 # set to 1 to enable sending metadata to kafka + #Type - 1=FakeSink 2=EglSink 3=File 4=UDPSink 5=nvoverlaysink 6=MsgConvBroker + type: 6 + msg-conv-config: /opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/configs/dstest5_msgconv_sample_config.txt + #(0): PAYLOAD_DEEPSTREAM - Deepstream schema payload + #(1): PAYLOAD_DEEPSTREAM_MINIMAL - Deepstream schema payload minimal + #(2): PAYLOAD_DEEPSTREAM_PROTOBUF - Deepstream schema payload protobuf + #(256): PAYLOAD_RESERVED - Reserved type + #(257): PAYLOAD_CUSTOM - Custom schema payload + msg-conv-payload-type: 1 + #(0): Create payload using NvdsEventMsgMeta + #(1): New Api to create payload using NvDsFrameMeta + msg-conv-msg2p-new-api: 0 + #Frame interval at which payload is generated + msg-conv-frame-interval: 1 + msg-broker-proto-lib: /opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so + #Provide your msg-broker-conn-str here + msg-broker-conn-str: 127.0.0.1;9092;test + topic: test + +osd: + enable: 1 + gpu-id: 0 + border-width: 1 + text-size: 15 + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Arial + show-clock: 0 + clock-x-offset: 800 + clock-y-offset: 820 + clock-text-size: 12 + clock-color: 1;0;0;0 + nvbuf-memory-type: 0 + +streammux: + gpu-id: 0 + ##Boolean property to inform muxer that sources are live + live-source: 0 + ## batch-size must be identical to the number of input sources. + batch-size: 1 + ##time out in usec, to wait after the first buffer is available + ##to push the batch even if the complete batch is not formed + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + ##Enable to maintain aspect ratio wrt source, and allow black borders, works + ##along with width, height properties + enable-padding: 0 + nvbuf-memory-type: 0 + ## If set to TRUE, system timestamp will be attached as ntp timestamp + ## If set to FALSE, ntp timestamp from rtspsrc, if available, will be attached + # attach-sys-ts-as-ntp: 1 + +primary-gie: + enable: 1 + plugin-type: 0 + #0:nvinfer, 1:nvinfeserver + gie-unique-id: 1 + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;1;1;1 + bbox-border-color3: 0;1;0;1 + config-file: ../nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml + #config-file: ../triton/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml + #config-file: ../triton-grpc/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml + +tracker: + enable: 1 + gpu-id: 0 + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 + +secondary-gie0: + enable: 1 + plugin-type: 0 + #0:nvinfer, 1:nvinfeserver + gie-unique-id: 2 + # operate-on-gie-id has to be specified in the main configuration file. + operate-on-gie-id: 1 + config-file: ../nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml + #config-file: ../triton/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml + #config-file: ../triton-grpc/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml + labelfile-path: ../../models/retail_object_recognition/retail_object_recognition_labels.txt + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/configs/app/seg_app_config.yml b/src/apps/tao_apps/configs/app/seg_app_config.yml new file mode 100644 index 00000000..a564caf9 --- /dev/null +++ b/src/apps/tao_apps/configs/app/seg_app_config.yml @@ -0,0 +1,61 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4 + +streammux: + batch-size: 1 + width: 1280 + height: 720 + batched-push-timeout: 40000 + +primary-gie: + plugin-type: 0 + #0:nvinfer, 1:nvinfeserver + + #citysemsegformer + config-file-path: ../nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml + #config-file-path: ../triton/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml + #config-file-path: ../triton-grpc/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml + +segvisual: + gpu-id: 0 + #The segvisual width and height should be set as the same as the segmentation modle output resolution + width: 1820 + height: 1024 + orig_background: 1 + alpha: 0.5 + +eglsink: + enable: 0 + +filesink: + enable: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + +fakesink: + enable: 0 + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/configs/app/sources.csv b/src/apps/tao_apps/configs/app/sources.csv new file mode 100644 index 00000000..8a117242 --- /dev/null +++ b/src/apps/tao_apps/configs/app/sources.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,nvbuf-memory-type +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_720p.mp4,1,0,0 diff --git a/src/apps/tao_apps/configs/app/sources_retail_object.csv b/src/apps/tao_apps/configs/app/sources_retail_object.csv new file mode 100644 index 00000000..17dee354 --- /dev/null +++ b/src/apps/tao_apps/configs/app/sources_retail_object.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,nvbuf-memory-type +1,3,file://../../sample/streams/retail_object_h264.mp4,1,0,0 diff --git a/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt b/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt new file mode 100644 index 00000000..e1e00991 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt @@ -0,0 +1,89 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-color-format=0 +labelfile-path=../../../models/LPD_ch/ccpd_label.txt +model-engine-file=../../../models/LPD_ch/LPDNet_CCPD_pruned_tao5.onnx_b16_gpu0_fp16.engine +onnx-file=../../../models/LPD_ch/LPDNet_CCPD_pruned_tao5.onnx +uff-input-dims=3;1168;720;0 +uff-input-blob-name=input_1 +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=1 +##1 Primary 2 Secondary +process-mode=2 +interval=0 +gie-unique-id=2 +#0 detector 1 classifier 2 segmentatio 3 instance segmentation +network-type=0 +operate-on-gie-id=1 +operate-on-class-ids=0 +#no cluster +cluster-mode=3 +output-blob-names=output_cov/Sigmoid:0;output_bbox/BiasAdd:0 +#if scaling-compute-hw = VIC, input-object-min-height need to be even and greater than or equal to (model height)/16 +input-object-min-height=74 +#if scaling-compute-hw = VIC, input-object-min-width need to be even and greater than or equal to( model width)/16 +input-object-min-width=46 +#GPU:1 VIC:2(Jetson only) +scaling-compute-hw=1 +#enable-dla=1 + +[class-attrs-all] +pre-cluster-threshold=0.3 +roi-top-offset=0 +roi-bottom-offset=0 +detected-min-w=0 +detected-min-h=0 +detected-max-w=0 +detected-max-h=0 + diff --git a/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.yml b/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.yml new file mode 100644 index 00000000..50660555 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/LPD_ch_tao/sgie_lpd_DetectNet2_ch.yml @@ -0,0 +1,88 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + model-color-format: 0 + labelfile-path: ../../../models/LPD_ch/ccpd_label.txt + model-engine-file: ../../../models/LPD_ch/LPDNet_CCPD_pruned_tao5.onnx_b16_gpu0_fp16.engine + onnx-file: ../../../models/LPD_ch/LPDNet_CCPD_pruned_tao5.onnx + infer-dims: 3;1168;720 + uff-input-order: 0 + uff-input-blob-name: input_1 + batch-size: 16 + ## 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 2 + num-detected-classes: 1 + ##1 Primary 2 Secondary + process-mode: 2 + interval: 0 + gie-unique-id: 2 + #0 detector 1 classifier 2 segmentatio 3 instance segmentation + network-type: 0 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + #no cluster + cluster-mode: 3 + output-blob-names: output_cov/Sigmoid:0;output_bbox/BiasAdd:0 + input-object-min-height: 73 + input-object-min-width: 45 + #GPU:1 VIC:2(Jetson only) + scaling-compute-hw: 1 + #enable-dla: 1 + +class-attrs-all: + pre-cluster-threshold: 0.3 + roi-top-offset: 0 + roi-bottom-offset: 0 + detected-min-w: 0 + detected-min-h: 0 + detected-max-w: 0 + detected-max-h: 0 + diff --git a/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.txt b/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.txt new file mode 100644 index 00000000..319fbc6f --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.txt @@ -0,0 +1,87 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-color-format=0 +labelfile-path=../../../models/LPD_us/usa_lpd_label.txt +model-engine-file=../../../models/LPD_us/LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine +onnx-file=../../../models/LPD_us/LPDNet_usa_pruned_tao5.onnx +uff-input-dims=3;480;640;0 +uff-input-blob-name=input_1 +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=1 +##1 Primary 2 Secondary +process-mode=2 +interval=0 +gie-unique-id=2 +#0 detector 1 classifier 2 segmentatio 3 instance segmentation +network-type=0 +operate-on-gie-id=1 +operate-on-class-ids=0 +#no cluster +cluster-mode=3 +output-blob-names=output_cov/Sigmoid:0;output_bbox/BiasAdd:0 +input-object-min-height=30 +input-object-min-width=40 +#GPU:1 VIC:2(Jetson only) +scaling-compute-hw=1 +#enable-dla=1 + +[class-attrs-all] +pre-cluster-threshold=0.3 +roi-top-offset=0 +roi-bottom-offset=0 +detected-min-w=0 +detected-min-h=0 +detected-max-w=0 +detected-max-h=0 + diff --git a/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.yml b/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.yml new file mode 100644 index 00000000..9934d2fa --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/LPD_us_tao/sgie_lpd_DetectNet2_us.yml @@ -0,0 +1,88 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + model-color-format: 0 + labelfile-path: ../../../models/LPD_us/usa_lpd_label.txt + model-engine-file: ../../../models/LPD_us/LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine + onnx-file: ../../../models/LPD_us/LPDNet_usa_pruned_tao5.onnx + infer-dims: 3;480;640 + uff-input-order: 0 + uff-input-blob-name: input_1 + batch-size: 16 + ## 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 2 + num-detected-classes: 1 + ##1 Primary 2 Secondary + process-mode: 2 + interval: 0 + gie-unique-id: 2 + #0 detector 1 classifier 2 segmentatio 3 instance segmentation + network-type: 0 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + #no cluster + cluster-mode: 3 + output-blob-names: output_cov/Sigmoid:0;output_bbox/BiasAdd:0 + input-object-min-height: 30 + input-object-min-width: 40 + #GPU:1 VIC:2(Jetson only) + #scaling-compute-hw: 1 + #enable-dla: 1 + +class-attrs-all: + pre-cluster-threshold: 0.3 + roi-top-offset: 0 + roi-bottom-offset: 0 + detected-min-w: 0 + detected-min-h: 0 + detected-max-w: 0 + detected-max-h: 0 + diff --git a/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_bodypose_post_process_sgie.yml b/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_bodypose_post_process_sgie.yml new file mode 100644 index 00000000..d7cf8048 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_bodypose_post_process_sgie.yml @@ -0,0 +1,27 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +property: + gpu-id: 0 + process-mode: 2 + num-detected-classes: 17 + gie-unique-id: 1 #Operate on gie-unique-id's output + ## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) + output-blob-names: output_0 + #only support 4=Bodypose + network-type: 4 + #release-tensor-meta: 1 + +#Use the config params below for NMS clustering mode +class-attrs-all: + classifier-threshold: 0.2 + diff --git a/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_infer_secondary_bodypose3dnet.txt b/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_infer_secondary_bodypose3dnet.txt new file mode 100644 index 00000000..9fb7750c --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/bodypose3d_tao/config_infer_secondary_bodypose3dnet.txt @@ -0,0 +1,44 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.00392156 +## Accuracy mode: _mode0_; Performance mode: _mode1_ +onnx-file=../../../models/bodypose3dnet/bodypose3dnet_accuracy.onnx +model-engine-file=../../../models/bodypose3dnet/bodypose3dnet_accuracy.onnx_b8_gpu0_fp16.engine +infer-dims=3;256;192 +batch-size=8 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +## 0=Detection 1=Classifier 2=Segmentation 100=other +network-type=100 +interval=0 +gie-unique-id=1 +output-blob-names=pose2d;pose2d_org_img;pose25d;pose3d +classifier-threshold=0.7 +operate-on-class-ids=0 +## Integer 0:NCHW 1:NHWC +network-input-order=0 +# Enable tensor metadata output +output-tensor-meta=1 +## 1-Primary 2-Secondary +process-mode=2 +## 0=RGB 1=BGR 2=GRAY +model-color-format=1 +maintain-aspect-ratio=0 +symmetric-padding=0 +scaling-filter=1 +custom-lib-path=../../../apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/libnvdsinfer_custom_impl_BodyPose3DNet.so +scaling-compute-hw=1 + + diff --git a/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_infer_third_bodypose_classification.txt b/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_infer_third_bodypose_classification.txt new file mode 100644 index 00000000..d5137399 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_infer_third_bodypose_classification.txt @@ -0,0 +1,79 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. +# +# NOTE : Below config parameters will not get used when preprocessing plugin is used +# to form tensor : +# net-scale-factor, model-color-format, mean-file, offsets + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-engine-file=../../../models/poseclassificationnet/st-gcn_3dbp_nvidia.onnx_b2_gpu0_fp16.engine +onnx-file=../../../models/poseclassificationnet/st-gcn_3dbp_nvidia.onnx +batch-size=2 +#infer-dims=3;224;224 +network-mode=2 +process-mode=2 +gie-unique-id=4 +operate-on-gie-id=1 +operate-on-class-ids=0 +input-tensor-from-meta=1 +#output-blob-names=fc_pred +#0=Detection 1=Classifier 2=Segmentation +network-type=1 +parse-classifier-func-name=NvDsParseCustomPoseClassification +custom-lib-path=../../../apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/libnvdsinfer_pose_classfication_parser.so +classifier-threshold=0.51 +scaling-compute-hw=1 + +[class-attrs-all] +pre-cluster-threshold=0.2 +topk=20 +nms-iou-threshold=0.5 + diff --git a/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_preprocess_bodypose_classification.txt b/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_preprocess_bodypose_classification.txt new file mode 100644 index 00000000..7267446e --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/bodypose_classification_tao/config_preprocess_bodypose_classification.txt @@ -0,0 +1,59 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +enable=1 +unique-id=3 +target-unique-ids=4 + # 0=NCHW, 1=NHWC, 2=CUSTOM +network-input-order=0 + # tensor shape based on network-input-order +network-input-shape=2;3;300;34;1 +process-on-frame=0 +processing-width=34 +processing-height=300 +scaling-buf-pool-size=6 +tensor-buf-pool-size=6 + # 0=RGB, 1=BGR, 2=GRAY +network-color-format=1 + # 0=FP32, 1=UINT8, 2=INT8, 3=UINT32, 4=INT32, 5=FP16 +tensor-data-type=0 +tensor-name=input + # 0=NVBUF_MEM_DEFAULT 1=NVBUF_MEM_CUDA_PINNED 2=NVBUF_MEM_CUDA_DEVICE 3=NVBUF_MEM_CUDA_UNIFIED +scaling-pool-memory-type=0 + # 0=NvBufSurfTransformCompute_Default 1=NvBufSurfTransformCompute_GPU 2=NvBufSurfTransformCompute_VIC +scaling-pool-compute-hw=1 + # Scaling Interpolation method + # 0=NvBufSurfTransformInter_Nearest 1=NvBufSurfTransformInter_Bilinear 2=NvBufSurfTransformInter_Algo1 + # 3=NvBufSurfTransformInter_Algo2 4=NvBufSurfTransformInter_Algo3 5=NvBufSurfTransformInter_Algo4 + # 6=NvBufSurfTransformInter_Default +scaling-filter=0 +custom-lib-path=../../../apps/tao_others/deepstream-pose-classification/nvdspreprocess_lib/libcustom2d_preprocess.so +custom-tensor-preparation-function=CustomTensorPreparation + +[user-configs] +#actual sequence length of frames +frames-sequence-length=300 + +[group-0] +src-ids=0;1 +custom-input-transformation-function=CustomTransformation +process-on-roi=1 +process-on-all-objects=1 +roi-params-src-0=0;0;100;100 +roi-params-src-1=0;0;100;100 +input-object-min-width=100 +input-object-min-height=100 +draw-roi=0 diff --git a/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.txt b/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.txt new file mode 100644 index 00000000..5a74b1c8 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.txt @@ -0,0 +1,47 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.01735207357279195 +offsets=123.675;116.28;103.53 +labelfile-path=../../../models/citysemsegformer/labels.txt +model-engine-file=../../../models/citysemsegformer/citysemsegformer.onnx_b1_gpu0_fp16.engine +onnx-file=../../../models/citysemsegformer/citysemsegformer.onnx +model-color-format=0 +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +## workspace-size default to 1024 x 1024 MB +workspace-size=1048576 +interval=0 +gie-unique-id=1 +cluster-mode=2 +## 0=Detector, 1=Classifier, 2=Semantic Segmentation, 3=Instance Segmentation, 100=Other +network-type=2 +## num-detected-classes= is required to set NvDsInferSegmentationMeta::classes. +num-detected-classes=19 +##specify the output tensor order, 0(default value) for CHW and 1 for HWC +segmentation-output-order=1 +parse-segmentation-func-name=NvDsInferParseCustomSegformerTAO +custom-lib-path=../../../post_processor/libnvds_infercustomparser_tao.so +scaling-compute-hw=1 diff --git a/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml new file mode 100644 index 00000000..f70e6636 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml @@ -0,0 +1,50 @@ +################################################################################ +# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.01735207357279195 + offsets: 123.675;116.28;103.53 + labelfile-path: ../../../models/citysemsegformer/labels.txt + model-engine-file: ../../../models/citysemsegformer/citysemsegformer.onnx_b1_gpu0_fp16.engine + onnx-file: ../../../models/citysemsegformer/citysemsegformer.onnx + model-color-format: 0 + batch-size: 1 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + ## workspace-size 1024 x 1024 MB + workspace-size: 1048576 + interval: 0 + gie-unique-id: 1 + cluster-mode: 2 + ## 0=Detector, 1=Classifier, 2=Semantic Segmentation, 3=Instance Segmentation, 100=Other + network-type: 2 + ## num-detected-classes= is required to set NvDsInferSegmentationMeta::classes. + num-detected-classes: 19 + ## Specify the output tensor order, 0(default value) for CHW and 1 for HWC + segmentation-output-order: 1 + parse-segmentation-func-name: NvDsInferParseCustomSegformerTAO + custom-lib-path: ../../../post_processor/libnvds_infercustomparser_tao.so + scaling-compute-hw: 1 + +source-list: + list: file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h264.mp4 diff --git a/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.txt b/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.txt new file mode 100644 index 00000000..ed43364f --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.txt @@ -0,0 +1,74 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +model-engine-file=../../../models/LPR_ch/ch_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine +labelfile-path=../../../models/LPR_ch/labels_ch.txt +onnx-file=../../../models/LPR_ch/ch_lprnet_baseline18_deployable.onnx +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +gie-unique-id=3 +output-blob-names=tf_op_layer_ArgMax;tf_op_layer_Max +#0=Detection 1=Classifier 2=Segmentation +network-type=1 +parse-classifier-func-name=NvDsInferParseCustomNVPlate +custom-lib-path=../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so +process-mode=2 +operate-on-gie-id=2 +net-scale-factor=0.00392156862745098 +#net-scale-factor=1.0 +#0=RGB 1=BGR 2=GRAY +model-color-format=0 +maintain-aspect-ratio=0 +scaling-compute-hw=1 + +[class-attrs-all] +threshold=0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.yml b/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.yml new file mode 100644 index 00000000..216343b2 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/lpr_ch_tao/sgie_lpr_ch_config.yml @@ -0,0 +1,74 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +property: + gpu-id: 0 + model-engine-file: ../../../models/LPR_ch/ch_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine + labelfile-path: ../../../models/LPR_ch/labels_ch.txt + onnx-file: ../../../models/LPR_ch/ch_lprnet_baseline18_deployable.onnx + batch-size: 16 + ## 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 2 + num-detected-classes: 3 + gie-unique-id: 3 + output-blob-names: tf_op_layer_ArgMax;tf_op_layer_Max + #0: Detection 1: Classifier 2: Segmentation + network-type: 1 + parse-classifier-func-name: NvDsInferParseCustomNVPlate + custom-lib-path: ../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so + process-mode: 2 + operate-on-gie-id: 2 + net-scale-factor: 0.00392156862745098 + #net-scale-factor: 1.0 + #0: RGB 1: BGR 2: GRAY + model-color-format: 0 + maintain-aspect-ratio: 0 + scaling-compute-hw: 1 + +class-attrs-all: + threshold: 0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.txt b/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.txt new file mode 100644 index 00000000..9a1fd747 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.txt @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +model-engine-file=../../../models/LPR_us/us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine +labelfile-path=../../../models/LPR_us/labels_us.txt +onnx-file=../../../models/LPR_us/us_lprnet_baseline18_deployable.onnx +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +gie-unique-id=3 +output-blob-names=tf_op_layer_ArgMax;tf_op_layer_Max +#0=Detection 1=Classifier 2=Segmentation +network-type=1 +parse-classifier-func-name=NvDsInferParseCustomNVPlate +custom-lib-path=../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so +process-mode=2 +operate-on-gie-id=2 +net-scale-factor=0.00392156862745098 +#net-scale-factor=1.0 +#0=RGB 1=BGR 2=GRAY +model-color-format=0 +scaling-compute-hw=1 + +[class-attrs-all] +threshold=0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.yml b/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.yml new file mode 100644 index 00000000..a3587bab --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/lpr_us_tao/sgie_lpr_us_config.yml @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +property: + gpu-id: 0 + model-engine-file: ../../../models/LPR_us/us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine + labelfile-path: ../../../models/LPR_us/labels_us.txt + onnx-file: ../../../models/LPR_us/us_lprnet_baseline18_deployable.onnx + batch-size: 16 + ## 0: FP32, 1: INT8, 2: FP16 mode + network-mode: 2 + num-detected-classes: 3 + gie-unique-id: 3 + output-blob-names: tf_op_layer_ArgMax;tf_op_layer_Max + #0: Detection 1: Classifier 2: Segmentation + network-type: 1 + parse-classifier-func-name: NvDsInferParseCustomNVPlate + custom-lib-path: ../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so + process-mode: 2 + operate-on-gie-id: 2 + net-scale-factor: 0.00392156862745098 + #net-scale-factor: 1.0 + #0: RGB 1: BGR 2: GRAY + model-color-format: 0 + scaling-compute-hw: 1 + +class-attrs-all: + threshold: 0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/mask2former_tao/pgie_mask2former_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/mask2former_tao/pgie_mask2former_tao_config.yml new file mode 100644 index 00000000..0a61a635 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/mask2former_tao/pgie_mask2former_tao_config.yml @@ -0,0 +1,53 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.017507 + offsets: 123.675;116.280;103.53 + model-color-format: 0 + model-engine-file: ../../../models/mask2former/mask2former.onnx_b1_gpu0_fp16.engine + onnx-file: ../../../models/mask2former/mask2former.onnx + network-type: 3 ## 3 is for instance segmentation network + infer-dims: 3;800;800 + num-detected-classes: 2 + batch-size: 1 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + interval: 0 + gie-unique-id: 1 + #no cluster + ## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) + ## MRCNN supports only cluster-mode=4; Clustering is done by the model itself + cluster-mode: 4 + output-instance-mask: 1 + input-object-min-width: 32 + input-object-min-height: 32 + output-blob-names: pred_masks;pred_scores;pred_classes + parse-bbox-instance-mask-func-name: NvDsInferParseCustomMask2Former + custom-lib-path: ../../../post_processor/libnvds_infercustomparser_tao.so + scaling-compute-hw: 1 + layer-device-precision: /post_processor/Div:fp32:gpu;/post_processor/ReduceSum:fp32:gpu;/post_processor/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Cast_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Where:fp32:gpu;sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.0/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.1/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Cast_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.2/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.3/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Cast_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.4/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Cast_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.5/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.6/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.7/norm/LayerNormalization:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/Add:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_2:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Div:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_6:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_9:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Cast:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Cast_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Div_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_3:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_7:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Where:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Mul_8:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Add_5:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Softmax:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/MatMul_4:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/multihead_attn/Gemm:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/Add_1:fp32:gpu;/sem_seg_head/predictor/transformer_cross_attention_layers.8/norm/LayerNormalization:fp32:gpu + +class-attrs-all: + pre-cluster-threshold: 0.6 diff --git a/src/apps/tao_apps/configs/nvinfer/peoplenet_tao/config_infer_primary_peoplenet.txt b/src/apps/tao_apps/configs/nvinfer/peoplenet_tao/config_infer_primary_peoplenet.txt new file mode 100644 index 00000000..2b4b9f5b --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/peoplenet_tao/config_infer_primary_peoplenet.txt @@ -0,0 +1,54 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=../../../models/peoplenet/resnet34_peoplenet_int8.onnx +labelfile-path=../../../models/peoplenet/labels.txt +model-engine-file=../../../models/peoplenet/resnet34_peoplenet_int8.onnx_b2_gpu0_fp16.engine +infer-dims=3;544;960 +batch-size=2 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +cluster-mode=2 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 +scaling-compute-hw=1 + +#Use the config params below for dbscan clustering mode +#[class-attrs-all] +#detected-min-w=4 +#detected-min-h=4 +#minBoxes=3 +#eps=0.7 + +#Use the config params below for NMS clustering mode +[class-attrs-all] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 + +#[class-attrs-1] +#pre-cluster-threshold=0.05 +#eps=0.7 +#dbscan-min-score=0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt b/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt new file mode 100644 index 00000000..8fdcc975 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.txt @@ -0,0 +1,46 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +offsets=123.675;116.28;103.53 +net-scale-factor=0.0173520735728 +labelfile-path=../../../models/peoplenet_transformer/labels.txt +model-engine-file=../../../models/peoplenet_transformer/resnet50_peoplenet_transformer_op17.onnx_b1_gpu0_fp16.engine +onnx-file=../../../models/peoplenet_transformer/resnet50_peoplenet_transformer_op17.onnx +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=pred_boxes;pred_logits +infer-dims=3;544;960 +workspace-size=1048576 +cluster-mode=4 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +custom-lib-path=../../../post_processor/libnvds_infercustomparser_tao.so +scaling-compute-hw=1 + +[class-attrs-all] +pre-cluster-threshold=0.5 +topk=20 diff --git a/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml new file mode 100644 index 00000000..df6ab035 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml @@ -0,0 +1,46 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + offsets: 123.675;116.28;103.53 + net-scale-factor: 0.0173520735728 + labelfile-path: ../../../models/peoplenet_transformer/labels.txt + model-engine-file: ../../../models/peoplenet_transformer/resnet50_peoplenet_transformer_op17.onnx_b1_gpu0_fp16.engine + onnx-file: ../../../models/peoplenet_transformer/resnet50_peoplenet_transformer_op17.onnx + batch-size: 1 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: pred_boxes;pred_logits + infer-dims: 3;544;960 + workspace-size: 1048576 + cluster-mode: 4 + parse-bbox-func-name: NvDsInferParseCustomDDETRTAO + custom-lib-path: ../../../post_processor/libnvds_infercustomparser_tao.so + scaling-compute-hw: 1 + +class-attrs-all: + pre-cluster-threshold: 0.5 + topk: 20 diff --git a/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.txt b/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.txt new file mode 100644 index 00000000..a4136fd9 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.txt @@ -0,0 +1,46 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.01735207357279195 +offsets=123.675;116.28;103.53 +model-color-format=0 +onnx-file=../../../models/reidentificationnet/resnet50_market1501_aicity156.onnx +model-engine-file=../../../models/reidentificationnet/resnet50_market1501_aicity156.onnx_b16_gpu0_fp16.engine +infer-dims=3;256;128 +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +network-type=100 +interval=0 +## Infer Processing Mode 1=Primary Mode 2=Secondary Mode +process-mode=2 +# ## Clustering algorithm=0=GroupRectangle 1=DBSCAN 2=NMS 3=Hybrid 4=NoClustering +# cluster-mode=3 +classifier-threshold=0 +output-tensor-meta=1 +maintain-aspect-ratio=0 +## Extract embedding on "people" class only +operate-on-class-ids=1 +scaling-compute-hw=1 +gie-unique-id=2 diff --git a/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml new file mode 100644 index 00000000..d6a553da --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml @@ -0,0 +1,46 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.01735207357279195 + offsets: 123.675;116.28;103.53 + model-color-format: 0 + onnx-file: ../../../models/reidentificationnet/resnet50_market1501_aicity156.onnx + model-engine-file: ../../../models/reidentificationnet/resnet50_market1501_aicity156.onnx_b16_gpu0_fp16.engine + infer-dims: 3;256;128 + batch-size: 16 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + network-type: 100 + interval: 0 + ## Infer Processing Mode 1=Primary Mode 2=Secondary Mode + process-mode: 2 + # ## Clustering algorithm: 0=GroupRectangle 1=DBSCAN 2=NMS 3=Hybrid 4=NoClustering + # cluster-mode: 3 + classifier-threshold: 0 + output-tensor-meta: 1 + maintain-aspect-ratio: 0 + ## Extract embedding on "people" class only + operate-on-class-ids: 1 + scaling-compute-hw: 1 + gie-unique-id: 2 diff --git a/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.txt b/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.txt new file mode 100644 index 00000000..832cde9c --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.txt @@ -0,0 +1,28 @@ +[property] +net-scale-factor=0.01735207357279195 +offsets=123.675;116.28;103.53 +model-color-format=1 +onnx-file=../../../models/retail_object_detection_binary_dino/retail_object_detection_dino_binary.onnx +model-engine-file=../../../models/retail_object_detection_binary_dino/retail_object_detection_dino_binary.onnx_b1_gpu0_fp32.engine +labelfile-path=../../../models/retail_object_detection_binary_dino/class_map.txt +network-input-order=0 +infer-dims=3;540;960 +maintain-aspect-ratio=1 +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=0 +num-detected-classes=2 +interval=0 +gie-unique-id=1 +cluster-mode=4 +output-blob-names=pred_boxes;pred_logits +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +custom-lib-path=../../../post_processor/libnvds_infercustomparser_tao.so +scaling-compute-hw=1 + + +#Use the config params below for NMS clustering mode +[class-attrs-all] +pre-cluster-threshold=0.5 +topk=300 + diff --git a/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml new file mode 100644 index 00000000..0bc9ba28 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml @@ -0,0 +1,28 @@ +property: + net-scale-factor: 0.01735207357279195 + offsets: 123.675;116.28;103.53 + model-color-format: 1 + onnx-file: ../../../models/retail_object_detection_binary_dino/retail_object_detection_dino_binary.onnx + model-engine-file: ../../../models/retail_object_detection_binary_dino/retail_object_detection_dino_binary.onnx_b1_gpu0_fp32.engine + labelfile-path: ../../../models/retail_object_detection_binary_dino/class_map.txt + network-input-order: 0 + infer-dims: 3;540;960 + maintain-aspect-ratio: 1 + batch-size: 1 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 0 + num-detected-classes: 2 + interval: 0 + gie-unique-id: 1 + cluster-mode: 4 + output-blob-names: pred_boxes;pred_logits + parse-bbox-func-name: NvDsInferParseCustomDDETRTAO + custom-lib-path: ../../../post_processor/libnvds_infercustomparser_tao.so + scaling-compute-hw: 1 + + +#Use the config params below for NMS clustering mode +class-attrs-all: + pre-cluster-threshold: 0.5 + topk: 300 + diff --git a/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.txt b/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.txt new file mode 100644 index 00000000..b22638ad --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.txt @@ -0,0 +1,47 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.01460167693791089023079776340069 +offsets=122.7709383;116.7460125;104.09373615 +model-color-format=0 +onnx-file=../../../models/retail_object_recognition/retail_object_recognition.onnx +model-engine-file=../../../models/retail_object_recognition/retail_object_recognition.onnx_b16_gpu0_fp16.engine +labelfile-path=../../../models/retail_object_recognition/retail_object_recognition_labels.txt +infer-dims=3;224;224 +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +network-type=100 +interval=0 +## Infer Processing Mode 1=Primary Mode 2=Secondary Mode +process-mode=2 +# ## Clustering algorithm=0=GroupRectangle 1=DBSCAN 2=NMS 3=Hybrid 4=NoClustering +# cluster-mode=3 +classifier-threshold=0 +output-tensor-meta=1 +maintain-aspect-ratio=0 +# operate-on-class-ids=0;1;2;3 +output-blob-names=probs;embeddings +scaling-compute-hw=1 +gie-unique-id=2 diff --git a/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml b/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml new file mode 100644 index 00000000..05911c4d --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml @@ -0,0 +1,47 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.01460167693791089023079776340069 + offsets: 122.7709383;116.7460125;104.09373615 + model-color-format: 0 + onnx-file: ../../../models/retail_object_recognition/retail_object_recognition.onnx + model-engine-file: ../../../models/retail_object_recognition/retail_object_recognition.onnx_b16_gpu0_fp16.engine + labelfile-path: ../../../models/retail_object_recognition/retail_object_recognition_labels.txt + infer-dims: 3;224;224 + batch-size: 16 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + network-type: 100 + interval: 0 + ## Infer Processing Mode 1=Primary Mode 2=Secondary Mode + process-mode: 2 + # ## Clustering algorithm: 0=GroupRectangle 1=DBSCAN 2=NMS 3=Hybrid 4=NoClustering + # cluster-mode: 3 + classifier-threshold: 0 + output-tensor-meta: 1 + maintain-aspect-ratio: 0 + # operate-on-class-ids: 0;1;2;3 + output-blob-names: probs;embeddings + scaling-compute-hw: 1 + gie-unique-id: 2 diff --git a/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.txt b/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.txt new file mode 100644 index 00000000..7777006e --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.txt @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2020-2025 NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=../../../models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx +labelfile-path=../../../models/trafficcamnet/labels_trafficnet.txt +model-engine-file=../../../models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +input-dims=3;544;960;0 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 +cluster-mode=2 +scaling-compute-hw=1 + +#Use the config params below for dbscan clustering mode +#[class-attrs-all] +#detected-min-w=4 +#detected-min-h=4 +#minBoxes=3 +#eps=0.7 + +#Use the config params below for NMS clustering mode +[class-attrs-all] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 + +#[class-attrs-1] +#pre-cluster-threshold=0.05 +#eps=0.7 +#dbscan-min-score=0.5 diff --git a/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.yml b/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.yml new file mode 100644 index 00000000..ec481a19 --- /dev/null +++ b/src/apps/tao_apps/configs/nvinfer/trafficcamnet_tao/pgie_trafficcamnet_config.yml @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + onnx-file: ../../../models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx + labelfile-path: ../../../models/trafficcamnet/labels_trafficnet.txt + model-engine-file: ../../../models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine + infer-dims: 3;544;960 + uff-input-blob-name: input_1 + batch-size: 1 + process-mode: 1 + model-color-format: 0 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: output_cov/Sigmoid:0;output_bbox/BiasAdd:0 + cluster-mode: 2 + scaling-compute-hw: 1 + +#Use the config params below for dbscan clustering mode +#class-attrs-all: + #detected-min-w: 4 + #detected-min-h: 4 + #minBoxes: 3 + #eps: 0.7 + +#Use the config params below for NMS clustering mode +class-attrs-all: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.2 + +## Per class configurations +class-attrs-0: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.4 + +#class-attrs-1: + #pre-cluster-threshold: 0.05 + #eps: 0.7 + #dbscan-min-score: 0.5 diff --git a/src/apps/tao_apps/configs/triton-grpc/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt b/src/apps/tao_apps/configs/triton-grpc/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt new file mode 100644 index 00000000..09184f15 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt @@ -0,0 +1,85 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "LPD_ch" + version: -1 + grpc { + url: "127.0.0.1:10001" + } + } + } + + preprocess { + #network_format: IMAGE_FORMAT_BGR + network_format: IMAGE_FORMAT_RGB + #tensor_order: TENSOR_ORDER_LINEAR + tensor_order: TENSOR_ORDER_NONE + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + } + } + + postprocess { + labelfile_path: "../../../models/LPD_ch/ccpd_label.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + secondary_reinfer_interval: 0 + async_mode: false + object_control { + bbox_filter { + #if scaling-compute-hw = VIC, input-object-min-width need to be even and greater than or equal to( model width)/16 + min_width: 46 + #if scaling-compute-hw = VIC, input-object-min-height need to be even and greater than or equal to (model height)/16 + min_height: 74 + } + } +} + +output_control { + output_tensor_meta: true +} diff --git a/src/apps/tao_apps/configs/triton-grpc/LPD_us_tao/sgie_lpd_DetectNet2_us.txt b/src/apps/tao_apps/configs/triton-grpc/LPD_us_tao/sgie_lpd_DetectNet2_us.txt new file mode 100644 index 00000000..69ef248f --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/LPD_us_tao/sgie_lpd_DetectNet2_us.txt @@ -0,0 +1,83 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "LPD_us" + version: -1 + grpc { + url: "127.0.0.1:10001" + } + } + } + + preprocess { + #network_format: IMAGE_FORMAT_BGR + network_format: IMAGE_FORMAT_RGB + #tensor_order: TENSOR_ORDER_LINEAR + tensor_order: TENSOR_ORDER_NONE + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + } + } + + postprocess { + labelfile_path: "../../../models/LPD_us/usa_lpd_label.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + secondary_reinfer_interval: 0 + async_mode: false + object_control { + bbox_filter { + min_width: 64 + min_height: 64 + } + } +} + +output_control { + output_tensor_meta: true +} diff --git a/src/apps/tao_apps/configs/triton-grpc/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml b/src/apps/tao_apps/configs/triton-grpc/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml new file mode 100644 index 00000000..586fb4ef --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 8 + backend { + inputs: [ + {name: "input0"}, + {name: "k_inv"}, + {name: "t_form_inv"}, + {name: "scale_normalized_mean_limb_lengths"}, + {name: "mean_limb_lengths"} + ] + outputs: [ + {name: "pose2d"}, + {name: "pose2d_org_img"}, + {name: "pose25d"}, + {name: "pose3d"} + ] + triton { + model_name: "bodypose3dnet" + version: -1 + grpc { + url: "localhost:10001" + } + } + output_mem_type: MEMORY_TYPE_DEFAULT + } + + preprocess { + network_format: IMAGE_FORMAT_BGR + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + symmetric_padding: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156 + } + } + postprocess { + other {} + } + + custom_lib { + path: "../../../apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/libnvdsinfer_custom_impl_BodyPose3DNet.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/bodypose_classification_tao/config_infer_third_bodypose_classification.yml b/src/apps/tao_apps/configs/triton-grpc/bodypose_classification_tao/config_infer_third_bodypose_classification.yml new file mode 100644 index 00000000..f1c3b9d2 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/bodypose_classification_tao/config_infer_third_bodypose_classification.yml @@ -0,0 +1,51 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: 0 + max_batch_size: 4 + backend { + triton { + model_name: "poseclassificationnet" + version: -1 + grpc { + url: "localhost:10001" + } + } + output_mem_type: MEMORY_TYPE_DEFAULT + } + + input_tensor_from_meta { + is_first_dim_batch: true + } + + postprocess { + classification { + custom_parse_classifier_func: "NvDsParseCustomPoseClassification" + threshold: 0.51 + } + } + + custom_lib { + path: "../../../apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/libnvdsinfer_pose_classfication_parser.so" + } + + extra { + output_buffer_pool_size: 4 + } +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/labels.txt b/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/labels.txt new file mode 100644 index 00000000..76fb476f --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/labels.txt @@ -0,0 +1,19 @@ +road +sidewalk +building +wall +fence +pole +traffic light +traffic sign +vegetation +terrain +sky +person +rider +car +truck +bus +train +motorcycle +bicycle diff --git a/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml new file mode 100644 index 00000000..1cf08bea --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "citysemsegformer" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "../../../models/citysemsegformer/labels.txt" + segmentation { + num_segmentation_classes: 19 + custom_parse_segmentation_func: "NvDsInferParseCustomSegformerTAO" + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/lpr_ch_tao/sgie_lpr_ch_config.txt b/src/apps/tao_apps/configs/triton-grpc/lpr_ch_tao/sgie_lpr_ch_config.txt new file mode 100644 index 00000000..d02f1bf9 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/lpr_ch_tao/sgie_lpr_ch_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: 0 + max_batch_size: 16 + backend { + + triton { + model_name: "LPR_ch" + version: -1 + grpc { + url: "127.0.0.1:10001" + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_NONE + #maintain_aspect_ratio: 0 + #frame_scaling_hw: FRAME_SCALING_HW_GPU + #frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + #channel_offsets: [0, 0, 0] + #channel_offsets: [128, 128, 128] + } + } + + postprocess { + classification { + threshold:0.5 + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + } + } + custom_lib { + path: "../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 2 + operate_on_class_ids: [0] + interval: 0 + async_mode: false +} + +output_control { + output_tensor_meta: false +} diff --git a/src/apps/tao_apps/configs/triton-grpc/lpr_us_tao/sgie_lpr_us_config.txt b/src/apps/tao_apps/configs/triton-grpc/lpr_us_tao/sgie_lpr_us_config.txt new file mode 100644 index 00000000..19bb4fa9 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/lpr_us_tao/sgie_lpr_us_config.txt @@ -0,0 +1,72 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: 0 + max_batch_size: 16 + backend { + + triton { + model_name: "LPR_us" + version: -1 + grpc { + url: "127.0.0.1:10001" + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_NONE + #maintain_aspect_ratio: 0 + #frame_scaling_hw: FRAME_SCALING_HW_GPU + #frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + #channel_offsets: [0, 0, 0] + #channel_offsets: [128, 128, 128] + } + } + + postprocess { + classification { + threshold:0.5 + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + } + } + custom_lib { + path: "../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 2 + operate_on_class_ids: [0] + interval: 0 + async_mode: false +} + +output_control { + output_tensor_meta: false +} diff --git a/src/apps/tao_apps/configs/triton-grpc/mask2former_tao/pgie_mask2former_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/mask2former_tao/pgie_mask2former_tao_config.yml new file mode 100644 index 00000000..5593d92f --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/mask2former_tao/pgie_mask2former_tao_config.yml @@ -0,0 +1,76 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 4 + backend { + triton { + model_name: "mask2former" + version: -1 + grpc { + url: "127.0.0.1:10001" + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.017507 + channel_offsets: [123.675, 116.280, 103.53] + } + } + + postprocess { + detection { + per_class_params { + key: 0 + value { pre_threshold: 0.8 } + } + num_detected_classes: 2 + custom_parse_bbox_func: "NvDsInferParseCustomMask2Former" + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/peoplenet_tao/config_infer_primary_peoplenet.yml b/src/apps/tao_apps/configs/triton-grpc/peoplenet_tao/config_infer_primary_peoplenet.yml new file mode 100644 index 00000000..f9cf9624 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/peoplenet_tao/config_infer_primary_peoplenet.yml @@ -0,0 +1,72 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 2 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peoplenet" + version: -1 + grpc { + url: "localhost:10001" + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../../../models/peoplenet/labels.txt" + detection { + num_detected_classes: 3 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/labels.txt b/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/labels.txt new file mode 100644 index 00000000..fca8a60c --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/labels.txt @@ -0,0 +1,4 @@ +BG +Person +Face +Bag \ No newline at end of file diff --git a/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml new file mode 100644 index 00000000..7ef6de81 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "peoplenet_transformer" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0173520735728 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "./labels.txt" + detection { + num_detected_classes: 4 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/configs/triton-grpc/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml new file mode 100644 index 00000000..6d3acbce --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml @@ -0,0 +1,71 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "reidentificationnet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + interval: 0 + operate_on_class_ids: [1] +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml new file mode 100644 index 00000000..4554aa24 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml @@ -0,0 +1,72 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "retail_object_detection_binary_dino" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 1 + symmetric_padding: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675, 116.28, 103.53] + } + } + postprocess { + labelfile_path: "../../../models/retail_object_detection_binary_dino/class_map.txt" + detection { + num_detected_classes: 2 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/retail_detector_binary_labels.txt b/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/retail_detector_binary_labels.txt new file mode 100644 index 00000000..99fbfd61 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/retail_object_detection_tao/retail_detector_binary_labels.txt @@ -0,0 +1 @@ +ITEM diff --git a/src/apps/tao_apps/configs/triton-grpc/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml b/src/apps/tao_apps/configs/triton-grpc/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml new file mode 100644 index 00000000..122c7628 --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml @@ -0,0 +1,71 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "retail_object_recognition" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01460167693791089023079776340069 + channel_offsets: [122.7709383,116.7460125,104.09373615] + } + } + + postprocess { + labelfile_path: "../../../models/retail_object_recognition/retail_object_recognition_labels.txt" + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + interval: 0 +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton-grpc/trafficcamnet_tao/pgie_trafficcamnet_config.txt b/src/apps/tao_apps/configs/triton-grpc/trafficcamnet_tao/pgie_trafficcamnet_config.txt new file mode 100644 index 00000000..9dafa42a --- /dev/null +++ b/src/apps/tao_apps/configs/triton-grpc/trafficcamnet_tao/pgie_trafficcamnet_config.txt @@ -0,0 +1,76 @@ +################################################################################ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../../../models/trafficcamnet/labels_trafficnet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/configs/triton/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt b/src/apps/tao_apps/configs/triton/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt new file mode 100644 index 00000000..ef718a3d --- /dev/null +++ b/src/apps/tao_apps/configs/triton/LPD_ch_tao/sgie_lpd_DetectNet2_ch.txt @@ -0,0 +1,89 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "LPD_ch" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + #network_format: IMAGE_FORMAT_BGR + network_format: IMAGE_FORMAT_RGB + #tensor_order: TENSOR_ORDER_LINEAR + tensor_order: TENSOR_ORDER_NONE + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + } + } + + postprocess { + labelfile_path: "../../../models/LPD_ch/ccpd_label.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + output_buffer_pool_size: 8 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + secondary_reinfer_interval: 0 + async_mode: false + object_control { + bbox_filter { + #if scaling-compute-hw = VIC, input-object-min-width need to be even and greater than or equal to( model width)/16 + min_width: 46 + #if scaling-compute-hw = VIC, input-object-min-height need to be even and greater than or equal to (model height)/16 + min_height: 74 + } + } +} + +output_control { + output_tensor_meta: true +} diff --git a/src/apps/tao_apps/configs/triton/LPD_us_tao/sgie_lpd_DetectNet2_us.txt b/src/apps/tao_apps/configs/triton/LPD_us_tao/sgie_lpd_DetectNet2_us.txt new file mode 100644 index 00000000..4b614155 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/LPD_us_tao/sgie_lpd_DetectNet2_us.txt @@ -0,0 +1,87 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "LPD_us" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + #network_format: IMAGE_FORMAT_BGR + network_format: IMAGE_FORMAT_RGB + #tensor_order: TENSOR_ORDER_LINEAR + tensor_order: TENSOR_ORDER_NONE + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + } + } + + postprocess { + labelfile_path: "../../../models/LPD_us/usa_lpd_label.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + extra { + output_buffer_pool_size: 8 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + secondary_reinfer_interval: 0 + async_mode: false + object_control { + bbox_filter { + min_width: 64 + min_height: 64 + } + } +} + +output_control { + output_tensor_meta: true +} diff --git a/src/apps/tao_apps/configs/triton/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml b/src/apps/tao_apps/configs/triton/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml new file mode 100644 index 00000000..641fc06a --- /dev/null +++ b/src/apps/tao_apps/configs/triton/bodypose3d_tao/config_infer_secondary_bodypose3dnet.yml @@ -0,0 +1,77 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 8 + backend { + inputs: [ + {name: "input0"}, + {name: "k_inv"}, + {name: "t_form_inv"}, + {name: "scale_normalized_mean_limb_lengths"}, + {name: "mean_limb_lengths"} + ] + outputs: [ + {name: "pose2d"}, + {name: "pose2d_org_img"}, + {name: "pose25d"}, + {name: "pose3d"} + ] + triton { + model_name: "bodypose3dnet" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + output_mem_type: MEMORY_TYPE_DEFAULT + } + + preprocess { + network_format: IMAGE_FORMAT_BGR + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 0 + symmetric_padding: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156 + } + } + postprocess { + other {} + } + + custom_lib { + path: "../../../apps/tao_others/deepstream-pose-classification/nvdsinfer_custom_impl_BodyPose3DNet/libnvdsinfer_custom_impl_BodyPose3DNet.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton/bodypose_classification_tao/config_infer_third_bodypose_classification.yml b/src/apps/tao_apps/configs/triton/bodypose_classification_tao/config_infer_third_bodypose_classification.yml new file mode 100644 index 00000000..78b9f5af --- /dev/null +++ b/src/apps/tao_apps/configs/triton/bodypose_classification_tao/config_infer_third_bodypose_classification.yml @@ -0,0 +1,52 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 4 + gpu_ids: 0 + max_batch_size: 4 + backend { + triton { + model_name: "poseclassificationnet" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + output_mem_type: MEMORY_TYPE_DEFAULT + } + + input_tensor_from_meta { + is_first_dim_batch: true + } + + postprocess { + classification { + custom_parse_classifier_func: "NvDsParseCustomPoseClassification" + threshold: 0.51 + } + } + + custom_lib { + path: "../../../apps/tao_others/deepstream-pose-classification/infer_pose_classification_parser/libnvdsinfer_pose_classfication_parser.so" + } + + extra { + output_buffer_pool_size: 4 + } +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml b/src/apps/tao_apps/configs/triton/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml new file mode 100644 index 00000000..8e167148 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/citysemsegformer_tao/pgie_citysemsegformer_tao_config.yml @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "citysemsegformer" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "../../../models/citysemsegformer/labels.txt" + segmentation { + num_segmentation_classes: 19 + custom_parse_segmentation_func: "NvDsInferParseCustomSegformerTAO" + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton/lpr_ch_tao/sgie_lpr_ch_config.txt b/src/apps/tao_apps/configs/triton/lpr_ch_tao/sgie_lpr_ch_config.txt new file mode 100644 index 00000000..ea025b3e --- /dev/null +++ b/src/apps/tao_apps/configs/triton/lpr_ch_tao/sgie_lpr_ch_config.txt @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: 0 + max_batch_size: 16 + backend { + + triton { + model_name: "LPR_ch" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_NONE + #maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + #frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + #channel_offsets: [0, 0, 0] + #channel_offsets: [128, 128, 128] + } + } + + postprocess { + classification { + threshold:0.5 + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + } + } + custom_lib { + path: "../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 2 + operate_on_class_ids: [0] + interval: 0 + async_mode: false +} + +output_control { + output_tensor_meta: false +} diff --git a/src/apps/tao_apps/configs/triton/lpr_us_tao/sgie_lpr_us_config.txt b/src/apps/tao_apps/configs/triton/lpr_us_tao/sgie_lpr_us_config.txt new file mode 100644 index 00000000..dadaeaa3 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/lpr_us_tao/sgie_lpr_us_config.txt @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 3 + gpu_ids: 0 + max_batch_size: 16 + backend { + + triton { + model_name: "LPR_us" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_NONE + #maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + #frame_scaling_filter: 1 + normalize { + scale_factor: 0.00392156862745098 + #channel_offsets: [0, 0, 0] + #channel_offsets: [128, 128, 128] + } + } + + postprocess { + classification { + threshold:0.5 + custom_parse_classifier_func: "NvDsInferParseCustomNVPlate" + } + } + custom_lib { + path: "../../../apps/tao_others/deepstream_lpr_app/nvinfer_custom_lpr_parser/libnvdsinfer_custom_impl_lpr.so" + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 2 + operate_on_class_ids: [0] + interval: 0 + async_mode: false +} + +output_control { + output_tensor_meta: false +} diff --git a/src/apps/tao_apps/configs/triton/mask2former_tao/pgie_mask2former_tao_config.yml b/src/apps/tao_apps/configs/triton/mask2former_tao/pgie_mask2former_tao_config.yml new file mode 100644 index 00000000..6a173b57 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/mask2former_tao/pgie_mask2former_tao_config.yml @@ -0,0 +1,77 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 4 + backend { + triton { + model_name: "mask2former" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.017507 + channel_offsets: [123.675, 116.280, 103.53] + } + } + + postprocess { + detection { + per_class_params { + key: 0 + value { pre_threshold: 0.8 } + } + num_detected_classes: 2 + custom_parse_bbox_func: "NvDsInferParseCustomMask2Former" + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton/peoplenet_tao/config_infer_primary_peoplenet.yml b/src/apps/tao_apps/configs/triton/peoplenet_tao/config_infer_primary_peoplenet.yml new file mode 100644 index 00000000..8d993864 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/peoplenet_tao/config_infer_primary_peoplenet.yml @@ -0,0 +1,73 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 2 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peoplenet" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: MEDIA_FORMAT_NONE + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../../../models/peoplenet/labels.txt" + detection { + num_detected_classes: 3 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/configs/triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml b/src/apps/tao_apps/configs/triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml new file mode 100644 index 00000000..7a5d1dcd --- /dev/null +++ b/src/apps/tao_apps/configs/triton/peoplenet_transformer_tao/pgie_peoplenet_transformer_tao_config.yml @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "peoplenet_transformer" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0173520735728 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "../../../models/peoplenet_transformer/labels.txt" + detection { + num_detected_classes: 4 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/configs/triton/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml b/src/apps/tao_apps/configs/triton/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml new file mode 100644 index 00000000..95ca9d54 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/reidentificationnet_tao/sgie_reidentificationnet_tao_config.yml @@ -0,0 +1,71 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "reidentificationnet" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + interval: 0 + operate_on_class_ids: [1] +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml b/src/apps/tao_apps/configs/triton/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml new file mode 100644 index 00000000..842f6d3f --- /dev/null +++ b/src/apps/tao_apps/configs/triton/retail_object_detection_tao/pgie_retail_object_detection_binary_dino_tao_config.yml @@ -0,0 +1,72 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "retail_object_detection_binary_dino" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + maintain_aspect_ratio: 1 + symmetric_padding: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01735207357279195 + channel_offsets: [123.675, 116.28, 103.53] + } + } + postprocess { + labelfile_path: "../../../models/retail_object_detection_binary_dino/class_map.txt" + detection { + num_detected_classes: 2 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + custom_lib { + path: "../../../post_processor/libnvds_infercustomparser_tao.so" + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} + diff --git a/src/apps/tao_apps/configs/triton/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml b/src/apps/tao_apps/configs/triton/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml new file mode 100644 index 00000000..da3ed9f0 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/retail_object_recognition_tao/sgie_retail_object_recognition_tao_config.yml @@ -0,0 +1,71 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 2 + gpu_ids: 0 + max_batch_size: 16 + backend { + triton { + model_name: "retail_object_recognition" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.01460167693791089023079776340069 + channel_offsets: [122.7709383,116.7460125,104.09373615] + } + } + + postprocess { + labelfile_path: "../../../models/retail_object_recognition/retail_object_recognition_labels.txt" + other {} + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 64 + } +} + +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + interval: 0 +} + +output_control { + output_tensor_meta: true +} + diff --git a/src/apps/tao_apps/configs/triton/trafficcamnet_tao/pgie_trafficcamnet_config.txt b/src/apps/tao_apps/configs/triton/trafficcamnet_tao/pgie_trafficcamnet_config.txt new file mode 100644 index 00000000..27d9ade8 --- /dev/null +++ b/src/apps/tao_apps/configs/triton/trafficcamnet_tao/pgie_trafficcamnet_config.txt @@ -0,0 +1,75 @@ +################################################################################ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + model_repo { + root: "../../../models" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_GPU + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../../../models/trafficcamnet/labels_trafficnet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} \ No newline at end of file diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/README.md b/src/apps/tao_apps/deepstream_app_tao_configs/README.md new file mode 100644 index 00000000..6cbd8dcb --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/README.md @@ -0,0 +1,93 @@ +This document describes the procedure to download and run the TAO pre-trained purpose-built models in DeepStream. + +The following pre-trained models are provided: + +- VehicleMakeNet (https://ngc.nvidia.com/catalog/models/nvidia:tao:vehiclemakenet) +- VehicleTypeNet (https://ngc.nvidia.com/catalog/models/nvidia:tao:vehicletypenet) +- TrafficeCamNet (https://ngc.nvidia.com/catalog/models/nvidia:tao:trafficcamnet) +- PeopleNet (https://ngc.nvidia.com/catalog/models/nvidia:tao:peoplenet) +- PeopleNet Transformer v2 (https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2) + +******************************************************************************************* +## 1. Download the config files + +******************************************************************************************* +```bash +$ git clone https://github.com/NVIDIA/deepstream.git +$ cd deepstream/src/apps/tao_apps/deepstream_app_tao_configs +$ sudo apt install -y wget zip +``` + +******************************************************************************************* +## 2. Prepare Pretrained Models +******************************************************************************************* +Choose one of the following three inferencing methods: +- For TensorRT based inferencing, please run the following commands + +```bash +$ sudo cp -a * /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ +$ cd /opt/nvidia/deepstream/deepstream/samples/configs/tao_pretrained_models/ +$ sudo ./download_models.sh +``` + +- For Triton Inference Server based inferencing, the DeepStream application works as the Triton client: + * To set up the native Triton Inference Sever, please refer to [triton_server](triton_server.md). + + * To set up the separated Triton Inference Sever, please refer to [triton_server_grpc](triton_server_grpc.md) + +******************************************************************************* +## 3. Run the models in DeepStream +******************************************************************************* +```bash +$ sudo deepstream-app -c deepstream_app_source1_$MODEL.txt +``` +e.g. +```bash +$ sudo deepstream-app -c deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.txt +``` + +The yaml config files can also be used +```bash +$ sudo deepstream-app -c deepstream_app_source1_$MODEL.yml +``` +e.g. +```bash +$ sudo deepstream-app -c deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.yml +``` + +**Note:** + +1. For which model of the *deepstream_app_source1_$MODEL.txt* uses, please find from the **[primary-gie]** section in it, for example + + Below is the **[primary-gie]** config of deepstream_app_source1_peoplenet.txt, which indicates it uses peoplenet model by default, and user can change to peoplenet_transformer_v2 by commenting "config-file="config-file=nvinfer/config_infer_primary_peoplenet.txt" and uncommenting the corresponding "config-file=" . + + ```ini + [primary-gie] + enable=1 + #(0): nvinfer; (1): nvinferserver + plugin-type=0 + gpu-id=0 + # Modify as necessary + batch-size=1 + #Required by the app for OSD, not a plugin property + bbox-border-color0=1;0;0;1 + bbox-border-color1=0;1;1;1 + bbox-border-color2=0;0;1;1 + bbox-border-color3=0;1;0;1 + gie-unique-id=1 + config-file=nvinfer/config_infer_primary_peoplenet.txt + #config-file=nvinfer/config_infer_primary_peoplenet_transformer_v2.txt + #config-file=triton/config_infer_primary_peoplenet.txt + #config-file=triton/config_infer_primary_peoplenet_transformer_v2.txt + #config-file=triton-grpc/config_infer_primary_peoplenet.txt + #config-file=triton-grpc/config_infer_primary_peoplenet_transformer_v2.txt + ``` + +2. The GIE can be set to nvinfer or nvinferserver with the configuration file. For Triton grpc mode, the DeepStream application should run in different machine or terminal from the server. + +******************************************************************************* +## 4. Related Links +******************************************************************************* +deepstream-tao-app : https://github.com/DeepStream/deepstream/-/tree/main/src/apps/tao_apps + +TAO Toolkit Guide : https://docs.nvidia.com/tao/tao-toolkit/index.html diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.txt new file mode 100644 index 00000000..2a72f455 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.txt @@ -0,0 +1,139 @@ +################################################################################ +# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=1 + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +num-sources=1 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 +gpu-id=0 + +[streammux] +gpu-id=0 +batch-size=1 +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 + +[sink0] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File +type=2 +sync=1 +source-id=0 +gpu-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=3 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial + +[primary-gie] +enable=1 +#(0): nvinfer; (1): nvinferserver +plugin-type=0 +gpu-id=0 +# Modify as necessary +batch-size=1 +#Required by the app for OSD, not a plugin property +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +config-file=nvinfer/config_infer_primary_peoplenet.txt +#config-file=nvinfer/config_infer_primary_peoplenet_transformer_v2.txt +#config-file=triton/config_infer_primary_peoplenet.txt +#config-file=triton/config_infer_primary_peoplenet_transformer_v2.txt +#config-file=triton-grpc/config_infer_primary_peoplenet.txt +#config-file=triton-grpc/config_infer_primary_peoplenet_transformer_v2.txt + +[sink1] +enable=0 +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=2000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +output-file=out.mp4 +source-id=0 + +[sink2] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay +type=4 +#1=h264 2=h265 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=4000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +# set below properties in case of RTSPStreaming +rtsp-port=8554 +udp-port=5400 + +[tracker] +enable=1 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 + +[tests] +file-loop=0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.yml new file mode 100644 index 00000000..9b18bf39 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_peoplenet.yml @@ -0,0 +1,128 @@ +################################################################################ +# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 1 + +tiled-display: + enable: 1 + rows: 1 + columns: 1 + width: 1280 + height: 720 + gpu-id: 0 + +source: + csv-file-path: source_tao_app.csv + +streammux: + gpu-id: 0 + batch-size: 1 + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + +sink0: + enable: 1 + #Type - 1=FakeSink 2=EglSink 3=File + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + +osd: + enable: 1 + gpu-id: 0 + border-width: 3 + text-size: 15 + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Arial + +primary-gie: + enable: 1 + gpu-id: 0 + # Modify as necessary + batch-size: 1 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + gie-unique-id: 1 + config-file: nvinfer/config_infer_primary_peoplenet.yml + #config-file: nvinfer/config_infer_primary_peoplenet_transformer_v2.yml + +sink1: + enable: 0 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 3=mpeg4 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1: Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +tracker: + enable: 1 + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.txt new file mode 100644 index 00000000..f2540246 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.txt @@ -0,0 +1,136 @@ +################################################################################ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=1 + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +num-sources=1 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 +gpu-id=0 + +[streammux] +gpu-id=0 +batch-size=1 +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 + +[sink0] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File +type=2 +sync=1 +source-id=0 +gpu-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=3 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial + +[primary-gie] +enable=1 +#(0): nvinfer; (1): nvinferserver +plugin-type=0 +gpu-id=0 +# Modify as necessary +batch-size=1 +#Required by the app for OSD, not a plugin property +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +config-file=nvinfer/config_infer_primary_trafficcamnet.txt +#config-file=triton/config_infer_primary_trafficcamnet.txt +#config-file=triton-grpc/config_infer_primary_trafficcamnet.txt + +[sink1] +enable=0 +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=2000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +output-file=out.mp4 +source-id=0 + +[sink2] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay +type=4 +#1=h264 2=h265 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=4000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +# set below properties in case of RTSPStreaming +rtsp-port=8554 +udp-port=5400 + +[tracker] +enable=1 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 + +[tests] +file-loop=0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.yml new file mode 100644 index 00000000..211accef --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet.yml @@ -0,0 +1,127 @@ +################################################################################ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 1 + +tiled-display: + enable: 1 + rows: 1 + columns: 1 + width: 1280 + height: 720 + gpu-id: 0 + +source: + csv-file-path: source_tao_app.csv + +streammux: + gpu-id: 0 + batch-size: 1 + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + +sink0: + enable: 1 + #Type - 1=FakeSink 2=EglSink 3=File + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + +osd: + enable: 1 + gpu-id: 0 + border-width: 3 + text-size: 15 + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Arial + +primary-gie: + enable: 1 + gpu-id: 0 + # Modify as necessary + batch-size: 1 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + gie-unique-id: 1 + config-file: nvinfer/config_infer_primary_trafficcamnet.yml + +sink1: + enable: 0 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 3=mpeg4 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +tracker: + enable: 1 + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.txt new file mode 100644 index 00000000..eaa0a94d --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.txt @@ -0,0 +1,163 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=1 + +[tiled-display] +enable=1 +rows=1 +columns=1 +width=1280 +height=720 +gpu-id=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +num-sources=1 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 +gpu-id=0 + +[streammux] +gpu-id=0 +batch-size=1 +batched-push-timeout=40000 +## Set muxer output width and height +width=1920 +height=1080 + +[sink0] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File +type=2 +sync=1 +source-id=0 +gpu-id=0 + +[osd] +enable=1 +gpu-id=0 +border-width=3 +text-size=15 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Arial + +[primary-gie] +enable=1 +gpu-id=0 +#(0): nvinfer; (1): nvinferserver +plugin-type=0 +# Modify as necessary +batch-size=1 +#Required by the app for OSD, not a plugin property +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +gie-unique-id=1 +config-file=nvinfer/config_infer_primary_trafficcamnet.txt +#config-file=triton/config_infer_primary_trafficcamnet.txt +#config-file=triton-grpc/config_infer_primary_trafficcamnet.txt + +[sink1] +enable=0 +type=3 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 3=mpeg4 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=2000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +output-file=out.mp4 +source-id=0 + +[sink2] +enable=0 +#Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay +type=4 +#1=h264 2=h265 +codec=1 +#encoder type 0=Hardware 1=Software +enc-type=0 +sync=0 +bitrate=4000000 +#H264 Profile - 0=Baseline 2=Main 4=High +#H265 Profile - 0=Main 1=Main10 +profile=0 +# set below properties in case of RTSPStreaming +rtsp-port=8554 +udp-port=5400 + +[tracker] +enable=1 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 +compute-hw=1 + +[secondary-gie0] +enable=1 +#(0): nvinfer; (1): nvinferserver +plugin-type=0 +gpu-id=0 +batch-size=4 +gie-unique-id=4 +operate-on-gie-id=1 +operate-on-class-ids=0; +config-file=nvinfer/config_infer_secondary_vehiclemakenet.txt +#config-file=triton/config_infer_secondary_vehiclemakenet.txt +#config-file=triton-grpc/config_infer_secondary_vehiclemakenet.txt + +[secondary-gie1] +enable=1 +#(0): nvinfer; (1): nvinferserver +plugin-type=0 +gpu-id=0 +batch-size=4 +gie-unique-id=5 +operate-on-gie-id=1 +operate-on-class-ids=0; +config-file=nvinfer/config_infer_secondary_vehicletypenet.txt +#config-file=triton/config_infer_secondary_vehicletypenet.txt +#config-file=triton-grpc/config_infer_secondary_vehicletypenet.txt + +[tests] +file-loop=0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.yml new file mode 100644 index 00000000..45b97272 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/deepstream_app_source1_trafficcamnet_vehiclemakenet_vehicletypenet.yml @@ -0,0 +1,149 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +application: + enable-perf-measurement: 1 + perf-measurement-interval-sec: 1 + +tiled-display: + enable: 1 + rows: 1 + columns: 1 + width: 1280 + height: 720 + gpu-id: 0 + +source: + csv-file-path: source_tao_app.csv + +streammux: + gpu-id: 0 + batch-size: 1 + batched-push-timeout: 40000 + ## Set muxer output width and height + width: 1920 + height: 1080 + +sink0: + enable: 1 + #Type - 1: FakeSink 2: EglSink 3: File + type: 2 + sync: 1 + source-id: 0 + gpu-id: 0 + +osd: + enable: 1 + gpu-id: 0 + border-width: 3 + text-size: 15 + text-color: 1;1;1;1 + text-bg-color: 0.3;0.3;0.3;1 + font: Arial + +primary-gie: + enable: 1 + gpu-id: 0 + # Modify as necessary + model-engine-file: ../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_int8.engine + batch-size: 1 + #Required by the app for OSD, not a plugin property + bbox-border-color0: 1;0;0;1 + bbox-border-color1: 0;1;1;1 + bbox-border-color2: 0;0;1;1 + bbox-border-color3: 0;1;0;1 + gie-unique-id: 1 + config-file: nvinfer/config_infer_primary_trafficcamnet.yml + +sink1: + enable: 0 + type: 3 + #1=mp4 2=mkv + container: 1 + #1=h264 2=h265 3=mpeg4 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 2000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + output-file: out.mp4 + source-id: 0 + +sink2: + enable: 0 + #Type - 1=FakeSink 2=EglSink 3=File 4=RTSPStreaming 5=Overlay + type: 4 + #1=h264 2=h265 + codec: 1 + #encoder type 0=Hardware 1=Software + enc-type: 0 + sync: 0 + bitrate: 4000000 + #H264 Profile - 0=Baseline 2=Main 4=High + #H265 Profile - 0=Main 1=Main10 + profile: 0 + # set below properties in case of RTSPStreaming + rtsp-port: 8554 + udp-port: 5400 + +tracker: + enable: 1 + # For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively + tracker-width: 640 + tracker-height: 384 + ll-lib-file: /opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so + # ll-config-file required to set different tracker types + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml + ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml + # ll-config-file: /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml + gpu-id: 0 + enable-batch-process: 1 + enable-past-frame: 1 + display-tracking-id: 1 + compute-hw: 1 + +secondary-gie0: + enable: 1 + model-engine-file: ../../models/tao_pretrained_models/vehiclemakenet/resnet18_vehiclemakenet_pruned.onnx_b4_gpu0_int8.engine + gpu-id: 0 + batch-size: 4 + gie-unique-id: 4 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: nvinfer/config_infer_secondary_vehiclemakenet.yml + +secondary-gie1: + enable: 1 + model-engine-file: ../../models/tao_pretrained_models/vehicletypenet/resnet18_vehicletypenet_pruned.onnx_b4_gpu0_int8.engine + gpu-id: 0 + batch-size: 4 + gie-unique-id: 5 + operate-on-gie-id: 1 + operate-on-class-ids: 0 + config-file: nvinfer/config_infer_secondary_vehicletypenet.yml + +tests: + file-loop: 0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/download_models.sh b/src/apps/tao_apps/deepstream_app_tao_configs/download_models.sh new file mode 100755 index 00000000..0eec942a --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/download_models.sh @@ -0,0 +1,73 @@ +#!/bin/sh +################################################################################ +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Check following part for how to download the TAO models: +# https://docs.nvidia.com/tao/tao-toolkit/text/ds_tao/deepstream_tao_integration.html + +echo "===================================================================" +echo "begin download models for vehiclemakenet / vehicletypenet" +echo " / trafficcamnet" +echo "===================================================================" +mkdir -p ../../models/tao_pretrained_models/vehiclemakenet && \ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ +-O ../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned_int8.txt' \ +-O ../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ +-O ../../models/tao_pretrained_models/vehiclemakenet/labels.txt +mkdir -p ../../models/tao_pretrained_models/vehicletypenet && \ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ +-O ../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ +-O ../../models/tao_pretrained_models/vehicletypenet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned_int8.txt' \ +-O ../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned_int8.txt +mkdir -p ../../models/tao_pretrained_models/trafficcamnet && \ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=labels.txt' \ +-O ../../models/tao_pretrained_models/trafficcamnet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned.onnx' \ +-O ../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned_int8.txt' \ +-O ../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned_int8.txt + +echo "===================================================================" +echo "begin download models for peopleNet " +echo "===================================================================" +mkdir -p ../../models/tao_pretrained_models/peopleNet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' \ +-O ../../models/tao_pretrained_models/peopleNet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' \ +-O ../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.txt' \ +-O ../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.txt + +echo "===================================================================" +echo "begin download models for peoplenet_transformer_v2 " +echo "===================================================================" +mkdir -p ../../models/tao_pretrained_models/peoplenet_transformer_v2/ +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet_transformer/versions/deployable_v1.0/files/labels.txt -O ../../models/tao_pretrained_models/peoplenet_transformer_v2/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O ../../models/tao_pretrained_models/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx + +echo "===================================================================" +echo "Download models successfully " +echo "===================================================================" diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet.txt new file mode 100644 index 00000000..ef12c0ae --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet.txt @@ -0,0 +1,3 @@ +Person +Bag +Face diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet_transform_v2.txt b/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet_transform_v2.txt new file mode 100644 index 00000000..4b02dbbc --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/labels_peoplenet_transform_v2.txt @@ -0,0 +1,4 @@ +BG +Person +Face +Bag diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/labels_trafficnet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/labels_trafficnet.txt new file mode 100644 index 00000000..3f7921af --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/labels_trafficnet.txt @@ -0,0 +1,4 @@ +Car +Bicycle +Person +Roadsign diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehiclemakenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehiclemakenet.txt new file mode 100644 index 00000000..7a047ed7 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehiclemakenet.txt @@ -0,0 +1 @@ +acura;audi;bmw;chevrolet;chrysler;dodge;ford;gmc;honda;hyundai;infiniti;jeep;kia;lexus;mazda;mercedes;nissan;subaru;toyota;volkswagen diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehicletypenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehicletypenet.txt new file mode 100644 index 00000000..48960874 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/labels_vehicletypenet.txt @@ -0,0 +1 @@ +coupe;largevehicle;sedan;suv;truck;van diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.txt new file mode 100644 index 00000000..45d4c707 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.txt @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#tlt-model-key=tlt_encode +onnx-file=../../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx +labelfile-path=../labels_peoplenet.txt +model-engine-file=../../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine +input-dims=3;544;960;0 +uff-input-blob-name=input_1:0 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=3 +cluster-mode=2 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 + +#Use the config params below for dbscan clustering mode +#[class-attrs-all] +#detected-min-w=4 +#detected-min-h=4 +#minBoxes=3 +#eps=0.7 + +#Use the config params below for NMS clustering mode +[class-attrs-all] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 + +#[class-attrs-1] +#pre-cluster-threshold=0.05 +#eps=0.7 +#dbscan-min-score=0.5 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.yml new file mode 100644 index 00000000..e7d2cdbd --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet.yml @@ -0,0 +1,64 @@ +################################################################################ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + onnx-file: ../../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx + labelfile-path: ../labels_peoplenet.txt + model-engine-file: ../../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine + infer-dims: 3;544;960 + uff-input-blob-name: input_1:0 + batch-size: 1 + process-mode: 1 + model-color-format: 0 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + num-detected-classes: 3 + cluster-mode: 2 + interval: 0 + gie-unique-id: 1 + output-blob-names: output_bbox/BiasAdd:0;output_cov/Sigmoid:0 + +#Use the config params below for dbscan clustering mode +#class-attrs-all: + #detected-min-w: 4 + #detected-min-h: 4 + #minBoxes: 3 + #eps: 0.7 + +#Use the config params below for NMS clustering mode +class-attrs-all: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.2 + +## Per class configurations +class-attrs-0: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.4 + +#class-attrs-1: + #pre-cluster-threshold: 0.05 + #eps: 0.7 + #dbscan-min-score: 0.5 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.txt b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.txt new file mode 100644 index 00000000..ac3411c5 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.txt @@ -0,0 +1,45 @@ +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +offsets=123.675;116.28;103.53 +net-scale-factor=0.0173520735728 +labelfile-path=../../../models/tao_pretrained_models/peoplenet_transformer_v2/labels.txt +model-engine-file=../../../models/tao_pretrained_models/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx_b1_gpu0_fp16.engine +onnx-file=../../../models/tao_pretrained_models/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx +batch-size=1 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=pred_boxes;pred_logits +infer-dims=3;544;960 +workspace-size=1048576 +cluster-mode=4 +parse-bbox-func-name=NvDsInferParseCustomDDETRTAO +custom-lib-path=/opt/nvidia/deepstream/deepstream/lib/libnvds_infercustomparser.so + +[class-attrs-all] +pre-cluster-threshold=0.5 +topk=20 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.yml b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.yml new file mode 100644 index 00000000..28d01fed --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_peoplenet_transformer_v2.yml @@ -0,0 +1,45 @@ +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + offsets: 123.675;116.28;103.53 + net-scale-factor: 0.0173520735728 + labelfile-path: ../../../models/tao_pretrained_models/peoplenet_transformer_v2/labels.txt + model-engine-file: ../../../models/tao_pretrained_models/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx_b1_gpu0_fp16.engine + onnx-file: ../../../models/tao_pretrained_models/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx + batch-size: 1 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: pred_boxes;pred_logits + infer-dims: 3;544;960 + workspace-size: 1048576 + cluster-mode: 4 + parse-bbox-func-name: NvDsInferParseCustomDDETRTAO + custom-lib-path: /opt/nvidia/deepstream/deepstream/lib/libnvds_infercustomparser.so + +class-attrs-all: + pre-cluster-threshold: 0.5 + topk: 20 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.txt new file mode 100644 index 00000000..5e3dbf19 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.txt @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +onnx-file=../../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx +labelfile-path=../labels_trafficnet.txt +model-engine-file=../../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine +input-dims=3;544;960;0 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0 +cluster-mode=2 +scaling-compute-hw=1 + +#Use the config params below for dbscan clustering mode +#[class-attrs-all] +#detected-min-w=4 +#detected-min-h=4 +#minBoxes=3 +#eps=0.7 + +#Use the config params below for NMS clustering mode +[class-attrs-all] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.2 + +## Per class configurations +[class-attrs-0] +topk=20 +nms-iou-threshold=0.5 +pre-cluster-threshold=0.4 + +#[class-attrs-1] +#pre-cluster-threshold=0.05 +#eps=0.7 +#dbscan-min-score=0.5 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.yml new file mode 100644 index 00000000..2d44874c --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_primary_trafficcamnet.yml @@ -0,0 +1,65 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 0.0039215697906911373 + onnx-file: ../../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx + labelfile-path: ../labels_trafficnet.txt + model-engine-file: ../../../models/tao_pretrained_models/trafficcamnet/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine + infer-dims: 3;544;960 + uff-input-blob-name: input_1 + batch-size: 1 + process-mode: 1 + model-color-format: 0 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: output_cov/Sigmoid:0;output_bbox/BiasAdd:0 + cluster-mode: 2 + scaling-compute-hw: 1 + +#Use the config params below for dbscan clustering mode +#class-attrs-all: + #detected-min-w: 4 + #detected-min-h: 4 + #minBoxes: 3 + #eps: 0.7 + +#Use the config params below for NMS clustering mode +class-attrs-all: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.2 + +## Per class configurations +class-attrs-0: + topk: 20 + nms-iou-threshold: 0.5 + pre-cluster-threshold: 0.4 + +#class-attrs-1: + #pre-cluster-threshold: 0.05 + #eps: 0.7 + #dbscan-min-score: 0.5 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.txt new file mode 100644 index 00000000..f5dbaac2 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.txt @@ -0,0 +1,43 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=1 +offsets=124;117;104 +onnx-file=../../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned.onnx +labelfile-path=../labels_vehiclemakenet.txt +model-engine-file=../../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned.onnx_b4_gpu0_fp16.engine +input-dims=3;224;224;0 +uff-input-blob-name=input_1 +batch-size=4 +process-mode=2 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +network-type=1 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=predictions/Softmax:0 +classifier-threshold=0.2 +scaling-compute-hw=1 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.yml new file mode 100644 index 00000000..af0225cc --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehiclemakenet.yml @@ -0,0 +1,43 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 1 + offsets: 124;117;104 + onnx-file: ../../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned.onnx + labelfile-path: ../labels_vehiclemakenet.txt + model-engine-file: ../../../models/tao_pretrained_models/vehiclemakenet/resnet18_pruned.onnx_b4_gpu0_fp16.engine + infer-dims: 3;224;224 + uff-input-blob-name: input_1 + batch-size: 4 + process-mode: 2 + model-color-format: 0 + ## 0=FP32, 1=INT8, 2=FP16 mode + network-mode: 2 + network-type: 1 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: predictions/Softmax:0 + classifier-threshold: 0.2 + scaling-compute-hw: 1 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.txt new file mode 100644 index 00000000..2344f4fc --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.txt @@ -0,0 +1,43 @@ +################################################################################ +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[property] +gpu-id=0 +net-scale-factor=1 +offsets=124;117;104 +onnx-file=../../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned.onnx +labelfile-path=../labels_vehicletypenet.txt +model-engine-file=../../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned.onnx_b4_gpu0_fp16.engine +input-dims=3;224;224;0 +uff-input-blob-name=input_1 +batch-size=4 +process-mode=2 +model-color-format=0 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +network-type=1 +num-detected-classes=4 +interval=0 +gie-unique-id=1 +output-blob-names=predictions/Softmax:0 +classifier-threshold=0.2 +scaling-compute-hw=1 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.yml b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.yml new file mode 100644 index 00000000..668e683a --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/nvinfer/config_infer_secondary_vehicletypenet.yml @@ -0,0 +1,43 @@ +################################################################################ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +property: + gpu-id: 0 + net-scale-factor: 1 + offsets: 124;117;104 + onnx-file: ../../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned.onnx + labelfile-path: ../labels_vehicletypenet.txt + model-engine-file: ../../../models/tao_pretrained_models/vehicletypenet/resnet18_pruned.onnx_b4_gpu0_fp16.engine + infer-dims: 3;224;224 + uff-input-blob-name: input_1 + batch-size: 4 + process-mode: 2 + model-color-format: 0 + ## 0:FP32, 1:INT8, 2:FP16 mode + network-mode: 2 + network-type: 1 + num-detected-classes: 4 + interval: 0 + gie-unique-id: 1 + output-blob-names: predictions/Softmax:0 + classifier-threshold: 0.2 + scaling-compute-hw: 1 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_test.sh b/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_test.sh new file mode 100755 index 00000000..26ff91dd --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_test.sh @@ -0,0 +1,38 @@ +#!/bin/sh +################################################################################ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +#This script is to download peoplenet pretrained model and run peoplenet DeepStream application + +echo "===================================================================" +echo "begin download models for peopleNet " +echo "===================================================================" +mkdir -p ../../models/tao_pretrained_models/peopleNet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' \ +-O ../../models/tao_pretrained_models/peopleNet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' \ +-O ../../models/tao_pretrained_models/peopleNet/resnet34_peoplenet_int8.onnx + +echo "===================================================================" +echo "Run deepstream-app" +echo "===================================================================" +deepstream-app -c deepstream_app_source1_peoplenet.txt diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_triton_test.sh b/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_triton_test.sh new file mode 100755 index 00000000..c96a6028 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/peoplenet_triton_test.sh @@ -0,0 +1,42 @@ +#!/bin/sh +################################################################################ +# Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +#This script is to download peoplenet pretrained model and run peoplenet DeepStream application + +echo "===================================================================" +echo "begin download models for peopleNet " +echo "===================================================================" +mkdir -p ./triton/peopleNet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' \ +-O ./triton/peopleNet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' \ +-O ./triton/peopleNet/resnet34_peoplenet_int8.onnx + +mkdir -p ./triton/peopleNet/1 +trtexec --onnx=./triton/peopleNet/resnet34_peoplenet_int8.onnx --fp16 --saveEngine=./triton/peopleNet/1/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 --optShapes="input_1:0":1x3x544x960 --maxShapes="input_1:0":1x3x544x960 +cp triton/peopleNet_config.pbtxt ./triton/peopleNet/config.pbtxt + +echo "===================================================================" +echo "Run deepstream-app" +echo "===================================================================" +deepstream-app -c deepstream_app_source1_peoplenet.txt diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/prepare_triton_models.sh b/src/apps/tao_apps/deepstream_app_tao_configs/prepare_triton_models.sh new file mode 100755 index 00000000..48051c27 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/prepare_triton_models.sh @@ -0,0 +1,86 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +IS_JETSON_PLATFORM=`uname -i | grep aarch64` + +export PATH=$PATH:/usr/src/tensorrt/bin + +mkdir -p ./triton/vehiclemakenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ +-O ./triton/vehiclemakenet/resnet18_pruned.onnx && \ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehiclemakenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ +-O ./triton/vehiclemakenet/labels.txt + +mkdir -p ./triton/vehiclemakenet/1 +trtexec --onnx=./triton/vehiclemakenet/resnet18_pruned.onnx --fp16 \ + --saveEngine=./triton/vehiclemakenet/1/resnet18_pruned.onnx_b4_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":4x3x224x224 --maxShapes="input_1:0":4x3x224x224 +cp triton/vehiclemakenet_config.pbtxt ./triton/vehiclemakenet/config.pbtxt + +mkdir -p ./triton/vehicletypenet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=resnet18_pruned.onnx' \ +-O ./triton/vehicletypenet/resnet18_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/vehicletypenet/pruned_onnx_v1.1.0/files?redirect=true&path=labels.txt' \ +-O ./triton/vehicletypenet/labels.txt + + +mkdir -p ./triton/vehicletypenet/1 +trtexec --onnx=./triton/vehicletypenet/resnet18_pruned.onnx --fp16 \ + --saveEngine=./triton/vehicletypenet/1/resnet18_pruned.onnx_b4_gpu0_fp16.engine --minShapes="input_1:0":1x3x224x224 \ + --optShapes="input_1:0":4x3x224x224 --maxShapes="input_1:0":4x3x224x224 +cp triton/vehicletypenet_config.pbtxt ./triton/vehicletypenet/config.pbtxt + +mkdir -p ./triton/peopleNet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' \ +-O ./triton/peopleNet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' \ +-O ./triton/peopleNet/resnet34_peoplenet_int8.onnx + + +mkdir -p ./triton/peopleNet/1 +trtexec --onnx=./triton/peopleNet/resnet34_peoplenet_int8.onnx --fp16 --saveEngine=./triton/peopleNet/1/resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 --optShapes="input_1:0":1x3x544x960 --maxShapes="input_1:0":1x3x544x960& +cp triton/peopleNet_config.pbtxt ./triton/peopleNet/config.pbtxt + +mkdir -p ./triton/trafficcamnet/ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=labels.txt' \ +-O ./triton/trafficcamnet/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned.onnx' \ +-O ./triton/trafficcamnet/resnet18_trafficcamnet_pruned.onnx + + +mkdir -p ./triton/trafficcamnet/1 +trtexec --onnx=./triton/trafficcamnet/resnet18_trafficcamnet_pruned.onnx --fp16 \ + --saveEngine=./triton/trafficcamnet/1/resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":1x3x544x960 --maxShapes="input_1:0":1x3x544x960 +cp triton/trafficcamnet_config.pbtxt ./triton/trafficcamnet/config.pbtxt + +mkdir -p ./triton/peoplenet_transformer_v2/ +wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet_transformer/versions/deployable_v1.0/files/labels.txt -O ./triton/peoplenet_transformer_v2/labels.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer_v2/deployable_v1.0/files?redirect=true&path=dino_fan_small_astro_delta.onnx' -O ./triton/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx + +mkdir -p ./triton/peoplenet_transformer_v2/1 +trtexec --onnx=./triton/peoplenet_transformer_v2/dino_fan_small_astro_delta.onnx --fp16 \ + --saveEngine=./triton/peoplenet_transformer_v2/1/dino_fan_small_astro_delta.onnx_b1_gpu0_fp16.engine \ + --minShapes="inputs":1x3x544x960 --optShapes="inputs":1x3x544x960 --maxShapes="inputs":1x3x544x960& +cp triton/peoplenet_transformer_v2_config.pbtxt ./triton/peoplenet_transformer_v2/config.pbtxt + diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/source_tao_app.csv b/src/apps/tao_apps/deepstream_app_tao_configs/source_tao_app.csv new file mode 100644 index 00000000..7d0b2521 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/source_tao_app.csv @@ -0,0 +1,2 @@ +enable,type,uri,num-sources,gpu-id,cudadec-memtype +1,3,file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4,1,0,0 diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet.txt new file mode 100644 index 00000000..aad933f2 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet.txt @@ -0,0 +1,82 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peopleNet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../labels_peoplenet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet_transformer_v2.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet_transformer_v2.txt new file mode 100644 index 00000000..e55aa634 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_peoplenet_transformer_v2.txt @@ -0,0 +1,78 @@ +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "peoplenet_transformer_v2" + version: -1 + grpc { + url: "localhost:8001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0173520735728 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "../labels_peoplenet_transform_v2.txt" + detection { + num_detected_classes: 4 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + + custom_lib { + path: "/opt/nvidia/deepstream/deepstream/lib/libnvds_infercustomparser.so" + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_trafficcamnet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_trafficcamnet.txt new file mode 100644 index 00000000..2b71a4e5 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_primary_trafficcamnet.txt @@ -0,0 +1,75 @@ +################################################################################ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: false + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../labels_trafficnet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehiclemakenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehiclemakenet.txt new file mode 100644 index 00000000..70e7a982 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehiclemakenet.txt @@ -0,0 +1,80 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 4 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "vehiclemakenet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../labels_vehiclemakenet.txt" + classification { + threshold: 0.2 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehicletypenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehicletypenet.txt new file mode 100644 index 00000000..e282c639 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton-grpc/config_infer_secondary_vehicletypenet.txt @@ -0,0 +1,80 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 4 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "vehicletypenet" + version: -1 + grpc { + url: "localhost:10001" + enable_cuda_buffer_sharing: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../labels_vehicletypenet.txt" + classification { + threshold: 0.2 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet.txt new file mode 100644 index 00000000..60390a17 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet.txt @@ -0,0 +1,82 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "output_bbox/BiasAdd:0"}, + {name: "output_cov/Sigmoid:0"} + ] + triton { + model_name: "peopleNet" + version: -1 + model_repo { + root: "./" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../labels_peoplenet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet_transformer_v2.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet_transformer_v2.txt new file mode 100644 index 00000000..462df6a8 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_peoplenet_transformer_v2.txt @@ -0,0 +1,73 @@ +################################################################################ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +infer_config { + unique_id: 1 + gpu_ids: 0 + max_batch_size: 1 + backend { + triton { + model_name: "peoplenet_transformer_v2" + version: -1 + model_repo { + root: "./" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "inputs" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0173520735728 + channel_offsets: [123.675,116.28,103.53] + } + } + + postprocess { + labelfile_path: "./peoplenet_transformer_v2/labels.txt" + detection { + num_detected_classes: 4 + custom_parse_bbox_func: "NvDsInferParseCustomDDETRTAO" + nms { + confidence_threshold: 0.5 + iou_threshold: 0.5 + topk: 20 + } + } + } + + custom_lib { + path: "/opt/nvidia/deepstream/deepstream/lib/libnvds_infercustomparser.so" + } +} + +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_trafficcamnet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_trafficcamnet.txt new file mode 100644 index 00000000..ff96dd4f --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_primary_trafficcamnet.txt @@ -0,0 +1,75 @@ +################################################################################ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 1 + gpu_ids: [0] + max_batch_size: 1 + backend { + triton { + model_name: "trafficcamnet" + version: -1 + model_repo { + root: "./" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 0.0039215697906911373 + channel_offsets: [0, 0, 0] + } + } + + postprocess { + labelfile_path: "../labels_trafficnet.txt" + detection { + num_detected_classes: 4 + per_class_params { + key: 0 + value { pre_threshold: 0.4 } + } + nms { + confidence_threshold:0.2 + topk:20 + iou_threshold:0.5 + } + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_FULL_FRAME + operate_on_gie_id: -1 + interval: 0 +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehiclemakenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehiclemakenet.txt new file mode 100644 index 00000000..09579425 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehiclemakenet.txt @@ -0,0 +1,80 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 2 + gpu_ids: [0] + max_batch_size: 4 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "vehiclemakenet" + version: -1 + model_repo { + root: "./" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../labels_vehiclemakenet.txt" + classification { + threshold: 0.2 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehicletypenet.txt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehicletypenet.txt new file mode 100644 index 00000000..cf6cdbc9 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/config_infer_secondary_vehicletypenet.txt @@ -0,0 +1,80 @@ +################################################################################ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +infer_config { + unique_id: 3 + gpu_ids: [0] + max_batch_size: 4 + backend { + inputs: [ { + name: "input_1:0" + }] + outputs: [ + {name: "predictions/Softmax:0"} + ] + triton { + model_name: "vehicletypenet" + version: -1 + model_repo { + root: "./" + strict_model_config: true + } + } + } + + preprocess { + network_format: IMAGE_FORMAT_RGB + tensor_order: TENSOR_ORDER_LINEAR + tensor_name: "input_1:0" + maintain_aspect_ratio: 0 + frame_scaling_hw: FRAME_SCALING_HW_DEFAULT + frame_scaling_filter: 1 + normalize { + scale_factor: 1.0 + channel_offsets: [124, 117, 104] + } + } + + postprocess { + labelfile_path: "../labels_vehicletypenet.txt" + classification { + threshold: 0.2 + } + } + + extra { + copy_input_to_host_buffers: false + output_buffer_pool_size: 2 + } +} +input_control { + process_mode: PROCESS_MODE_CLIP_OBJECTS + operate_on_gie_id: 1 + operate_on_class_ids: [0] + interval: 0 + async_mode: true + object_control { + bbox_filter { + min_width: 14 + min_height: 14 + } + } +} diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/peopleNet_config.pbtxt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/peopleNet_config.pbtxt new file mode 100644 index 00000000..54759074 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/peopleNet_config.pbtxt @@ -0,0 +1,53 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "peopleNet" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "resnet34_peoplenet_int8.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 12, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 3, 34, 60 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/peoplenet_transformer_v2_config.pbtxt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/peoplenet_transformer_v2_config.pbtxt new file mode 100644 index 00000000..4bd59237 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/peoplenet_transformer_v2_config.pbtxt @@ -0,0 +1,52 @@ +################################################################################ +# Copyright (c) 2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "peoplenet_transformer_v2" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "dino_fan_small_astro_delta.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "inputs" + data_type: TYPE_FP32 + dims: [ 3, 544, 960] + } +] +output [ + { + name: "pred_logits" + data_type: TYPE_FP32 + dims: [ 900, 4 ] + }, + { + name: "pred_boxes" + data_type: TYPE_FP32 + dims: [ 900, 4 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/trafficcamnet_config.pbtxt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/trafficcamnet_config.pbtxt new file mode 100644 index 00000000..0b74e561 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/trafficcamnet_config.pbtxt @@ -0,0 +1,53 @@ +################################################################################ +# Copyright (c) 2023-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "trafficcamnet" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 16, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 4, 34, 60 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehiclemakenet_config.pbtxt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehiclemakenet_config.pbtxt new file mode 100644 index 00000000..d0cf1d90 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehiclemakenet_config.pbtxt @@ -0,0 +1,48 @@ +################################################################################ +# Copyright (c) 2023-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "vehiclemakenet" +platform: "tensorrt_plan" +max_batch_size: 4 +default_model_filename: "resnet18_pruned.onnx_b4_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 224, 224 ] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [ 20 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehicletypenet_config.pbtxt b/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehicletypenet_config.pbtxt new file mode 100644 index 00000000..b6c38766 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton/vehicletypenet_config.pbtxt @@ -0,0 +1,48 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "vehicletypenet" +platform: "tensorrt_plan" +max_batch_size: 4 +default_model_filename: "resnet18_pruned.onnx_b4_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 224, 224 ] + } +] +output [ + { + name: "predictions/Softmax:0" + data_type: TYPE_FP32 + dims: [ 6 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton_server.md b/src/apps/tao_apps/deepstream_app_tao_configs/triton_server.md new file mode 100644 index 00000000..50a1983e --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton_server.md @@ -0,0 +1,15 @@ +## Prepare Triton Server For Native Inferencing +As mentioned in the README, the DeepStream applications should work as Triton client with Triton Server running natively for cAPIs. So the [Triton Inference Server libraries](https://github.com/triton-inference-server/client) should be installed in the machine. An easier way is to run the DeepStream application in the [DeepStream Triton container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream). + +Running DeepStream Triton container, takes the DeepStream 7.1 GA container as the example: + +```bash +$ docker run --gpus all -it --ipc=host --rm --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -v $(pwd):/samples -e DISPLAY=$DISPLAY -w /samples nvcr.io/nvidia/deepstream:9.0-gc-triton-devel +``` +Inside the container, prepare model engines for Triton server, the [tao-converter links](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/resources/tao-converter) inside the prepare_triton_models.sh scripts can be changed to proper versions according to the actual TensorRT version: + +```bash +$ ./prepare_triton_models.sh +``` + +Then the DeepStream sample application can be build and run inside this container. diff --git a/src/apps/tao_apps/deepstream_app_tao_configs/triton_server_grpc.md b/src/apps/tao_apps/deepstream_app_tao_configs/triton_server_grpc.md new file mode 100644 index 00000000..b805de44 --- /dev/null +++ b/src/apps/tao_apps/deepstream_app_tao_configs/triton_server_grpc.md @@ -0,0 +1,41 @@ +# Triton Server +## [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server) Bring Up + +DeepStream applications can work as Triton Inference client. So the corresponding Triton Inference Server should be started before the Triton client start to work. + +An immediate way to start a corresponding Triton Server is to use Triton containers provided in [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tritonserver). Since every DeepStream version has its corresponding Triton Server version, so the reliable way is to use the [DeepStream Triton container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream). + +* The Triton Server can be started in the same machine which the DeepStream application works in, please make sure the Triton Server is started in a new terminal. + +* The Triton Server can be started in another machine as the server which is coonected to the machine for DeepStream applications through ehternet. + +## Prepare Triton Server For gRPC Connection +The following steps take the DeepStream 7.1 GA as an example, if you use other DeepStream versions, the corresponding DeepStream Triton [image](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream) can be used. + +To start Triton Server with DeepStream Triton container, the docker should be run in a new terminal and the following commands should be run in the same path as the deepstream_app_tao_configs codes are downloaded: +* Start the Triton Inferece Server with DeepStream Triton docker + +```bash +# start Triton docker, 10001:8001 is used to map docker container's 8000 port to host's 10000 port, these ports can be changed. +$ sudo docker run --gpus all -it --ipc=host --rm --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -p 10000:8000 -p 10001:8001 -p 10002:8002 -v $(pwd):/samples -e DISPLAY=$DISPLAY -w /samples nvcr.io/nvidia/deepstream:9.0-gc-triton-devel + +$ ./prepare_triton_models.sh +``` + +If the server is running in the same machine as the DeepStream application, the following command can be used directly. If it is not, please set the gRPC url as the IP address of the server machine in all the configuration files in deepstream_app_tao_configs/triton-grpc: + +The gRPC url setting looks like: + +```proto +grpc { + url: "192.168.0.51:10001" +} +``` + +Then the Triton Server service can be started with the following command: + +```bash +$ tritonserver --model-repository=/samples/triton --strict-model-config=false --grpc-infer-allocation-pool-size=16 --log-verbose=1 --exit-on-error=false +``` + +The DeepStream sample application should run in another terminal with the Triton Inference client libraries installed. It is recommend to run the application in the DeepStream Triton container, please refer to [triton_server](triton_server.md) for how to start a DeepStream Triton container. diff --git a/src/apps/tao_apps/download_models.sh b/src/apps/tao_apps/download_models.sh new file mode 100755 index 00000000..700c72d2 --- /dev/null +++ b/src/apps/tao_apps/download_models.sh @@ -0,0 +1,161 @@ +#!/bin/sh +################################################################################ +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Check following part for how to download the TAO models: + +set -e + +echo "===================================================================" +echo "begin download models for Mask2Former " +echo "===================================================================" +mkdir -p models/mask2former +cd ./models/mask2former +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/mask2former/mask2former_swint_deployable_v1.0/files?redirect=true&path=mask2former_swint.onnx' \ +-O mask2former.onnx + +echo "===================================================================" +echo "begin downloading CitySemSegFormer model " +echo "===================================================================" +cd - +mkdir -p ./models/citysemsegformer +cd ./models/citysemsegformer +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/citysemsegformer/deployable_onnx_v1.0/files?redirect=true&path=citysemsegformer.onnx' -O citysemsegformer.onnx && \ +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/citysemsegformer/deployable_onnx_v1.0/files?redirect=true&path=labels.txt' -O labels.txt + +echo "===================================================================" +echo "begin downloading PeopleNet Transformer model " +echo "===================================================================" +cd - +mkdir -p ./models/peoplenet_transformer +cd ./models/peoplenet_transformer +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet_transformer/deployable_v1.1/files?redirect=true&path=resnet50_peoplenet_transformer_op17.onnx' \ + -O resnet50_peoplenet_transformer_op17.onnx +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplenet_transformer/versions/deployable_v1.0/files/labels.txt -O labels.txt + +echo "===================================================================" +echo "begin downloading Re-Identification model " +echo "===================================================================" +cd - +mkdir -p ./models/reidentificationnet +cd ./models/reidentificationnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/reidentificationnet/deployable_v1.2/files?redirect=true&path=resnet50_market1501_aicity156.onnx' -O resnet50_market1501_aicity156.onnx + +echo "========================================================================" +echo "begin downloading Retail Object Detection DINO vdeployable_binary model " +echo "========================================================================" +cd - +mkdir -p ./models/retail_object_detection_binary_dino +cd ./models/retail_object_detection_binary_dino +wget 'https://api.ngc.nvidia.com/v2/models/nvidia/tao/retail_object_detection/versions/deployable_binary_v2.0/files/class_map.txt' -O class_map.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/retail_object_detection/deployable_retail_object_detection_binary_v2.2.2.3/files?redirect=true&path=retail_object_detection_binary_v2.2.2.3.onnx' -O retail_object_detection_dino_binary.onnx + +echo "===================================================================" +echo "begin downloading Retail Object Recognition model " +echo "===================================================================" +cd - +mkdir -p ./models/retail_object_recognition +cd ./models/retail_object_recognition +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/retail_object_recognition/deployable_v2.0/files?redirect=true&path=FANDualHead_Base_NVIN_op16.onnx' -O retail_object_recognition.onnx +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/retail_object_recognition/versions/deployable_v2.0/files/recognitionv2_name_list.txt -O retail_object_recognition_labels.txt + +echo "===================================================================" +echo "begin downloading PeopleNet model " +echo "===================================================================" +cd - +mkdir -p ./models/peoplenet +cd ./models/peoplenet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.onnx' -O resnet34_peoplenet_int8.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=resnet34_peoplenet_int8.txt' -O resnet34_peoplenet_int8.txt +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplenet/pruned_quantized_decrypted_v2.3.4/files?redirect=true&path=labels.txt' -O labels.txt + +echo "===================================================================" +echo "begin downloading BodyPose3DNet model " +echo "===================================================================" +cd - +mkdir -p ./models/bodypose3dnet +cd ./models/bodypose3dnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/bodypose3dnet/deployable_accuracy_onnx_1.0/files?redirect=true&path=bodypose3dnet_accuracy.onnx' -O bodypose3dnet_accuracy.onnx + +echo "===================================================================" +echo "begin downloading poseclassificationnet model " +echo "===================================================================" +cd - +mkdir -p ./models/poseclassificationnet +cd ./models/poseclassificationnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/poseclassificationnet/deployable_onnx_v1.0/files?redirect=true&path=st-gcn_3dbp_nvidia.onnx' -O st-gcn_3dbp_nvidia.onnx + +echo "===================================================================" +echo "begin downloading nvocdr model " +echo "===================================================================" +cd - +mkdir -p ./models/nvocdr +cd ./models/nvocdr +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/ocdnet/deployable_onnx_v2.4/files?redirect=true&path=ocdnet_fan_tiny_2x_icdar_pruned.onnx' -O ocdnet.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/ocdnet/deployable_onnx_v2.4/files?redirect=true&path=ocdnet_fan_tiny_2x_icdar_pruned.cal' -O ocdnet.cal +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/ocrnet/deployable_v2.1.1/files?redirect=true&path=ocrnet-vit-pcb.onnx' -O ocrnet.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/ocrnet/deployable_v2.0/files?redirect=true&path=character_list' -O character_list + +echo "===================================================================" +echo "begin downloading CAR Plate models " +echo "===================================================================" +cd - +mkdir -p ./models/trafficcamnet +cd ./models/trafficcamnet +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned.onnx' -O resnet18_trafficcamnet_pruned.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/trafficcamnet/pruned_onnx_v1.0.4/files?redirect=true&path=resnet18_trafficcamnet_pruned_int8.txt' -O resnet18_trafficcamnet_pruned_int8.txt + +cd - +mkdir -p ./models/LPD_us +cd ./models/LPD_us +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=LPDNet_usa_pruned_tao5.onnx' -O LPDNet_usa_pruned_tao5.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=usa_cal_10.1.0.bin' -O usa_cal_10.1.0.bin +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/lpdnet/versions/pruned_v1.0/files/usa_lpd_label.txt +cd - +mkdir -p ./models/LPD_ch +cd ./models/LPD_ch +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=LPDNet_CCPD_pruned_tao5.onnx' -O LPDNet_CCPD_pruned_tao5.onnx +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lpdnet/pruned_v2.3.1/files?redirect=true&path=ccpd_cal_10.1.0.bin' -O ccpd_cal_10.1.0.bin +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/lpdnet/versions/pruned_v1.0/files/ccpd_label.txt +cd - + +mkdir -p ./models/LPR_us +cd ./models/LPR_us +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lprnet/deployable_onnx_v1.1/files?redirect=true&path=us_lprnet_baseline18_deployable.onnx' -O us_lprnet_baseline18_deployable.onnx +touch labels_us.txt +cd - +mkdir -p ./models/LPR_ch +cd ./models/LPR_ch +wget --content-disposition 'https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/lprnet/deployable_onnx_v1.1/files?redirect=true&path=ch_lprnet_baseline18_deployable.onnx' -O ch_lprnet_baseline18_deployable.onnx +touch labels_ch.txt + +echo "===================================================================" +echo "begin downloading tracker model " +echo "===================================================================" +mkdir -p /opt/nvidia/deepstream/deepstream/samples/models/Tracker +cd /opt/nvidia/deepstream/deepstream/samples/models/Tracker +wget https://api.ngc.nvidia.com/v2/models/nvidia/tao/reidentificationnet/versions/deployable_v1.0/files/resnet50_market1501.etlt +# cp ./models/reidentificationnet/resnet50_market1501_aicity156.onnx /opt/nvidia/deepstream/deepstream/samples/models/Tracker + +echo "===================================================================" +echo "Download models successfully " +echo "===================================================================" diff --git a/src/apps/tao_apps/misc/CLA.md b/src/apps/tao_apps/misc/CLA.md new file mode 100644 index 00000000..389a4746 --- /dev/null +++ b/src/apps/tao_apps/misc/CLA.md @@ -0,0 +1,58 @@ +## Individual Contributor License Agreement (CLA) + +**Thank you for submitting your contributions to this project.** + +By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions +to the project. + +### License. + +You hereby represent that all present, past and future contributions are governed by the +[MIT License](https://opensource.org/licenses/MIT) +copyright statement. + +This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights +of the code or documents you contribute to the project itself or its maintainers. +Furthermore you also represent that you have the authority to perform the above waiver +with respect to the entirety of you contributions. + +### Moral Rights. + +To the fullest extent permitted under applicable law, you hereby waive, and agree not to +assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. + +### Third Party Content. + +If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, +specifications, documentation, data, materials, feedback, information or other works of authorship that were not +authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary +rights associated with your Contribution (“Third Party Rights”), +then you agree to include with the submission of your Contribution full details respecting such Third Party +Content and Third Party Rights, including, without limitation, identification of which aspects of your +Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the +Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable +third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater +certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights +do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. + +### Representations. + +You represent that, other than the Third Party Content and Third Party Rights identified by +you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled +to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were +created in the course of your employment with your past or present employer(s), you represent that such +employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer +(s) has waived all of their right, title or interest in or to your Contributions. + +### Disclaimer. + +To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" +basis, without any warranties or conditions, express or implied, including, without limitation, any implied +warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not +required to provide support for your Contributions, except to the extent you desire to provide support. + +### No Obligation. + +You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions +into the project. The decision to use or incorporate your contributions into the project will be made at the +sole discretion of the maintainers or their authorized delegates. diff --git a/src/apps/tao_apps/misc/LICENSE.md b/src/apps/tao_apps/misc/LICENSE.md new file mode 100644 index 00000000..87fcbbc2 --- /dev/null +++ b/src/apps/tao_apps/misc/LICENSE.md @@ -0,0 +1,9 @@ +The MIT License (MIT) + +Copyright (c) 2019-2021 NVIDIA CORPORATION + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/src/apps/tao_apps/misc/README.md b/src/apps/tao_apps/misc/README.md new file mode 100644 index 00000000..249fbfe2 --- /dev/null +++ b/src/apps/tao_apps/misc/README.md @@ -0,0 +1,3 @@ +TAO Toolkit helper files for TAO dev blogs of +- Training State-Of-The-Art Models for Classification and Object Detection with NVIDIA TAO Toolkit +- Real time vehicle license plate detection and recognition using NVIDIA TAO Toolkit diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_tfrecord.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_tfrecord.txt new file mode 100644 index 00000000..ad3132d7 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_tfrecord.txt @@ -0,0 +1,36 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +kitti_config { + root_directory_path: "/workspace/openalpr/lpd/data" + image_dir_name: "image" + label_dir_name: "label" + image_extension: ".jpg" + partition_mode: "random" + num_partitions: 2 + val_split: 20 + num_shards: 4 +} + +image_directory_path: "/workspace/openalpr/lpd/data" \ No newline at end of file diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_train.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_train.txt new file mode 100644 index 00000000..e26a467c --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/SPECS_train.txt @@ -0,0 +1,170 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +dataset_config { + data_sources { + tfrecords_path: "/workspace/openalpr/lpd_tfrecord/*" + image_directory_path: "/workspace/openalpr/lpd/data/" + } + image_extension: "jpg" + target_class_mapping { + key: "lpd" + value: "lpd" + } + validation_fold: 0 +} +augmentation_config { + preprocessing { + output_image_width: 640 + output_image_height: 480 + min_bbox_width: 1.0 + min_bbox_height: 1.0 + output_image_channel: 3 + } + spatial_augmentation { + hflip_probability: 0.5 + zoom_min: 1.0 + zoom_max: 1.0 + translate_max_x: 8.0 + translate_max_y: 8.0 + } + color_augmentation { + hue_rotation_max: 25.0 + saturation_shift_max: 0.20000000298 + contrast_scale_max: 0.10000000149 + contrast_center: 0.5 + } +} +postprocessing_config { + target_class_config { + key: "lpd" + value { + clustering_config { + coverage_threshold: 0.00499999988824 + dbscan_eps: 0.20000000298 + dbscan_min_samples: 0.0500000007451 + minimum_bounding_box_height: 4 + } + } + } +} +model_config { + pretrained_model_file: "/workspace/openalpr/usa_unpruned.tlt" + num_layers: 18 + use_batch_norm: true + objective_set { + bbox { + scale: 35.0 + offset: 0.5 + } + cov { + } + } + training_precision { + backend_floatx: FLOAT32 + } + arch: "resnet" +} +evaluation_config { + validation_period_during_training: 10 + first_validation_epoch: 1 + minimum_detection_ground_truth_overlap { + key: "lpd" + value: 0.699999988079 + } + evaluation_box_config { + key: "lpd" + value { + minimum_height: 10 + maximum_height: 9999 + minimum_width: 10 + maximum_width: 9999 + } + } + average_precision_mode: INTEGRATE +} +cost_function_config { + target_classes { + name: "lpd" + class_weight: 1.0 + coverage_foreground_weight: 0.0500000007451 + objectives { + name: "cov" + initial_weight: 1.0 + weight_target: 1.0 + } + objectives { + name: "bbox" + initial_weight: 10.0 + weight_target: 10.0 + } + } + enable_autoweighting: true + max_objective_weight: 0.999899983406 + min_objective_weight: 9.99999974738e-05 +} +training_config { + batch_size_per_gpu: 4 + num_epochs: 120 + enable_qat: False + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 5e-06 + max_learning_rate: 5e-04 + soft_start: 0.10000000149 + annealing: 0.699999988079 + } + } + regularizer { + type: L1 + weight: 3.00000002618e-09 + } + optimizer { + adam { + epsilon: 9.99999993923e-09 + beta1: 0.899999976158 + beta2: 0.999000012875 + } + } + cost_scaling { + initial_exponent: 20.0 + increment: 0.005 + decrement: 1.0 + } + checkpoint_interval: 10 +} +bbox_rasterizer_config { + target_class_config { + key: "lpd" + value { + cov_center_x: 0.5 + cov_center_y: 0.5 + cov_radius_x: 0.40000000596 + cov_radius_y: 0.40000000596 + bbox_min_radius: 1.0 + } + } + deadzone_radius: 0.400000154972 +} \ No newline at end of file diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/lpd_prepare_data.py b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/lpd_prepare_data.py new file mode 100644 index 00000000..8414bd4e --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpd/lpd_prepare_data.py @@ -0,0 +1,177 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +"""Script to prepare resized images/labels for LPD.""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +import argparse +import os +import cv2 +from PIL import Image + + +def parse_args(args=None): + """parse the arguments.""" + parser = argparse.ArgumentParser(description='Prepare resized images/labels dataset for LPD') + + + parser.add_argument( + "--input_dir", + type=str, + required=True, + help="Input directory to OpenALPR's benchmark end2end us license plates." + ) + + + parser.add_argument( + "--output_dir", + type=str, + required=True, + help="Ouput directory to resized images/labels." + ) + + + parser.add_argument( + "--target_width", + type=int, + required=True, + help="target width for resized images/labels." + ) + + + parser.add_argument( + "--target_height", + type=int, + required=True, + help="target height for resized images/labels." + ) + return parser.parse_args(args) + + + + +def prepare_data(input_dir, img_list, output_dir, output_size): + """Crop the license plates from the orginal images.""" + + + w, h = output_size + + + target_img_path = os.path.join(output_dir, "image") + target_label_path = os.path.join(output_dir, "label") + + + if not os.path.exists(target_img_path): + os.makedirs(target_img_path) + + + if not os.path.exists(target_label_path): + os.makedirs(target_label_path) + + + for img_name in img_list: + img_path = os.path.join(input_dir, img_name) + label_path = os.path.join(input_dir, + img_name.split(".")[0] + ".txt") + print("Resizing {} and its label".format(img_path)) + + + # resize labels + img = cv2.imread(img_path) + (height, width, _) = img.shape + ratio_w = float(float(w)/float(width) ) + ratio_h = float(float(h)/float(height) ) + + + with open(label_path, "r") as f: + label_lines = f.readlines() + assert len(label_lines) == 1 + label_items = label_lines[0].split() + + + assert img_name == label_items[0] + xmin = int(label_items[1]) + ymin = int(label_items[2]) + width = int(label_items[3]) + xmax = xmin + width + height = int(label_items[4]) + ymax = ymin + height + + + + + x1 = float(xmin) * ratio_w + y1 = float(ymin) * ratio_h + x2 = float(xmax) * ratio_w + y2 = float(ymax) * ratio_h + + + + + with open(os.path.join(target_label_path, + img_name.split(".")[0] + ".txt"), "w") as f: + f.write("lpd 0.0 0 0.0 {:.2f} {:.2f} {:.2f} {:.2f} 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n".format(x1, y1, x2, y2)) + + + # resize images + image = Image.open(img_path) + scale_image = image.resize(output_size, Image.ANTIALIAS) + scale_image.save(os.path.join(target_img_path, img_name)) + + +def main(args=None): + """Main function for data preparation.""" + + + args = parse_args(args) + + + w = args.target_width + h = args.target_height + output_size = (w,h) + + + img_files = [] + for file_name in os.listdir(args.input_dir): + if file_name.split(".")[-1] == "jpg": + img_files.append(file_name) + + + image_dir = os.path.join(args.output_dir, "data") + + + prepare_data(args.input_dir, img_files, image_dir, output_size) + + print("\nDone. Resized images/labels are saved at {} folder".format(args.output_dir)) + + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpd_config.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpd_config.txt new file mode 100644 index 00000000..74da76a8 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpd_config.txt @@ -0,0 +1,91 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +model-color-format=1 +labelfile-path=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPD/usa_lpd_label.txt +tlt-encoded-model=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPD/usa_pruned.etlt +tlt-model-key=nvidia_tlt +#model-engine-file=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPD/usa_pruned.etlt_b16_gpu0_int8.engine +int8-calib-file=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPD/usa_lpd_cal.bin +uff-input-dims=3;480;640;0 +uff-input-blob-name=input_1 +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=1 +num-detected-classes=1 +##1 Primary 2 Secondary +process-mode=2 +interval=0 +gie-unique-id=2 +#0 detector 1 classifier 2 segmentatio 3 instance segmentation +network-type=0 +operate-on-gie-id=1 +operate-on-class-ids=0 +#no cluster +cluster-mode=3 +output-blob-names=output_cov/Sigmoid;output_bbox/BiasAdd +input-object-min-height=30 +input-object-min-width=40 +#GPU: 1 VIC:2(Jetson only) +#scaling-compute-hw=2 +#enable-dla=1 + +[class-attrs-all] +pre-cluster-threshold=0.3 +roi-top-offset=0 +roi-bottom-offset=0 +detected-min-w=0 +detected-min-h=0 +detected-max-w=0 +detected-max-h=0 + diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpr_config_sgie_us.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpr_config_sgie_us.txt new file mode 100644 index 00000000..aa9e6737 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpdr_sample/lpr_config_sgie_us.txt @@ -0,0 +1,73 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8) +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# mean-file, gie-unique-id(Default=0), offsets, gie-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +model-engine-file=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPR/lpr_us_onnx_b16.engine +labelfile-path=/opt/nvidia/deepstream/deepstream-5.0/samples/models/LP/LPR/labels_us.txt +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=35 +gie-unique-id=3 +output-blob-names=output1 +#0=Detection 1=Classifier 2=Segmentation +network-type=1 +parse-classifier-func-name=NvDsInferParseCustomNVPlate +custom-lib-path=/opt/nvidia/deepstream/deepstream-5.0/lib/libnvdsinfer_custom_impl_lpr.so +process_mode=2 +operate-on-gie-id=2 +net-scale-factor=0.00392156862745098 +#net-scale-factor=1.0 +#0=RGB 1=BGR 2=GRAY +model-color-format=0 + +[class-attrs-all] +threshold=0.5 diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/preprocess_openalpr_benchmark.py b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/preprocess_openalpr_benchmark.py new file mode 100644 index 00000000..cbfcb398 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/preprocess_openalpr_benchmark.py @@ -0,0 +1,124 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +"""Script to prepare train/val dataset for LPRNet tutorial.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import os +import cv2 + + +def parse_args(args=None): + """parse the arguments.""" + parser = argparse.ArgumentParser(description='Prepare train/val dataset for LPRNet tutorial') + + parser.add_argument( + "--input_dir", + type=str, + required=True, + help="Input directory to OpenALPR's benchmark end2end us license plates." + ) + + parser.add_argument( + "--output_dir", + type=str, + required=True, + help="Ouput directory to TAO train/eval dataset." + ) + + return parser.parse_args(args) + + +def prepare_data(input_dir, img_list, output_dir): + """Crop the license plates from the orginal images.""" + + target_img_path = os.path.join(output_dir, "image") + target_label_path = os.path.join(output_dir, "label") + + if not os.path.exists(target_img_path): + os.makedirs(target_img_path) + + if not os.path.exists(target_label_path): + os.makedirs(target_label_path) + + for img_name in img_list: + img_path = os.path.join(input_dir, img_name) + label_path = os.path.join(input_dir, + img_name.split(".")[0] + ".txt") + + img = cv2.imread(img_path) + with open(label_path, "r") as f: + label_lines = f.readlines() + assert len(label_lines) == 1 + label_items = label_lines[0].split() + + assert img_name == label_items[0] + xmin = int(label_items[1]) + ymin = int(label_items[2]) + width = int(label_items[3]) + xmax = xmin + width + height = int(label_items[4]) + ymax = ymin + height + lp = label_items[5] + + cropped_lp = img[ymin:ymax, xmin:xmax, :] + + # save img and label + cv2.imwrite(os.path.join(target_img_path, img_name), cropped_lp) + with open(os.path.join(target_label_path, + img_name.split(".")[0] + ".txt"), "w") as f: + f.write(lp) + + +def main(args=None): + """Main function for data preparation.""" + + args = parse_args(args) + + img_files = [] + for file_name in os.listdir(args.input_dir): + if file_name.split(".")[-1] == "jpg": + img_files.append(file_name) + + total_cnt = len(img_files) + train_cnt = int(total_cnt * 0.8) + val_cnt = total_cnt - train_cnt + train_img_list = img_files[0:train_cnt] + val_img_list = img_files[train_cnt + 1:] + print("Total {} samples in benchmark dataset".format(total_cnt)) + print("{} for train and {} for val".format(train_cnt, val_cnt)) + + train_dir = os.path.join(args.output_dir, "train") + prepare_data(args.input_dir, train_img_list, train_dir) + + val_dir = os.path.join(args.output_dir, "val") + prepare_data(args.input_dir, val_img_list, val_dir) + + +if __name__ == "__main__": + main() diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec.txt new file mode 100644 index 00000000..91ccc162 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec.txt @@ -0,0 +1,70 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +lpr_config { + hidden_units: 512 + max_label_length: 8 + arch: "baseline" + nlayers: 18 #setting nlayers to be 10 to use baseline10 model +} +training_config { + batch_size_per_gpu: 32 + num_epochs: 24 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 1e-6 + max_learning_rate: 1e-5 + soft_start: 0.001 + annealing: 0.5 + } + } + regularizer { + type: L2 + weight: 5e-4 + } +} +eval_config { + validation_period_during_training: 5 + batch_size: 1 +} +augmentation_config { + output_width: 96 + output_height: 48 + output_channel: 3 + keep_original_prob: 0.3 + transform_prob: 0.5 + rotate_degree: 5 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/openalpr/train/label" + image_directory_path: "/workspace/tao-experiments/data/openalpr/train/image" + } + characters_list_file: "/workspace/tao-experiments/lprnet/specs/us_lp_characters.txt" + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/openalpr/val/label" + image_directory_path: "/workspace/tao-experiments/data/openalpr/val/image" + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec_scratch.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec_scratch.txt new file mode 100644 index 00000000..6a25455e --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/tutorial_spec_scratch.txt @@ -0,0 +1,69 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +random_seed: 42 +lpr_config { + hidden_units: 512 + max_label_length: 8 + arch: "baseline" + nlayers: 18 #setting nlayers to be 10 to use baseline10 model +} +training_config { + batch_size_per_gpu: 32 + num_epochs: 100 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 1e-6 + max_learning_rate: 1e-4 + soft_start: 0.001 + annealing: 0.5 + } + } + regularizer { + type: L2 + weight: 5e-4 + } +} +eval_config { + validation_period_during_training: 5 + batch_size: 1 +} +augmentation_config { + output_width: 96 + output_height: 48 + output_channel: 3 + keep_original_prob: 0.3 + transform_prob: 0.5 + rotate_degree: 5 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/openalpr/train/label" + image_directory_path: "/workspace/tao-experiments/data/openalpr/train/image" + } + characters_list_file: "/workspace/tao-experiments/lprnet/specs/us_lp_characters.txt" + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/openalpr/val/label" + image_directory_path: "/workspace/tao-experiments/data/openalpr/val/image" + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/us_lp_characters.txt b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/us_lp_characters.txt new file mode 100644 index 00000000..ce7f2546 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/LPDR/lpr/us_lp_characters.txt @@ -0,0 +1,35 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +A +B +C +D +E +F +G +H +I +J +K +L +M +N +P +Q +R +S +T +U +V +W +X +Y +Z diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/darknet/darknet53.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/darknet/darknet53.txt new file mode 100644 index 00000000..23e68d25 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/darknet/darknet53.txt @@ -0,0 +1,70 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "darknet" + n_layers: 53 + use_batch_norm: True + use_bias: False + use_imagenet_head: True + all_projections: False + use_pooling: True + input_image_size: "3,224,224" +} +train_config { + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + preprocess_mode: "torch" + enable_random_crop: True + enable_center_crop: True + label_smoothing: 0.0 + batch_size_per_gpu: 64 + n_epochs: 200 + mixup_alpha: 0.2 + + # Number of CPU cores for loading data + n_workers: 40 + + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 0.00003 + } + + lr_config { + cosine{ + learning_rate: 0.05 + soft_start: 0.0 + min_lr_ratio: 0.001 + } + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/efficientnetB0/spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/efficientnetB0/spec.txt new file mode 100644 index 00000000..4693f96a --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/efficientnetB0/spec.txt @@ -0,0 +1,64 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "efficientnet_b0" + use_bias: False + use_imagenet_head: True + resize_interpolation_method: BICUBIC + input_image_size: "3,224,224" +} +train_config { + preprocess_mode: "caffe" + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 200 + n_epochs: 500 + n_workers: 16 + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 5e-5 + } + lr_config { + cosine { + learning_rate: 0.05 + min_lr_ratio: 0.001 + } + } + enable_random_crop: True + enable_center_crop: True + enable_color_augmentation: True + mixup_alpha: 0.2 + label_smoothing: 0.1 +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet101/spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet101/spec.txt new file mode 100644 index 00000000..c12646a2 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet101/spec.txt @@ -0,0 +1,76 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "resnet" + n_layers: 101 + use_batch_norm: True + use_bias: False + all_projections: False + use_pooling: True + use_imagenet_head: True + resize_interpolation_method: BICUBIC + input_image_size: "3,224,224" +} +train_config { + preprocess_mode: "caffe" + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 50 + n_epochs: 150 + n_workers: 16 + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 0.000015 + } + lr_config { + cosine { + learning_rate: 0.04 + soft_start: 0.0 + } + } + enable_random_crop: True + enable_center_crop: True + enable_color_augmentation: True + mixup_alpha: 0.2 + label_smoothing: 0.1 +} +eval_config { + eval_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + model_path: "/workspace/tao-experiments/classification/resnet101/weights/resnet_150.tlt" + top_k: 1 + batch_size: 32 + n_workers: 8 + enable_center_crop: True +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_for_retinanet.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_for_retinanet.txt new file mode 100644 index 00000000..48f83f49 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_for_retinanet.txt @@ -0,0 +1,72 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "resnet" + n_layers: 50 + use_batch_norm: True + use_bias: False + all_projections: True + use_pooling: False + use_imagenet_head: True + input_image_size: "3,224,224" +} +train_config { + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.05 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 64 + n_epochs: 120 + n_workers: 16 + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 0.00003 + } + lr_config { + cosine { + learning_rate: 0.05 + soft_start: 0.0 + } + } + preprocess_mode: "caffe" + enable_random_crop: True + enable_center_crop: True + label_smoothing: 0.0 +} +eval_config { + eval_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + model_path: "/workspace/tao-experiments/classification/resnet50/weights/resnet50_120.tlt" + top_k: 1 + batch_size: 64 + n_workers: 8 + enable_center_crop: True +} \ No newline at end of file diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_imagenet.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_imagenet.txt new file mode 100644 index 00000000..490d55fd --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/resnet50/resnet50_imagenet.txt @@ -0,0 +1,74 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "resnet" + n_layers: 50 + use_batch_norm: True + use_bias: False + all_projections: False + use_pooling: True + use_imagenet_head: True + resize_interpolation_method: BICUBIC + input_image_size: "3,224,224" +} +train_config { + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.05 + decay: 0.0 + momentum: 0.9 + nesterov: False + } + } + batch_size_per_gpu: 64 + n_epochs: 120 + n_workers: 16 + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 0.000015 + } + lr_config { + cosine { + learning_rate: 0.05 + soft_start: 0.0 + } + } + preprocess_mode: "torch" + enable_random_crop: True + enable_center_crop: True + label_smoothing: 0.0 +} +eval_config { + eval_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + model_path: "/workspace/tao-experiments/classification/resnet50/weights/resnet_120.tlt" + top_k: 1 + batch_size: 64 + n_workers: 8 + enable_center_crop: True +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/classification/vgg16/spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/vgg16/spec.txt new file mode 100644 index 00000000..dcd88390 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/classification/vgg16/spec.txt @@ -0,0 +1,77 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +model_config { + arch: "vgg" + n_layers: 16 + use_batch_norm: True + use_bias: False + use_pooling: True + dropout: 0.0 + use_imagenet_head: True + resize_interpolation_method: BICUBIC + input_image_size: "3,224,224" +} +train_config { + preprocess_mode: "caffe" + train_dataset_path: "/workspace/tao-experiments/data/imagenet2012/train" + val_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + optimizer { + sgd { + lr: 0.01 + decay: 0.0 + momentum: 0.9 + nesterov: False + } +} + batch_size_per_gpu: 64 + n_epochs: 80 + n_workers: 16 + reg_config { + type: "L2" + scope: "Conv2D,Dense" + weight_decay: 0.00005 + } + lr_config { + soft_anneal { + learning_rate: 0.05 + soft_start: 0.056 + annealing_points: 0.3 + annealing_points: 0.6 + annealing_points: 0.8 + annealing_divider: 10 + } + } + enable_random_crop: True + enable_center_crop: True +} +eval_config { + eval_dataset_path: "/workspace/tao-experiments/data/imagenet2012/val" + model_path: "/workspace/tao-experiments/classification/vgg16/weights/vgg_080.tlt" + top_k: 1 + batch_size: 32 + n_workers: 8 + enable_center_crop: True +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/clean_dataset.sh b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/clean_dataset.sh new file mode 100644 index 00000000..a6c5608b --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/clean_dataset.sh @@ -0,0 +1,47 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +#!/bin/bash +for img in $1/*.jpg +do +kitti_name=$2/$(basename $img .jpg).txt +if [ ! -f $kitti_name ]; then +echo "$kitti_name not exist, will remove the pair." +rm $img +fi +done + +for kit in $2/*.txt +do +img_name=$1/$(basename $kit .txt).jpg +if [ ! -f $img_name ]; then +echo "$img_name not exist, will remove the pair." +rm $kit +fi +done + +echo "$1: $(ls $1|wc -l)" +echo "$2: $(ls $2|wc -l)" +echo Done! + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/coco2kitti.py b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/coco2kitti.py new file mode 100644 index 00000000..38aa655a --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/coco2kitti.py @@ -0,0 +1,83 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +import os +import sys +from pycocotools.coco import COCO + + +def coco2kitti(catNms, annFile): + # initialize COCO api for instance annotations + coco = COCO(annFile) + # Create an index for the category names + cats = coco.loadCats(coco.getCatIds()) + cat_idx = {} + for c in cats: + cat_idx[c['id']] = c['name'] + for img in coco.imgs: + # Get all annotation IDs for the image + catIds = coco.getCatIds(catNms=catNms) + annIds = coco.getAnnIds(imgIds=[img], catIds=catIds) + # If there are annotations, create a label file + if len(annIds) > 0: + # Get image filename + img_fname = coco.imgs[img]['file_name'] + # open text file + with open('./labels/' + img_fname.split('.')[0] + '.txt','w') as label_file: + anns = coco.loadAnns(annIds) + for a in anns: + bbox = a['bbox'] + # Convert COCO bbox coords to Kitti ones + bbox = [bbox[0], bbox[1], bbox[2] + bbox[0], bbox[3] + bbox[1]] + bbox = [str(b) for b in bbox] + catname = cat_idx[a['category_id']] + # Format line in label file + # Note: all whitespace will be removed from class names + out_str = [catname.replace(" ","") + + ' ' + ' '.join(['0.0']*1) + + ' ' + ' '.join(['0']*1) + + ' ' + ' '.join(['0.0']*1) + + ' ' + ' '.join([b for b in bbox]) + + ' ' + ' '.join(['0.0']*7) + +'\n'] + label_file.write(out_str[0]) +if __name__ == '__main__': + # These settings assume this script is in the annotations directory + dataDir = sys.argv[1] + dataType = sys.argv[2] + annFile = '%s/annotations/instances_%s.json' % (dataDir, dataType) + # If this list is populated then label files will only be produced + # for images containing the listed classes and only the listed classes + # will be in the label file + # EXAMPLE: + #catNms = ['person', 'dog', 'skateboard'] + catNms = [] + # Check if a labels file exists and, if not, make one + # If it exists already, exit to avoid overwriting + if os.path.isdir('./labels'): + print('Labels folder already exists - exiting to prevent badness') + else: + os.mkdir('./labels') + coco2kitti(catNms, annFile) + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/download_pascal_voc.sh b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/download_pascal_voc.sh new file mode 100644 index 00000000..48495350 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/download_pascal_voc.sh @@ -0,0 +1,44 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# download +cd ~/tao-experiments/data +wget http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar +wget http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar +wget http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar +tar xvf VOCtrainval_06-Nov-2007.tar +tar xvf VOCtrainval_11-May-2012.tar +tar xvf VOCtest_06-Nov-2007.tar +# Splitting datasets +mkdir -p ~/tao-experiments/data/voc0712trainval/images +mkdir -p ~/tao-experiments/data/voc0712trainval/labels +mkdir -p ~/tao-experiments/data/voc07test/images +mkdir -p ~/tao-experiments/data/voc07test/labels +cat ~/tao-experiments/data/VOCdevkit/VOC2007/ImageSets/Main/test.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc07test/images ~/tao-experiments/data/VOCdevkit/VOC2007/JPEGImages/'{}'.jpg +cat ~/tao-experiments/data/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc0712trainval/images ~/tao-experiments/data/VOCdevkit/VOC2007/JPEGImages/'{}'.jpg +cat ~/tao-experiments/data/VOCdevkit/VOC2012/ImageSets/Main/trainval.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc0712trainval/images ~/tao-experiments/data/VOCdevkit/VOC2012/JPEGImages/'{}'.jpg +cat ~/tao-experiments/data/VOCdevkit/VOC2007/ImageSets/Main/test.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc07test/labels ~/tao-experiments/data/VOCdevkit/VOC2007/Annotations/'{}'.xml +cat ~/tao-experiments/data/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc0712trainval/labels ~/tao-experiments/data/VOCdevkit/VOC2007/Annotations/'{}'.xml +cat ~/tao-experiments/data/VOCdevkit/VOC2012/ImageSets/Main/trainval.txt | xargs -I'{}' mv -t ~/tao-experiments/data/voc0712trainval/labels ~/tao-experiments/data/VOCdevkit/VOC2012/Annotations/'{}'.xml + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet.py b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet.py new file mode 100644 index 00000000..6600d063 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet.py @@ -0,0 +1,94 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +"""Prepare the ImageNet dataset""" +import os +import argparse +import tarfile +import pickle +import gzip +import subprocess +from tqdm import tqdm + +_TRAIN_TAR = 'ILSVRC2012_img_train.tar' +_VAL_TAR = 'ILSVRC2012_img_val.tar' + +def parse_args(): + parser = argparse.ArgumentParser( + description='Setup the ImageNet dataset.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--download-dir', required=True, + help="The directory that contains downloaded tar files") + parser.add_argument('--target-dir', required=True, + help="The directory to store extracted images") + parser.add_argument('--num-thread', type=int, default=1, + help="Number of threads to use when building image record file.") + args = parser.parse_args() + return args + +def extract_train(tar_fname, target_dir, num_thread=1): + os.makedirs(target_dir) + with tarfile.open(tar_fname) as tar: + print("Extracting "+tar_fname+"...") + # extract each class one-by-one + pbar = tqdm(total=len(tar.getnames())) + for class_tar in tar: + pbar.set_description('Extract '+class_tar.name) + tar.extract(class_tar, target_dir) + class_fname = os.path.join(target_dir, class_tar.name) + class_dir = os.path.splitext(class_fname)[0] + os.mkdir(class_dir) + with tarfile.open(class_fname) as f: + f.extractall(class_dir) + os.remove(class_fname) + pbar.update(1) + pbar.close() + +def extract_val(tar_fname, target_dir, num_thread=1): + os.makedirs(target_dir) + print('Extracting ' + tar_fname) + with tarfile.open(tar_fname) as tar: + tar.extractall(target_dir) + # move images to proper subfolders + val_maps_file = os.path.join(os.path.dirname(__file__), 'imagenet_val_maps.pklz') + with gzip.open(val_maps_file, 'rb') as f: + dirs, mappings = pickle.load(f) + for d in dirs: + os.makedirs(os.path.join(target_dir, d)) + for m in mappings: + os.rename(os.path.join(target_dir, m[0]), os.path.join(target_dir, m[1], m[0])) + +def main(): + args = parse_args() + target_dir = os.path.expanduser(args.target_dir) + if os.path.exists(target_dir): + raise ValueError('Target dir ['+target_dir+'] exists. Remove it first') + download_dir = os.path.expanduser(args.download_dir) + train_tar_fname = os.path.join(download_dir, _TRAIN_TAR) + val_tar_fname = os.path.join(download_dir, _VAL_TAR) + extract_train(train_tar_fname, os.path.join(target_dir, 'train'), args.num_thread) + extract_val(val_tar_fname, os.path.join(target_dir, 'val'), args.num_thread) + +if __name__ == '__main__': + main() diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet_val_maps.pklz b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet_val_maps.pklz new file mode 100644 index 00000000..fcf09283 Binary files /dev/null and b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/imagenet_val_maps.pklz differ diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/xml_to_kitti.py b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/xml_to_kitti.py new file mode 100644 index 00000000..60707f09 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/dataset_tools/xml_to_kitti.py @@ -0,0 +1,109 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +import argparse +import os +import xml.etree.ElementTree as ET + +def parse_args(args=None): + parser = argparse.ArgumentParser('Converting xml labels to KITTI format.') + parser.add_argument('-i', '--input_label_dir', type=str, required=True, help='directory of the input xml labels') + parser.add_argument('-o', '--output_label_dir', type=str, required=True, help='directory of the output KITTI labels') + parser.add_argument('-d', '--encode_difficult', action="store_true", required=False, help='Whether or not to encode the difficult object into KITTI labels') + args, _ = parser.parse_known_args(args) + return args + + +def xml_to_kitti(input_dir, output_dir, encode_difficult, classes): + if not os.path.exists(input_dir): + raise ValueError('input_dir not found.') + if not os.path.exists(output_dir): + raise ValueError('output_dir not found.') + for annot in os.listdir(input_dir): + et = ET.parse(os.path.join(input_dir, annot)) + element = et.getroot() + element_objs = element.findall('object') + element_width = int(element.find('size').find('width').text) + element_height = int(element.find('size').find('height').text) + element_depth = int(element.find('size').find('depth').text) + assert element_depth == 3 + assert len(element_objs) > 0, 'No objects in {}.'.format(os.path.join(input_dir, annot)) + lines = [] + for element_obj in element_objs: + difficulty = int(element_obj.find('difficult').text) == 1 + if difficulty and encode_difficult: + dif = '1' + else: + dif = '0' + line = '' + class_name = element_obj.find('name').text + assert class_name in classes + line += class_name + line += ' ' + line += '0 {} 0 '.format(dif) + obj_bbox = element_obj.find('bndbox') + x1 = int(round(float(obj_bbox.find('xmin').text))) + y1 = int(round(float(obj_bbox.find('ymin').text))) + x2 = int(round(float(obj_bbox.find('xmax').text))) + y2 = int(round(float(obj_bbox.find('ymax').text))) + line += str(x1) + line += ' ' + line += str(y1) + line += ' ' + line += str(x2) + line += ' ' + line += str(y2) + line += ' ' + line += '0 0 0 0 0 0 0\n' + lines.append(line) + with open(os.path.join(output_dir, os.path.basename(annot).split('.')[0]+'.txt'), 'w') as f: + f.writelines(lines) + + +if __name__ =='__main__': + classes = ['horse', + "pottedplant", + "train", + "person", + "bird", + "car", + "chair", + "tvmonitor", + "bus", + "sofa", + "dog", + "motorbike", + "bicycle", + "sheep", + "boat", + "cat", + "bottle", + "diningtable", + "cow", + "aeroplane", + "background", + ] + args = parse_args() + xml_to_kitti(args.input_label_dir, args.output_label_dir, args.encode_difficult, classes) + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_train/coco14_dataset_spec_train.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_train/coco14_dataset_spec_train.txt new file mode 100644 index 00000000..0502bdc7 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_train/coco14_dataset_spec_train.txt @@ -0,0 +1,36 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +kitti_config { + root_directory_path: "/workspace/tao-experiments/data/coco2014" + image_dir_name: "images/train2014" + label_dir_name: "KITTI/train2014" + image_extension: ".jpg" + partition_mode: "random" + num_partitions: 2 + val_split: 10 + num_shards: 10 +} +image_directory_path: "/workspace/tao-experiments/data/coco2014" + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_val/coco14_dataset_spec_val.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_val/coco14_dataset_spec_val.txt new file mode 100644 index 00000000..7a0c75bb --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/dataset_val/coco14_dataset_spec_val.txt @@ -0,0 +1,36 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +kitti_config { + root_directory_path: "/workspace/tao-experiments/data/coco2014" + image_dir_name: "images/val2014" + label_dir_name: "KITTI/val2014" + image_extension: ".jpg" + partition_mode: "random" + num_partitions: 2 + val_split: 10 + num_shards: 10 +} +image_directory_path: "/workspace/tao-experiments/data/coco2014" + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/spec.txt new file mode 100644 index 00000000..0f32c856 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_coco/spec.txt @@ -0,0 +1,521 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +enc_key: 'tao' +verbose: True +model_config { +input_image_config { +image_type: RGB +image_channel_order: 'bgr' +size_min { +min: 600 +} + image_channel_mean { + key: 'b' + value: 103.939 +} + image_channel_mean { + key: 'g' + value: 116.779 +} + image_channel_mean { + key: 'r' + value: 123.68 +} +image_scaling_factor: 1.0 +max_objects_num_per_image: 100 +} +arch: "resnet:101" +anchor_box_config { +scale: 32.0 +scale: 128.0 +scale: 256.0 +scale: 512.0 +ratio: 1.0 +ratio: 0.5 +ratio: 2.0 +} +freeze_bn: True +freeze_blocks: 1 +freeze_blocks: 2 +roi_mini_batch: 256 +rpn_stride: 16 +use_bias: False +roi_pooling_config { +pool_size: 7 +pool_size_2x: True +} +all_projections: False +use_pooling:True +} +dataset_config { + data_sources: { + tfrecords_path: "/workspace/tao-experiments/data/coco2014/tfrecords/train2014/*" + image_directory_path: "/workspace/tao-experiments/data/coco2014" + } +image_extension: 'jpg' + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "car" + value: "car" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "book" + value: "book" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "cup" + value: "cup" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "bowl" + value: "bowl" + } + target_class_mapping { + key: "trafficlight" + value: "trafficlight" + } + target_class_mapping { + key: "handbag" + value: "handbag" + } + target_class_mapping { + key: "umbrella" + value: "umbrella" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "truck" + value: "truck" + } + target_class_mapping { + key: "banana" + value: "banana" + } + target_class_mapping { + key: "bench" + value: "bench" + } + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "kite" + value: "kite" + } + target_class_mapping { + key: "backpack" + value: "backpack" + } + target_class_mapping { + key: "motorcycle" + value: "motorcycle" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "wineglass" + value: "wineglass" + } + target_class_mapping { + key: "knife" + value: "knife" + } + target_class_mapping { + key: "carrot" + value: "carrot" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "broccoli" + value: "broccoli" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "donut" + value: "donut" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "skis" + value: "skis" + } + target_class_mapping { + key: "vase" + value: "vase" + } + target_class_mapping { + key: "cake" + value: "cake" + } + target_class_mapping { + key: "orange" + value: "orange" + } + target_class_mapping { + key: "cellphone" + value: "cellphone" + } + target_class_mapping { + key: "sportsball" + value: "sportsball" + } + target_class_mapping { + key: "tie" + value: "tie" + } + target_class_mapping { + key: "apple" + value: "apple" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "spoon" + value: "spoon" + } + target_class_mapping { + key: "suitcase" + value: "suitcase" + } + target_class_mapping { + key: "clock" + value: "clock" + } + target_class_mapping { + key: "surfboard" + value: "surfboard" + } + target_class_mapping { + key: "couch" + value: "couch" + } + target_class_mapping { + key: "remote" + value: "remote" + } + target_class_mapping { + key: "skateboard" + value: "skateboard" + } + target_class_mapping { + key: "tv" + value: "tv" + } + target_class_mapping { + key: "pizza" + value: "pizza" + } + target_class_mapping { + key: "sink" + value: "sink" + } + target_class_mapping { + key: "elephant" + value: "elephant" + } + target_class_mapping { + key: "fork" + value: "fork" + } + target_class_mapping { + key: "airplane" + value: "airplane" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "giraffe" + value: "giraffe" + } + target_class_mapping { + key: "zebra" + value: "zebra" + } + target_class_mapping { + key: "teddybear" + value: "teddybear" + } + target_class_mapping { + key: "tennisracket" + value: "tennisracket" + } + target_class_mapping { + key: "laptop" + value: "laptop" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "sandwich" + value: "sandwich" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "toilet" + value: "toilet" + } + target_class_mapping { + key: "bed" + value: "bed" + } + target_class_mapping { + key: "baseballglove" + value: "baseballglove" + } + target_class_mapping { + key: "baseballbat" + value: "baseballbat" + } + target_class_mapping { + key: "oven" + value: "oven" + } + target_class_mapping { + key: "hotdog" + value: "hotdog" + } + target_class_mapping { + key: "keyboard" + value: "keyboard" + } + target_class_mapping { + key: "snowboard" + value: "snowboard" + } + target_class_mapping { + key: "refrigerator" + value: "refrigerator" + } + target_class_mapping { + key: "frisbee" + value: "frisbee" + } + target_class_mapping { + key: "mouse" + value: "mouse" + } + target_class_mapping { + key: "stopsign" + value: "stopsign" + } + target_class_mapping { + key: "firehydrant" + value: "firehydrant" + } + target_class_mapping { + key: "toothbrush" + value: "toothbrush" + } + target_class_mapping { + key: "microwave" + value: "microwave" + } + target_class_mapping { + key: "scissors" + value: "scissors" + } + target_class_mapping { + key: "bear" + value: "bear" + } + target_class_mapping { + key: "parkingmeter" + value: "parkingmeter" + } + target_class_mapping { + key: "toaster" + value: "toaster" + } + target_class_mapping { + key: "hairdrier" + value: "hairdrier" + } +validation_data_source: { + tfrecords_path: "/workspace/tao-experiments/data/coco2014/tfrecords/val2014/*" + image_directory_path: "/workspace/tao-experiments/data/coco2014" +} +} +augmentation_config { +preprocessing { +output_image_channel: 3 +output_image_min: 600 +output_image_max: 1000 +min_bbox_width: 1.0 +min_bbox_height: 1.0 +} +spatial_augmentation { +hflip_probability: 0.5 +vflip_probability: 0.0 +zoom_min: 1.0 +zoom_max: 1.0 +translate_max_x: 0 +translate_max_y: 0 +} +color_augmentation { +hue_rotation_max: 0.0 +saturation_shift_max: 0.0 +contrast_scale_max: 0.0 +contrast_center: 0.5 +} +} +training_config { +enable_augmentation: True +enable_qat: False +batch_size_per_gpu: 1 +num_epochs: 20 +pretrained_weights: "/workspace/tao-experiments/classification/resnet101/weights/resnet_150.tlt" +output_model: "/workspace/tao-experiments/coco2014/frcnn_resnet101.tlt" +rpn_min_overlap: 0.3 +rpn_max_overlap: 0.7 +classifier_min_overlap: 0.0 +classifier_max_overlap: 0.5 +gt_as_roi: False +std_scaling: 1.0 +classifier_regr_std { +key: 'x' +value: 10.0 +} +classifier_regr_std { +key: 'y' +value: 10.0 +} +classifier_regr_std { +key: 'w' +value: 5.0 +} +classifier_regr_std { +key: 'h' +value: 5.0 +} + +rpn_mini_batch: 256 +rpn_pre_nms_top_N: 12000 +rpn_nms_max_boxes: 2000 +rpn_nms_overlap_threshold: 0.7 + +regularizer { +type: L2 +weight: 1e-4 +} + +optimizer { +adam { +lr: 0.00001 +beta_1: 0.9 +beta_2: 0.999 +decay: 0.0 +} +} + +learning_rate { +step { +base_lr: 0.00001 +gamma: 0.1 +step_size: 70 +} +} + +lambda_rpn_regr: 1.0 +lambda_rpn_class: 1.0 +lambda_cls_regr: 1.0 +lambda_cls_class: 1.0 +} +inference_config { +images_dir: '/workspace/tao-experiments/data/coco2014/images/val2014' +model: '/workspace/tao-experiments/coco2014/frcnn_resnet101.epoch20.tlt' +batch_size: 1 +detection_image_output_dir: '/workspace/tao-experiments/coco2014/inference_results_imgs' +labels_dump_dir: '/workspace/tao-experiments/coco2014/inference_dump_labels' +rpn_pre_nms_top_N: 6000 +rpn_nms_max_boxes: 300 +rpn_nms_overlap_threshold: 0.7 +object_confidence_thres: 0.0001 +bbox_visualize_threshold: 0.6 +classifier_nms_max_boxes: 100 +classifier_nms_overlap_threshold: 0.3 +} + +evaluation_config { +model: '/workspace/tao-experiments/coco2014/frcnn_resnet101.epoch20.tlt' +batch_size: 1 +validation_period_during_training: 1 +rpn_pre_nms_top_N: 6000 +rpn_nms_max_boxes: 300 +rpn_nms_overlap_threshold: 0.7 +classifier_nms_max_boxes: 100 +classifier_nms_overlap_threshold: 0.5 +object_confidence_thres: 0.0001 +use_voc07_11point_metric: False +gt_matching_iou_threshold_range { +start: 0.5 +end: 1.0 +step: 0.05 +} +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_test/dataset_spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_test/dataset_spec.txt new file mode 100644 index 00000000..ec2c10c7 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_test/dataset_spec.txt @@ -0,0 +1,36 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +kitti_config { + root_directory_path: "/workspace/tao-experiments/data/voc07test" + image_dir_name: "images" + label_dir_name: "labels_kitti" + image_extension: ".jpg" + partition_mode: "random" + num_partitions: 2 + val_split: 10 + num_shards: 10 +} +image_directory_path: "/workspace/tao-experiments/data/voc07test" + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_train/dataset_spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_train/dataset_spec.txt new file mode 100644 index 00000000..9d358697 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/dataset_train/dataset_spec.txt @@ -0,0 +1,36 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +kitti_config { + root_directory_path: "/workspace/tao-experiments/data/voc0712trainval" + image_dir_name: "images" + label_dir_name: "labels_kitti" + image_extension: ".jpg" + partition_mode: "random" + num_partitions: 2 + val_split: 10 + num_shards: 10 +} +image_directory_path: "/workspace/tao-experiments/data/voc0712trainval" + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/spec.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/spec.txt new file mode 100644 index 00000000..86d70ff4 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/faster_rcnn/faster-rcnn_pascal_voc/spec.txt @@ -0,0 +1,278 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +enc_key: 'tao' +verbose: True +model_config { +input_image_config { +image_type: RGB +image_channel_order: 'bgr' +size_min { +min: 600 +} + image_channel_mean { + key: 'b' + value: 103.939 +} + image_channel_mean { + key: 'g' + value: 116.779 +} + image_channel_mean { + key: 'r' + value: 123.68 +} +image_scaling_factor: 1.0 +max_objects_num_per_image: 100 +} +arch: "vgg:16" +anchor_box_config { +scale: 32.0 +scale: 128.0 +scale: 256.0 +scale: 512.0 +ratio: 1.0 +ratio: 0.5 +ratio: 2.0 +} +freeze_bn: True +freeze_blocks: 1 +freeze_blocks: 2 +roi_mini_batch: 256 +rpn_stride: 16 +use_bias: False +roi_pooling_config { +pool_size: 7 +pool_size_2x: True +} +all_projections: False +use_pooling:True +} +dataset_config { + data_sources: { + tfrecords_path: "/workspace/tao-experiments/data/voc0712trainval/tfrecords*" + image_directory_path: "/workspace/tao-experiments/data/voc0712trainval" + } +image_extension: 'jpg' +target_class_mapping { +key: "horse" +value: "horse" +} +target_class_mapping { +key: "pottedplant" +value: "pottedplant" +} +target_class_mapping { +key: "train" +value: "train" +} +target_class_mapping { +key: "person" +value: "person" +} +target_class_mapping { +key: "bird" +value: "bird" +} +target_class_mapping { +key: "car" +value: "car" +} +target_class_mapping { +key: "chair" +value: "chair" +} +target_class_mapping { +key: "tvmonitor" +value: "tvmonitor" +} +target_class_mapping { +key: "bus" +value: "bus" +} +target_class_mapping { +key: "sofa" +value: "sofa" +} +target_class_mapping { +key: "dog" +value: "dog" +} +target_class_mapping { +key: "motorbike" +value: "motorbike" +} +target_class_mapping { +key: "bicycle" +value: "bicycle" +} +target_class_mapping { +key: "sheep" +value: "sheep" +} +target_class_mapping { +key: "boat" +value: "boat" +} +target_class_mapping { +key: "cat" +value: "cat" +} +target_class_mapping { +key: "bottle" +value: "bottle" +} +target_class_mapping { +key: "diningtable" +value: "diningtable" +} +target_class_mapping { +key: "cow" +value: "cow" +} +target_class_mapping { +key: "aeroplane" +value: "aeroplane" +} +validation_data_source: { + tfrecords_path: "/workspace/tao-experiments/data/voc07test/tfrecords*" + image_directory_path: "/workspace/tao-experiments/data/voc07test" +} +} +augmentation_config { +preprocessing { +output_image_channel: 3 +output_image_min: 600 +output_image_max: 1000 +min_bbox_width: 1.0 +min_bbox_height: 1.0 +} +spatial_augmentation { +hflip_probability: 0.5 +vflip_probability: 0.0 +zoom_min: 1.0 +zoom_max: 1.0 +translate_max_x: 0 +translate_max_y: 0 +} +color_augmentation { +hue_rotation_max: 0.0 +saturation_shift_max: 0.0 +contrast_scale_max: 0.0 +contrast_center: 0.5 +} +} +training_config { +enable_augmentation: True +enable_qat: False +batch_size_per_gpu: 1 +num_epochs: 20 +pretrained_weights: "/workspace/tao-experiments/classification/vgg16/weights/vgg_080.tlt" +output_model: "/workspace/tao-experiments/faster_rcnn/pascal_voc/vgg16_frcnn.tlt" +rpn_min_overlap: 0.3 +rpn_max_overlap: 0.7 +classifier_min_overlap: 0.0 +classifier_max_overlap: 0.5 +gt_as_roi: False +std_scaling: 1.0 +classifier_regr_std { +key: 'x' +value: 10.0 +} +classifier_regr_std { +key: 'y' +value: 10.0 +} +classifier_regr_std { +key: 'w' +value: 5.0 +} +classifier_regr_std { +key: 'h' +value: 5.0 +} + +rpn_mini_batch: 256 +rpn_pre_nms_top_N: 12000 +rpn_nms_max_boxes: 2000 +rpn_nms_overlap_threshold: 0.7 + +regularizer { +type: L2 +weight: 1e-4 +} + +optimizer { +adam { +lr: 0.00001 +beta_1: 0.9 +beta_2: 0.999 +decay: 0.0 +} +} + +learning_rate { +step { +base_lr: 0.00001 +gamma: 0.1 +step_size: 70 +} +} + +lambda_rpn_regr: 1.0 +lambda_rpn_class: 1.0 +lambda_cls_regr: 1.0 +lambda_cls_class: 1.0 +} +inference_config { +images_dir: '/workspace/tao-experiments/data/voc07test/images' +model: '/workspace/tao-experiments/faster_rcnn/pascal_voc/vgg16_frcnn.epoch20.tlt' +batch_size: 1 +detection_image_output_dir: '/workspace/tao-experiments/faster_rcnn/pascal_voc/inference_results_imgs' +labels_dump_dir: '/workspace/tao-experiments/faster_rcnn/pascal_voc/inference_dump_labels' +rpn_pre_nms_top_N: 6000 +rpn_nms_max_boxes: 300 +rpn_nms_overlap_threshold: 0.7 +object_confidence_thres: 0.0001 +bbox_visualize_threshold: 0.6 +classifier_nms_max_boxes: 100 +classifier_nms_overlap_threshold: 0.3 +bbox_caption_on: True +} + +evaluation_config { +model: '/workspace/tao-experiments/faster_rcnn/pascal_voc/vgg16_frcnn.epoch20.tlt' +batch_size: 1 +validation_period_during_training: 1 +rpn_pre_nms_top_N: 6000 +rpn_nms_max_boxes: 300 +rpn_nms_overlap_threshold: 0.7 +classifier_nms_max_boxes: 100 +classifier_nms_overlap_threshold: 0.5 +object_confidence_thres: 0.0001 +use_voc07_11point_metric: True +gt_matching_iou_threshold: 0.5 +} + diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/retinanet/retinanet_res50_coco.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/retinanet/retinanet_res50_coco.txt new file mode 100644 index 00000000..1b8c6f58 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/retinanet/retinanet_res50_coco.txt @@ -0,0 +1,409 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +augmentation_config { + output_width: 608 + output_height: 608 + output_channel: 3 +} +retinanet_config { + aspect_ratios_global: "[1.0, 2.0, 0.5]" + scales: "[0.05, 0.15, 0.3, 0.45, 0.6, 0.75]" + variances: "[0.1, 0.1, 0.2, 0.2]" + arch: "resnet" + loss_loc_weight: 1.0 + focal_loss_alpha: 0.25 + focal_loss_gamma: 2 + freeze_blocks: 0 + nlayers: 50 + n_kernels: 2 + n_anchor_levels: 1 + feature_size: 256 +} +training_config { + batch_size_per_gpu: 16 + num_epochs: 150 + pretrain_model_path: "/workspace/tlt-experiments/classification/resnet50/weights/resnet50_120.tlt" + checkpoint_interval: 1 + optimizer { + sgd { + momentum: 0.9 + nesterov: True + } + } + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 5e-5 + max_learning_rate: 1e-2 + soft_start: 0.08 + annealing: 0.32 + } + } + regularizer { + type: L1 + weight: 1e-6 + } +} +eval_config { + batch_size: 16 + validation_period_during_training: 50 + matching_iou_threshold: 0.5 +} +nms_config { + confidence_threshold: 0.01 + clustering_iou_threshold: 0.5 + top_k: 200 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tlt-experiments/data/coco2017/KITTI/train2017" + image_directory_path: "/workspace/tlt-experiments/data/coco2017/images/train2017" + } + include_difficult_in_training: true + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "car" + value: "car" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "book" + value: "book" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "cup" + value: "cup" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "bowl" + value: "bowl" + } + target_class_mapping { + key: "trafficlight" + value: "trafficlight" + } + target_class_mapping { + key: "handbag" + value: "handbag" + } + target_class_mapping { + key: "umbrella" + value: "umbrella" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "truck" + value: "truck" + } + target_class_mapping { + key: "banana" + value: "banana" + } + target_class_mapping { + key: "bench" + value: "bench" + } + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "kite" + value: "kite" + } + target_class_mapping { + key: "backpack" + value: "backpack" + } + target_class_mapping { + key: "motorcycle" + value: "motorcycle" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "wineglass" + value: "wineglass" + } + target_class_mapping { + key: "knife" + value: "knife" + } + target_class_mapping { + key: "carrot" + value: "carrot" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "broccoli" + value: "broccoli" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "donut" + value: "donut" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "skis" + value: "skis" + } + target_class_mapping { + key: "vase" + value: "vase" + } + target_class_mapping { + key: "cake" + value: "cake" + } + target_class_mapping { + key: "orange" + value: "orange" + } + target_class_mapping { + key: "cellphone" + value: "cellphone" + } + target_class_mapping { + key: "sportsball" + value: "sportsball" + } + target_class_mapping { + key: "tie" + value: "tie" + } + target_class_mapping { + key: "apple" + value: "apple" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "spoon" + value: "spoon" + } + target_class_mapping { + key: "suitcase" + value: "suitcase" + } + target_class_mapping { + key: "clock" + value: "clock" + } + target_class_mapping { + key: "surfboard" + value: "surfboard" + } + target_class_mapping { + key: "couch" + value: "couch" + } + target_class_mapping { + key: "remote" + value: "remote" + } + target_class_mapping { + key: "skateboard" + value: "skateboard" + } + target_class_mapping { + key: "tv" + value: "tv" + } + target_class_mapping { + key: "pizza" + value: "pizza" + } + target_class_mapping { + key: "sink" + value: "sink" + } + target_class_mapping { + key: "elephant" + value: "elephant" + } + target_class_mapping { + key: "fork" + value: "fork" + } + target_class_mapping { + key: "airplane" + value: "airplane" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "giraffe" + value: "giraffe" + } + target_class_mapping { + key: "zebra" + value: "zebra" + } + target_class_mapping { + key: "teddybear" + value: "teddybear" + } + target_class_mapping { + key: "tennisracket" + value: "tennisracket" + } + target_class_mapping { + key: "laptop" + value: "laptop" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "sandwich" + value: "sandwich" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "toilet" + value: "toilet" + } + target_class_mapping { + key: "bed" + value: "bed" + } + target_class_mapping { + key: "baseballglove" + value: "baseballglove" + } + target_class_mapping { + key: "baseballbat" + value: "baseballbat" + } + target_class_mapping { + key: "oven" + value: "oven" + } + target_class_mapping { + key: "hotdog" + value: "hotdog" + } + target_class_mapping { + key: "keyboard" + value: "keyboard" + } + target_class_mapping { + key: "snowboard" + value: "snowboard" + } + target_class_mapping { + key: "refrigerator" + value: "refrigerator" + } + target_class_mapping { + key: "frisbee" + value: "frisbee" + } + target_class_mapping { + key: "mouse" + value: "mouse" + } + target_class_mapping { + key: "stopsign" + value: "stopsign" + } + target_class_mapping { + key: "firehydrant" + value: "firehydrant" + } + target_class_mapping { + key: "toothbrush" + value: "toothbrush" + } + target_class_mapping { + key: "microwave" + value: "microwave" + } + target_class_mapping { + key: "scissors" + value: "scissors" + } + target_class_mapping { + key: "bear" + value: "bear" + } + target_class_mapping { + key: "parkingmeter" + value: "parkingmeter" + } + target_class_mapping { + key: "toaster" + value: "toaster" + } + target_class_mapping { + key: "hairdrier" + value: "hairdrier" + } + validation_data_sources: { + label_directory_path: "/workspace/tlt-experiments/data/coco2017/KITTI/val2017" + image_directory_path: "/workspace/tlt-experiments/data/coco2017/images/val2017" + } +} \ No newline at end of file diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_coco17.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_coco17.txt new file mode 100644 index 00000000..6a2793b1 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_coco17.txt @@ -0,0 +1,397 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +ssd_config { + aspect_ratios: "[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]]" + scales: "[0.07, 0.15, 0.33, 0.51, 0.69, 0.87, 1.05]" + two_boxes_for_ar1: true + clip_boxes: true + variances: "[0.1, 0.1, 0.2, 0.2]" + arch: "vgg" + nlayers: 16 + freeze_bn: false +} +training_config { + batch_size_per_gpu: 32 + num_epochs: 95 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 1e-5 + max_learning_rate: 0.001 + soft_start: 0.01 + annealing: 0.7 + } + } + regularizer { + type: L2 + weight: 0.0005 + } +} +eval_config { + validation_period_during_training: 10 + average_precision_mode: SAMPLE + batch_size: 1 + matching_iou_threshold: 0.5 +} +nms_config { + confidence_threshold: 0.01 + clustering_iou_threshold: 0.45 + top_k: 200 +} +augmentation_config { + output_channel: 3 + output_height: 300 + output_width: 300 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/coco2017/KITTI/train2017" + image_directory_path: "/workspace/tao-experiments/data/coco2017/images/train2017" + } + include_difficult_in_training: true + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "car" + value: "car" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "book" + value: "book" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "cup" + value: "cup" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "bowl" + value: "bowl" + } + target_class_mapping { + key: "trafficlight" + value: "trafficlight" + } + target_class_mapping { + key: "handbag" + value: "handbag" + } + target_class_mapping { + key: "umbrella" + value: "umbrella" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "truck" + value: "truck" + } + target_class_mapping { + key: "banana" + value: "banana" + } + target_class_mapping { + key: "bench" + value: "bench" + } + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "kite" + value: "kite" + } + target_class_mapping { + key: "backpack" + value: "backpack" + } + target_class_mapping { + key: "motorcycle" + value: "motorcycle" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "wineglass" + value: "wineglass" + } + target_class_mapping { + key: "knife" + value: "knife" + } + target_class_mapping { + key: "carrot" + value: "carrot" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "broccoli" + value: "broccoli" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "donut" + value: "donut" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "skis" + value: "skis" + } + target_class_mapping { + key: "vase" + value: "vase" + } + target_class_mapping { + key: "cake" + value: "cake" + } + target_class_mapping { + key: "orange" + value: "orange" + } + target_class_mapping { + key: "cellphone" + value: "cellphone" + } + target_class_mapping { + key: "sportsball" + value: "sportsball" + } + target_class_mapping { + key: "tie" + value: "tie" + } + target_class_mapping { + key: "apple" + value: "apple" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "spoon" + value: "spoon" + } + target_class_mapping { + key: "suitcase" + value: "suitcase" + } + target_class_mapping { + key: "clock" + value: "clock" + } + target_class_mapping { + key: "surfboard" + value: "surfboard" + } + target_class_mapping { + key: "couch" + value: "couch" + } + target_class_mapping { + key: "remote" + value: "remote" + } + target_class_mapping { + key: "skateboard" + value: "skateboard" + } + target_class_mapping { + key: "tv" + value: "tv" + } + target_class_mapping { + key: "pizza" + value: "pizza" + } + target_class_mapping { + key: "sink" + value: "sink" + } + target_class_mapping { + key: "elephant" + value: "elephant" + } + target_class_mapping { + key: "fork" + value: "fork" + } + target_class_mapping { + key: "airplane" + value: "airplane" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "giraffe" + value: "giraffe" + } + target_class_mapping { + key: "zebra" + value: "zebra" + } + target_class_mapping { + key: "teddybear" + value: "teddybear" + } + target_class_mapping { + key: "tennisracket" + value: "tennisracket" + } + target_class_mapping { + key: "laptop" + value: "laptop" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "sandwich" + value: "sandwich" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "toilet" + value: "toilet" + } + target_class_mapping { + key: "bed" + value: "bed" + } + target_class_mapping { + key: "baseballglove" + value: "baseballglove" + } + target_class_mapping { + key: "baseballbat" + value: "baseballbat" + } + target_class_mapping { + key: "oven" + value: "oven" + } + target_class_mapping { + key: "hotdog" + value: "hotdog" + } + target_class_mapping { + key: "keyboard" + value: "keyboard" + } + target_class_mapping { + key: "snowboard" + value: "snowboard" + } + target_class_mapping { + key: "refrigerator" + value: "refrigerator" + } + target_class_mapping { + key: "frisbee" + value: "frisbee" + } + target_class_mapping { + key: "mouse" + value: "mouse" + } + target_class_mapping { + key: "stopsign" + value: "stopsign" + } + target_class_mapping { + key: "firehydrant" + value: "firehydrant" + } + target_class_mapping { + key: "toothbrush" + value: "toothbrush" + } + target_class_mapping { + key: "microwave" + value: "microwave" + } + target_class_mapping { + key: "scissors" + value: "scissors" + } + target_class_mapping { + key: "bear" + value: "bear" + } + target_class_mapping { + key: "parkingmeter" + value: "parkingmeter" + } + target_class_mapping { + key: "toaster" + value: "toaster" + } + target_class_mapping { + key: "hairdrier" + value: "hairdrier" + } + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/coco2017/KITTI/val2017" + image_directory_path: "/workspace/tao-experiments/data/coco2017/images/val2017" + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_voc.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_voc.txt new file mode 100644 index 00000000..86b97290 --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/ssd/ssd_vgg16_voc.txt @@ -0,0 +1,161 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +ssd_config { + aspect_ratios: "[[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5, 3.0, 1.0/3.0], [1.0, 2.0, 0.5], [1.0, 2.0, 0.5]]" + scales: "[0.1, 0.2, 0.37, 0.54, 0.71, 0.88, 1.05]" + two_boxes_for_ar1: true + clip_boxes: true + variances: "[0.1, 0.1, 0.2, 0.2]" + arch: "vgg" + nlayers: 16 + freeze_bn: false + # uncomment in retrain + # freeze_bn: True + # freeze_blocks: 0 + # freeze_blocks: 1 +} +training_config { + batch_size_per_gpu: 32 + num_epochs: 240 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 1e-5 + max_learning_rate: 0.001 + soft_start: 0.001 + annealing: 0.65 + } + } + regularizer { + type: L2 + weight: 0.0005 + } +} +eval_config { + validation_period_during_training: 5 + average_precision_mode: SAMPLE + batch_size: 1 + matching_iou_threshold: 0.5 +} +nms_config { + confidence_threshold: 0.01 + clustering_iou_threshold: 0.45 + top_k: 200 +} +augmentation_config { + output_width: 300 + output_height: 300 + output_channel: 3 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/voc0712trainval/labels_kitti" + image_directory_path: "/workspace/tao-experiments/data/voc0712trainval/images" + } + include_difficult_in_training: true + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "motorbike" + value: "motorbike" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "sofa" + value: "sofa" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "aeroplane" + value: "aeroplane" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "tvmonitor" + value: "tvmonitor" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "car" + value: "car" + } + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/voc07test/labels_kitti" + image_directory_path: "/workspace/tao-experiments/data/voc07test/images" + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_coco.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_coco.txt new file mode 100644 index 00000000..8b6d112f --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_coco.txt @@ -0,0 +1,421 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +yolov3_config { +big_anchor_shape: "[(116,90), (156,198), (373,326)]" +mid_anchor_shape: "[(30,61), (62,45), (59,119)]" +small_anchor_shape: "[(10,13), (16,30), (33,23)]" +matching_neutral_box_iou: 0.5 +arch: "darknet" +nlayers: 53 +arch_conv_blocks: 2 + +loss_loc_weight: 0.8 +loss_neg_obj_weights: 80.0 +loss_class_weights: 1.0 + +freeze_bn: false +#freeze_blocks: 0 +force_relu: false +} + +training_config { + batch_size_per_gpu: 8 + num_epochs: 200 + enable_qat: false + checkpoint_interval: 10 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 5e-6 + max_learning_rate: 3e-4 + soft_start: 0.1 + annealing: 0.5 + } + } + regularizer { + type: L1 + weight: 3e-8 + } + optimizer { + adam { + epsilon: 1e-7 + beta1: 0.9 + beta2: 0.999 + amsgrad: false + } + } + pretrain_model_path: "/workspace/tao-experiments/classification/darknet53/weights/darknet_$EPOCH.tlt" +} +eval_config { + average_precision_mode: SAMPLE + batch_size: 8 + matching_iou_threshold: 0.5 +} +nms_config { + confidence_threshold: 0.001 + clustering_iou_threshold: 0.5 + top_k: 200 +} +augmentation_config { + hue: 0.1 + saturation: 1.5 + exposure:1.5 + vertical_flip:0 + horizontal_flip: 0.5 + jitter: 0.3 + output_width: 416 + output_height: 416 + randomize_input_shape_period: 10 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/coco2014/KITTI/train2014" + image_directory_path: "/workspace/tao-experiments/data/coco2014/images/train2014" + } + target_class_mapping { + key: "apple" + value: "apple" + } + target_class_mapping { + key: "book" + value: "book" + } + target_class_mapping { + key: "handbag" + value: "handbag" + } + target_class_mapping { + key: "car" + value: "car" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "backpack" + value: "backpack" + } + target_class_mapping { + key: "clock" + value: "clock" + } + target_class_mapping { + key: "truck" + value: "truck" + } + target_class_mapping { + key: "knife" + value: "knife" + } + target_class_mapping { + key: "cup" + value: "cup" + } + target_class_mapping { + key: "snowboard" + value: "snowboard" + } + target_class_mapping { + key: "suitcase" + value: "suitcase" + } + target_class_mapping { + key: "umbrella" + value: "umbrella" + } + target_class_mapping { + key: "bowl" + value: "bowl" + } + target_class_mapping { + key: "carrot" + value: "carrot" + } + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "fork" + value: "fork" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "pizza" + value: "pizza" + } + target_class_mapping { + key: "couch" + value: "couch" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "skis" + value: "skis" + } + target_class_mapping { + key: "keyboard" + value: "keyboard" + } + target_class_mapping { + key: "firehydrant" + value: "firehydrant" + } + target_class_mapping { + key: "tennisracket" + value: "tennisracket" + } + target_class_mapping { + key: "sandwich" + value: "sandwich" + } + target_class_mapping { + key: "toothbrush" + value: "toothbrush" + } + target_class_mapping { + key: "motorcycle" + value: "motorcycle" + } + target_class_mapping { + key: "remote" + value: "remote" + } + target_class_mapping { + key: "frisbee" + value: "frisbee" + } + target_class_mapping { + key: "mouse" + value: "mouse" + } + target_class_mapping { + key: "trafficlight" + value: "trafficlight" + } + target_class_mapping { + key: "oven" + value: "oven" + } + target_class_mapping { + key: "scissors" + value: "scissors" + } + target_class_mapping { + key: "airplane" + value: "airplane" + } + target_class_mapping { + key: "teddybear" + value: "teddybear" + } + target_class_mapping { + key: "refrigerator" + value: "refrigerator" + } + target_class_mapping { + key: "stopsign" + value: "stopsign" + } + target_class_mapping { + key: "bed" + value: "bed" + } + target_class_mapping { + key: "orange" + value: "orange" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "sink" + value: "sink" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "broccoli" + value: "broccoli" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "elephant" + value: "elephant" + } + target_class_mapping { + key: "tie" + value: "tie" + } + target_class_mapping { + key: "banana" + value: "banana" + } + target_class_mapping { + key: "donut" + value: "donut" + } + target_class_mapping { + key: "baseballglove" + value: "baseballglove" + } + target_class_mapping { + key: "surfboard" + value: "surfboard" + } + target_class_mapping { + key: "hotdog" + value: "hotdog" + } + target_class_mapping { + key: "skateboard" + value: "skateboard" + } + target_class_mapping { + key: "zebra" + value: "zebra" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "vase" + value: "vase" + } + target_class_mapping { + key: "baseballbat" + value: "baseballbat" + } + target_class_mapping { + key: "hairdrier" + value: "hairdrier" + } + target_class_mapping { + key: "cake" + value: "cake" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "laptop" + value: "laptop" + } + target_class_mapping { + key: "wineglass" + value: "wineglass" + } + target_class_mapping { + key: "bear" + value: "bear" + } + target_class_mapping { + key: "parkingmeter" + value: "parkingmeter" + } + target_class_mapping { + key: "tv" + value: "tv" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "toilet" + value: "toilet" + } + target_class_mapping { + key: "sportsball" + value: "sportsball" + } + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "microwave" + value: "microwave" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "bench" + value: "bench" + } + target_class_mapping { + key: "giraffe" + value: "giraffe" + } + target_class_mapping { + key: "spoon" + value: "spoon" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "toaster" + value: "toaster" + } + target_class_mapping { + key: "cellphone" + value: "cellphone" + } + target_class_mapping { + key: "kite" + value: "kite" + } + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/coco2014/KITTI/val2014" + image_directory_path: "/workspace/tao-experiments/data/coco2014/images/val2014" + } +} diff --git a/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_voc.txt b/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_voc.txt new file mode 100644 index 00000000..7a68281b --- /dev/null +++ b/src/apps/tao_apps/misc/dev_blog/SOTA/yolov3/v3_voc.txt @@ -0,0 +1,182 @@ +################################################################################ +# The MIT License (MIT) +# +# Copyright (c) 2019-2021 NVIDIA CORPORATION +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +random_seed: 42 +yolov3_config { +big_anchor_shape: "[(116,90), (156,198), (373,326)]" +mid_anchor_shape: "[(30,61), (62,45), (59,119)]" +small_anchor_shape: "[(10,13), (16,30), (33,23)]" +matching_neutral_box_iou: 0.5 +arch: "darknet" +nlayers: 53 +arch_conv_blocks: 2 + +loss_loc_weight: 0.8 +loss_neg_obj_weights: 100.0 +loss_class_weights: 1.0 + +freeze_bn: false +#freeze_blocks: 0 +force_relu: false +} + +training_config { + batch_size_per_gpu: 8 + num_epochs: 180 + enable_qat: false + checkpoint_interval: 10 + learning_rate { + soft_start_annealing_schedule { + min_learning_rate: 1e-6 + max_learning_rate: 1e-4 + soft_start: 0.1 + annealing: 0.5 + } + } + regularizer { + type: L1 + weight: 3e-8 + } + optimizer { + adam { + epsilon: 1e-7 + beta1: 0.9 + beta2: 0.999 + amsgrad: false + } + } + pretrain_model_path: "/workspace/tao-experiments/classification/darknet53/weights/darknet_$EPOCH.tlt" +} +eval_config { + average_precision_mode: SAMPLE + batch_size: 8 + matching_iou_threshold: 0.5 +} +nms_config { + confidence_threshold: 0.001 + clustering_iou_threshold: 0.5 + top_k: 200 +} +augmentation_config { + hue: 0.1 + saturation: 1.5 + exposure:1.5 + vertical_flip:0 + horizontal_flip: 0.5 + jitter: 0.3 + output_width: 416 + output_height: 416 + randomize_input_shape_period: 10 +} +dataset_config { + data_sources: { + label_directory_path: "/workspace/tao-experiments/data/voc0712trainval/labels_kitti" + image_directory_path: "/workspace/tao-experiments/data/voc0712trainval/images" + } + target_class_mapping { + key: "car" + value: "car" + } + target_class_mapping { + key: "person" + value: "person" + } + target_class_mapping { + key: "bicycle" + value: "bicycle" + } + target_class_mapping { + key: "dog" + value: "dog" + } + target_class_mapping { + key: "cat" + value: "cat" + } + target_class_mapping { + key: "aeroplane" + value: "aeroplane" + } + target_class_mapping { + key: "bird" + value: "bird" + } + target_class_mapping { + key: "boat" + value: "boat" + } + target_class_mapping { + key: "bottle" + value: "bottle" + } + target_class_mapping { + key: "bus" + value: "bus" + } + target_class_mapping { + key: "chair" + value: "chair" + } + target_class_mapping { + key: "cow" + value: "cow" + } + target_class_mapping { + key: "diningtable" + value: "diningtable" + } + target_class_mapping { + key: "horse" + value: "horse" + } + target_class_mapping { + key: "motorbike" + value: "motorbike" + } + target_class_mapping { + key: "pottedplant" + value: "pottedplant" + } + target_class_mapping { + key: "sheep" + value: "sheep" + } + target_class_mapping { + key: "sofa" + value: "sofa" + } + target_class_mapping { + key: "train" + value: "train" + } + target_class_mapping { + key: "tvmonitor" + value: "tvmonitor" + } + + validation_data_sources: { + label_directory_path: "/workspace/tao-experiments/data/voc07test/labels_kitti" + image_directory_path: "/workspace/tao-experiments/data/voc07test/images" + } +} diff --git a/src/apps/tao_apps/models/LPD_ch/config.pbtxt b/src/apps/tao_apps/models/LPD_ch/config.pbtxt new file mode 100644 index 00000000..9afd4185 --- /dev/null +++ b/src/apps/tao_apps/models/LPD_ch/config.pbtxt @@ -0,0 +1,55 @@ +################################################################################ +# Copyright (c) 2022 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "LPD_ch" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "LPDNet_CCPD_pruned_tao5.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 1168, 720] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [4, 73, 45] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [1, 73, 45] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/LPD_us/config.pbtxt b/src/apps/tao_apps/models/LPD_us/config.pbtxt new file mode 100644 index 00000000..8fcc7823 --- /dev/null +++ b/src/apps/tao_apps/models/LPD_us/config.pbtxt @@ -0,0 +1,55 @@ +################################################################################ +# Copyright (c) 2022 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "LPD_us" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "LPDNet_usa_pruned_tao5.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 480, 640] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [4, 30, 40] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [1, 30, 40] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/LPR_ch/config.pbtxt b/src/apps/tao_apps/models/LPR_ch/config.pbtxt new file mode 100644 index 00000000..14f5adb9 --- /dev/null +++ b/src/apps/tao_apps/models/LPR_ch/config.pbtxt @@ -0,0 +1,55 @@ +################################################################################ +# Copyright (c) 2022-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "LPR_ch" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "ch_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "image_input" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 48, 96] + } +] +output [ + { + name: "tf_op_layer_ArgMax" + data_type: TYPE_INT32 + dims: [24] + }, + + { + name: "tf_op_layer_Max" + data_type: TYPE_FP32 + dims: [ 24 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/LPR_us/config.pbtxt b/src/apps/tao_apps/models/LPR_us/config.pbtxt new file mode 100644 index 00000000..6e3eadc4 --- /dev/null +++ b/src/apps/tao_apps/models/LPR_us/config.pbtxt @@ -0,0 +1,55 @@ +################################################################################ +# Copyright (c) 2022-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "LPR_us" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "us_lprnet_baseline18_deployable.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "image_input" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 48, 96] + } +] +output [ + { + name: "tf_op_layer_ArgMax" + data_type: TYPE_INT32 + dims: [24] + }, + + { + name: "tf_op_layer_Max" + data_type: TYPE_FP32 + dims: [ 24 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/bodypose3dnet/config.pbtxt b/src/apps/tao_apps/models/bodypose3dnet/config.pbtxt new file mode 100644 index 00000000..baf787b3 --- /dev/null +++ b/src/apps/tao_apps/models/bodypose3dnet/config.pbtxt @@ -0,0 +1,83 @@ +################################################################################ +# Copyright (c) 2023-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "bodypose3dnet" +platform: "tensorrt_plan" +max_batch_size: 8 +default_model_filename: "bodypose3dnet_accuracy.onnx_b8_gpu0_fp16.engine" +input [ + { + name: "input0" + data_type: TYPE_FP32 + dims: [ 3, 256, 192 ] + }, + { + name: "k_inv" + data_type: TYPE_FP32 + dims: [ 3, 3 ] + }, + { + name: "t_form_inv" + data_type: TYPE_FP32 + dims: [ 3, 3 ] + }, + { + name: "scale_normalized_mean_limb_lengths" + data_type: TYPE_FP32 + dims: [ 36 ] + }, + { + name: "mean_limb_lengths" + data_type: TYPE_FP32 + dims: [ 36 ] + } +] +output [ + { + name: "pose2d" + data_type: TYPE_FP32 + dims: [34, 3] + }, + { + name: "pose2d_org_img" + data_type: TYPE_FP32 + dims: [34, 3] + }, + { + name: "pose25d" + data_type: TYPE_FP32 + dims: [34, 4] + }, + { + name: "pose3d" + data_type: TYPE_FP32 + dims: [34, 3] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/citysemsegformer/config.pbtxt b/src/apps/tao_apps/models/citysemsegformer/config.pbtxt new file mode 100644 index 00000000..f4d23e75 --- /dev/null +++ b/src/apps/tao_apps/models/citysemsegformer/config.pbtxt @@ -0,0 +1,48 @@ +################################################################################ +# Copyright (c) 2023-2024 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "citysemsegformer" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "citysemsegformer.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 3, 1024, 1820 ] + } +] +output [ + { + name: "output" + data_type: TYPE_INT64 + dims: [ 1024, 1820, 1 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/mask2former/config.pbtxt b/src/apps/tao_apps/models/mask2former/config.pbtxt new file mode 100644 index 00000000..4434e5a8 --- /dev/null +++ b/src/apps/tao_apps/models/mask2former/config.pbtxt @@ -0,0 +1,59 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "mask2former" +platform: "tensorrt_plan" +max_batch_size: 4 +default_model_filename: "mask2former.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "inputs" + data_type: TYPE_FP32 + dims: [ 3, 800, 800 ] + } +] +output [ + { + name: "pred_masks" + data_type: TYPE_FP32 + dims: [ 100, 800, 800 ] + }, + { + name: "pred_scores" + data_type: TYPE_FP32 + dims: [ 100 ] + }, + { + name: "pred_classes" + data_type: TYPE_INT64 + dims: [ 100 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/peoplenet/config.pbtxt b/src/apps/tao_apps/models/peoplenet/config.pbtxt new file mode 100644 index 00000000..55a73d25 --- /dev/null +++ b/src/apps/tao_apps/models/peoplenet/config.pbtxt @@ -0,0 +1,43 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +name: "peoplenet" +platform: "tensorrt_plan" +max_batch_size: 2 +default_model_filename: "resnet34_peoplenet_int8.onnx_b2_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + dims: [ 3, 544, 960 ] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [ 12, 34, 60 ] + }, + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [ 3, 34, 60 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/peoplenet_transformer/config.pbtxt b/src/apps/tao_apps/models/peoplenet_transformer/config.pbtxt new file mode 100644 index 00000000..8f0d47aa --- /dev/null +++ b/src/apps/tao_apps/models/peoplenet_transformer/config.pbtxt @@ -0,0 +1,52 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "peoplenet_transformer" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "resnet50_peoplenet_transformer_op17.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "inputs" + data_type: TYPE_FP32 + dims: [ 3, 544, 960] + } +] +output [ + { + name: "pred_logits" + data_type: TYPE_FP32 + dims: [ 300, 4 ] + }, + { + name: "pred_boxes" + data_type: TYPE_FP32 + dims: [ 300, 4 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/poseclassificationnet/config.pbtxt b/src/apps/tao_apps/models/poseclassificationnet/config.pbtxt new file mode 100644 index 00000000..a71420eb --- /dev/null +++ b/src/apps/tao_apps/models/poseclassificationnet/config.pbtxt @@ -0,0 +1,38 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +################################################################################ + +name: "poseclassificationnet" +platform: "tensorrt_plan" +max_batch_size: 4 +default_model_filename: "st-gcn_3dbp_nvidia.onnx_b4_gpu0_fp16.engine" +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 3, 300, 34, 1 ] + } +] +output [ + { + name: "fc_pred" + data_type: TYPE_FP32 + dims: [ 6 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/reidentificationnet/config.pbtxt b/src/apps/tao_apps/models/reidentificationnet/config.pbtxt new file mode 100644 index 00000000..100c91f2 --- /dev/null +++ b/src/apps/tao_apps/models/reidentificationnet/config.pbtxt @@ -0,0 +1,49 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "reidentificationnet" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "resnet50_market1501_aicity156.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 3, 256, 128 ] + } +] +output [ + { + name: "fc_pred" + data_type: TYPE_FP32 + dims: [ 256 ] + } +] + + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/retail_object_detection_binary_dino/config.pbtxt b/src/apps/tao_apps/models/retail_object_detection_binary_dino/config.pbtxt new file mode 100644 index 00000000..40b15065 --- /dev/null +++ b/src/apps/tao_apps/models/retail_object_detection_binary_dino/config.pbtxt @@ -0,0 +1,52 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "retail_object_detection_binary_dino" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "retail_object_detection_dino_binary.onnx_b1_gpu0_fp32.engine" +input [ + { + name: "inputs" + data_type: TYPE_FP32 + dims: [ 3, 540, 960 ] + } +] +output [ + { + name: "pred_logits" + data_type: TYPE_FP32 + dims: [ 900, 2 ] + }, + { + name: "pred_boxes" + data_type: TYPE_FP32 + dims: [ 900, 4 ] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/retail_object_recognition/config.pbtxt b/src/apps/tao_apps/models/retail_object_recognition/config.pbtxt new file mode 100644 index 00000000..f98768f1 --- /dev/null +++ b/src/apps/tao_apps/models/retail_object_recognition/config.pbtxt @@ -0,0 +1,53 @@ +################################################################################ +# Copyright (c) 2023 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "retail_object_recognition" +platform: "tensorrt_plan" +max_batch_size: 16 +default_model_filename: "retail_object_recognition.onnx_b16_gpu0_fp16.engine" +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 3, 224, 224 ] + } +] +output [ + { + name: "probs" + data_type: TYPE_FP32 + dims: [ 315 ] + }, + { + name: "embeddings" + data_type: TYPE_FP32 + dims: [ 448 ] + } +] + +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/trafficcamnet/config.pbtxt b/src/apps/tao_apps/models/trafficcamnet/config.pbtxt new file mode 100644 index 00000000..50165fdc --- /dev/null +++ b/src/apps/tao_apps/models/trafficcamnet/config.pbtxt @@ -0,0 +1,54 @@ +################################################################################ +# Copyright (c) 2022 NVIDIA Corporation. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +name: "trafficcamnet" +platform: "tensorrt_plan" +max_batch_size: 1 +default_model_filename: "resnet18_trafficcamnet_pruned.onnx_b1_gpu0_fp16.engine" +input [ + { + name: "input_1:0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [3, 544, 960] + } +] +output [ + { + name: "output_bbox/BiasAdd:0" + data_type: TYPE_FP32 + dims: [16, 34, 60] + }, + + { + name: "output_cov/Sigmoid:0" + data_type: TYPE_FP32 + dims: [4, 34, 60] + } +] +instance_group [ + { + kind: KIND_GPU + count: 1 + gpus: 0 + } +] diff --git a/src/apps/tao_apps/models/trafficcamnet/labels_trafficnet.txt b/src/apps/tao_apps/models/trafficcamnet/labels_trafficnet.txt new file mode 100644 index 00000000..1a200952 --- /dev/null +++ b/src/apps/tao_apps/models/trafficcamnet/labels_trafficnet.txt @@ -0,0 +1,4 @@ +car +bicycle +person +road_sign diff --git a/src/apps/tao_apps/post_processor/Makefile b/src/apps/tao_apps/post_processor/Makefile new file mode 100644 index 00000000..975dd062 --- /dev/null +++ b/src/apps/tao_apps/post_processor/Makefile @@ -0,0 +1,81 @@ +################################################################################ +# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +CUDA_VER?= +ifeq ($(CUDA_VER),) + $(error "CUDA_VER is not set") +endif + +LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream/lib/ + +CC:= g++ + +# Base CFLAGS +CFLAGS+= -Wall -std=c++11 -shared -fPIC + +# Deepstream SDK includes and CUDA includes +# Change to your path if needed +CFLAGS+= -I../../../../includes \ + -I/usr/local/cuda-$(CUDA_VER)/include + +# Debug logger header +CFLAGS+= -I. + +# Libraries +LIBS+= -lnvinfer -L/usr/local/cuda-$(CUDA_VER)/lib64 -lcudart -lcublas +LFLAGS:= -Wl,--start-group $(LIBS) -Wl,--end-group + +# Source files and targets +SRCFILES:= nvdsinfer_custombboxparser_tao.cpp +SRCFILES+= debug_logger_raii.cpp debug_logger_tensor.hpp +TARGET_LIB:= libnvds_infercustomparser_tao.so +TARGET_LIB_DEBUG:= libnvds_infercustomparser_tao_debug.so + +# Release build flags (default) +# = $(CFLAGS) -O2 +CFLAGS_RELEASE = $(CFLAGS) + +# Debug build flags +CFLAGS_DEBUG = $(CFLAGS) -g -DDEBUG -O0 + +# Default target +all: $(TARGET_LIB) + +# Debug target +debug: $(TARGET_LIB_DEBUG) + +# Release build +$(TARGET_LIB) : $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS_RELEASE) $(LFLAGS) + +# Debug build +$(TARGET_LIB_DEBUG): $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS_DEBUG) $(LFLAGS) + +install: $(TARGET_LIB) + @mkdir -p $(LIB_INSTALL_DIR) + cp -rv $(TARGET_LIB) $(LIB_INSTALL_DIR) + +clean: + rm -rf $(TARGET_LIB) $(TARGET_LIB_DEBUG) + +.PHONY: all debug install clean diff --git a/src/apps/tao_apps/post_processor/README.md b/src/apps/tao_apps/post_processor/README.md new file mode 100644 index 00000000..b368a95f --- /dev/null +++ b/src/apps/tao_apps/post_processor/README.md @@ -0,0 +1,95 @@ +# DeepStream Post-Processor + +## Build +```bash + $ export CUDA_VER= + For x86, CUDA_VER=13.1 + For Jetson, CUDA_VER=13.0 +``` + +### Release build +```bash + $ sudo make +``` +libnvds_infercustomparser_tao.so will be generated in current directory + +### Debug build +```bash + $ sudo make debug +``` +libnvds_infercustomparser_tao_debug.so will be generated in current directory + +Both builds have below debug utilities, that can be enabled/disabled by setting DEBUG environment variable. The difference is debug build has gdb symbol. + +## Debug Utilities +### Overview +This directory contains debug utilities for DeepStream post-processing: +- RAII-based file logger for detailed debugging +- Tensor-specific debug utilities for visualizing tensor data +- Zero-overhead when disabled (single environment variable check) +- Thread-safe implementation +- These src files are self independent, can be easily integrated into any Deepstream lib + +### Debug Macros + +#### 1. DEBUG_DUMP_SECTION and DEBUG_DUMP +RAII-based file logger for detailed debugging: +```cpp +{ + DEBUG_DUMP_SECTION(); // Creates a scoped logger + DEBUG_DUMP("Processing %zu elements", data.size()); + for (size_t i = 0; i < data.size(); i++) { + DEBUG_DUMP("Element %zu: %f", i, data[i]); + } +} // Logger automatically closes file +``` +- Creates timestamped log file in /tmp +- Zero overhead when disabled +- Automatically closes file when scope ends +- Thread-safe (each instance has its own file) + +#### 2. DEBUG_DS_TENSOR +DeepStream tensor visualization: +```cpp +DEBUG_DS_TENSOR(layer_info); // Prints truncated tensor preview +``` +- Prints dimensions and first few elements +- Full tensor data dumped to file +- Supports FLOAT, INT32, INT64 types +- Shows tensor shape and memory layout + +#### 3. DEBUG_TENSOR +Generic tensor debug utility: +```cpp +DEBUG_TENSOR("input", data.data(), data.size(), float); +``` +- Prints first 100 elements +- Full data dumped to file +- Type-safe implementation + +#### 4. DEBUG_PRINT +Simple debug print utility: +```cpp +DEBUG_PRINT("Processing data: %d", value); +``` +- Lightweight console output +- Includes timestamp +- No file I/O + +### Usage + +#### Enable/Disable Debug + +```bash +# Enable all debug output +export DEBUG=1 + +# Disable all debug output +unset DEBUG +``` + +#### Performance Impact +- Disabled: Zero overhead (single static bool check) +- Enabled: File I/O only occurs within DEBUG_DUMP_SECTION scope +- Thread-safe with minimal locking +- Static initialization occurs once at program start \ No newline at end of file diff --git a/src/apps/tao_apps/post_processor/debug_logger_raii.cpp b/src/apps/tao_apps/post_processor/debug_logger_raii.cpp new file mode 100644 index 00000000..c0984128 --- /dev/null +++ b/src/apps/tao_apps/post_processor/debug_logger_raii.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "debug_logger_raii.hpp" +#include +#include + + +const bool DebugConfig::enabled = []() { + const char* debug_env = getenv(ENABLE_ENV_NAME); + if (!debug_env) { + std::cout << "[" << __FILE__ << "] " + << "Environment variable " << ENABLE_ENV_NAME << " not set, DEBUG_DUMP disabled" + << std::endl; + return false; + } + bool is_enabled = (strcmp(debug_env, "1") == 0 || + strcmp(debug_env, "true") == 0 || + strcmp(debug_env, "TRUE") == 0); + std::cout << "[" << __FILE__ << "] " + << "Environment variable " << ENABLE_ENV_NAME << "=" << debug_env + << ", DEBUG_DUMP " << (is_enabled ? "enabled" : "disabled") + << std::endl; + return is_enabled; +}(); + +std::string DebugLoggerRAII::GetTimestamp() { + auto now = std::chrono::system_clock::now(); + auto now_c = std::chrono::system_clock::to_time_t(now); + auto now_tm = std::localtime(&now_c); + + std::stringstream ss; + ss << std::put_time(now_tm, "%Y-%m-%d %H:%M:%S"); + return ss.str(); +} + +std::string DebugLoggerRAII::GetLogHeader() { + return "[" + GetTimestamp() + "] [" + func_name + "] "; +} + +DebugLoggerRAII::DebugLoggerRAII(const char* func, int line, bool is_enabled) + : func_name(func), start_line(line), enabled(is_enabled) { + if (!enabled) return; + + filename = std::string("/tmp/debug_") + func_name + "_" + + std::to_string(std::time(nullptr)) + ".log"; + log_file.open(filename); + if (log_file.is_open()) { + std::cout << GetLogHeader() << "DEBUG_DUMP started at file: " << filename << std::endl; + log_file << GetLogHeader() << "DEBUG_DUMP started at line " << line << std::endl; + } else { + std::cerr << GetLogHeader() << "Failed to create debug log file: " << filename << std::endl; + } +} + +DebugLoggerRAII::~DebugLoggerRAII() { + if (enabled && log_file.is_open()) { + log_file << GetLogHeader() << "DEBUG_DUMP ended" << std::endl; + log_file.close(); + std::cout << GetLogHeader() << "DEBUG_DUMP closed log file: " << filename << std::endl; + } +} + +void DebugLoggerRAII::log(const char* format, ...) { + if (!enabled || !log_file.is_open()) return; + + va_list args; + va_start(args, format); + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), format, args); + va_end(args); + log_file << GetLogHeader() << buffer << std::endl; +} diff --git a/src/apps/tao_apps/post_processor/debug_logger_raii.hpp b/src/apps/tao_apps/post_processor/debug_logger_raii.hpp new file mode 100644 index 00000000..6924e206 --- /dev/null +++ b/src/apps/tao_apps/post_processor/debug_logger_raii.hpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef DEBUG_LOGGER_RAII_H +#define DEBUG_LOGGER_RAII_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#define ENABLE_ENV_NAME "DEBUG" + +/** + * @brief Debug logger class using RAII (Resource Acquisition Is Initialization) + * to automatically manage the log file. + * + * This class is used to log debug messages to a file. + * The log file is automatically closed when the object is destroyed. + * With the help of the DEBUG_DUMP_SECTION macro, we can start and end a debug section. + * Advantages of using this class: + * - Automatically closes the log file even if there's an early return or exception. + * - clearly defines the scope of debug logging using {} block + * - No need to remember to close anything + * - Thread-safe as each instance has its own file handle + * - Object construction and destruction has minimal overhead when it is disabled + * + * @param func The function name + * @param line The line number + * @param is_enabled Whether the logger is enabled + * + * @note The log file is created in the /tmp directory. + */ +class DebugLoggerRAII { +public: + DebugLoggerRAII(const char* func, int line, bool is_enabled); + ~DebugLoggerRAII(); + void log(const char* format, ...); + +private: + std::string filename; + std::ofstream log_file; + const char* func_name; + int start_line; + bool enabled; + + // Helper function to get formatted timestamp + static std::string GetTimestamp(); + // Helper function to get log header with timestamp and file info + std::string GetLogHeader(); +}; + + +class DebugConfig { +private: + static const bool enabled; +public: + static bool IsEnabled() { return enabled; } +}; + +// Debug macros +#define DEBUG_DUMP_SECTION() \ + DebugLoggerRAII debug_logger(__func__, __LINE__, DebugConfig::IsEnabled()) + +#define DEBUG_DUMP(...) \ + debug_logger.log(__VA_ARGS__) + +#endif // DEBUG_LOGGER_RAII_H \ No newline at end of file diff --git a/src/apps/tao_apps/post_processor/debug_logger_tensor.hpp b/src/apps/tao_apps/post_processor/debug_logger_tensor.hpp new file mode 100644 index 00000000..db71a626 --- /dev/null +++ b/src/apps/tao_apps/post_processor/debug_logger_tensor.hpp @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef DEBUG_LOGGER_TENSOR_H +#define DEBUG_LOGGER_TENSOR_H + +#include +#include +#include "nvdsinfer_custom_impl.h" + +#include +#include +#include +#include +#include + +/** + * @brief Debug utilities for tensor operations + * + * This file contains specialized debug macros for tensor operations: + * - DEBUG_DS_TENSOR: For DeepStream tensor debugging + * - DEBUG_TENSOR: For general tensor debugging + * + * These macros can be used independently or within a RAII debug logger scope. + */ + +#define ENABLE_ENV_NAME "DEBUG" + +/** + * @brief Get formatted timestamp + */ +inline std::string GetTimestamp() { + auto now = std::chrono::system_clock::now(); + auto now_c = std::chrono::system_clock::to_time_t(now); + auto now_tm = std::localtime(&now_c); + + std::stringstream ss; + ss << std::put_time(now_tm, "%Y-%m-%d %H:%M:%S"); + return ss.str(); +} + +/** + * @brief Get header string with timestamp and file name + */ +inline std::string GetLogHeader() { + return "[" + GetTimestamp() + "] " + std::string(__FILE__) + "] "; +} + +/** + * @brief Global variable to check if debug is enabled. + * @note This is a static const variable that is initialized once and then reused. + */ +static const bool ENABLE_TENSOR_DEBUG = []() { + const char* debug_env = getenv(ENABLE_ENV_NAME); + if (!debug_env) { + std::cout << "[" << __FILE__ << "] " + << "Environment variable " << ENABLE_ENV_NAME << " not set, DEBUG_DS_TENSOR disabled" + << std::endl; + return false; + } + bool is_enabled = (strcmp(debug_env, "1") == 0 || + strcmp(debug_env, "true") == 0 || + strcmp(debug_env, "TRUE") == 0); + std::cout << "[" << __FILE__ << "] " + << "Environment variable " << ENABLE_ENV_NAME << "=" << debug_env + << ", DEBUG_DS_TENSOR " << (is_enabled ? "enabled" : "disabled") + << std::endl; + return is_enabled; +}(); + +/** + * @brief Debug print with timestamp + * @param ... Arguments to printf + */ +#define DEBUG_PRINT(...) \ + do { \ + if (ENABLE_TENSOR_DEBUG) { \ + std::cout << "[" << GetTimestamp() << "] DEBUG_PRINT: "; \ + printf(__VA_ARGS__); \ + std::cout << std::endl; \ + } \ + } while(0) + +/** + * @brief Calculate total elements from DS Infer dimensions + * @param dims DS Infer dimensions to calculate total elements from + * @return Total number of elements + */ +inline size_t GetTotalElements(const NvDsInferDims& dims) { + size_t total = 1; + for(unsigned int i = 0; i < dims.numDims; i++) { + total *= dims.d[i]; + } + return total; +} + +/** + * @brief Calculate coordinates for a block index + * @param block_idx Block index to calculate coordinates for + * @param dims DS Infer dimensions to calculate coordinates from + * @param coords Vector to store coordinates + */ +inline void GetBlockCoordinates(size_t block_idx, const NvDsInferDims& dims, std::vector& coords) { + coords.clear(); + size_t remaining = block_idx; + // Skip the last dimension as it represents the values within each block + for(unsigned int d = 0; d < dims.numDims-1; d++) { // Changed int to unsigned int + coords.push_back(remaining % dims.d[d]); + remaining /= dims.d[d]; + } +} + +/** + * @brief Format coordinates as string + * @param coords Vector of coordinates to format + * @return Formatted coordinates string + */ +inline std::string FormatCoordinates(const std::vector& coords) { + std::stringstream ss; + ss << "["; + for(size_t i = 0; i < coords.size(); i++) { + ss << coords[i]; + if(i < coords.size()-1) ss << ","; + } + ss << "]"; + return ss.str(); +} + +/** + * @brief Debug print and dump for tensors + * @param tensor_name Name of the tensor + * @param data_type Data type of the tensor + * @param layer_ref DS Infer layer reference + * @param data Typed pointer to the tensor data + * @param num_blocks Number of blocks in the tensor + * @param last_dim_size Size of the last dimension + * @note This function is used to dump the tensor data to a file. + * It prints the first 5 blocks of the tensor to the console and then dumps the full tensor to a file. + * + * @example + * [2025-03-15 00:57:30] DEBUG: Entering NvDsInferParseCustomDDETRTAO + * [2025-03-15 00:57:30] DEBUG: DS Tensor 'pred_boxes' + * Dimensions (2D): [300, 4] + * Block 0 [0]: 0.5762 0.5010 0.0685 0.0754 + * Block 1 [1]: 0.1716 0.3838 0.0967 0.0970 + * Block 2 [2]: 0.6582 0.5220 0.0612 0.0753 + * Block 3 [3]: 0.4597 0.4663 0.0687 0.0688 + * Block 4 [4]: 0.1991 0.5483 0.1510 0.2424 + * ... (truncated) + * [2025-03-15 00:57:30] DEBUG: Full tensor dumped to /tmp/tensor_dump_pred_boxes_1742000250.txt + * [2025-03-15 00:57:30] DEBUG: DS Tensor 'pred_logits' + * Dimensions (2D): [300, 5] + * Block 0 [0]: -7.3555 -4.0586 2.3086 -3.8867 -4.6758 + * Block 1 [1]: -7.1875 -4.2344 2.5039 -4.1758 -3.6934 + * Block 2 [2]: -6.8984 -4.0039 1.7812 -3.5215 -4.7070 + * Block 3 [3]: -7.3516 -4.2617 2.3906 -3.4824 -4.5156 + * Block 4 [4]: -7.3906 -3.9648 2.1016 -3.6211 -3.6543 + * ... (truncated) + * [2025-03-15 00:57:30] DEBUG: Full tensor dumped to /tmp/tensor_dump_pred_logits_1742000250.txt + * + * For 3D tensor [8, 4, 2]: + * [2024-03-14 15:30:45] DEBUG: DS Tensor 'example' + * Dimensions (3D): [8, 4, 2] + * Block 0 [0,0]: 0.1 0.2 + * Block 1 [0,1]: 0.3 0.4 + * Block 2 [0,2]: 0.5 0.6 + * Block 3 [0,3]: 0.7 0.8 + * Block 4 [1,0]: 0.9 1.0 + * ... + */ +template +void DumpTensorData(const char* tensor_name, const char* data_type, + const NvDsInferLayerInfo& layer_ref, + const T* data, size_t num_blocks, size_t last_dim_size) { + const size_t blocks_to_show = std::min((size_t)5, num_blocks); + std::vector coords; + + // Print preview to console + for(size_t block = 0; block < blocks_to_show; block++) { + // Calculate and show coordinates for this block + GetBlockCoordinates(block, layer_ref.inferDims, coords); + std::cout << "Block " << block << " " << FormatCoordinates(coords) << ": "; + + // Print the actual values + for(size_t i = 0; i < last_dim_size; i++) { + std::cout << data[block * last_dim_size + i] << " "; + } + std::cout << std::endl; + } + if(num_blocks > blocks_to_show) std::cout << "... (truncated)\n"; + + // Dump to file + std::string filename = std::string("/tmp/tensor_dump_") + tensor_name + "_" + + std::to_string(std::time(nullptr)) + ".txt"; + std::ofstream outfile(filename); + if (!outfile.is_open()) { + std::cerr << "ERROR: Failed to open file for writing: " << filename << std::endl; + return; + } + + // Set higher precision for file output + outfile.precision(6); + outfile << std::fixed; + + outfile << "Tensor name: " << tensor_name << "\n"; + outfile << "Data type: " << data_type << "\n"; + outfile << "Dimensions (" << layer_ref.inferDims.numDims << "D): ["; + for(unsigned int i = 0; i < layer_ref.inferDims.numDims; i++) { + outfile << layer_ref.inferDims.d[i]; + if(i < layer_ref.inferDims.numDims-1) outfile << ", "; + } + outfile << "]\n\n"; + + // Write all blocks with coordinates + for(size_t block = 0; block < num_blocks; block++) { + GetBlockCoordinates(block, layer_ref.inferDims, coords); + outfile << "Block " << block << " " << FormatCoordinates(coords) << ":\n"; + for(size_t i = 0; i < last_dim_size; i++) { + outfile << data[block * last_dim_size + i] << " "; + if((i + 1) % 10 == 0) outfile << "\n"; + } + outfile << "\n\n"; + } + outfile.close(); + std::cout << "[" << GetTimestamp() << "] DEBUG_DS_TENSOR: Full tensor dumped to " << filename << std::endl; +} + +/** + * @brief Debug print and dump for DS Infer tensors + * @param layer_arg DS Infer layer reference + * @note This macro is used to print and dump the tensor data for DS Infer. + * It validates the tensor, prints the dimensions, and then prints the tensor data based on the data type. + */ +#define DEBUG_DS_TENSOR(layer_arg) \ + do { \ + if (ENABLE_TENSOR_DEBUG) { \ + const NvDsInferLayerInfo& layer_ref = layer_arg; \ + const char* tensor_name = layer_ref.layerName; \ + std::cout << "[" << GetTimestamp() << "] DEBUG_DS_TENSOR: '" << tensor_name << "'\n"; \ + \ + /* Validate tensor */ \ + if (!layer_ref.buffer) { \ + std::cerr << "ERROR: Null tensor buffer\n"; \ + break; \ + } \ + if (layer_ref.inferDims.numDims == 0) { \ + std::cerr << "ERROR: Invalid tensor dimensions\n"; \ + break; \ + } \ + \ + /* Print dimensions */ \ + std::cout << "Dimensions (" << layer_ref.inferDims.numDims << "D): ["; \ + for(unsigned int i = 0; i < layer_ref.inferDims.numDims; i++) { \ + std::cout << layer_ref.inferDims.d[i]; \ + if(i < layer_ref.inferDims.numDims-1) std::cout << ", "; \ + } \ + std::cout << "]\n"; \ + \ + /* Calculate layout */ \ + const size_t total_elements = GetTotalElements(layer_ref.inferDims); \ + const size_t last_dim_size = layer_ref.inferDims.d[layer_ref.inferDims.numDims-1]; \ + const size_t num_blocks = total_elements / last_dim_size; \ + \ + /* Set output precision for floating point */ \ + std::cout.precision(4); \ + std::cout << std::fixed; \ + \ + /* Print tensor data based on type */ \ + switch(layer_ref.dataType) { \ + case FLOAT: { \ + const float* data = static_cast(layer_ref.buffer); \ + DumpTensorData(tensor_name, "FLOAT", layer_ref, data, num_blocks, last_dim_size); \ + break; \ + } \ + case INT32: { \ + const int32_t* data = static_cast(layer_ref.buffer); \ + DumpTensorData(tensor_name, "INT32", layer_ref, data, num_blocks, last_dim_size); \ + break; \ + } \ + case INT64: { \ + const int64_t* data = static_cast(layer_ref.buffer); \ + DumpTensorData(tensor_name, "INT64", layer_ref, data, num_blocks, last_dim_size); \ + break; \ + } \ + default: \ + std::cerr << "ERROR: Unsupported data type: " << layer_ref.dataType << std::endl; \ + } \ + } \ + } while(0) + +/** + * @brief Debug print and dump for generic tensors + * @param tensor_name Name of the tensor + * @param data Pointer to the tensor data + * @param size Size of the tensor + * @param type Data type of the tensor + * + * @note This macro is used to print and dump the tensor data. + * It prints the first 100 values of the tensor to the console and then dumps the full tensor to a file. + */ +#define DEBUG_TENSOR(tensor_name, data, size, type) \ + if (ENABLE_TENSOR_DEBUG) { \ + /* Console print first 100 values */ \ + std::cout << "[" << GetTimestamp() << "] DEBUG_TENSOR: '" << tensor_name \ + << "' (showing first 100 of " << size << " values):" << std::endl; \ + const type* typed_data = static_cast(data); \ + for (size_t i = 0; i < size && i < 100; i++) { \ + std::cout << typed_data[i] << " "; \ + if ((i + 1) % 10 == 0) std::cout << std::endl; \ + } \ + if (size > 100) std::cout << "... (truncated)" << std::endl; \ + std::cout << std::endl; \ + /* Dump complete tensor to file */ \ + std::string filename = std::string("/tmp/tensor_dump_") + tensor_name + "_" + \ + std::to_string(std::time(nullptr)) + ".txt"; \ + std::ofstream outfile(filename); \ + if (outfile.is_open()) { \ + outfile << "Tensor name: " << tensor_name << "\n"; \ + outfile << "Data type: " << #type << "\n"; \ + outfile << "Size: " << size << "\n"; \ + outfile << "Values:\n"; \ + for (size_t i = 0; i < size; i++) { \ + outfile << typed_data[i] << " "; \ + if ((i + 1) % 10 == 0) outfile << "\n"; \ + } \ + outfile.close(); \ + std::cout << "[" << GetTimestamp() << "] DEBUG_TENSOR: Full tensor dumped to " << filename << std::endl; \ + } \ + } + +#endif // DEBUG_LOGGER_TENSOR_H \ No newline at end of file diff --git a/src/apps/tao_apps/post_processor/logger_sample/Makefile b/src/apps/tao_apps/post_processor/logger_sample/Makefile new file mode 100644 index 00000000..83910927 --- /dev/null +++ b/src/apps/tao_apps/post_processor/logger_sample/Makefile @@ -0,0 +1,21 @@ +CC := g++ +CFLAGS := -Wall -std=c++11 -I.. + +# Debug flags +CFLAGS_DEBUG := $(CFLAGS) -g -DDEBUG + +# Source files +SRCFILES := debug_logger_raii_sample.cpp +SRCFILES += ../debug_logger_raii.cpp +TARGET := debug_logger_raii_sample + +# Default target +all: $(TARGET) + +$(TARGET): $(SRCFILES) + $(CC) -o $@ $^ $(CFLAGS_DEBUG) + +clean: + rm -rf $(TARGET) + +.PHONY: all clean diff --git a/src/apps/tao_apps/post_processor/logger_sample/debug_logger_raii_sample.cpp b/src/apps/tao_apps/post_processor/logger_sample/debug_logger_raii_sample.cpp new file mode 100644 index 00000000..4d40ad02 --- /dev/null +++ b/src/apps/tao_apps/post_processor/logger_sample/debug_logger_raii_sample.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "../debug_logger_raii.hpp" +#include + +// Sample function demonstrating debug logger usage +void processData(const std::vector& data) { + // Start debug section + { + DEBUG_DUMP_SECTION(); + + DEBUG_DUMP("Processing %zu elements", data.size()); + + for (size_t i = 0; i < data.size(); i++) { + if (data[i] > 0.5f) { + DEBUG_DUMP("Element %zu: %f exceeds threshold", i, data[i]); + } + } + + DEBUG_DUMP("Processing complete"); + } // Debug logger automatically closes here +} + +// Sample main function for testing +int main() { + std::vector test_data = {0.1f, 0.6f, 0.3f, 0.8f, 0.2f}; + // expose DEBUG=1 + // unset DEBUG + processData(test_data); + return 0; +} diff --git a/src/apps/tao_apps/post_processor/nvdsinfer_custombboxparser_tao.cpp b/src/apps/tao_apps/post_processor/nvdsinfer_custombboxparser_tao.cpp new file mode 100644 index 00000000..f0e58b91 --- /dev/null +++ b/src/apps/tao_apps/post_processor/nvdsinfer_custombboxparser_tao.cpp @@ -0,0 +1,845 @@ +/* + * Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "nvdsinfer_custom_impl.h" +#include +#include +#include +#include + +// DEBUG print includes +#include "debug_logger_raii.hpp" +#include "debug_logger_tensor.hpp" +// end DEBUG print includes + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CLIP(a,min,max) (MAX(MIN(a, max), min)) +#define DIVIDE_AND_ROUND_UP(a, b) ((a + b - 1) / b) + + +struct MrcnnRawDetection { + float y1, x1, y2, x2, class_id, score; +}; +/* This is a sample bounding box parsing function for the sample FasterRCNN + * + * detector model provided with the SDK. */ + +/* C-linkage to prevent name-mangling */ +extern "C" bool NvDsInferInitializeInputLayers (std::vector const &inputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + unsigned int maxBatchSize); + +extern "C" +bool NvDsInferParseCustomNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedYoloV5NMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferParseCustomSegformerTAO( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + float segmentationThreshold, + unsigned int numClasses, + int* classificationMap, + float*& classProbabilityMap); + +extern "C" +bool NvDsInferParseCustomChangeNet( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + float segmentationThreshold, + unsigned int numClasses, + int* classificationMap, + float*& classProbabilityMap); + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList); + +extern "C" +bool NvDsInferClassiferParseNonSoftmax (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + float classifierThreshold, + std::vector &attrList, + std::string &descString); +/** Implementation of a custom processor for DeepStream nvinfer + * plugin to process additonal inputs. + */ +extern "C" { + +bool NvDsInferInitializeInputLayers (std::vector const &inputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + unsigned int maxBatchSize) +{ + /* Nothing to do, no input layers with static values are expected*/ + return true; +} +} + +extern "C" +bool NvDsInferParseCustomNMSTLT (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 2) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 2 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + // Host memory for "nms" which has 2 output bindings: + // the order is bboxes and keep_count + float* out_nms = (float *) outputLayersInfo[0].buffer; + int * p_keep_count = (int *) outputLayersInfo[1].buffer; + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + float* det; + + for (int i = 0; i < p_keep_count[0]; i++) { + det = out_nms + i * 7; + + // Output format for each detection is stored in the below order + // [image_id, label, confidence, xmin, ymin, xmax, ymax] + if ( det[2] < threshold) continue; + assert((int) det[1] < out_class_size); + +#if 0 + std::cout << "id/label/conf/ x/y x/y -- " + << det[0] << " " << det[1] << " " << det[2] << " " + << det[3] << " " << det[4] << " " << det[5] << " " << det[6] << std::endl; +#endif + NvDsInferObjectDetectionInfo object; + object.classId = (int) det[1]; + object.detectionConfidence = det[2]; + object.rotation_angle = 0.0f; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(det[3] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(det[4] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP((det[5] - det[3]) * networkInfo.width, 0, networkInfo.width - 1); + object.height = CLIP((det[6] - det[4]) * networkInfo.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedYoloV5NMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep cout" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + object.rotation_angle = 0.0f; + + object.left = CLIP(p_bboxes[4*i], 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1], 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2], 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3], 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomBatchedNMSTLT ( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + /* Host memory for "BatchedNMS" + BatchedNMS has 4 output bindings, the order is: + keepCount, bboxes, scores, classes + */ + int* p_keep_count = (int *) outputLayersInfo[0].buffer; + float* p_bboxes = (float *) outputLayersInfo[1].buffer; + float* p_scores = (float *) outputLayersInfo[2].buffer; + float* p_classes = (float *) outputLayersInfo[3].buffer; + + const float threshold = detectionParams.perClassThreshold[0]; + + const int keep_top_k = 200; + const char* log_enable = std::getenv("ENABLE_DEBUG"); + + if(log_enable != NULL && std::stoi(log_enable)) { + std::cout <<"keep count" + <= detectionParams.numClassesConfigured) continue; + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) continue; + + NvDsInferObjectDetectionInfo object; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + object.rotation_angle = 0.0f; + + /* Clip object box co-ordinates to network resolution */ + object.left = CLIP(p_bboxes[4*i] * networkInfo.width, 0, networkInfo.width - 1); + object.top = CLIP(p_bboxes[4*i+1] * networkInfo.height, 0, networkInfo.height - 1); + object.width = CLIP(p_bboxes[4*i+2] * networkInfo.width, 0, networkInfo.width - 1) - object.left; + object.height = CLIP(p_bboxes[4*i+3] * networkInfo.height, 0, networkInfo.height - 1) - object.top; + + if(object.height < 0 || object.width < 0) + continue; + objectList.push_back(object); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomMrcnnTLTV2 (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *detectionLayer = layerFinder("generate_detections"); + const NvDsInferLayerInfo *maskLayer = layerFinder("mask_fcn_logits/BiasAdd"); + + if (!detectionLayer || !maskLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + return false; + } + + if(maskLayer->inferDims.numDims != 4U) { + std::cerr << "Network output number of dims is : " << + maskLayer->inferDims.numDims << " expect is 4"<< std::endl; + return false; + } + + const unsigned int det_max_instances = maskLayer->inferDims.d[0]; + const unsigned int num_classes = maskLayer->inferDims.d[1]; + if(num_classes != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" << + detectionParams.numClassesConfigured << ", detected by network: " << + num_classes << std::endl; + } + const unsigned int mask_instance_height= maskLayer->inferDims.d[2]; + const unsigned int mask_instance_width = maskLayer->inferDims.d[3]; + + auto out_det = reinterpret_cast( detectionLayer->buffer); + auto out_mask = reinterpret_cast(maskLayer->buffer); + + for(auto i = 0U; i < det_max_instances; i++) { + MrcnnRawDetection &rawDec = out_det[i]; + + if(rawDec.score < detectionParams.perClassPreclusterThreshold[0]) + continue; + + NvDsInferInstanceMaskInfo obj; + obj.left = CLIP(rawDec.x1, 0, networkInfo.width - 1); + obj.top = CLIP(rawDec.y1, 0, networkInfo.height - 1); + obj.width = CLIP(rawDec.x2, 0, networkInfo.width - 1) - rawDec.x1; + obj.height = CLIP(rawDec.y2, 0, networkInfo.height - 1) - rawDec.y1; + if(obj.width <= 0 || obj.height <= 0) + continue; + obj.classId = static_cast(rawDec.class_id); + obj.detectionConfidence = rawDec.score; + + obj.mask_size = sizeof(float)*mask_instance_width*mask_instance_height; + obj.mask = new float[mask_instance_width*mask_instance_height]; + obj.mask_width = mask_instance_width; + obj.mask_height = mask_instance_height; + + float *rawMask = reinterpret_cast(out_mask + i + * detectionParams.numClassesConfigured + obj.classId); + memcpy (obj.mask, rawMask, sizeof(float)*mask_instance_width*mask_instance_height); + + objectList.push_back(obj); + } + + return true; + +} + +void getMaskDimension(float* buf, int w, int h, int& left, int& top, int& width, int& height) +{ + int right, bottom; + right = bottom = 0; + left = top = width = height = 0; + for(int i = 0; i < w; i++) { + for(int j = 0; j < h; j++) { + if(*(buf + j*w + i) == 1) { + if(left == 0) left = i; + if(i < left) left = i; + if(i > right) right = i; + if(top == 0) top = j; + if(j < top) top = j; + if(j > bottom) bottom = j; + } + width = right - left; + height = bottom - top; + } + } +} + +void copy_mask(float* dst, float* src, int w, int h, + int mask_left, int mask_top, int mask_width, int mask_height) { + int j = 0; + for(int i = mask_top; i < mask_top + mask_height; i++){ + float* pSrc = src + i*w + mask_left; + memcpy(dst + (j++)*mask_width, pSrc, mask_width*sizeof(float)); + } +} + +extern "C" +bool NvDsInferParseCustomMask2Former (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if ((layer.dataType == FLOAT || layer.dataType == INT32 || layer.dataType == INT64) && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *pred_classes = layerFinder("pred_classes"); + const NvDsInferLayerInfo *pred_masks = layerFinder("pred_masks"); + const NvDsInferLayerInfo *pred_scores = layerFinder("pred_scores"); + const unsigned int det_max_instances = pred_masks->inferDims.d[0]; + + int width = pred_masks->inferDims.d[1]; + int height = pred_masks->inferDims.d[2]; + int* pclass = (int*)pred_classes->buffer; + float* pmask = (float*)pred_masks->buffer; + float* pscore = (float*)pred_scores->buffer; + assert(pclass != NULL && pmask != NULL && pscore != NULL); + float* tmp_pmask = NULL; + int mask_left, mask_top, mask_width, mask_height; + + for(auto i = 0U; i < det_max_instances; i++) { + if(std::isnan(pscore[i]) || pscore[i] < detectionParams.perClassPreclusterThreshold[0]) + continue; + mask_left = mask_top = mask_width = mask_height = 0; + tmp_pmask = pmask + i * width * height; + /* get rect from mask*/ + getMaskDimension(tmp_pmask, width, height, mask_left, mask_top, mask_width, mask_height); + NvDsInferInstanceMaskInfo obj; + obj.left = mask_left; + obj.top = mask_top; + obj.width = mask_width; + obj.height = mask_height; + if(obj.width <= 0 || obj.height <= 0 || mask_width < 0|| mask_height <= 0) + continue; + obj.classId = pclass[i]; + obj.detectionConfidence = pscore[i]; + obj.mask_size = sizeof(float)*mask_width*mask_height; + obj.mask = new float[mask_width*mask_height]; + obj.mask_width = mask_width; + obj.mask_height = mask_height; + copy_mask(obj.mask, tmp_pmask, width, height, mask_left, mask_top, mask_width, mask_height); + + objectList.push_back(obj); + } + return true; +} + +extern "C" +bool NvDsInferParseCustomEfficientDetTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + if(outputLayersInfo.size() != 4) + { + std::cerr << "Mismatch in the number of output buffers." + << "Expected 4 output buffers, detected in the network :" + << outputLayersInfo.size() << std::endl; + return false; + } + + int* p_keep_count = nullptr; + float* p_bboxes = nullptr; + float* p_scores = nullptr; + int* p_classes = nullptr; + + for (int i = 0; i < 4; i++){ + const char* layerName = outputLayersInfo[i].layerName; + if(!strcmp(layerName, "num_detections")) { + p_keep_count = (int *) outputLayersInfo[i].buffer; + } else if(!strcmp(layerName, "detection_boxes")) { + p_bboxes = (float *) outputLayersInfo[i].buffer; + } else if(!strcmp(layerName, "detection_scores")) { + p_scores = (float *) outputLayersInfo[i].buffer; + } else if(!strcmp(layerName, "detection_classes")) { + p_classes = (int *) outputLayersInfo[i].buffer; + } + } + + const int out_class_size = detectionParams.numClassesConfigured; + const float threshold = detectionParams.perClassThreshold[0]; + + if (p_keep_count[0] > 0) + { + for (int i = 0; i < p_keep_count[0]; i++) { + if ( p_scores[i] < threshold) continue; + //assert((int) p_classes[i] < out_class_size); + if(p_classes[i] >= out_class_size) + break; + + if(p_bboxes[4*i+2] < p_bboxes[4*i] || p_bboxes[4*i+3] < p_bboxes[4*i+1]) + continue; + + NvDsInferObjectDetectionInfo object; + object.rotation_angle = 0.0f; + object.classId = (int) p_classes[i]; + object.detectionConfidence = p_scores[i]; + + object.left=p_bboxes[4*i+1]; + object.top=p_bboxes[4*i]; + object.width=( p_bboxes[4*i+3] - object.left); + object.height= ( p_bboxes[4*i+2] - object.top); + + object.left=CLIP(object.left, 0, networkInfo.width - 1); + object.top=CLIP(object.top, 0, networkInfo.height - 1); + object.width=CLIP(object.width, 0, networkInfo.width - 1); + object.height=CLIP(object.height, 0, networkInfo.height - 1); + + objectList.push_back(object); + } + } + return true; +} + +extern "C" +bool NvDsInferParseCustomDDETRTAO (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + + DEBUG_DUMP_SECTION(); // scope to entire function {} + + // Code from NvDsInferParseCustomTfSSD for layer finding + auto layerFinder = [&outputLayersInfo](const std::string &name) + -> const NvDsInferLayerInfo *{ + for (auto &layer : outputLayersInfo) { + if (layer.dataType == FLOAT && + (layer.layerName && name == layer.layerName)) { + return &layer; + } + } + return nullptr; + }; + + const NvDsInferLayerInfo *boxLayer = layerFinder("pred_boxes"); // 1 x num_queries x 4 + const NvDsInferLayerInfo *classLayer = layerFinder("pred_logits"); // 1 x num_queries x num_classes + + if (!boxLayer || !classLayer) { + std::cerr << "ERROR: some layers missing or unsupported data types " + << "in output tensors" << std::endl; + DEBUG_PRINT("Exiting NvDsInferParseCustomDDETRTAO with error"); + return false; + } + + // Debug print both tensors + DEBUG_DS_TENSOR(*boxLayer); + DEBUG_DS_TENSOR(*classLayer); + + const int keep_top_k = 200; + unsigned int numDetections = classLayer->inferDims.d[0]; + unsigned int numClasses = classLayer->inferDims.d[1]; + std::map ordered_objects; + + size_t numClassesConfigured = detectionParams.perClassPreclusterThreshold.size(); + DEBUG_PRINT("detectionParams.perClassPreclusterThreshold.size(): %ld", numClassesConfigured); + if(numClassesConfigured != numClasses) { + std::cerr << "ERROR: numClassesConfigured: " << numClassesConfigured << " != numClasses from output tensor: " << numClasses << std::endl; + return false; + } + + for (unsigned int idx = 0; idx < numDetections; idx += 1) { + NvDsInferObjectDetectionInfo res; + + unsigned int class_layer_idx = idx * numClasses; + + res.classId = std::max_element(((float*)classLayer->buffer+class_layer_idx), ((float*)classLayer->buffer+class_layer_idx+numClasses)) - ((float*)classLayer->buffer+class_layer_idx); + res.detectionConfidence = ((float*)classLayer->buffer)[class_layer_idx+res.classId]; + + // If model does not have sigmoid layer, perform sigmoid calculation here + res.detectionConfidence = 1.0/(1.0 + exp(-res.detectionConfidence)); + + if(res.detectionConfidence < detectionParams.perClassPreclusterThreshold[res.classId]) { + // use DEBUG_DUMP for intensive logging inside for loop + DEBUG_DUMP("#%d classId: %d, detectionConfidence: %f, threshold: %f", idx, res.classId, res.detectionConfidence, detectionParams.perClassPreclusterThreshold[res.classId]); + continue; + } + // use DEBUG_DUMP for intensive logging inside for loop + DEBUG_DUMP("[add] #%d classId: %d, detectionConfidence: %f, threshold: %f", idx, res.classId, res.detectionConfidence, detectionParams.perClassPreclusterThreshold[res.classId]); + enum {cx, cy, w, h}; + float rectX1f, rectY1f, rectX2f, rectY2f; + + unsigned int box_layer_idx = idx * 4; + + rectX1f = (((float*)boxLayer->buffer)[box_layer_idx + cx] - (((float*)boxLayer->buffer)[box_layer_idx + w]/2)) * networkInfo.width; + rectY1f = (((float*)boxLayer->buffer)[box_layer_idx + cy] - (((float*)boxLayer->buffer)[box_layer_idx + h]/2)) * networkInfo.height; + rectX2f = rectX1f + ((float*)boxLayer->buffer)[box_layer_idx + w] * networkInfo.width; + rectY2f = rectY1f + ((float*)boxLayer->buffer)[box_layer_idx + h] * networkInfo.height; + + rectX1f = CLIP(rectX1f, 0.0f, networkInfo.width - 1); + rectX2f = CLIP(rectX2f, 0.0f, networkInfo.width - 1); + rectY1f = CLIP(rectY1f, 0.0f, networkInfo.height - 1); + rectY2f = CLIP(rectY2f, 0.0f, networkInfo.height - 1); + + res.left = rectX1f; + res.top = rectY1f; + res.width = rectX2f - rectX1f; + res.height = rectY2f - rectY1f; + + res.rotation_angle = 0.0f; + ordered_objects[res.detectionConfidence] = res; + } + + int jdx = 0; + for (auto iter=ordered_objects.rbegin(); iter!=ordered_objects.rend() && jdxsecond.classId != 0){ + objectList.emplace_back(iter->second);} + } + DEBUG_PRINT("Found %ld valid detections", objectList.size()); + return true; +} + +bool NvDsInferParseCustomSegformerTAO( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + float segmentationThreshold, + unsigned int numClasses, + int* classificationMap, + float*& classProbabilityMap) { + + DEBUG_DUMP_SECTION(); // scope to entire function {} + + for(auto layer : outputLayersInfo) { + if(layer.dataType == INT64 && !strcmp(layer.layerName, "output")) { + DEBUG_PRINT("Processing int64 output layer"); + DEBUG_DS_TENSOR(layer); + + for (unsigned int i = 0; i < networkInfo.height * networkInfo.width; i++) { + classificationMap[i] = ((int64_t*)layer.buffer)[i]; + } + + DEBUG_TENSOR("segformer_classification_map", + classificationMap, + networkInfo.height * networkInfo.width, + int); + } else { + std::cerr << "ERROR: Mismatched data type from output layer: " << layer.layerName << std::endl; + DEBUG_PRINT("Exiting NvDsInferParseCustomSegformerTAO with error"); + return false; + } + } + + return true; +} + +extern "C" +bool NvDsInferParseCustomSegformerTAO2( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + float segmentationThreshold, + unsigned int numClasses, + int* classificationMap, + float*& classProbabilityMap) { + + DEBUG_DUMP_SECTION(); // scope to entire function {} + + if (numClasses <= 0) { + DEBUG_PRINT("numClasses should be configured greater than 0, but got %d", numClasses); + return false; + } + + for(auto layer : outputLayersInfo) { + if (layer.dataType == FLOAT && !strcmp(layer.layerName, "output")) { + DEBUG_PRINT("Processing float32 output layer"); + DEBUG_DS_TENSOR(layer); + DEBUG_PRINT("numClasses=%d, height=%d, width=%d", numClasses, networkInfo.height, networkInfo.width); + + classProbabilityMap = (float*)layer.buffer; + // Create temporary buffer for argmax result before resize + unsigned int tensorHeight = layer.inferDims.d[1]; // 56 eg + unsigned int tensorWidth = layer.inferDims.d[2]; // 56 eg + std::vector tempClassMap(tensorHeight * tensorWidth); + + // Perform argmax along class dimension + float maxScore = -1; + int maxIdx = -1; + for (unsigned int y = 0; y < tensorHeight; y++) { + for (unsigned int x = 0; x < tensorWidth; x++) { + // find max score and max class index at each pixel + for (unsigned int c = 0; c < numClasses; c++) { + if (c == 0) { + // init maxScore and maxIdx with class 0 + maxScore = classProbabilityMap[c * tensorWidth * tensorHeight + y * tensorWidth + x]; + maxIdx = c; + } else { + float score = classProbabilityMap[c * tensorWidth * tensorHeight + y * tensorWidth + x]; + if (score > maxScore) { + maxScore = score; + maxIdx = c; + } + } + } + tempClassMap[y * tensorWidth + x] = maxIdx; + } + } + + // Nearest neighbor interpolation to resize from tensorHeight x tensorWidth to networkInfo.height x networkInfo.width + float scaleY = (float)tensorHeight / networkInfo.height; + float scaleX = (float)tensorWidth / networkInfo.width; + + for (unsigned int y = 0; y < networkInfo.height; y++) { + for (unsigned int x = 0; x < networkInfo.width; x++) { + // Find nearest source pixel + unsigned int srcY = std::min((unsigned int)(y * scaleY), tensorHeight - 1); + unsigned int srcX = std::min((unsigned int)(x * scaleX), tensorWidth - 1); + + // Direct copy of class index + classificationMap[y * networkInfo.width + x] = tempClassMap[srcY * tensorWidth + srcX]; + } + } + DEBUG_TENSOR("segformer_classification_map", + classificationMap, + networkInfo.height * networkInfo.width, + int); + } else { + std::cerr << "ERROR: Mismatched data type from output layer: " << layer.layerName << std::endl; + DEBUG_PRINT("Exiting NvDsInferParseCustomSegformerTAO2 with error"); + return false; + } + } + + return true; +} + +bool NvDsInferParseCustomChangeNet( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + float segmentationThreshold, + unsigned int numClasses, + int* classificationMap, + float*& classProbabilityMap) { + bool found = false; + for(auto layer : outputLayersInfo) { + if(layer.dataType == FLOAT && !strcmp(layer.layerName, "output_final")) { + classProbabilityMap = (float*)layer.buffer; + for (unsigned int y = 0; y < networkInfo.height; y++) { + for (unsigned int x = 0; x < networkInfo.width; x++) { + float max_prob = -1; + int& cls = classificationMap[y * networkInfo.width + x] = -1; + for (unsigned int c = 0; c < numClasses; c++) { + float prob = classProbabilityMap[c * networkInfo.width * networkInfo.height + y * networkInfo.width + x]; + if (prob > max_prob && prob > segmentationThreshold) { + cls = c; + max_prob = prob; + } + } + } + } + found = true; + break; + } + } + if (!found) { + std::cerr << "ERROR: output_final layer not found" << std::endl; + return false; + } + return true; +} + +extern "C" +bool NvDsInferClassiferParseNonSoftmax (std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + float classifierThreshold, + std::vector &attrList, + std::string &attrString) { + /* Get the number of attributes supported by the classifier. */ + unsigned int numAttributes = outputLayersInfo.size(); + /* Iterate through all the output coverage layers of the classifier. + */ + for (unsigned int l = 0; l < numAttributes; l++) + { + const NvDsInferLayerInfo &layerInfo = outputLayersInfo[l]; + /* Check if the layer is of type FLOAT. */ + if (layerInfo.dataType != FLOAT) + { + std::cerr << "ERROR: Mismatched data type from output layer: " << layerInfo.layerName << std::endl; + return false; + } + unsigned int numClasses = layerInfo.inferDims.d[0]; + float *outputCoverageBuffer = (float *)layerInfo.buffer; + float maxProbability = 0; + bool attrFound = false; + NvDsInferAttribute attr; + float sum = 0; + + for (unsigned int c = 0; c < numClasses; c++) { + sum += exp(outputCoverageBuffer[c]); + } + /* Iterate through all the probabilities that the object belongs to + * each class. Find the maximum probability and the corresponding class + * which meets the minimum threshold. */ + for (unsigned int c = 0; c < numClasses; c++) + { + // Calculate the softmax probability + float probability = exp(outputCoverageBuffer[c]) / sum; + if (probability > classifierThreshold + && probability > maxProbability) + { + maxProbability = probability; + attrFound = true; + attr.attributeIndex = l; + attr.attributeValue = c; + attr.attributeConfidence = probability; + attr.attributeLabel = nullptr; + } + } + if (attrFound) { + attrList.push_back(attr); + } + } + return true; +} + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomNMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedYoloV5NMSTLT); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomBatchedNMSTLT); +CHECK_CUSTOM_INSTANCE_MASK_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomMrcnnTLTV2); +CHECK_CUSTOM_INSTANCE_MASK_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomMask2Former); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomEfficientDetTAO); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomDDETRTAO); +CHECK_CUSTOM_SEM_SEGMENTATION_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomSegformerTAO); +CHECK_CUSTOM_SEM_SEGMENTATION_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomSegformerTAO2); +CHECK_CUSTOM_CLASSIFIER_PARSE_FUNC_PROTOTYPE(NvDsInferClassiferParseNonSoftmax); diff --git a/src/apps/tao_apps/pysm-apps/README.md b/src/apps/tao_apps/pysm-apps/README.md new file mode 100644 index 00000000..f547b8a3 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/README.md @@ -0,0 +1,155 @@ +- [1.Description](#1description) +- [2.Prerequisites](#2prerequisites) +- [3.Download Models](#3download-models) +- [4.Prepare Python Environment](#4prepare-python-environment) +- [5.Detection & Instance segmention Sample](#5detection--instance-segmention-sample) +- [6.Semantic Segmention Sample](#6semantic-segmention-sample) +- [7.Car License Recognization Sample](#7car-license-recognization-sample) +- [8.Optical Character Detection And Recognition Sample](#8optical-character-detection-and-recognition-sample) +- [9.MDX Perception Sample Application Sample](#9mdx-perception-sample-application-sample) + +===================================================================== + +### 1.Description + +This document describes how to use Pyservicemaker's Flow API to quickly build applications for TAO pre-trained models. + +The following pyservicemaker applications with TAO models are provided: + +- [Detection & Instance Segmention Sample](tao_detection/deepstream_det_app.py) + +- [Semantic Segmention Sample](tao_segmentation/deepstream_seg_app.py) + +- [MDX sample with embedding Models](tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.py) + +- [OCD/OCR sample](tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.py) + +- [Pose Classification Sample](tao_others/deepstream-pose-classification/deepstream_pose_classification_app.py) + +- [Car License Plate Recognition Sample](tao_others/deepstream_lpr_app/deepstream_lpr_app.py) + +### 2.Prerequisites + +* [DeepStream SDK 9.0 GA](https://developer.nvidia.com/deepstream-sdk) + + Make sure deepstream-test1 sample can run successful to verify your installation. According to the + [document](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html), + please run below command to install additional audio video packages. + +```bash + # install pyservicemaker wheel + /opt/nvidia/deepstream/deepstream/install.sh + # install deepstream dependencies + /opt/nvidia/deepstream/deepstream/user_additional_install.sh +``` + +* Eigen development packages +```bash + sudo apt install libeigen3-dev + cd /usr/include + sudo ln -sf eigen3/Eigen Eigen +``` + +### 3.Download Models + +Run below script to download models + +```bash +$ git clone https://github.com/NVIDIA/deepstream.git + +# rebuild if the code is modified. +#$ make + +# download tao models +$ ./download_models.sh +``` + +### 4.Prepare Python Environment + +```bash +# install uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Create a venv for pyservicemaker +uv venv pysmvenv + +# Activate the pysmvenv environment +source pysmvenv/bin/activate + +# install numpy & opencv-python +uv pip install numpy opencv-python cuda-python pyyaml + +# install pyservicemaker +uv pip install /opt/nvidia/deepstream/deepstream/service-maker/python/pyservicemaker*.whl +``` + +### 5.Detection & Instance Segmention Sample + +Run TAO Detection Pyservicemaker Application + +```bash +cd pysm-apps/tao_detection +python3 deepstream_det_app.py ../../configs/app/det_app_config.yml +``` + +Run TAO Instance Segmention Application + +```bash +SHOW_MASK=1 python3 deepstream_det_app.py ../../configs/app/ins_seg_app.yml +``` + +The application saves the detection results as `pysm_det_output.mp4` + +### 6.Semantic Segmention Sample + +Run TAO Semantic Segmention Application + +```bash +cd pysm-apps/tao_segmentation +python3 deepstream_seg_app.py ../../configs/app/seg_app_config.yml +``` + +The application will save the semantic segmentation results to `pysm_seg_output.mp4`, and save the segmentation meta as an image every 100 frames. At the same time, the `FPS` will be output in the terminal. + +### 7.Car License Recognization Sample + +Run Car License Recognization Application + +A sample of US car plate recognition: + +```shell +cd pysm-apps/tao_others/deepstream_lpr_app +cp ../../../apps/tao_others/deepstream_lpr_app/dict_us.txt dict.txt +python3 deepstream_lpr_app.py ../../../configs/app/lpr_app_us_config.yml +``` + +A sample of Chinese car plate recognition: + +```shell +cp ../../../apps/tao_others/deepstream_lpr_app/dict_ch.txt dict.txt +python3 deepstream_lpr_app.py ../../../configs/app/lpr_app_ch_config.yml +``` +Note: please refer to ../../../apps/tao_others/deepstream_lpr_app/README.md for more details. + +### 8.Optical Character Detection And Recognition Sample + +Run nvOCDR Application + +```shell +cd pysm-apps/deepstream-nvocdr-app +python3 deepstream_nvocdr_app.py ../../../configs/app/nvocdr_app_config.yml +``` + +Note: please refer to ../../../apps/tao_others/deepstream-nvocdr-app/README.md for more details. + +### 9.MDX Perception Sample Application Sample + +Run MDX Perception Application + +```shell +cd pysm-apps/deepstream-mdx-perception-app +python3 deepstream_mdx_perception_app.py ../../../configs/app/peoplenet_reidentification.yml +python3 deepstream_mdx_perception_app.py ../../../configs/app/retail_object_detection_recognition.yml +``` + +Note: please refer to ../../../apps/tao_others/deepstream-mdx-perception-app/README.md for more details. diff --git a/src/apps/tao_apps/pysm-apps/common/__init__.py b/src/apps/tao_apps/pysm-apps/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/apps/tao_apps/pysm-apps/common/utils.py b/src/apps/tao_apps/pysm-apps/common/utils.py new file mode 100644 index 00000000..aa38b248 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/common/utils.py @@ -0,0 +1,279 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +import os, yaml +from pyservicemaker import Flow +from pyservicemaker.logging import get_logger +from typing import Dict, Optional, Any +import subprocess + +logger = get_logger("tao_seg_app") +logger.setLevel("DEBUG") + + +class Config: + def __init__(self, config_path, config): + self._config_path = config_path + self._config = config + + @property + def config(self): + return self._config + + def __getattr__(self, name: str) -> Any: + match name: + case "stream_list": + return self._config["source-list"]["list"].split(";") + case "source_cvs": + cvs_config = self._config["source"]["csv-file-path"] + yaml_directory = os.path.dirname(os.path.abspath(self._config_path)) + cvs_abs_path = os.path.join(yaml_directory, cvs_config) + print("cvs_abs_path;", cvs_abs_path) + urls = [] + with open(cvs_abs_path, "r") as file: + for line in file: + items = line.split(",") + if items[2] == "uri": + continue + urls.append(items[2]) + print(urls) + return urls + case "streammux_width": + return self._config["streammux"]["width"] + case "streammux_height": + return self._config["streammux"]["height"] + case "pgie_infer_type": + return self._config["primary-gie"]["plugin-type"] + case "pgie_infer_config": + infer_config = self._config["primary-gie"]["config-file-path"] + yaml_directory = os.path.dirname(os.path.abspath(self._config_path)) + return os.path.join(yaml_directory, infer_config) + case "pgie_config_file": + infer_config = self._config["primary-gie"]["config-file"] + yaml_directory = os.path.dirname(os.path.abspath(self._config_path)) + return os.path.join(yaml_directory, infer_config) + case str() if name.startswith("sgie") and "_" in name: + print(name.split("_")) + index = int(name.split("_")[1]) # get sgie index + sgie_key = f"secondary-gie{index}" + print(f"sgie_key: {sgie_key}") + match name: + case _ if "infer_type" in name: + return self._config[sgie_key]["plugin-type"] + case _ if "infer_config" in name: + infer_config = self._config[sgie_key]["config-file-path"] + yaml_directory = os.path.dirname( + os.path.abspath(self._config_path) + ) + return os.path.join(yaml_directory, infer_config) + case _ if "config_file" in name: + infer_config = self._config[sgie_key]["config-file"] + yaml_directory = os.path.dirname( + os.path.abspath(self._config_path) + ) + return os.path.join(yaml_directory, infer_config) + case "eglsink": + return self._config["eglsink"].get("enable") or False + case "fakesink": + return self._config["fakesink"].get("enable") or False + case "filesink": + return self._config["filesink"].get("enable") or False + case "enc_type": + # 0 for hardware encoding, 1 for software encoding + # if the hardware doesn't supports encoding, force return 1 + if is_enc_hw_support() is False: + return 1 + return self._config["filesink"].get("enc-type") or 0 + #use filesink by default + + #preprocess + case str() if name.startswith("preprocess") and "_" in name: + print(name.split("_")) + index = int(name.split("_")[1]) # get sgie index + sgie_key = f"secondary-preprocess{index}" + print(f"preprocess_key: {sgie_key}") + match name: + case _ if "config_file_path" in name: + infer_config = self._config[sgie_key]["config-file-path"] + yaml_directory = os.path.dirname( + os.path.abspath(self._config_path) + ) + return os.path.join(yaml_directory, infer_config) + + #postprocess + case str() if name.startswith("postprocess") and "_" in name: + print(name.split("_")) + index = int(name.split("_")[1]) # get sgie index + sgie_key = f"secondary-postprocess{index}" + print(f"postprocess_key: {sgie_key}") + match name: + case _ if "config_file_path" in name: + tmp = self._config[sgie_key]["config-file-path"] + yaml_directory = os.path.dirname( + os.path.abspath(self._config_path) + ) + return os.path.join(yaml_directory, tmp) + case _ if "lib_name" in name: + tmp = self._config[sgie_key]["lib-name"] + yaml_directory = os.path.dirname( + os.path.abspath(self._config_path) + ) + return os.path.join(yaml_directory, tmp) + + #sink + case str() if name.startswith("sink") and "_" in name: + index = int(name.split("_")[1]) # get sgie index + print("index:", index) + sgie_key = f"sink{index}" + print(f"sgie_key: {sgie_key}") + match name: + case _ if "enable" in name: + return self._config[sgie_key].get("enable") or 0 + case _ if "type" in name: + return self._config[sgie_key].get("type") or 1 + case _ if "filename" in name: + return self._config[sgie_key].get("output-file") or "test" + case _ if "encType" in name: + return self._config[sgie_key].get("enc-type") or 0 + case _ if "bitrate" in name: + return self._config[sgie_key].get("bitrate") or 2000000 + case _ if "msg_conv_config" in name: + return self._config[sgie_key].get("msg-conv-config") or \ + "/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream-test5/configs/dstest5_msgconv_sample_config.txt" + case _ if "msg_broker_proto_lib" in name: + return self._config[sgie_key].get("msg-broker-proto-lib") or \ + "/opt/nvidia/deepstream/deepstream/lib/libnvds_kafka_proto.so" + case _ if "msg_broker_conn_str" in name: + return self._config[sgie_key].get("msg-broker-conn-str") or \ + "127.0.0.1;9092" + case _ if "msg_topic" in name: + return self._config[sgie_key].get("topic") or \ + "test" + + case "sinkType": + return self._config["sink"].get("sink-type") or 1 + case "sinkEncType": + return self._config["sink"].get("enc-type") or 0 + #output + case "output_type": + return self._config["output"].get("type") or 1 + case "output_enc_type": + return self._config["output"].get("enc-type") or 0 + case "output_filename": + return self._config["output"].get("filename") or "test" + case "output_bitrate": + return self._config["output"].get("bitrate") or 2000000 + case "output_codec": + return self._config["output"].get("codec") or 0 + + #tracker + case "tracker_ll_cfg_file": + return self._config["tracker"].get("ll-config-file") + case "tracker_ll_lib_file": + return self._config["tracker"].get("ll-lib-file") + + #analytics + case "analytics": + return self._config["analytics"].get("enable") or False + case "analytics_config_file": + config = self._config["analytics"]["config-file"] + yaml_directory = os.path.dirname(os.path.abspath(self._config_path)) + return os.path.join(yaml_directory, config) + + #segvisual + case "segvisual": + data = self._config["segvisual"] + segvisual_width = data["width"] + segvisual_height = data["height"] + segvisual_orig_background = data.get("orig_background") or False + segvisual_alpha = data.get("alpha") or 0.5 + return { + "width": segvisual_width, + "height": segvisual_height, + "original-background": segvisual_orig_background, + "alpha": float(segvisual_alpha), + } + + #video_template + case "video_template_customlib_name": + return self._config["video-template"].get("customlib-name") + case "video_template_customlib_props": + return self._config["video-template"].get("customlib-props") + case _: + if name in self._config: + return self._config[name] + else: + raise AttributeError(f"Config has no attribute '{name}'") + + +def dump_config(yaml_config_path: str) -> Optional[Config]: + """ + Load configuration from a YAML file. + Args: + config_file (str): Path to the YAML configuration file. + Returns: + Config: An instance of Config containing the loaded configuration, or None if loading fails. + """ + try: + with open(yaml_config_path, "r") as file: + yaml_data = yaml.safe_load(file) + return Config(yaml_config_path, yaml_data) + except FileNotFoundError: + logger.error(f"file not found: {yaml_config_path}") + return None + except PermissionError: + logger.error(f"no permission: {yaml_config_path}") + return None + except Exception as e: + logger.error(f"unknown error : {e}") + raise e + +def get_node_name(func, name) -> str: + if not hasattr(func, "name_counter"): + func.name_counter = 0 + else: + func.name_counter += 1 + base_name = func.__name__.split('.')[-1] + return f"{base_name}-{name}-{func.name_counter}" + +def flow_cls_hook(self, type_names: list, properties: list) -> "Flow": + """ + Dynamic add method to Flow class. For adding a new element to the pipeline + Args: + type_names(list(str)): list of element name + properties(list(dict)): list of property dict + Return: A derived flow + Raises: Upstream Exception + """ + last_name="" + for index, name in enumerate(type_names): + element_name = get_node_name(flow_cls_hook, name) + self._pipeline.add(name, element_name, properties[index]) + self._pipeline.link(self._streams[0], element_name) + self._streams=[element_name] + last_name=element_name + return Flow(self._pipeline, streams=[last_name], parent=self) + + +def is_enc_hw_support(): + enc_hw_support = True + + process = subprocess.Popen(['gst-inspect-1.0', 'nvv4l2h264enc'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + stdout, stderr = process.communicate() + + if process.returncode != 0: + print("enc_hw is not supported") + enc_hw_support = False + + return enc_hw_support diff --git a/src/apps/tao_apps/pysm-apps/tao_detection/deepstream_det_app.py b/src/apps/tao_apps/pysm-apps/tao_detection/deepstream_det_app.py new file mode 100644 index 00000000..cb934a2a --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_detection/deepstream_det_app.py @@ -0,0 +1,141 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import os, sys, math +import numpy as np +import cv2 + +sys.path.append("../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("tao_det_app") +logger.setLevel("DEBUG") +show_mask = os.environ.get("SHOW_MASK", 0) + + +class ObjectCounterMarker(BatchMetadataOperator): + def _save_instance_mask(self, img_path, ins_mask, target_width, target_height): + print(f"instance mask : {ins_mask.shape} {ins_mask.dtype}") + mask_uint8 = (ins_mask * 255).astype(np.uint8) + + mask_image = cv2.resize( + mask_uint8, + (target_width, target_height), + dst=mask_uint8, + interpolation=cv2.INTER_LINEAR, + ) + + cv2.imwrite(img_path, mask_image) # Save mask to image + + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + text = f"Frame Number={frame_meta.frame_number}, Object Count={len(list(frame_meta.object_items))}" + logger.debug(f"Object Counter: Pad Idx={frame_meta.pad_index}, {text}") + is_first_object = True + if show_mask and frame_meta.frame_number % 100 == 0: + for object_meta in frame_meta.object_items: + if is_first_object: + print( + f"Object Rect {object_meta.rect_params} Mask {object_meta.mask_params}" + ) + is_first_object = False + ins_mask = object_meta.mask_params.mask_array + img_path = f"frame-{frame_meta.frame_number}-obj-mask.jpg" + self._save_instance_mask( + img_path, + ins_mask, + math.floor(object_meta.rect_params.width), + math.floor(object_meta.rect_params.height), + ) + + display_meta = batch_meta.acquire_display_meta() + label = osd.Text() + label.display_text = text.encode("ascii") + label.x_offset = 10 + label.y_offset = 12 + label.font.name = osd.FontFamily.Serif + label.font.size = 12 + label.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + label.set_bg_color = True + label.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(label) + frame_meta.append(display_meta) + + +# alias flow_cls_hook as overlay +overlay = flow_cls_hook + + +def deepstream_det_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + osd_properties = {} + if show_mask: + osd_properties = { + "display-mask": True, + "display-bbox": False, + "display-text": False, + "process-mode": 0, + } + pipeline = Pipeline("deepstream_det_app") + # add Monkey patch for osd + Flow.overlay = overlay # type: ignore + flow = Flow(pipeline) + flow = flow.batch_capture( + conf.stream_list, width=conf.streammux_width, height=conf.streammux_height + ) + flow = flow.infer(conf.pgie_infer_config, with_triton=conf.pgie_infer_type) + flow = flow.attach(what=Probe("counter", ObjectCounterMarker())) + flow = flow.attach(what="measure_fps_probe", name="fps_probe") + # Add OSD overlay to the pipeline for encoder can output bbox and mask + type_names = [] + properties = [] + type_names.append("nvdsosd") + properties.append(osd_properties) + flow = flow.overlay(type_names, properties) + + # Fork the pipeline for parallel processing + # This allows simultaneous video rendering and encode to files + flow = flow.fork() + + if conf.eglsink: + flow = flow.render(enable_osd=False) + if conf.filesink: + flow = flow.encode(dest="pysm_det_output.mp4", use_sw_codec=conf.enc_type) + if conf.fakesink: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_det_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/pysm-apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.py b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.py new file mode 100644 index 00000000..85f57062 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-mdx-perception-app/deepstream_mdx_perception_app.py @@ -0,0 +1,149 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import os, sys +import torch +from enum import Enum + +sys.path.append("../../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("deepstream_mdx_perception_app") +logger.setLevel("DEBUG") + +class Sink_Type(Enum): + fakesink = 1 + displaysink = 2 + filesink = 3 + msgbrokersink = 6 + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + for object_meta in frame_meta.object_items: + for user_meta in object_meta.tensor_items: + output_layers=user_meta.as_tensor_output().get_layers() + layer_name="embeddings" + vector_data = output_layers.pop(layer_name, None) + embedding_vector = torch.utils.dlpack.from_dlpack(vector_data).to('cpu') + print(embedding_vector) + + +# alias flow_cls_hook as overlay +make_link_element = flow_cls_hook + +def sink_out(flow, properties: list): + print("properties", properties) + sink_type=properties["type"] + if sink_type == Sink_Type.fakesink.value: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + elif sink_type == Sink_Type.displaysink.value: + flow = flow.render(enable_osd=False) + elif sink_type == Sink_Type.filesink.value: + flow = flow.encode(dest=properties["name"], use_sw_codec=properties["encType"], bitrate=properties["bitrate"]) + elif sink_type == Sink_Type.msgbrokersink.value: + # Configure message publishing to Kafka + # Sends processed data to external systems + flow.publish( + msg_broker_proto_lib=properties["msg_broker_proto_lib"], + msg_broker_conn_str=properties["msg_broker_conn_str"], + topic=properties["msg_topic"], + msg_conv_config=properties["msg_conv_config"], + sync=False, # Asynchronous publishing for better performance + ) + +def deepstream_mdx_perception_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + + pipeline = Pipeline("deepstream_mdx_perception_app") + # add Monkey patch for osd + Flow.make_link_element = make_link_element + flow = Flow(pipeline) + + flow = flow.batch_capture( + conf.source_cvs, width=1280, height=720 + ) + flow = flow.infer(conf.pgie_config_file, with_triton=conf.pgie_infer_type) + flow = flow.track(ll_config_file=conf.tracker_ll_cfg_file, ll_lib_file=conf.tracker_ll_lib_file) + flow = flow.infer(conf.sgie_0_config_file, with_triton=conf.sgie_0_infer_type) + flow = flow.attach(what=Probe("counter", ObjectCounterMarker())) + # Message metadata generation probe - prepares data for Kafka publishing + flow = flow.attach( + what="add_message_meta_probe", + name="message_generator" + ) + # Add OSD overlay to the pipeline for encoder can output bbox and mask + osd_properties = { + "display-mask": False, + "display-bbox": True, + "display-text": True, + "process-mode": 0, + } + flow = flow.make_link_element(["nvdsosd"], [osd_properties]) + + # Fork the pipeline for parallel processing + # This allows simultaneous video rendering and encode to files + flow = flow.fork() + + #sink0 + if conf.sink_0_enable == 1: + sink_properties = { + "type": conf.sink_0_type, + } + sink_out(flow, sink_properties) + #sink1 + if conf.sink_1_enable == 1: + sink_properties = { + "type": conf.sink_1_type, + "name": conf.sink_1_filename, + "encType": conf.sink_1_encType, + "bitrate": conf.sink_1_bitrate, + } + sink_out(flow, sink_properties) + #sink2 + if conf.sink_2_enable == 1: + sink_properties = { + "type": conf.sink_2_type, + "msg_conv_config": conf.sink_2_msg_conv_config, + "msg_broker_proto_lib": conf.sink_2_msg_broker_proto_lib, + "msg_broker_conn_str": conf.sink_2_msg_broker_conn_str, + "msg_topic": conf.sink_2_msg_topic, + } + sink_out(flow, sink_properties) + + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_mdx_perception_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/pysm-apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.py b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.py new file mode 100644 index 00000000..a3cb9bea --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-nvocdr-app/deepstream_nvocdr_app.py @@ -0,0 +1,88 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import os, sys + +sys.path.append("../../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("deepstream_nvocdr_app") +logger.setLevel("DEBUG") + +# alias flow_cls_hook as overlay +make_link_element = flow_cls_hook + +def deepstream_nvocdr_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + + pipeline = Pipeline("deepstream_nvocdr_app") + # add Monkey patch for osd + Flow.make_link_element = make_link_element # type: ignore + flow = Flow(pipeline) + flow = flow.batch_capture( + conf.stream_list, width=conf.streammux_width, height=conf.streammux_height + ) + + # Add nvdsvideotemplate to the pipeline + list_props = conf.video_template_customlib_props + vt_properties = { + "customlib_name": conf.video_template_customlib_name + } + type_names = [] + properties = [] + type_names.append("nvdsvideotemplate") + properties.append(vt_properties) + flow = flow.make_link_element(type_names, properties) + for prop in list_props: + my_dict = {"customlib-props": prop} + flow.pipeline[flow._streams[0]].set(my_dict) + + # Add OSD overlay to the pipeline for encoder can output bbox and mask + osd_properties = {} + flow = flow.make_link_element(["nvdsosd"], [osd_properties]) + + if conf.output_type == 1: + filename=conf.output_filename+".mp4" + flow = flow.encode(dest=filename, use_sw_codec=conf.output_enc_type, bitrate=conf.output_bitrate) + elif conf.output_type == 2: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + elif conf.output_type == 3: + flow = flow.render(enable_osd=False) + else: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_nvocdr_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/pysm-apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.py b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.py new file mode 100644 index 00000000..340c7332 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_others/deepstream-pose-classification/deepstream_pose_classification_app.py @@ -0,0 +1,181 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import os, sys +from enum import Enum +import math + +sys.path.append("../../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("deepstream_pose_classification_app") +logger.setLevel("DEBUG") + +class Sink_Type(Enum): + fakesink = 0 + filesink = 1 + rtspsink = 2 + displaysink = 3 +#Padding due to AR SDK model requires bigger bboxes +PAD_DIM=128 +_pad_dim=PAD_DIM +MUXER_OUTPUT_WIDTH=1280 +MUXER_OUTPUT_HEIGHT=720 +_image_width=MUXER_OUTPUT_WIDTH +_image_height=MUXER_OUTPUT_HEIGHT + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + # Padding due to AR SDK model requires bigger bounding boxes + muxer_output_width_pad = _pad_dim * 2 + _image_width + muxer_output_height_pad = _pad_dim * 2 + _image_height + + for frame_meta in batch_meta.frame_items: + for obj_meta in frame_meta.object_items: + sizex = obj_meta.rect_params.width * 0.5 + sizey = obj_meta.rect_params.height * 0.5 + centrx = obj_meta.rect_params.left + sizex + centry = obj_meta.rect_params.top + sizey + sizex *= 1.25 + sizey *= 1.25 + if sizex < sizey: + sizex = sizey + else: + sizey = sizex + obj_meta.rect_params.width = round(2.0 * sizex) + obj_meta.rect_params.height = round(2.0 * sizey) + obj_meta.rect_params.left = round(centrx - obj_meta.rect_params.width / 2.0) + obj_meta.rect_params.top = round(centry - obj_meta.rect_params.height / 2.0) + + sizex = obj_meta.rect_params.width * 0.5 + sizey = obj_meta.rect_params.height * 0.5 + centrx = obj_meta.rect_params.left + sizex + centry = obj_meta.rect_params.top + sizey + + x_scale = 192.0 / sizex + y_scale = 256.0 / sizey + + if x_scale < y_scale: + sizey = 256.0 / x_scale # Expand height + else: + sizex = 192.0 / y_scale # Expand width + + obj_meta.rect_params.width = round(2.0 * sizex) + obj_meta.rect_params.height = round(2.0 * sizey) + obj_meta.rect_params.left = round(centrx - obj_meta.rect_params.width / 2.0) + obj_meta.rect_params.top = round(centry - obj_meta.rect_params.height / 2.0) + + if obj_meta.rect_params.left < 0.0: + obj_meta.rect_params.left = 0.0 + if obj_meta.rect_params.top < 0.0: + obj_meta.rect_params.top = 0.0 + if obj_meta.rect_params.left + obj_meta.rect_params.width > muxer_output_width_pad - 1: + obj_meta.rect_params.width = muxer_output_width_pad - 1 - obj_meta.rect_params.left + if obj_meta.rect_params.top + obj_meta.rect_params.height > muxer_output_height_pad - 1: + obj_meta.rect_params.height = muxer_output_height_pad - 1 - obj_meta.rect_params.top + + +# alias flow_cls_hook as overlay +make_link_element = flow_cls_hook + +def sink_out(flow, sink_type, enc_name="test.mp4", enc_type=1, enc_bitrate=2000000): + print("sink_out", sink_type, enc_name, enc_type, enc_bitrate) + if sink_type == Sink_Type.fakesink.value: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + elif sink_type == Sink_Type.filesink.value: + flow = flow.encode(dest=enc_name, use_sw_codec=enc_type, bitrate=enc_bitrate) + elif sink_type == Sink_Type.rtspsink.value: + flow = flow.make_link_element(["nvrtspoutsinkbin"], [{"enc-type": enc_type}]) + elif sink_type == Sink_Type.displaysink.value: + flow = flow.render(enable_osd=False) + +def deepstream_pose_classification_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + + pipeline = Pipeline("deepstream_pose_classification_app") + # add Monkey patch for osd + Flow.make_link_element = make_link_element + flow = Flow(pipeline) + + #the pipeline is ......->nvvideoconvert->capsfilter->pgie->tracker->sgie0->nvdspostprocess-> + #->nvdspreprocess-> + flow = flow.batch_capture( + conf.stream_list, width=1280, height=720 + ) + + _pad_dim = PAD_DIM * _image_width / MUXER_OUTPUT_WIDTH + muxer_output_width_pad = int(_pad_dim * 2 + _image_width) + muxer_output_height_pad = int(_pad_dim * 2 + _image_height) + flow = flow.make_link_element(["nvvideoconvert", "capsfilter"], + [{ + "dest-crop": f"{_pad_dim}:{_pad_dim}:{_image_width}:{_image_height}", + "interpolation-method": 1}, + {"caps": f"video/x-raw(memory:NVMM),width={muxer_output_width_pad},height={muxer_output_height_pad}"} + ] + ) + + flow = flow.infer(conf.pgie_infer_config, with_triton=conf.pgie_infer_type) + flow = flow.track(ll_config_file=conf.tracker_ll_cfg_file, ll_lib_file=conf.tracker_ll_lib_file) + flow = flow.attach(what=Probe("counter", ObjectCounterMarker())) + flow = flow.infer(conf.sgie_0_infer_config, with_triton=conf.sgie_0_infer_type) + print(conf.postprocess_0_config_file_path, conf.postprocess_0_lib_name) + flow = flow.make_link_element(["nvdspreprocess", "nvdspostprocess"], + [{"config-file": conf.preprocess_1_config_file_path}, + {"postprocesslib-config-file": conf.postprocess_0_config_file_path, + "postprocesslib-name": conf.postprocess_0_lib_name}] + ) + flow = flow.infer(conf.sgie_1_infer_config, with_triton=conf.sgie_1_infer_type) + + source_num = len(conf.stream_list) + tiler_rows = int(math.sqrt(source_num)) + tiler_columns = int(math.ceil(source_num / tiler_rows)) + flow = flow.make_link_element( + ["nvdslogger", "nvvideoconvert", "capsfilter", "nvmultistreamtiler", "nvvideoconvert", "nvdsosd"], + [{"fps-measurement-interval-sec": 1}, + { "src-crop": f"video/x-raw(memory:NVMM),width={muxer_output_width_pad},height={muxer_output_height_pad}"}, + { "caps": f"video/x-raw(memory:NVMM),width={_image_width},height={_image_height}"}, + { "rows": tiler_rows, "columns": tiler_columns, "width": MUXER_OUTPUT_WIDTH, "height": MUXER_OUTPUT_HEIGHT}, + {}, + {"display-mask": False, "display-bbox": True, "display-text": True, "process-mode": 0}, + ] + ) + + #sink + sink_out(flow, conf.sinkType, "out.mp4", conf.sinkEncType) + + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_pose_classification_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/pysm-apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.py b/src/apps/tao_apps/pysm-apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.py new file mode 100644 index 00000000..9b5dcd13 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_others/deepstream_lpr_app/deepstream_lpr_app.py @@ -0,0 +1,111 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe, osd +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import os, sys + +sys.path.append("../../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("deepstream_lpr_app") +logger.setLevel("DEBUG") + +class ObjectCounterMarker(BatchMetadataOperator): + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + # TODO: missing binding of obj_meta.mask_params + text = f"Frame Number={frame_meta.frame_number}, Object Count={len(list(frame_meta.object_items))}" + logger.debug(f"Object Counter: Pad Idx={frame_meta.pad_index}, {text}") + + display_meta = batch_meta.acquire_display_meta() + label = osd.Text() + label.display_text = text.encode("ascii") + label.x_offset = 10 + label.y_offset = 12 + label.font.name = osd.FontFamily.Serif + label.font.size = 12 + label.font.color = osd.Color(1.0, 1.0, 1.0, 1.0) + label.set_bg_color = True + label.bg_color = osd.Color(0.0, 0.0, 0.0, 1.0) + display_meta.add_text(label) + frame_meta.append(display_meta) + + +# alias flow_cls_hook as overlay +make_link_element = flow_cls_hook + + +def deepstream_lpr_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + + pipeline = Pipeline("deepstream_lpr_app") + # add Monkey patch for osd + Flow.make_link_element = make_link_element + flow = Flow(pipeline) + flow = flow.batch_capture( + conf.stream_list, width=1280, height=720 + ) + flow = flow.infer(conf.pgie_infer_config, with_triton=conf.pgie_infer_type) + flow = flow.attach(what=Probe("counter", ObjectCounterMarker())) + flow = flow.attach(what="measure_fps_probe", name="fps_probe") + flow = flow.track(ll_config_file=conf.tracker_ll_cfg_file, ll_lib_file=conf.tracker_ll_lib_file) + flow = flow.infer(conf.sgie_0_infer_config, with_triton=conf.sgie_0_infer_type) + flow = flow.infer(conf.sgie_1_infer_config, with_triton=conf.sgie_1_infer_type) + # Add OSD overlay to the pipeline for encoder can output bbox and mask + type_names = [] + properties = [] + if conf.analytics: + analytics_properties = { + "config-file": conf.analytics_config_file + } + type_names.append("nvdsanalytics") + properties.append(analytics_properties) + type_names.append("nvdsosd") + osd_properties = {} + properties.append(osd_properties) + flow = flow.make_link_element(type_names, properties) + + if conf.output_type == 1: + filename=conf.output_filename+".mp4" + flow = flow.encode(dest=filename, use_sw_codec=conf.output_enc_type, bitrate=conf.output_bitrate) + elif conf.output_type == 2: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + elif conf.output_type == 3: + flow = flow.render(enable_osd=False) + else: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_lpr_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/pysm-apps/tao_segmentation/deepstream_seg_app.py b/src/apps/tao_apps/pysm-apps/tao_segmentation/deepstream_seg_app.py new file mode 100644 index 00000000..41847109 --- /dev/null +++ b/src/apps/tao_apps/pysm-apps/tao_segmentation/deepstream_seg_app.py @@ -0,0 +1,142 @@ +# Copyright and license information +# +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +# + +# Import required libraries +from pyservicemaker import Pipeline, Flow, BatchMetadataOperator, Probe +from pyservicemaker.logging import get_logger +from pyservicemaker.flow import RenderMode +from multiprocessing import Process +import numpy as np +import cv2 + +import sys + +sys.path.append("../") +from common.utils import flow_cls_hook, dump_config + +logger = get_logger("tao_seg_app") +logger.setLevel("DEBUG") + +COLORS = np.array( + [ + [128, 128, 64], + [0, 0, 128], + [0, 128, 128], + [128, 0, 0], + [128, 0, 128], + [128, 128, 0], + [0, 128, 0], + [0, 0, 64], + [0, 0, 192], + [0, 128, 64], + [0, 128, 192], + [128, 0, 64], + [128, 0, 192], + [128, 128, 128], + [128, 64, 128], + [128, 64, 0], + [0, 64, 128], + [192, 128, 0], + [192, 128, 64], + ], + dtype=np.uint8, +) + + +class SegmentationMaskReceiver(BatchMetadataOperator): + def _map_mask_as_display_bgr(self, mask): + """ + Assigning multiple colors as image output using the information + contained in mask. (BGR is OpenCV standard.) + """ + # get unique class indices from the mask + unique_classes = np.unique(mask) + # create BGR image with zeros + bgr = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=np.uint8) + + # use numpy advanced indexing to assign colors + for idx in unique_classes: + bgr[mask == idx] = COLORS[idx] + + return bgr + + def handle_metadata(self, batch_meta): + for frame_meta in batch_meta.frame_items: + logger.debug(f"Frame Number={frame_meta.frame_number}") + for seg_meta_it in frame_meta.segmentation_items: + if frame_meta.frame_number % 100 == 0: + seg_meta = seg_meta_it.as_segmentation() + masks = seg_meta.class_map + masks = np.array(masks, copy=True, order='C') + masks=masks.reshape(seg_meta.height, seg_meta.width) + print(f"type(masks)={type(masks)}, shape={masks.shape}") + frame_image = self._map_mask_as_display_bgr(masks) + cv2.imwrite(f"frame-{frame_meta.frame_number}.jpg", frame_image) + + +# alias flow_cls_hook as segvisual +segvisual = flow_cls_hook + + +def deepstream_seg_app(yaml_config_path): + conf = dump_config(yaml_config_path) + if conf is None: + logger.error(f"Failed to load configuration from {yaml_config_path}") + sys.exit(1) + pipeline = Pipeline("deepstream_seg_app") + # add Monkey patch for segvisual + Flow.segvisual = segvisual # type: ignore + flow = Flow(pipeline) + flow = flow.batch_capture( + conf.stream_list, width=conf.streammux_width, height=conf.streammux_height + ) + flow = flow.infer(conf.pgie_infer_config, with_triton=conf.pgie_infer_type) + + #add segvisual element + type_names = [] + properties = [] + type_names.append("nvsegvisual") + properties.append(conf.segvisual) + flow = flow.segvisual(type_names, properties) + + flow = flow.attach(what=Probe("segmentation_mask", SegmentationMaskReceiver())) + flow = flow.attach(what="measure_fps_probe", name="fps_probe") + # Fork the pipeline for parallel processing + # This allows simultaneous video rendering and encode to files + flow = flow.fork() + + if conf.eglsink: + flow = flow.render(enable_osd=False) + if conf.filesink: + flow = flow.encode(dest="pysm_seg_output.mp4", use_sw_codec=conf.enc_type) + if conf.fakesink: + flow = flow.render(mode=RenderMode.DISCARD, enable_osd=False, sync=True) + # Execute the pipeline + flow() + + +if __name__ == "__main__": + # Check input arguments + if len(sys.argv) != 2: + logger.error(f"usage: {sys.argv[0]} ") + sys.exit(1) + + # Flow()() is a blocking call due to which the KeyboardInterrupt may not be processed immediately. + # we use Process from multiprocessing which runs the main function in a different process and processes KeyboardInterrupt immediately. + process = Process(target=deepstream_seg_app, args=(sys.argv[1],)) + try: + process.start() + process.join() + except KeyboardInterrupt: + logger.debug("\nCtrl+C detected. Terminating process...") + process.terminate() diff --git a/src/apps/tao_apps/sample/streams/retail_object_h264.mp4 b/src/apps/tao_apps/sample/streams/retail_object_h264.mp4 new file mode 100644 index 00000000..5c1c2bb7 --- /dev/null +++ b/src/apps/tao_apps/sample/streams/retail_object_h264.mp4 @@ -0,0 +1,3 @@ +version https://git.lfs.github.com/spec/v1 +oid sha256:4fe46d3ba9117b5b892e6c629971fd9570210924f8e4c1e72ec8dd71531fe948 +size 4961629 diff --git a/src/apps/tao_apps/triton_server.md b/src/apps/tao_apps/triton_server.md new file mode 100644 index 00000000..24714bd6 --- /dev/null +++ b/src/apps/tao_apps/triton_server.md @@ -0,0 +1,14 @@ +## Prepare Triton Server For Native Inferencing +As mentioned in the README, the DeepStream applications should work as Triton client with Triton Server running natively for cAPIs. So the [Triton Inference Server libraries](https://github.com/triton-inference-server/client) should be installed in the machine. An easier way is to run the DeepStream application in the [DeepStream Triton container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream). + +Running DeepStream Triton container, takes the DeepStream 9.0 container as the example: +``` + docker run --gpus all -it --ipc=host --rm --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -v $(pwd):/samples -e DISPLAY=$DISPLAY -w /samples nvcr.io/nvidia/deepstream:9.0-triton-devel +``` +Inside the container, prepare model engines for Triton server: +``` + ./build_triton_engine.sh + +``` + +Then the DeepStream sample application can be build and run inside this container. diff --git a/src/apps/tao_apps/triton_server_grpc.md b/src/apps/tao_apps/triton_server_grpc.md new file mode 100644 index 00000000..a2da20cc --- /dev/null +++ b/src/apps/tao_apps/triton_server_grpc.md @@ -0,0 +1,45 @@ +# Triton Server +## [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server) Bring Up + +DeepStream applications can work as Triton Inference client. So the corresponding Triton Inference Server should be started before the Triton client start to work. + +An immediate way to start a corresponding Triton Server is to use Triton containers provided in [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tritonserver). Since every DeepStream version has its corresponding Triton Server version, so the reliable way is to use the [DeepStream Triton container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream). + +* The Triton Server can be started in the same machine which the DeepStream application works in, please make sure the Triton Server is started in a new terminal. + +* The Triton Server can be started in another machine as the server which is coonected to the machine for DeepStream applications through ehternet. + +## Prepare Triton Server For gRPC Connection +The following steps take the DeepStream 9.0 GA as an example, if you use other DeepStream versions, the corresponding DeepStream Triton [image](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/deepstream) can be used. + +To start Triton Server with DeepStream Triton container, the docker should be run in a new terminal and the following commands should be run in the same path as the deepstream_app_tao_configs codes are downloaded: +* Start the Triton Inferece Server with DeepStream Triton docker under ./deepstream_tao_apps directory +``` + //start Triton docker, 10001:8001 is used to map docker container's 8000 port to host's 10000 port, these ports can be changed. + docker run --gpus all -it --ipc=host --rm --privileged -v /tmp/.X11-unix:/tmp/.X11-unix -p 10000:8000 -p 10001:8001 -p 10002:8002 -v $(pwd):/samples -e DISPLAY=$DISPLAY -w /samples nvcr.io/nvidia/deepstream:9.0-triton-devel +``` + +For TensorRT 8.6, the OSS libnvinfer_plugin.so should be updated according to [OSS TRT x86 instruction](TRT-OSS/x86/README.md) or [OSS TRT Jetson instruction](TRT-OSS/Jetson/README.md). + +Then the model engines should be generated in the server: + +``` + ./build_triton_engine.sh +``` + +If the server is running in the same machine as the DeepStream application, the following command can be used directly. If it is not, please set the gRPC url as the IP address of the server machine in all the configuration files in deepstream_app_tao_configs/triton-grpc: + +The gRPC url setting looks like: +``` +grpc { + url: "192.168.0.51:10001" + } +``` + +Then the Triton Server service can be started with the following command: +``` + tritonserver --model-repository=/samples/models --strict-model-config=false --grpc-infer-allocation-pool-size=16 --log-verbose=1 --exit-on-error=false + +``` + +The DeepStream sample application should run in another terminal with the Triton Inference client libraries installed. It is recommend to run the application in the DeepStream Triton container, please refer to [triton_server.md](triton_server.md) for how to start a DeepStream Triton container. diff --git a/tools/sam2-onnx-tensorrt/.gitkeep b/tools/sam2-onnx-tensorrt/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/sam2-onnx-tensorrt/LICENSE b/tools/sam2-onnx-tensorrt/LICENSE new file mode 100644 index 00000000..e4234b29 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/LICENSE @@ -0,0 +1,14 @@ +SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/tools/sam2-onnx-tensorrt/README.md b/tools/sam2-onnx-tensorrt/README.md new file mode 100644 index 00000000..f35d2a8a --- /dev/null +++ b/tools/sam2-onnx-tensorrt/README.md @@ -0,0 +1,99 @@ +# SAM2-ONNX-TRT +Export all the Segment Anything 2 (SAM2) network modules to ONNX files, so TensorRT can create inference +engines for accelerated inference. [DeepStream MaskTracker](https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps/tree/master/deepstream-masktracker) uses them to perform +multi-object tracking. The output includes below modules: + +## Table of Contents +- [Overview](#overview) +- [Network Architecture](#network-architecture) + - [Image Encoder](#image-encoder) + - [Mask Decoder](#mask-decoder) + - [Memory Attention](#memory-attention) + - [Memory Encoder](#memory-encoder) +- [Run](#run) +- [References](#references) + +## Overview + +The SAM2-ONNX-TRT project exports four key network modules from Segment Anything 2 (SAM2) to ONNX format for TensorRT acceleration: + +### 1. Image Encoder +A transformer-based encoder network that extracts visual features from each frame. This network processes the frame once and shares the extracted features across all targets in that frame. + +### 2. Memory Attention +This module generates conditional features for each target by combining the current frame's image encoder features with spatial memory features and object pointers from previous frames, which are stored in the memory bank. + +### 3. Mask Decoder (including prompt encoder) +This component takes bounding box prompts (if provided) and conditional features from memory attention to generate segmentation masks and object pointers. The object pointers capture high-level semantic information to uniquely identify each target. + +### 4. Memory Encoder +This module downsamples and fuses the segmentation mask with the image encoder features to create spatial memory features for the current frame, which are then stored in the memory bank for future reference. + +## Network Architecture + +The following tables show the input/output specifications for each network module. All inputs and outputs across all networks use float data type. + +### Image Encoder +| Index | Type | Name | Size | Description | +|-------|------|------|------|-------------| +| 0 | Input | image | [1, 3, 1024, 1024] | Input RGB image | +| 1 | Output | pix_feat | [1, 256, 64, 64] | Pixel-level features | +| 2 | Output | high_res_feat0 | [1, 32, 256, 256] | High-resolution features (level 0) | +| 3 | Output | high_res_feat1 | [1, 64, 128, 128] | High-resolution features (level 1) | +| 4 | Output | vision_feats | [1, 256, 64, 64] | Vision features | +| 5 | Output | vision_pos_embed | [4096, 1, 256] | Vision positional embeddings | + +### Mask Decoder +| Index | Type | Name | Size | Description | +|-------|------|------|------|-------------| +| 0 | Input | point_coords | [-1, -1, 2] | Point coordinates (dynamic batch) | +| 1 | Input | point_labels | [-1, -1] | Point labels (dynamic batch) | +| 2 | Input | image_embed | [-1, 256, 64, 64] | Image embeddings (dynamic batch) | +| 3 | Input | high_res_feats_0 | [1, 32, 256, 256] | High-resolution features (level 0) | +| 4 | Input | high_res_feats_1 | [1, 64, 128, 128] | High-resolution features (level 1) | +| 5 | Output | obj_ptr | [-1, 256] | Object pointer (dynamic batch) | +| 6 | Output | mask_for_mem | [-1, 1, 1024, 1024] | Mask for memory storage (dynamic batch) | +| 7 | Output | pred_mask | [-1, 1, 256, 256] | Predicted segmentation mask (dynamic batch) | +| 8 | Output | iou | [-1, 1] | Intersection over Union score (dynamic batch) | +| 9 | Output | occ_logit | [-1, 1] | Occlusion logit (dynamic batch) | + +### Memory Attention +| Index | Type | Name | Size | Description | +|-------|------|------|------|-------------| +| 0 | Input | current_vision_feat | [1, 256, 64, 64] | Current frame vision features | +| 1 | Input | current_vision_pos_embed | [4096, 1, 256] | Current frame positional embeddings | +| 2 | Input | memory_0 | [1, -1, 256] | Memory bank features (dynamic) | +| 3 | Input | memory_1 | [1, -1, 64, 64, 64] | Memory bank spatial features (dynamic) | +| 4 | Input | memory_pos_embed | [1, -1, 64] | Memory positional embeddings (dynamic) | +| 5 | Input | cond_frame_id_diff | [] | Conditional frame ID difference (scalar) | +| 6 | Output | image_embed | [1, 256, 64, 64] | Enhanced image embeddings | + +### Memory Encoder +| Index | Type | Name | Size | Description | +|-------|------|------|------|-------------| +| 0 | Input | mask_for_mem | [1, 1, 1024, 1024] | Mask for memory encoding | +| 1 | Input | pix_feat | [1, 256, 64, 64] | Pixel features | +| 2 | Input | occ_logit | [1, 1] | Occlusion logit | +| 3 | Output | maskmem_features | [1, 64, 64, 64] | Mask memory features | +| 4 | Output | maskmem_pos_enc | [1, 4096, 64] | Mask memory positional encodings | +| 5 | Output | temporal_code | [7, 1, 1, 64] | Temporal encoding | + +## Run + +Below script will install SAM2 dependencies, download the SAM2 model checkpoints and export them to ONNX files. The ONNX files are saved in `checkpoints/${MODEL_TYPE}` directory. + +```bash +# this script will install Python dependencies and export ONNX models +# better to run it in a Python virtual environment or docker container. +bash run.sh +``` + +> **Note:** It is normal to see TracerWarning messages during the export process. These warnings can be ignored as they don't affect the functionality of the exported ONNX models. + +## References + +This project uses information from the following repositories: + +* SAM2 Meta Inc Repo: https://github.com/facebookresearch/segment-anything-2.git +* SAM2Export: https://github.com/Aimol-l/SAM2Export +* SAM2 ONNX Export Fixes: https://github.com/axinc-ai/segment-anything-2.git diff --git a/tools/sam2-onnx-tensorrt/THIRD_PARTY_LICENSE b/tools/sam2-onnx-tensorrt/THIRD_PARTY_LICENSE new file mode 100644 index 00000000..0acdd034 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/THIRD_PARTY_LICENSE @@ -0,0 +1,268 @@ +SAM2-ONNX-TRT use third-aprty packages that may be distributed under +different licensing terms from SAM2-ONNX-TRT license. + +------------------------------------- +SAM2Export is licensed under MIT. +------------------------------------- + +MIT License + +Copyright (c) 2024 子菲鱼 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------- +Segment Anything Model 2 (SAM 2) is licensed under Apache2.0. +------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------- +cctorch is licensed under BSD 3-Clause. +------------------------------------- + +BSD 3-Clause License + +Copyright (c) 2020, the respective contributors, as shown by the AUTHORS file. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/sam2-onnx-tensorrt/checkpoints/download_ckpts.sh b/tools/sam2-onnx-tensorrt/checkpoints/download_ckpts.sh new file mode 100644 index 00000000..eedee8ee --- /dev/null +++ b/tools/sam2-onnx-tensorrt/checkpoints/download_ckpts.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# Use either wget or curl to download the checkpoints +if command -v wget &> /dev/null; then + CMD="wget" +elif command -v curl &> /dev/null; then + CMD="curl -L -O" +else + echo "Please install wget or curl to download the checkpoints." + exit 1 +fi + +# Define the URLs for SAM 2 checkpoints +# SAM2_BASE_URL="https://dl.fbaipublicfiles.com/segment_anything_2/072824" +# sam2_hiera_t_url="${SAM2_BASE_URL}/sam2_hiera_tiny.pt" +# sam2_hiera_s_url="${SAM2_BASE_URL}/sam2_hiera_small.pt" +# sam2_hiera_b_plus_url="${SAM2_BASE_URL}/sam2_hiera_base_plus.pt" +# sam2_hiera_l_url="${SAM2_BASE_URL}/sam2_hiera_large.pt" + +# Download each of the four checkpoints using wget +# echo "Downloading sam2_hiera_tiny.pt checkpoint..." +# $CMD $sam2_hiera_t_url || { echo "Failed to download checkpoint from $sam2_hiera_t_url"; exit 1; } + +# echo "Downloading sam2_hiera_small.pt checkpoint..." +# $CMD $sam2_hiera_s_url || { echo "Failed to download checkpoint from $sam2_hiera_s_url"; exit 1; } + +# echo "Downloading sam2_hiera_base_plus.pt checkpoint..." +# $CMD $sam2_hiera_b_plus_url || { echo "Failed to download checkpoint from $sam2_hiera_b_plus_url"; exit 1; } + +# echo "Downloading sam2_hiera_large.pt checkpoint..." +# $CMD $sam2_hiera_l_url || { echo "Failed to download checkpoint from $sam2_hiera_l_url"; exit 1; } + +# Define the URLs for SAM 2.1 checkpoints +SAM2p1_BASE_URL="https://dl.fbaipublicfiles.com/segment_anything_2/092824" +sam2p1_hiera_t_url="${SAM2p1_BASE_URL}/sam2.1_hiera_tiny.pt" +sam2p1_hiera_s_url="${SAM2p1_BASE_URL}/sam2.1_hiera_small.pt" +sam2p1_hiera_b_plus_url="${SAM2p1_BASE_URL}/sam2.1_hiera_base_plus.pt" +sam2p1_hiera_l_url="${SAM2p1_BASE_URL}/sam2.1_hiera_large.pt" + +# SAM 2.1 checkpoints +echo "Downloading sam2.1_hiera_tiny.pt checkpoint..." +$CMD $sam2p1_hiera_t_url || { echo "Failed to download checkpoint from $sam2p1_hiera_t_url"; exit 1; } + +echo "Downloading sam2.1_hiera_small.pt checkpoint..." +$CMD $sam2p1_hiera_s_url || { echo "Failed to download checkpoint from $sam2p1_hiera_s_url"; exit 1; } + +echo "Downloading sam2.1_hiera_base_plus.pt checkpoint..." +$CMD $sam2p1_hiera_b_plus_url || { echo "Failed to download checkpoint from $sam2p1_hiera_b_plus_url"; exit 1; } + +echo "Downloading sam2.1_hiera_large.pt checkpoint..." +$CMD $sam2p1_hiera_l_url || { echo "Failed to download checkpoint from $sam2p1_hiera_l_url"; exit 1; } + +echo "All checkpoints are downloaded successfully." diff --git a/tools/sam2-onnx-tensorrt/export_sam2_onnx.py b/tools/sam2-onnx-tensorrt/export_sam2_onnx.py new file mode 100644 index 00000000..a67fc96b --- /dev/null +++ b/tools/sam2-onnx-tensorrt/export_sam2_onnx.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import onnx +import argparse +from src.Module import ImageEncoder +from src.Module import MemAttention +from src.Module import MemEncoder +from src.Module import MaskDecoder +from sam2.build_sam import build_sam2 + + +def export_image_encoder(model,onnx_path): + print(">>> Exporting Image Encoder...") + input_img = torch.randn(1, 3,1024, 1024).cpu() + out = model(input_img) + output_names = ["pix_feat","high_res_feat0","high_res_feat1","vision_feats","vision_pos_embed"] + torch.onnx.export( + model, + input_img, + onnx_path+"image_encoder.onnx", + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names=["image"], + output_names=output_names, + ) + onnx_model = onnx.load(onnx_path+"image_encoder.onnx") + onnx.checker.check_model(onnx_model) + print("[SUCCESS] Image Encoder exported successfully!") + + +def export_memory_attention(model,onnx_path): + print(">>> Exporting Memory Attention...") + batch_size = 1 + current_vision_feat = torch.randn(1,256,64,64) #[1, 256, 64, 64] + current_vision_pos_embed = torch.randn(4096,1,256) #[4096, 1, 256] + memory_0 = torch.randn(batch_size,16,256) # [batch size, num obj ptr, feature size] + memory_1 = torch.randn(batch_size,7,64,64,64) + memory_pos_embed = torch.randn(batch_size,7*4096+64,64) #[y*4096,1,64] + cond_frame_id_diff = torch.tensor(10.0) + out = model( + current_vision_feat = current_vision_feat, + current_vision_pos_embed = current_vision_pos_embed, + memory_0 = memory_0, + memory_1 = memory_1, + memory_pos_embed = memory_pos_embed, + cond_frame_id_diff = cond_frame_id_diff, + ) + input_name = ["current_vision_feat", + "current_vision_pos_embed", + "memory_0", + "memory_1", + "memory_pos_embed", + "cond_frame_id_diff",] + dynamic_axes = { + "memory_0": {0: "batch_size", 1: "num"}, + "memory_1": {0: "batch_size", 1: "buff_size"}, + "memory_pos_embed": {0: "batch_size", 1: "buff_size_embed"} + } + torch.onnx.export( + model, + (current_vision_feat,current_vision_pos_embed,memory_0,memory_1,memory_pos_embed,cond_frame_id_diff), + onnx_path+"memory_attention.onnx", + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names= input_name, + output_names=["image_embed"], + dynamic_axes = dynamic_axes + ) + # original_model = onnx.load(onnx_path+"memory_attention.onnx") + # simplified_model, check = simplify(original_model) + # onnx.save(simplified_model, onnx_path+"memory_attention.onnx") + onnx_model = onnx.load(onnx_path+"memory_attention.onnx") + onnx.checker.check_model(onnx_model) + print("[SUCCESS] Memory Attention exported successfully!") + + +def export_mask_decoder(model,onnx_path): + print(">>> Exporting Mask Decoder...") + batch_size = 20 + point_coords = torch.randn(batch_size,2,2).cpu() + point_labels = torch.randn(batch_size,2).cpu() + # point_coords = torch.randn(1,2,2).cpu() + # point_labels = torch.randn(1,2).cpu() + # frame_size = torch.tensor([1024,1024],dtype=torch.int64) + image_embed = torch.randn(batch_size,256,64,64).cpu() + high_res_feats_0 = torch.randn(1,32,256,256).cpu() + high_res_feats_1 = torch.randn(1,64,128,128).cpu() + + out = model( + point_coords = point_coords, + point_labels = point_labels, + # frame_size = frame_size, + image_embed = image_embed, + high_res_feats_0 = high_res_feats_0, + high_res_feats_1 = high_res_feats_1 + ) + # input_name = ["point_coords","point_labels","frame_size","image_embed","high_res_feats_0","high_res_feats_1"] + input_name = ["point_coords","point_labels","image_embed","high_res_feats_0","high_res_feats_1"] + output_name = ["obj_ptr","mask_for_mem","pred_mask", "iou", "occ_logit"] + dynamic_axes = { + "point_coords":{0: "batch_size",1:"num_points"}, + "point_labels": {0: "batch_size",1:"num_points"}, + "image_embed": {0: "batch_size"}, + # "obj_ptr": {0: "batch_size"}, + # "mask_for_mem": {0: "batch_size"}, + # "pred_mask": {0: "batch_size"} + } + torch.onnx.export( + model, + # (point_coords,point_labels,frame_size,image_embed,high_res_feats_0,high_res_feats_1), + (point_coords,point_labels,image_embed,high_res_feats_0,high_res_feats_1), + onnx_path+"mask_decoder.onnx", + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names= input_name, + output_names=output_name, + dynamic_axes = dynamic_axes + ) + onnx_model = onnx.load(onnx_path+"mask_decoder.onnx") + onnx.checker.check_model(onnx_model) + print("[SUCCESS] Mask Decoder exported successfully!") + + +def export_memory_encoder(model,onnx_path): + print(">>> Exporting Memory Encoder...") + batch_size = 1 + mask_for_mem = torch.randn(batch_size,1,1024,1024) + pix_feat = torch.randn(1,256,64,64) + occ_logit = torch.randn(1,1) + dynamic_axes = { + "mask_for_mem":{0: "batch_size"}, + "occ_logit":{0: "batch_size"} + } + + out = model(mask_for_mem = mask_for_mem,pix_feat = pix_feat,occ_logit = occ_logit) + + input_names = ["mask_for_mem","pix_feat","occ_logit"] + output_names = ["maskmem_features","maskmem_pos_enc","temporal_code"] + torch.onnx.export( + model, + (mask_for_mem,pix_feat,occ_logit), + onnx_path+"memory_encoder.onnx", + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names= input_names, + output_names= output_names, + dynamic_axes = dynamic_axes + ) + onnx_model = onnx.load(onnx_path+"memory_encoder.onnx") + onnx.checker.check_model(onnx_model) + print("[SUCCESS] Memory Encoder exported successfully!") + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="export SAM2.1 to onnx") + parser.add_argument("--model",type=str,choices=["tiny", "small", "base_plus", "large"], + default="tiny",required=False,help="SAM2 model type. Choose one of: tiny, small, base_plus, large") + args = parser.parse_args() + + config_suffix_dict = { + "tiny": "t", + "small": "s", + "base_plus": "b+", + "large": "l" + } + + model_type = args.model + outdir = "checkpoints/{}/".format(model_type) + config = "configs/sam2.1/sam2.1_hiera_{}.yaml".format(config_suffix_dict[model_type]) + checkpoint = "checkpoints/sam2.1_hiera_{}.pt".format(model_type) + + sam2_model = build_sam2(config, checkpoint, device="cpu") + + image_encoder = ImageEncoder(sam2_model).cpu() + export_image_encoder(image_encoder,outdir) + + mask_decoder = MaskDecoder(sam2_model).cpu() + export_mask_decoder(mask_decoder,outdir) + + mem_attention = MemAttention(sam2_model).cpu() + export_memory_attention(mem_attention,outdir) + + mem_encoder = MemEncoder(sam2_model).cpu() + export_memory_encoder(mem_encoder,outdir) diff --git a/tools/sam2-onnx-tensorrt/run.sh b/tools/sam2-onnx-tensorrt/run.sh new file mode 100755 index 00000000..152551b7 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/run.sh @@ -0,0 +1,13 @@ +pip3 install -e . +cd checkpoints; bash download_ckpts.sh; cd .. + +# --model can be chosen from tiny, small, base_plus, large +MODEL_TYPE="large" +mkdir -p checkpoints/${MODEL_TYPE} +python3 export_sam2_onnx.py --model ${MODEL_TYPE} +echo "SAM2 ONNX models are exported to checkpoints/${MODEL_TYPE}" + +# Copy models to DeepStream tracker path +mkdir -p /opt/nvidia/deepstream/deepstream/samples/models/Tracker/ +cp checkpoints/${MODEL_TYPE}/*.onnx /opt/nvidia/deepstream/deepstream/samples/models/Tracker/ +echo "SAM2 ONNX models are copied to /opt/nvidia/deepstream/deepstream/samples/models/Tracker/" diff --git a/tools/sam2-onnx-tensorrt/sam2/__init__.py b/tools/sam2-onnx-tensorrt/sam2/__init__.py new file mode 100644 index 00000000..0712dd03 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from hydra import initialize_config_module +from hydra.core.global_hydra import GlobalHydra + +if not GlobalHydra.instance().is_initialized(): + initialize_config_module("sam2", version_base="1.2") diff --git a/tools/sam2-onnx-tensorrt/sam2/automatic_mask_generator.py b/tools/sam2-onnx-tensorrt/sam2/automatic_mask_generator.py new file mode 100644 index 00000000..065e469e --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/automatic_mask_generator.py @@ -0,0 +1,454 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# Adapted from https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/automatic_mask_generator.py +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +import torch +from torchvision.ops.boxes import batched_nms, box_area # type: ignore + +from sam2.modeling.sam2_base import SAM2Base +from sam2.sam2_image_predictor import SAM2ImagePredictor +from sam2.utils.amg import ( + area_from_rle, + batch_iterator, + batched_mask_to_box, + box_xyxy_to_xywh, + build_all_layer_point_grids, + calculate_stability_score, + coco_encode_rle, + generate_crop_boxes, + is_box_near_crop_edge, + mask_to_rle_pytorch, + MaskData, + remove_small_regions, + rle_to_mask, + uncrop_boxes_xyxy, + uncrop_masks, + uncrop_points, +) + + +class SAM2AutomaticMaskGenerator: + def __init__( + self, + model: SAM2Base, + points_per_side: Optional[int] = 32, + points_per_batch: int = 64, + pred_iou_thresh: float = 0.8, + stability_score_thresh: float = 0.95, + stability_score_offset: float = 1.0, + mask_threshold: float = 0.0, + box_nms_thresh: float = 0.7, + crop_n_layers: int = 0, + crop_nms_thresh: float = 0.7, + crop_overlap_ratio: float = 512 / 1500, + crop_n_points_downscale_factor: int = 1, + point_grids: Optional[List[np.ndarray]] = None, + min_mask_region_area: int = 0, + output_mode: str = "binary_mask", + use_m2m: bool = False, + multimask_output: bool = True, + **kwargs, + ) -> None: + """ + Using a SAM 2 model, generates masks for the entire image. + Generates a grid of point prompts over the image, then filters + low quality and duplicate masks. The default settings are chosen + for SAM 2 with a HieraL backbone. + + Arguments: + model (Sam): The SAM 2 model to use for mask prediction. + points_per_side (int or None): The number of points to be sampled + along one side of the image. The total number of points is + points_per_side**2. If None, 'point_grids' must provide explicit + point sampling. + points_per_batch (int): Sets the number of points run simultaneously + by the model. Higher numbers may be faster but use more GPU memory. + pred_iou_thresh (float): A filtering threshold in [0,1], using the + model's predicted mask quality. + stability_score_thresh (float): A filtering threshold in [0,1], using + the stability of the mask under changes to the cutoff used to binarize + the model's mask predictions. + stability_score_offset (float): The amount to shift the cutoff when + calculated the stability score. + mask_threshold (float): Threshold for binarizing the mask logits + box_nms_thresh (float): The box IoU cutoff used by non-maximal + suppression to filter duplicate masks. + crop_n_layers (int): If >0, mask prediction will be run again on + crops of the image. Sets the number of layers to run, where each + layer has 2**i_layer number of image crops. + crop_nms_thresh (float): The box IoU cutoff used by non-maximal + suppression to filter duplicate masks between different crops. + crop_overlap_ratio (float): Sets the degree to which crops overlap. + In the first crop layer, crops will overlap by this fraction of + the image length. Later layers with more crops scale down this overlap. + crop_n_points_downscale_factor (int): The number of points-per-side + sampled in layer n is scaled down by crop_n_points_downscale_factor**n. + point_grids (list(np.ndarray) or None): A list over explicit grids + of points used for sampling, normalized to [0,1]. The nth grid in the + list is used in the nth crop layer. Exclusive with points_per_side. + min_mask_region_area (int): If >0, postprocessing will be applied + to remove disconnected regions and holes in masks with area smaller + than min_mask_region_area. Requires opencv. + output_mode (str): The form masks are returned in. Can be 'binary_mask', + 'uncompressed_rle', or 'coco_rle'. 'coco_rle' requires pycocotools. + For large resolutions, 'binary_mask' may consume large amounts of + memory. + use_m2m (bool): Whether to add a one step refinement using previous mask predictions. + multimask_output (bool): Whether to output multimask at each point of the grid. + """ + + assert (points_per_side is None) != ( + point_grids is None + ), "Exactly one of points_per_side or point_grid must be provided." + if points_per_side is not None: + self.point_grids = build_all_layer_point_grids( + points_per_side, + crop_n_layers, + crop_n_points_downscale_factor, + ) + elif point_grids is not None: + self.point_grids = point_grids + else: + raise ValueError("Can't have both points_per_side and point_grid be None.") + + assert output_mode in [ + "binary_mask", + "uncompressed_rle", + "coco_rle", + ], f"Unknown output_mode {output_mode}." + if output_mode == "coco_rle": + try: + from pycocotools import mask as mask_utils # type: ignore # noqa: F401 + except ImportError as e: + print("Please install pycocotools") + raise e + + self.predictor = SAM2ImagePredictor( + model, + max_hole_area=min_mask_region_area, + max_sprinkle_area=min_mask_region_area, + ) + self.points_per_batch = points_per_batch + self.pred_iou_thresh = pred_iou_thresh + self.stability_score_thresh = stability_score_thresh + self.stability_score_offset = stability_score_offset + self.mask_threshold = mask_threshold + self.box_nms_thresh = box_nms_thresh + self.crop_n_layers = crop_n_layers + self.crop_nms_thresh = crop_nms_thresh + self.crop_overlap_ratio = crop_overlap_ratio + self.crop_n_points_downscale_factor = crop_n_points_downscale_factor + self.min_mask_region_area = min_mask_region_area + self.output_mode = output_mode + self.use_m2m = use_m2m + self.multimask_output = multimask_output + + @classmethod + def from_pretrained(cls, model_id: str, **kwargs) -> "SAM2AutomaticMaskGenerator": + """ + Load a pretrained model from the Hugging Face hub. + + Arguments: + model_id (str): The Hugging Face repository ID. + **kwargs: Additional arguments to pass to the model constructor. + + Returns: + (SAM2AutomaticMaskGenerator): The loaded model. + """ + from sam2.build_sam import build_sam2_hf + + sam_model = build_sam2_hf(model_id, **kwargs) + return cls(sam_model, **kwargs) + + @torch.no_grad() + def generate(self, image: np.ndarray) -> List[Dict[str, Any]]: + """ + Generates masks for the given image. + + Arguments: + image (np.ndarray): The image to generate masks for, in HWC uint8 format. + + Returns: + list(dict(str, any)): A list over records for masks. Each record is + a dict containing the following keys: + segmentation (dict(str, any) or np.ndarray): The mask. If + output_mode='binary_mask', is an array of shape HW. Otherwise, + is a dictionary containing the RLE. + bbox (list(float)): The box around the mask, in XYWH format. + area (int): The area in pixels of the mask. + predicted_iou (float): The model's own prediction of the mask's + quality. This is filtered by the pred_iou_thresh parameter. + point_coords (list(list(float))): The point coordinates input + to the model to generate this mask. + stability_score (float): A measure of the mask's quality. This + is filtered on using the stability_score_thresh parameter. + crop_box (list(float)): The crop of the image used to generate + the mask, given in XYWH format. + """ + + # Generate masks + mask_data = self._generate_masks(image) + + # Encode masks + if self.output_mode == "coco_rle": + mask_data["segmentations"] = [ + coco_encode_rle(rle) for rle in mask_data["rles"] + ] + elif self.output_mode == "binary_mask": + mask_data["segmentations"] = [rle_to_mask(rle) for rle in mask_data["rles"]] + else: + mask_data["segmentations"] = mask_data["rles"] + + # Write mask records + curr_anns = [] + for idx in range(len(mask_data["segmentations"])): + ann = { + "segmentation": mask_data["segmentations"][idx], + "area": area_from_rle(mask_data["rles"][idx]), + "bbox": box_xyxy_to_xywh(mask_data["boxes"][idx]).tolist(), + "predicted_iou": mask_data["iou_preds"][idx].item(), + "point_coords": [mask_data["points"][idx].tolist()], + "stability_score": mask_data["stability_score"][idx].item(), + "crop_box": box_xyxy_to_xywh(mask_data["crop_boxes"][idx]).tolist(), + } + curr_anns.append(ann) + + return curr_anns + + def _generate_masks(self, image: np.ndarray) -> MaskData: + orig_size = image.shape[:2] + crop_boxes, layer_idxs = generate_crop_boxes( + orig_size, self.crop_n_layers, self.crop_overlap_ratio + ) + + # Iterate over image crops + data = MaskData() + for crop_box, layer_idx in zip(crop_boxes, layer_idxs): + crop_data = self._process_crop(image, crop_box, layer_idx, orig_size) + data.cat(crop_data) + + # Remove duplicate masks between crops + if len(crop_boxes) > 1: + # Prefer masks from smaller crops + scores = 1 / box_area(data["crop_boxes"]) + scores = scores.to(data["boxes"].device) + keep_by_nms = batched_nms( + data["boxes"].float(), + scores, + torch.zeros_like(data["boxes"][:, 0]), # categories + iou_threshold=self.crop_nms_thresh, + ) + data.filter(keep_by_nms) + data.to_numpy() + return data + + def _process_crop( + self, + image: np.ndarray, + crop_box: List[int], + crop_layer_idx: int, + orig_size: Tuple[int, ...], + ) -> MaskData: + # Crop the image and calculate embeddings + x0, y0, x1, y1 = crop_box + cropped_im = image[y0:y1, x0:x1, :] + cropped_im_size = cropped_im.shape[:2] + self.predictor.set_image(cropped_im) + + # Get points for this crop + points_scale = np.array(cropped_im_size)[None, ::-1] + points_for_image = self.point_grids[crop_layer_idx] * points_scale + + # Generate masks for this crop in batches + data = MaskData() + for (points,) in batch_iterator(self.points_per_batch, points_for_image): + batch_data = self._process_batch( + points, cropped_im_size, crop_box, orig_size, normalize=True + ) + data.cat(batch_data) + del batch_data + self.predictor.reset_predictor() + + # Remove duplicates within this crop. + keep_by_nms = batched_nms( + data["boxes"].float(), + data["iou_preds"], + torch.zeros_like(data["boxes"][:, 0]), # categories + iou_threshold=self.box_nms_thresh, + ) + data.filter(keep_by_nms) + + # Return to the original image frame + data["boxes"] = uncrop_boxes_xyxy(data["boxes"], crop_box) + data["points"] = uncrop_points(data["points"], crop_box) + data["crop_boxes"] = torch.tensor([crop_box for _ in range(len(data["rles"]))]) + + return data + + def _process_batch( + self, + points: np.ndarray, + im_size: Tuple[int, ...], + crop_box: List[int], + orig_size: Tuple[int, ...], + normalize=False, + ) -> MaskData: + orig_h, orig_w = orig_size + + # Run model on this batch + points = torch.as_tensor( + points, dtype=torch.float32, device=self.predictor.device + ) + in_points = self.predictor._transforms.transform_coords( + points, normalize=normalize, orig_hw=im_size + ) + in_labels = torch.ones( + in_points.shape[0], dtype=torch.int, device=in_points.device + ) + masks, iou_preds, low_res_masks = self.predictor._predict( + in_points[:, None, :], + in_labels[:, None], + multimask_output=self.multimask_output, + return_logits=True, + ) + + # Serialize predictions and store in MaskData + data = MaskData( + masks=masks.flatten(0, 1), + iou_preds=iou_preds.flatten(0, 1), + points=points.repeat_interleave(masks.shape[1], dim=0), + low_res_masks=low_res_masks.flatten(0, 1), + ) + del masks + + if not self.use_m2m: + # Filter by predicted IoU + if self.pred_iou_thresh > 0.0: + keep_mask = data["iou_preds"] > self.pred_iou_thresh + data.filter(keep_mask) + + # Calculate and filter by stability score + data["stability_score"] = calculate_stability_score( + data["masks"], self.mask_threshold, self.stability_score_offset + ) + if self.stability_score_thresh > 0.0: + keep_mask = data["stability_score"] >= self.stability_score_thresh + data.filter(keep_mask) + else: + # One step refinement using previous mask predictions + in_points = self.predictor._transforms.transform_coords( + data["points"], normalize=normalize, orig_hw=im_size + ) + labels = torch.ones( + in_points.shape[0], dtype=torch.int, device=in_points.device + ) + masks, ious = self.refine_with_m2m( + in_points, labels, data["low_res_masks"], self.points_per_batch + ) + data["masks"] = masks.squeeze(1) + data["iou_preds"] = ious.squeeze(1) + + if self.pred_iou_thresh > 0.0: + keep_mask = data["iou_preds"] > self.pred_iou_thresh + data.filter(keep_mask) + + data["stability_score"] = calculate_stability_score( + data["masks"], self.mask_threshold, self.stability_score_offset + ) + if self.stability_score_thresh > 0.0: + keep_mask = data["stability_score"] >= self.stability_score_thresh + data.filter(keep_mask) + + # Threshold masks and calculate boxes + data["masks"] = data["masks"] > self.mask_threshold + data["boxes"] = batched_mask_to_box(data["masks"]) + + # Filter boxes that touch crop boundaries + keep_mask = ~is_box_near_crop_edge( + data["boxes"], crop_box, [0, 0, orig_w, orig_h] + ) + if not torch.all(keep_mask): + data.filter(keep_mask) + + # Compress to RLE + data["masks"] = uncrop_masks(data["masks"], crop_box, orig_h, orig_w) + data["rles"] = mask_to_rle_pytorch(data["masks"]) + del data["masks"] + + return data + + @staticmethod + def postprocess_small_regions( + mask_data: MaskData, min_area: int, nms_thresh: float + ) -> MaskData: + """ + Removes small disconnected regions and holes in masks, then reruns + box NMS to remove any new duplicates. + + Edits mask_data in place. + + Requires open-cv as a dependency. + """ + if len(mask_data["rles"]) == 0: + return mask_data + + # Filter small disconnected regions and holes + new_masks = [] + scores = [] + for rle in mask_data["rles"]: + mask = rle_to_mask(rle) + + mask, changed = remove_small_regions(mask, min_area, mode="holes") + unchanged = not changed + mask, changed = remove_small_regions(mask, min_area, mode="islands") + unchanged = unchanged and not changed + + new_masks.append(torch.as_tensor(mask).unsqueeze(0)) + # Give score=0 to changed masks and score=1 to unchanged masks + # so NMS will prefer ones that didn't need postprocessing + scores.append(float(unchanged)) + + # Recalculate boxes and remove any new duplicates + masks = torch.cat(new_masks, dim=0) + boxes = batched_mask_to_box(masks) + keep_by_nms = batched_nms( + boxes.float(), + torch.as_tensor(scores), + torch.zeros_like(boxes[:, 0]), # categories + iou_threshold=nms_thresh, + ) + + # Only recalculate RLEs for masks that have changed + for i_mask in keep_by_nms: + if scores[i_mask] == 0.0: + mask_torch = masks[i_mask].unsqueeze(0) + mask_data["rles"][i_mask] = mask_to_rle_pytorch(mask_torch)[0] + mask_data["boxes"][i_mask] = boxes[i_mask] # update res directly + mask_data.filter(keep_by_nms) + + return mask_data + + def refine_with_m2m(self, points, point_labels, low_res_masks, points_per_batch): + new_masks = [] + new_iou_preds = [] + + for cur_points, cur_point_labels, low_res_mask in batch_iterator( + points_per_batch, points, point_labels, low_res_masks + ): + best_masks, best_iou_preds, _ = self.predictor._predict( + cur_points[:, None, :], + cur_point_labels[:, None], + mask_input=low_res_mask[:, None, :], + multimask_output=False, + return_logits=True, + ) + new_masks.append(best_masks) + new_iou_preds.append(best_iou_preds) + masks = torch.cat(new_masks, dim=0) + return masks, torch.cat(new_iou_preds, dim=0) diff --git a/tools/sam2-onnx-tensorrt/sam2/benchmark.py b/tools/sam2-onnx-tensorrt/sam2/benchmark.py new file mode 100644 index 00000000..6519534c --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/benchmark.py @@ -0,0 +1,92 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import os +import time + +import numpy as np +import torch +from tqdm import tqdm + +from sam2.build_sam import build_sam2_video_predictor + +# Only cuda supported +assert torch.cuda.is_available() +device = torch.device("cuda") + +torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() +if torch.cuda.get_device_properties(0).major >= 8: + # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + +# Config and checkpoint +sam2_checkpoint = "checkpoints/sam2.1_hiera_base_plus.pt" +model_cfg = "configs/sam2.1/sam2.1_hiera_b+.yaml" + +# Build video predictor with vos_optimized=True setting +predictor = build_sam2_video_predictor( + model_cfg, sam2_checkpoint, device=device, vos_optimized=True +) + + +# Initialize with video +video_dir = "notebooks/videos/bedroom" +# scan all the JPEG frame names in this directory +frame_names = [ + p + for p in os.listdir(video_dir) + if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"] +] +frame_names.sort(key=lambda p: int(os.path.splitext(p)[0])) +inference_state = predictor.init_state(video_path=video_dir) + + +# Number of runs, warmup etc +warm_up, runs = 5, 25 +verbose = True +num_frames = len(frame_names) +total, count = 0, 0 +torch.cuda.empty_cache() + +# We will select an object with a click. +# See video_predictor_example.ipynb for more detailed explanation +ann_frame_idx, ann_obj_id = 0, 1 +# Add a positive click at (x, y) = (210, 350) +# For labels, `1` means positive click +points = np.array([[210, 350]], dtype=np.float32) +labels = np.array([1], np.int32) + +_, out_obj_ids, out_mask_logits = predictor.add_new_points_or_box( + inference_state=inference_state, + frame_idx=ann_frame_idx, + obj_id=ann_obj_id, + points=points, + labels=labels, +) + +# Warmup and then average FPS over several runs +with torch.autocast("cuda", torch.bfloat16): + with torch.inference_mode(): + for i in tqdm(range(runs), disable=not verbose, desc="Benchmarking"): + start = time.time() + # Start tracking + for ( + out_frame_idx, + out_obj_ids, + out_mask_logits, + ) in predictor.propagate_in_video(inference_state): + pass + + end = time.time() + total += end - start + count += 1 + if i == warm_up - 1: + print("Warmup FPS: ", count * num_frames / total) + total = 0 + count = 0 + +print("FPS: ", count * num_frames / total) diff --git a/tools/sam2-onnx-tensorrt/sam2/build_sam.py b/tools/sam2-onnx-tensorrt/sam2/build_sam.py new file mode 100644 index 00000000..3a3bef1e --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/build_sam.py @@ -0,0 +1,174 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import logging +import os + +import torch +from hydra import compose +from hydra.utils import instantiate +from omegaconf import OmegaConf + +import sam2 + +# Check if the user is running Python from the parent directory of the sam2 repo +# (i.e. the directory where this repo is cloned into) -- this is not supported since +# it could shadow the sam2 package and cause issues. +if os.path.isdir(os.path.join(sam2.__path__[0], "sam2")): + # If the user has "sam2/sam2" in their path, they are likey importing the repo itself + # as "sam2" rather than importing the "sam2" python package (i.e. "sam2/sam2" directory). + # This typically happens because the user is running Python from the parent directory + # that contains the sam2 repo they cloned. + raise RuntimeError( + "You're likely running Python from the parent directory of the sam2 repository " + "(i.e. the directory where https://github.com/facebookresearch/sam2 is cloned into). " + "This is not supported since the `sam2` Python package could be shadowed by the " + "repository name (the repository is also named `sam2` and contains the Python package " + "in `sam2/sam2`). Please run Python from another directory (e.g. from the repo dir " + "rather than its parent dir, or from your home directory) after installing SAM 2." + ) + + +HF_MODEL_ID_TO_FILENAMES = { + "facebook/sam2-hiera-tiny": ( + "configs/sam2/sam2_hiera_t.yaml", + "sam2_hiera_tiny.pt", + ), + "facebook/sam2-hiera-small": ( + "configs/sam2/sam2_hiera_s.yaml", + "sam2_hiera_small.pt", + ), + "facebook/sam2-hiera-base-plus": ( + "configs/sam2/sam2_hiera_b+.yaml", + "sam2_hiera_base_plus.pt", + ), + "facebook/sam2-hiera-large": ( + "configs/sam2/sam2_hiera_l.yaml", + "sam2_hiera_large.pt", + ), + "facebook/sam2.1-hiera-tiny": ( + "configs/sam2.1/sam2.1_hiera_t.yaml", + "sam2.1_hiera_tiny.pt", + ), + "facebook/sam2.1-hiera-small": ( + "configs/sam2.1/sam2.1_hiera_s.yaml", + "sam2.1_hiera_small.pt", + ), + "facebook/sam2.1-hiera-base-plus": ( + "configs/sam2.1/sam2.1_hiera_b+.yaml", + "sam2.1_hiera_base_plus.pt", + ), + "facebook/sam2.1-hiera-large": ( + "configs/sam2.1/sam2.1_hiera_l.yaml", + "sam2.1_hiera_large.pt", + ), +} + + +def build_sam2( + config_file, + ckpt_path=None, + device="cuda", + mode="eval", + hydra_overrides_extra=[], + apply_postprocessing=True, + **kwargs, +): + + if apply_postprocessing: + hydra_overrides_extra = hydra_overrides_extra.copy() + hydra_overrides_extra += [ + # dynamically fall back to multi-mask if the single mask is not stable + "++model.sam_mask_decoder_extra_args.dynamic_multimask_via_stability=true", + "++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_delta=0.05", + "++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_thresh=0.98", + ] + # Read config and init model + cfg = compose(config_name=config_file, overrides=hydra_overrides_extra) + OmegaConf.resolve(cfg) + model = instantiate(cfg.model, _recursive_=True) + _load_checkpoint(model, ckpt_path) + model = model.to(device) + if mode == "eval": + model.eval() + return model + + +def build_sam2_video_predictor( + config_file, + ckpt_path=None, + device="cuda", + mode="eval", + hydra_overrides_extra=[], + apply_postprocessing=True, + vos_optimized=False, + **kwargs, +): + hydra_overrides = [ + "++model._target_=sam2.sam2_video_predictor.SAM2VideoPredictor", + ] + if vos_optimized: + hydra_overrides = [ + "++model._target_=sam2.sam2_video_predictor.SAM2VideoPredictorVOS", + "++model.compile_image_encoder=True", # Let sam2_base handle this + ] + + if apply_postprocessing: + hydra_overrides_extra = hydra_overrides_extra.copy() + hydra_overrides_extra += [ + # dynamically fall back to multi-mask if the single mask is not stable + "++model.sam_mask_decoder_extra_args.dynamic_multimask_via_stability=true", + "++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_delta=0.05", + "++model.sam_mask_decoder_extra_args.dynamic_multimask_stability_thresh=0.98", + # the sigmoid mask logits on interacted frames with clicks in the memory encoder so that the encoded masks are exactly as what users see from clicking + "++model.binarize_mask_from_pts_for_mem_enc=true", + # fill small holes in the low-res masks up to `fill_hole_area` (before resizing them to the original video resolution) + "++model.fill_hole_area=8", + ] + hydra_overrides.extend(hydra_overrides_extra) + + # Read config and init model + cfg = compose(config_name=config_file, overrides=hydra_overrides) + OmegaConf.resolve(cfg) + model = instantiate(cfg.model, _recursive_=True) + _load_checkpoint(model, ckpt_path) + model = model.to(device) + if mode == "eval": + model.eval() + return model + + +def _hf_download(model_id): + from huggingface_hub import hf_hub_download + + config_name, checkpoint_name = HF_MODEL_ID_TO_FILENAMES[model_id] + ckpt_path = hf_hub_download(repo_id=model_id, filename=checkpoint_name) + return config_name, ckpt_path + + +def build_sam2_hf(model_id, **kwargs): + config_name, ckpt_path = _hf_download(model_id) + return build_sam2(config_file=config_name, ckpt_path=ckpt_path, **kwargs) + + +def build_sam2_video_predictor_hf(model_id, **kwargs): + config_name, ckpt_path = _hf_download(model_id) + return build_sam2_video_predictor( + config_file=config_name, ckpt_path=ckpt_path, **kwargs + ) + + +def _load_checkpoint(model, ckpt_path): + if ckpt_path is not None: + sd = torch.load(ckpt_path, map_location="cpu", weights_only=True)["model"] + missing_keys, unexpected_keys = model.load_state_dict(sd) + if missing_keys: + logging.error(missing_keys) + raise RuntimeError() + if unexpected_keys: + logging.error(unexpected_keys) + raise RuntimeError() + logging.info("Loaded checkpoint sucessfully") diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_b+.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_b+.yaml new file mode 100644 index 00000000..d7172f9b --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_b+.yaml @@ -0,0 +1,116 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 112 + num_heads: 2 + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [896, 448, 224, 112] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_l.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_l.yaml new file mode 100644 index 00000000..23073ea7 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_l.yaml @@ -0,0 +1,120 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 144 + num_heads: 2 + stages: [2, 6, 36, 4] + global_att_blocks: [23, 33, 43] + window_pos_embed_bkg_spatial_size: [7, 7] + window_spec: [8, 4, 16, 8] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [1152, 576, 288, 144] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_s.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_s.yaml new file mode 100644 index 00000000..fd8d4046 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_s.yaml @@ -0,0 +1,119 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 96 + num_heads: 1 + stages: [1, 2, 11, 2] + global_att_blocks: [7, 10, 13] + window_pos_embed_bkg_spatial_size: [7, 7] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [768, 384, 192, 96] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_t.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_t.yaml new file mode 100644 index 00000000..e762aec9 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1/sam2.1_hiera_t.yaml @@ -0,0 +1,121 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 96 + num_heads: 1 + stages: [1, 2, 7, 2] + global_att_blocks: [5, 7, 9] + window_pos_embed_bkg_spatial_size: [7, 7] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [768, 384, 192, 96] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + # SAM decoder + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + # HieraT does not currently support compilation, should always be set to False + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1_training/sam2.1_hiera_b+_MOSE_finetune.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1_training/sam2.1_hiera_b+_MOSE_finetune.yaml new file mode 100644 index 00000000..9b6faa79 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2.1_training/sam2.1_hiera_b+_MOSE_finetune.yaml @@ -0,0 +1,339 @@ +# @package _global_ + +scratch: + resolution: 1024 + train_batch_size: 1 + num_train_workers: 10 + num_frames: 8 + max_num_objects: 3 + base_lr: 5.0e-6 + vision_lr: 3.0e-06 + phases_per_epoch: 1 + num_epochs: 40 + +dataset: + # PATHS to Dataset + img_folder: null # PATH to MOSE JPEGImages folder + gt_folder: null # PATH to MOSE Annotations folder + file_list_txt: training/assets/MOSE_sample_train_list.txt # Optional PATH to filelist containing a subset of videos to be used for training + multiplier: 2 + +# Video transforms +vos: + train_transforms: + - _target_: training.dataset.transforms.ComposeAPI + transforms: + - _target_: training.dataset.transforms.RandomHorizontalFlip + consistent_transform: True + - _target_: training.dataset.transforms.RandomAffine + degrees: 25 + shear: 20 + image_interpolation: bilinear + consistent_transform: True + - _target_: training.dataset.transforms.RandomResizeAPI + sizes: ${scratch.resolution} + square: true + consistent_transform: True + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: True + brightness: 0.1 + contrast: 0.03 + saturation: 0.03 + hue: null + - _target_: training.dataset.transforms.RandomGrayscale + p: 0.05 + consistent_transform: True + - _target_: training.dataset.transforms.ColorJitter + consistent_transform: False + brightness: 0.1 + contrast: 0.05 + saturation: 0.05 + hue: null + - _target_: training.dataset.transforms.ToTensorAPI + - _target_: training.dataset.transforms.NormalizeAPI + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + +trainer: + _target_: training.trainer.Trainer + mode: train_only + max_epochs: ${times:${scratch.num_epochs},${scratch.phases_per_epoch}} + accelerator: cuda + seed_value: 123 + + model: + _target_: training.model.sam2.SAM2Train + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 112 + num_heads: 2 + drop_path_rate: 0.1 + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [896, 448, 224, 112] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: ${scratch.resolution} + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + # compile_image_encoder: False + + ####### Training specific params ####### + # box/point input and corrections + prob_to_use_pt_input_for_train: 0.5 + prob_to_use_pt_input_for_eval: 0.0 + prob_to_use_box_input_for_train: 0.5 # 0.5*0.5 = 0.25 prob to use box instead of points + prob_to_use_box_input_for_eval: 0.0 + prob_to_sample_from_gt_for_train: 0.1 # with a small prob, sampling correction points from GT mask instead of prediction errors + num_frames_to_correct_for_train: 2 # iteratively sample on random 1~2 frames (always include the first frame) + num_frames_to_correct_for_eval: 1 # only iteratively sample on first frame + rand_frames_to_correct_for_train: True # random #init-cond-frame ~ 2 + add_all_frames_to_correct_as_cond: True # when a frame receives a correction click, it becomes a conditioning frame (even if it's not initially a conditioning frame) + # maximum 2 initial conditioning frames + num_init_cond_frames_for_train: 2 + rand_init_cond_frames_for_train: True # random 1~2 + num_correction_pt_per_frame: 7 + use_act_ckpt_iterative_pt_sampling: false + + + + num_init_cond_frames_for_eval: 1 # only mask on the first frame + forward_backbone_per_frame_for_eval: True + + + data: + train: + _target_: training.dataset.sam2_datasets.TorchTrainMixedDataset + phases_per_epoch: ${scratch.phases_per_epoch} + batch_sizes: + - ${scratch.train_batch_size} + + datasets: + - _target_: training.dataset.utils.RepeatFactorWrapper + dataset: + _target_: training.dataset.utils.ConcatDataset + datasets: + - _target_: training.dataset.vos_dataset.VOSDataset + transforms: ${vos.train_transforms} + training: true + video_dataset: + _target_: training.dataset.vos_raw_dataset.PNGRawDataset + img_folder: ${dataset.img_folder} + gt_folder: ${dataset.gt_folder} + file_list_txt: ${dataset.file_list_txt} + sampler: + _target_: training.dataset.vos_sampler.RandomUniformSampler + num_frames: ${scratch.num_frames} + max_num_objects: ${scratch.max_num_objects} + multiplier: ${dataset.multiplier} + shuffle: True + num_workers: ${scratch.num_train_workers} + pin_memory: True + drop_last: True + collate_fn: + _target_: training.utils.data_utils.collate_fn + _partial_: true + dict_key: all + + optim: + amp: + enabled: True + amp_dtype: bfloat16 + + optimizer: + _target_: torch.optim.AdamW + + gradient_clip: + _target_: training.optimizer.GradientClipper + max_norm: 0.1 + norm_type: 2 + + param_group_modifiers: + - _target_: training.optimizer.layer_decay_param_modifier + _partial_: True + layer_decay_value: 0.9 + apply_to: 'image_encoder.trunk' + overrides: + - pattern: '*pos_embed*' + value: 1.0 + + options: + lr: + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: ${scratch.base_lr} + end_value: ${divide:${scratch.base_lr},10} + - scheduler: + _target_: fvcore.common.param_scheduler.CosineParamScheduler + start_value: ${scratch.vision_lr} + end_value: ${divide:${scratch.vision_lr},10} + param_names: + - 'image_encoder.*' + weight_decay: + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.1 + - scheduler: + _target_: fvcore.common.param_scheduler.ConstantParamScheduler + value: 0.0 + param_names: + - '*bias*' + module_cls_names: ['torch.nn.LayerNorm'] + + loss: + all: + _target_: training.loss_fns.MultiStepMultiMasksAndIous + weight_dict: + loss_mask: 20 + loss_dice: 1 + loss_iou: 1 + loss_class: 1 + supervise_all_iou: true + iou_use_l1_loss: true + pred_obj_scores: true + focal_gamma_obj_score: 0.0 + focal_alpha_obj_score: -1.0 + + distributed: + backend: nccl + find_unused_parameters: True + + logging: + tensorboard_writer: + _target_: training.utils.logger.make_tensorboard_logger + log_dir: ${launcher.experiment_log_dir}/tensorboard + flush_secs: 120 + should_log: True + log_dir: ${launcher.experiment_log_dir}/logs + log_freq: 10 + + # initialize from a SAM 2 checkpoint + checkpoint: + save_dir: ${launcher.experiment_log_dir}/checkpoints + save_freq: 0 # 0 only last checkpoint is saved. + model_weight_initializer: + _partial_: True + _target_: training.utils.checkpoint_utils.load_state_dict_into_model + strict: True + ignore_unexpected_keys: null + ignore_missing_keys: null + + state_dict: + _target_: training.utils.checkpoint_utils.load_checkpoint_and_apply_kernels + checkpoint_path: ./checkpoints/sam2.1_hiera_base_plus.pt # PATH to SAM 2.1 checkpoint + ckpt_state_dict_keys: ['model'] + +launcher: + num_nodes: 1 + gpus_per_node: 8 + experiment_log_dir: null # Path to log directory, defaults to ./sam2_logs/${config_name} + +# SLURM args if running on a cluster +submitit: + partition: null + account: null + qos: null + cpus_per_task: 10 + use_cluster: false + timeout_hour: 24 + name: null + port_range: [10000, 65000] + diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_b+.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_b+.yaml new file mode 100644 index 00000000..0f435af0 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_b+.yaml @@ -0,0 +1,113 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 112 + num_heads: 2 + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [896, 448, 224, 112] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: false + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_l.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_l.yaml new file mode 100644 index 00000000..1092802b --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_l.yaml @@ -0,0 +1,117 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 144 + num_heads: 2 + stages: [2, 6, 36, 4] + global_att_blocks: [23, 33, 43] + window_pos_embed_bkg_spatial_size: [7, 7] + window_spec: [8, 4, 16, 8] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [1152, 576, 288, 144] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: false + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_s.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_s.yaml new file mode 100644 index 00000000..174e414f --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_s.yaml @@ -0,0 +1,116 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 96 + num_heads: 1 + stages: [1, 2, 11, 2] + global_att_blocks: [7, 10, 13] + window_pos_embed_bkg_spatial_size: [7, 7] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [768, 384, 192, 96] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: false + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_t.yaml b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_t.yaml new file mode 100644 index 00000000..121447aa --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/configs/sam2/sam2_hiera_t.yaml @@ -0,0 +1,118 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 96 + num_heads: 1 + stages: [1, 2, 7, 2] + global_att_blocks: [5, 7, 9] + window_pos_embed_bkg_spatial_size: [7, 7] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [768, 384, 192, 96] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + # SAM decoder + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: false + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + # HieraT does not currently support compilation, should always be set to False + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/csrc/connected_components.cu b/tools/sam2-onnx-tensorrt/sam2/csrc/connected_components.cu new file mode 100644 index 00000000..ced21eb3 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/csrc/connected_components.cu @@ -0,0 +1,289 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. + +// This source code is licensed under the license found in the +// LICENSE file in the root directory of this source tree. + +// adapted from https://github.com/zsef123/Connected_components_PyTorch +// with license found in the LICENSE_cctorch file in the root directory. +#include +#include +#include +#include +#include +#include + +// 2d +#define BLOCK_ROWS 16 +#define BLOCK_COLS 16 + +namespace cc2d { + +template +__device__ __forceinline__ unsigned char hasBit(T bitmap, unsigned char pos) { + return (bitmap >> pos) & 1; +} + +__device__ int32_t find(const int32_t* s_buf, int32_t n) { + while (s_buf[n] != n) + n = s_buf[n]; + return n; +} + +__device__ int32_t find_n_compress(int32_t* s_buf, int32_t n) { + const int32_t id = n; + while (s_buf[n] != n) { + n = s_buf[n]; + s_buf[id] = n; + } + return n; +} + +__device__ void union_(int32_t* s_buf, int32_t a, int32_t b) { + bool done; + do { + a = find(s_buf, a); + b = find(s_buf, b); + + if (a < b) { + int32_t old = atomicMin(s_buf + b, a); + done = (old == b); + b = old; + } else if (b < a) { + int32_t old = atomicMin(s_buf + a, b); + done = (old == a); + a = old; + } else + done = true; + + } while (!done); +} + +__global__ void +init_labeling(int32_t* label, const uint32_t W, const uint32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y) * 2; + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x) * 2; + const uint32_t idx = row * W + col; + + if (row < H && col < W) + label[idx] = idx; +} + +__global__ void +merge(uint8_t* img, int32_t* label, const uint32_t W, const uint32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y) * 2; + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x) * 2; + const uint32_t idx = row * W + col; + + if (row >= H || col >= W) + return; + + uint32_t P = 0; + + if (img[idx]) + P |= 0x777; + if (row + 1 < H && img[idx + W]) + P |= 0x777 << 4; + if (col + 1 < W && img[idx + 1]) + P |= 0x777 << 1; + + if (col == 0) + P &= 0xEEEE; + if (col + 1 >= W) + P &= 0x3333; + else if (col + 2 >= W) + P &= 0x7777; + + if (row == 0) + P &= 0xFFF0; + if (row + 1 >= H) + P &= 0xFF; + + if (P > 0) { + // If need check about top-left pixel(if flag the first bit) and hit the + // top-left pixel + if (hasBit(P, 0) && img[idx - W - 1]) { + union_(label, idx, idx - 2 * W - 2); // top left block + } + + if ((hasBit(P, 1) && img[idx - W]) || (hasBit(P, 2) && img[idx - W + 1])) + union_(label, idx, idx - 2 * W); // top bottom block + + if (hasBit(P, 3) && img[idx + 2 - W]) + union_(label, idx, idx - 2 * W + 2); // top right block + + if ((hasBit(P, 4) && img[idx - 1]) || (hasBit(P, 8) && img[idx + W - 1])) + union_(label, idx, idx - 2); // just left block + } +} + +__global__ void compression(int32_t* label, const int32_t W, const int32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y) * 2; + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x) * 2; + const uint32_t idx = row * W + col; + + if (row < H && col < W) + find_n_compress(label, idx); +} + +__global__ void final_labeling( + const uint8_t* img, + int32_t* label, + const int32_t W, + const int32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y) * 2; + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x) * 2; + const uint32_t idx = row * W + col; + + if (row >= H || col >= W) + return; + + int32_t y = label[idx] + 1; + + if (img[idx]) + label[idx] = y; + else + label[idx] = 0; + + if (col + 1 < W) { + if (img[idx + 1]) + label[idx + 1] = y; + else + label[idx + 1] = 0; + + if (row + 1 < H) { + if (img[idx + W + 1]) + label[idx + W + 1] = y; + else + label[idx + W + 1] = 0; + } + } + + if (row + 1 < H) { + if (img[idx + W]) + label[idx + W] = y; + else + label[idx + W] = 0; + } +} + +__global__ void init_counting( + const int32_t* label, + int32_t* count_init, + const int32_t W, + const int32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y); + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x); + const uint32_t idx = row * W + col; + + if (row >= H || col >= W) + return; + + int32_t y = label[idx]; + if (y > 0) { + int32_t count_idx = y - 1; + atomicAdd(count_init + count_idx, 1); + } +} + +__global__ void final_counting( + const int32_t* label, + const int32_t* count_init, + int32_t* count_final, + const int32_t W, + const int32_t H) { + const uint32_t row = (blockIdx.y * blockDim.y + threadIdx.y); + const uint32_t col = (blockIdx.x * blockDim.x + threadIdx.x); + const uint32_t idx = row * W + col; + + if (row >= H || col >= W) + return; + + int32_t y = label[idx]; + if (y > 0) { + int32_t count_idx = y - 1; + count_final[idx] = count_init[count_idx]; + } else { + count_final[idx] = 0; + } +} + +} // namespace cc2d + +std::vector get_connected_componnets( + const torch::Tensor& inputs) { + AT_ASSERTM(inputs.is_cuda(), "inputs must be a CUDA tensor"); + AT_ASSERTM(inputs.ndimension() == 4, "inputs must be [N, 1, H, W] shape"); + AT_ASSERTM( + inputs.scalar_type() == torch::kUInt8, "inputs must be a uint8 type"); + + const uint32_t N = inputs.size(0); + const uint32_t C = inputs.size(1); + const uint32_t H = inputs.size(2); + const uint32_t W = inputs.size(3); + + AT_ASSERTM(C == 1, "inputs must be [N, 1, H, W] shape"); + AT_ASSERTM((H % 2) == 0, "height must be an even number"); + AT_ASSERTM((W % 2) == 0, "width must be an even number"); + + // label must be uint32_t + auto label_options = + torch::TensorOptions().dtype(torch::kInt32).device(inputs.device()); + torch::Tensor labels = torch::zeros({N, C, H, W}, label_options); + torch::Tensor counts_init = torch::zeros({N, C, H, W}, label_options); + torch::Tensor counts_final = torch::zeros({N, C, H, W}, label_options); + + dim3 grid = dim3( + ((W + 1) / 2 + BLOCK_COLS - 1) / BLOCK_COLS, + ((H + 1) / 2 + BLOCK_ROWS - 1) / BLOCK_ROWS); + dim3 block = dim3(BLOCK_COLS, BLOCK_ROWS); + dim3 grid_count = + dim3((W + BLOCK_COLS) / BLOCK_COLS, (H + BLOCK_ROWS) / BLOCK_ROWS); + dim3 block_count = dim3(BLOCK_COLS, BLOCK_ROWS); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + for (int n = 0; n < N; n++) { + uint32_t offset = n * H * W; + + cc2d::init_labeling<<>>( + labels.data_ptr() + offset, W, H); + cc2d::merge<<>>( + inputs.data_ptr() + offset, + labels.data_ptr() + offset, + W, + H); + cc2d::compression<<>>( + labels.data_ptr() + offset, W, H); + cc2d::final_labeling<<>>( + inputs.data_ptr() + offset, + labels.data_ptr() + offset, + W, + H); + + // get the counting of each pixel + cc2d::init_counting<<>>( + labels.data_ptr() + offset, + counts_init.data_ptr() + offset, + W, + H); + cc2d::final_counting<<>>( + labels.data_ptr() + offset, + counts_init.data_ptr() + offset, + counts_final.data_ptr() + offset, + W, + H); + } + + // returned values are [labels, counts] + std::vector outputs; + outputs.push_back(labels); + outputs.push_back(counts_final); + return outputs; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def( + "get_connected_componnets", + &get_connected_componnets, + "get_connected_componnets"); +} diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/__init__.py b/tools/sam2-onnx-tensorrt/sam2/modeling/__init__.py new file mode 100644 index 00000000..5277f461 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/__init__.py b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/__init__.py new file mode 100644 index 00000000..5277f461 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/hieradet.py b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/hieradet.py new file mode 100644 index 00000000..19ac77b6 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/hieradet.py @@ -0,0 +1,317 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import logging +from functools import partial +from typing import List, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from iopath.common.file_io import g_pathmgr + +from sam2.modeling.backbones.utils import ( + PatchEmbed, + window_partition, + window_unpartition, +) + +from sam2.modeling.sam2_utils import DropPath, MLP + + +def do_pool(x: torch.Tensor, pool: nn.Module, norm: nn.Module = None) -> torch.Tensor: + if pool is None: + return x + # (B, H, W, C) -> (B, C, H, W) + x = x.permute(0, 3, 1, 2) + x = pool(x) + # (B, C, H', W') -> (B, H', W', C) + x = x.permute(0, 2, 3, 1) + if norm: + x = norm(x) + + return x + + +class MultiScaleAttention(nn.Module): + def __init__( + self, + dim: int, + dim_out: int, + num_heads: int, + q_pool: nn.Module = None, + ): + super().__init__() + + self.dim = dim + self.dim_out = dim_out + self.num_heads = num_heads + self.q_pool = q_pool + self.qkv = nn.Linear(dim, dim_out * 3) + self.proj = nn.Linear(dim_out, dim_out) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, H, W, _ = x.shape + # qkv with shape (B, H * W, 3, nHead, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1) + # q, k, v with shape (B, H * W, nheads, C) + q, k, v = torch.unbind(qkv, 2) + + # Q pooling (for downsample at stage changes) + if self.q_pool: + q = do_pool(q.reshape(B, H, W, -1), self.q_pool) + H, W = q.shape[1:3] # downsampled shape + q = q.reshape(B, H * W, self.num_heads, -1) + + # Torch's SDPA expects [B, nheads, H*W, C] so we transpose + x = F.scaled_dot_product_attention( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), + ) + # Transpose back + x = x.transpose(1, 2) + x = x.reshape(B, H, W, -1) + + x = self.proj(x) + + return x + + +class MultiScaleBlock(nn.Module): + def __init__( + self, + dim: int, + dim_out: int, + num_heads: int, + mlp_ratio: float = 4.0, + drop_path: float = 0.0, + norm_layer: Union[nn.Module, str] = "LayerNorm", + q_stride: Tuple[int, int] = None, + act_layer: nn.Module = nn.GELU, + window_size: int = 0, + ): + super().__init__() + + if isinstance(norm_layer, str): + norm_layer = partial(getattr(nn, norm_layer), eps=1e-6) + + self.dim = dim + self.dim_out = dim_out + self.norm1 = norm_layer(dim) + + self.window_size = window_size + + self.pool, self.q_stride = None, q_stride + if self.q_stride: + self.pool = nn.MaxPool2d( + kernel_size=q_stride, stride=q_stride, ceil_mode=False + ) + + self.attn = MultiScaleAttention( + dim, + dim_out, + num_heads=num_heads, + q_pool=self.pool, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim_out) + self.mlp = MLP( + dim_out, + int(dim_out * mlp_ratio), + dim_out, + num_layers=2, + activation=act_layer, + ) + + if dim != dim_out: + self.proj = nn.Linear(dim, dim_out) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + shortcut = x # B, H, W, C + x = self.norm1(x) + + # Skip connection + if self.dim != self.dim_out: + shortcut = do_pool(self.proj(x), self.pool) + + # Window partition + window_size = self.window_size + if window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, window_size) + + # Window Attention + Q Pooling (if stage change) + x = self.attn(x) + if self.q_stride: + # Shapes have changed due to Q pooling + window_size = self.window_size // self.q_stride[0] + H, W = shortcut.shape[1:3] + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + pad_hw = (H + pad_h, W + pad_w) + + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, window_size, pad_hw, (H, W)) + + x = shortcut + self.drop_path(x) + # MLP + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class Hiera(nn.Module): + """ + Reference: https://arxiv.org/abs/2306.00989 + """ + + def __init__( + self, + embed_dim: int = 96, # initial embed dim + num_heads: int = 1, # initial number of heads + drop_path_rate: float = 0.0, # stochastic depth + q_pool: int = 3, # number of q_pool stages + q_stride: Tuple[int, int] = (2, 2), # downsample stride bet. stages + stages: Tuple[int, ...] = (2, 3, 16, 3), # blocks per stage + dim_mul: float = 2.0, # dim_mul factor at stage shift + head_mul: float = 2.0, # head_mul factor at stage shift + window_pos_embed_bkg_spatial_size: Tuple[int, int] = (14, 14), + # window size per stage, when not using global att. + window_spec: Tuple[int, ...] = ( + 8, + 4, + 14, + 7, + ), + # global attn in these blocks + global_att_blocks: Tuple[int, ...] = ( + 12, + 16, + 20, + ), + weights_path=None, + return_interm_layers=True, # return feats from every stage + ): + super().__init__() + + assert len(stages) == len(window_spec) + self.window_spec = window_spec + + depth = sum(stages) + self.q_stride = q_stride + self.stage_ends = [sum(stages[:i]) - 1 for i in range(1, len(stages) + 1)] + assert 0 <= q_pool <= len(self.stage_ends[:-1]) + self.q_pool_blocks = [x + 1 for x in self.stage_ends[:-1]][:q_pool] + self.return_interm_layers = return_interm_layers + + self.patch_embed = PatchEmbed( + embed_dim=embed_dim, + ) + # Which blocks have global att? + self.global_att_blocks = global_att_blocks + + # Windowed positional embedding (https://arxiv.org/abs/2311.05613) + self.window_pos_embed_bkg_spatial_size = window_pos_embed_bkg_spatial_size + self.pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, *self.window_pos_embed_bkg_spatial_size) + ) + self.pos_embed_window = nn.Parameter( + torch.zeros(1, embed_dim, self.window_spec[0], self.window_spec[0]) + ) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + + cur_stage = 1 + self.blocks = nn.ModuleList() + + for i in range(depth): + dim_out = embed_dim + # lags by a block, so first block of + # next stage uses an initial window size + # of previous stage and final window size of current stage + window_size = self.window_spec[cur_stage - 1] + + if self.global_att_blocks is not None: + window_size = 0 if i in self.global_att_blocks else window_size + + if i - 1 in self.stage_ends: + dim_out = int(embed_dim * dim_mul) + num_heads = int(num_heads * head_mul) + cur_stage += 1 + + block = MultiScaleBlock( + dim=embed_dim, + dim_out=dim_out, + num_heads=num_heads, + drop_path=dpr[i], + q_stride=self.q_stride if i in self.q_pool_blocks else None, + window_size=window_size, + ) + + embed_dim = dim_out + self.blocks.append(block) + + self.channel_list = ( + [self.blocks[i].dim_out for i in self.stage_ends[::-1]] + if return_interm_layers + else [self.blocks[-1].dim_out] + ) + + if weights_path is not None: + with g_pathmgr.open(weights_path, "rb") as f: + chkpt = torch.load(f, map_location="cpu") + logging.info("loading Hiera", self.load_state_dict(chkpt, strict=False)) + + def _get_pos_embed(self, hw: Tuple[int, int]) -> torch.Tensor: + h, w = hw + window_embed = self.pos_embed_window + pos_embed = F.interpolate(self.pos_embed, size=(h, w), mode="bicubic") + pos_embed = pos_embed + window_embed.tile( + [x // y for x, y in zip(pos_embed.shape, window_embed.shape)] + ) + pos_embed = pos_embed.permute(0, 2, 3, 1) + return pos_embed + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + x = self.patch_embed(x) + # x: (B, H, W, C) + + # Add pos embed + x = x + self._get_pos_embed(x.shape[1:3]) + + outputs = [] + for i, blk in enumerate(self.blocks): + x = blk(x) + if (i == self.stage_ends[-1]) or ( + i in self.stage_ends and self.return_interm_layers + ): + feats = x.permute(0, 3, 1, 2) + outputs.append(feats) + + return outputs + + def get_layer_id(self, layer_name): + # https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L33 + num_layers = self.get_num_layers() + + if layer_name.find("rel_pos") != -1: + return num_layers + 1 + elif layer_name.find("pos_embed") != -1: + return 0 + elif layer_name.find("patch_embed") != -1: + return 0 + elif layer_name.find("blocks") != -1: + return int(layer_name.split("blocks")[1].split(".")[1]) + 1 + else: + return num_layers + 1 + + def get_num_layers(self) -> int: + return len(self.blocks) diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/image_encoder.py b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/image_encoder.py new file mode 100644 index 00000000..37e9266b --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/image_encoder.py @@ -0,0 +1,134 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ImageEncoder(nn.Module): + def __init__( + self, + trunk: nn.Module, + neck: nn.Module, + scalp: int = 0, + ): + super().__init__() + self.trunk = trunk + self.neck = neck + self.scalp = scalp + assert ( + self.trunk.channel_list == self.neck.backbone_channel_list + ), f"Channel dims of trunk and neck do not match. Trunk: {self.trunk.channel_list}, neck: {self.neck.backbone_channel_list}" + + def forward(self, sample: torch.Tensor): + # Forward through backbone + features, pos = self.neck(self.trunk(sample)) + if self.scalp > 0: + # Discard the lowest resolution features + features, pos = features[: -self.scalp], pos[: -self.scalp] + + src = features[-1] + output = { + "vision_features": src, + "vision_pos_enc": pos, + "backbone_fpn": features, + } + return output + + +class FpnNeck(nn.Module): + """ + A modified variant of Feature Pyramid Network (FPN) neck + (we remove output conv and also do bicubic interpolation similar to ViT + pos embed interpolation) + """ + + def __init__( + self, + position_encoding: nn.Module, + d_model: int, + backbone_channel_list: List[int], + kernel_size: int = 1, + stride: int = 1, + padding: int = 0, + fpn_interp_model: str = "bilinear", + fuse_type: str = "sum", + fpn_top_down_levels: Optional[List[int]] = None, + ): + """Initialize the neck + :param trunk: the backbone + :param position_encoding: the positional encoding to use + :param d_model: the dimension of the model + :param neck_norm: the normalization to use + """ + super().__init__() + self.position_encoding = position_encoding + self.convs = nn.ModuleList() + self.backbone_channel_list = backbone_channel_list + self.d_model = d_model + for dim in backbone_channel_list: + current = nn.Sequential() + current.add_module( + "conv", + nn.Conv2d( + in_channels=dim, + out_channels=d_model, + kernel_size=kernel_size, + stride=stride, + padding=padding, + ), + ) + + self.convs.append(current) + self.fpn_interp_model = fpn_interp_model + assert fuse_type in ["sum", "avg"] + self.fuse_type = fuse_type + + # levels to have top-down features in its outputs + # e.g. if fpn_top_down_levels is [2, 3], then only outputs of level 2 and 3 + # have top-down propagation, while outputs of level 0 and level 1 have only + # lateral features from the same backbone level. + if fpn_top_down_levels is None: + # default is to have top-down features on all levels + fpn_top_down_levels = range(len(self.convs)) + self.fpn_top_down_levels = list(fpn_top_down_levels) + + def forward(self, xs: List[torch.Tensor]): + + out = [None] * len(self.convs) + pos = [None] * len(self.convs) + assert len(xs) == len(self.convs) + # fpn forward pass + # see https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/fpn.py + prev_features = None + # forward in top-down order (from low to high resolution) + n = len(self.convs) - 1 + for i in range(n, -1, -1): + x = xs[i] + lateral_features = self.convs[n - i](x) + if i in self.fpn_top_down_levels and prev_features is not None: + top_down_features = F.interpolate( + prev_features.to(dtype=torch.float32), + scale_factor=2.0, + mode=self.fpn_interp_model, + align_corners=( + None if self.fpn_interp_model == "nearest" else False + ), + antialias=False, + ) + prev_features = lateral_features + top_down_features + if self.fuse_type == "avg": + prev_features /= 2 + else: + prev_features = lateral_features + x_out = prev_features + out[i] = x_out + pos[i] = self.position_encoding(x_out).to(x_out.dtype) + + return out, pos diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/utils.py b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/utils.py new file mode 100644 index 00000000..930b1b76 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/backbones/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Some utilities for backbones, in particular for windowing""" + +from typing import Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def window_partition(x, window_size): + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).reshape(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition(windows, window_size, pad_hw, hw): + """ + Window unpartition into original sequences and removing padding. + Args: + x (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.reshape( + B, Hp // window_size, Wp // window_size, window_size, window_size, -1 + ) + x = x.permute(0, 1, 3, 2, 4, 5).reshape(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :] + return x + + +class PatchEmbed(nn.Module): + """ + Image to Patch Embedding. + """ + + def __init__( + self, + kernel_size: Tuple[int, ...] = (7, 7), + stride: Tuple[int, ...] = (4, 4), + padding: Tuple[int, ...] = (3, 3), + in_chans: int = 3, + embed_dim: int = 768, + ): + """ + Args: + kernel_size (Tuple): kernel size of the projection layer. + stride (Tuple): stride of the projection layer. + padding (Tuple): padding size of the projection layer. + in_chans (int): Number of input image channels. + embed_dim (int): embed_dim (int): Patch embedding dimension. + """ + super().__init__() + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + # B C H W -> B H W C + x = x.permute(0, 2, 3, 1) + return x diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/memory_attention.py b/tools/sam2-onnx-tensorrt/sam2/modeling/memory_attention.py new file mode 100644 index 00000000..0b07f9d8 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/memory_attention.py @@ -0,0 +1,169 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional + +import torch +from torch import nn, Tensor + +from sam2.modeling.sam.transformer import RoPEAttention + +from sam2.modeling.sam2_utils import get_activation_fn, get_clones + + +class MemoryAttentionLayer(nn.Module): + + def __init__( + self, + activation: str, + cross_attention: nn.Module, + d_model: int, + dim_feedforward: int, + dropout: float, + pos_enc_at_attn: bool, + pos_enc_at_cross_attn_keys: bool, + pos_enc_at_cross_attn_queries: bool, + self_attention: nn.Module, + ): + super().__init__() + self.d_model = d_model + self.dim_feedforward = dim_feedforward + self.dropout_value = dropout + self.self_attn = self_attention + self.cross_attn_image = cross_attention + + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation_str = activation + self.activation = get_activation_fn(activation) + + # Where to add pos enc + self.pos_enc_at_attn = pos_enc_at_attn + self.pos_enc_at_cross_attn_queries = pos_enc_at_cross_attn_queries + self.pos_enc_at_cross_attn_keys = pos_enc_at_cross_attn_keys + + def _forward_sa(self, tgt, query_pos): + # Self-Attention + tgt2 = self.norm1(tgt) + q = k = tgt2 + query_pos if self.pos_enc_at_attn else tgt2 + tgt2 = self.self_attn(q, k, v=tgt2) + tgt = tgt + self.dropout1(tgt2) + return tgt + + def _forward_ca(self, tgt, memory, query_pos, pos, num_k_exclude_rope=0): + kwds = {} + if num_k_exclude_rope > 0: + assert isinstance(self.cross_attn_image, RoPEAttention) + kwds = {"num_k_exclude_rope": num_k_exclude_rope} + + # Cross-Attention + tgt2 = self.norm2(tgt) + tgt2 = self.cross_attn_image( + q=tgt2 + query_pos if self.pos_enc_at_cross_attn_queries else tgt2, + k=memory + pos if self.pos_enc_at_cross_attn_keys else memory, + v=memory, + **kwds, + ) + tgt = tgt + self.dropout2(tgt2) + return tgt + + def forward( + self, + tgt, + memory, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None, + num_k_exclude_rope: int = 0, + ) -> torch.Tensor: + + # Self-Attn, Cross-Attn + tgt = self._forward_sa(tgt, query_pos) + tgt = self._forward_ca(tgt, memory, query_pos, pos, num_k_exclude_rope) + # MLP + tgt2 = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout3(tgt2) + return tgt + + +class MemoryAttention(nn.Module): + def __init__( + self, + d_model: int, + pos_enc_at_input: bool, + layer: nn.Module, + num_layers: int, + batch_first: bool = True, # Do layers expect batch first input? + ): + super().__init__() + self.d_model = d_model + self.layers = get_clones(layer, num_layers) + self.num_layers = num_layers + self.norm = nn.LayerNorm(d_model) + self.pos_enc_at_input = pos_enc_at_input + self.batch_first = batch_first + + def forward( + self, + curr: torch.Tensor, # self-attention inputs + memory: torch.Tensor, # cross-attention inputs + curr_pos: Optional[Tensor] = None, # pos_enc for self-attention inputs + memory_pos: Optional[Tensor] = None, # pos_enc for cross-attention inputs + num_obj_ptr_tokens: int = 0, # number of object pointer *tokens* + ): + if isinstance(curr, list): + assert isinstance(curr_pos, list) + assert len(curr) == len(curr_pos) == 1 + curr, curr_pos = ( + curr[0], + curr_pos[0], + ) + + assert ( + curr.shape[1] == memory.shape[1] + ), "Batch size must be the same for curr and memory" + + output = curr + if self.pos_enc_at_input and curr_pos is not None: + output = output + 0.1 * curr_pos + + if self.batch_first: + # Convert to batch first + output = output.transpose(0, 1) + curr_pos = curr_pos.transpose(0, 1) + memory = memory.transpose(0, 1) + memory_pos = memory_pos.transpose(0, 1) + + for layer in self.layers: + kwds = {} + if isinstance(layer.cross_attn_image, RoPEAttention): + kwds = {"num_k_exclude_rope": num_obj_ptr_tokens} + + output = layer( + tgt=output, + memory=memory, + pos=memory_pos, + query_pos=curr_pos, + **kwds, + ) + normed_output = self.norm(output) + + if self.batch_first: + # Convert back to seq first + normed_output = normed_output.transpose(0, 1) + curr_pos = curr_pos.transpose(0, 1) + + return normed_output diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/memory_encoder.py b/tools/sam2-onnx-tensorrt/sam2/modeling/memory_encoder.py new file mode 100644 index 00000000..f60202df --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/memory_encoder.py @@ -0,0 +1,181 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sam2.modeling.sam2_utils import DropPath, get_clones, LayerNorm2d + + +class MaskDownSampler(nn.Module): + """ + Progressively downsample a mask by total_stride, each time by stride. + Note that LayerNorm is applied per *token*, like in ViT. + + With each downsample (by a factor stride**2), channel capacity increases by the same factor. + In the end, we linearly project to embed_dim channels. + """ + + def __init__( + self, + embed_dim=256, + kernel_size=4, + stride=4, + padding=0, + total_stride=16, + activation=nn.GELU, + ): + super().__init__() + num_layers = int(math.log2(total_stride) // math.log2(stride)) + assert stride**num_layers == total_stride + self.encoder = nn.Sequential() + mask_in_chans, mask_out_chans = 1, 1 + for _ in range(num_layers): + mask_out_chans = mask_in_chans * (stride**2) + self.encoder.append( + nn.Conv2d( + mask_in_chans, + mask_out_chans, + kernel_size=kernel_size, + stride=stride, + padding=padding, + ) + ) + self.encoder.append(LayerNorm2d(mask_out_chans)) + self.encoder.append(activation()) + mask_in_chans = mask_out_chans + + self.encoder.append(nn.Conv2d(mask_out_chans, embed_dim, kernel_size=1)) + + def forward(self, x): + return self.encoder(x) + + +# Lightly adapted from ConvNext (https://github.com/facebookresearch/ConvNeXt) +class CXBlock(nn.Module): + r"""ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + + def __init__( + self, + dim, + kernel_size=7, + padding=3, + drop_path=0.0, + layer_scale_init_value=1e-6, + use_dwconv=True, + ): + super().__init__() + self.dwconv = nn.Conv2d( + dim, + dim, + kernel_size=kernel_size, + padding=padding, + groups=dim if use_dwconv else 1, + ) # depthwise conv + self.norm = LayerNorm2d(dim, eps=1e-6) + self.pwconv1 = nn.Linear( + dim, 4 * dim + ) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = ( + nn.Parameter(layer_scale_init_value * torch.ones((dim)), requires_grad=True) + if layer_scale_init_value > 0 + else None + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = self.norm(x) + x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) + + x = input + self.drop_path(x) + return x + + +class Fuser(nn.Module): + def __init__(self, layer, num_layers, dim=None, input_projection=False): + super().__init__() + self.proj = nn.Identity() + self.layers = get_clones(layer, num_layers) + + if input_projection: + assert dim is not None + self.proj = nn.Conv2d(dim, dim, kernel_size=1) + + def forward(self, x): + # normally x: (N, C, H, W) + x = self.proj(x) + for layer in self.layers: + x = layer(x) + return x + + +class MemoryEncoder(nn.Module): + def __init__( + self, + out_dim, + mask_downsampler, + fuser, + position_encoding, + in_dim=256, # in_dim of pix_feats + ): + super().__init__() + + self.mask_downsampler = mask_downsampler + + self.pix_feat_proj = nn.Conv2d(in_dim, in_dim, kernel_size=1) + self.fuser = fuser + self.position_encoding = position_encoding + self.out_proj = nn.Identity() + if out_dim != in_dim: + self.out_proj = nn.Conv2d(in_dim, out_dim, kernel_size=1) + + def forward( + self, + pix_feat: torch.Tensor, + masks: torch.Tensor, + skip_mask_sigmoid: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + ## Process masks + # sigmoid, so that less domain shift from gt masks which are bool + if not skip_mask_sigmoid: + masks = F.sigmoid(masks) + masks = self.mask_downsampler(masks) + + ## Fuse pix_feats and downsampled masks + # in case the visual features are on CPU, cast them to CUDA + pix_feat = pix_feat.to(masks.device) + + x = self.pix_feat_proj(pix_feat) + x = x + masks + x = self.fuser(x) + x = self.out_proj(x) + + pos = self.position_encoding(x).to(x.dtype) + + return {"vision_features": x, "vision_pos_enc": [pos]} diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/position_encoding.py b/tools/sam2-onnx-tensorrt/sam2/modeling/position_encoding.py new file mode 100644 index 00000000..21b4367e --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/position_encoding.py @@ -0,0 +1,279 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import Any, Optional, Tuple + +import numpy as np + +import torch +from torch import nn + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention Is All You Need paper, generalized to work on images. + """ + + def __init__( + self, + num_pos_feats, + temperature: int = 10000, + normalize: bool = True, + scale: Optional[float] = None, + # Following settings only relevant + # for warmping up cache for compilation + warmup_cache: bool = True, + image_size: int = 1024, + strides: Tuple[int] = (4, 8, 16, 32), + ): + super().__init__() + assert num_pos_feats % 2 == 0, "Expecting even model width" + self.num_pos_feats = num_pos_feats // 2 + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + self.cache = {} + if warmup_cache and torch.cuda.is_available(): + # Warmup cache for cuda, to help with compilation + device = torch.device("cuda") + for stride in strides: + cache_key = (image_size // stride, image_size // stride) + self._pe(1, device, *cache_key) + + def _encode_xy(self, x, y): + # The positions are expected to be normalized + assert len(x) == len(y) and x.ndim == y.ndim == 1 + x_embed = x * self.scale + y_embed = y * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, None] / dim_t + pos_y = y_embed[:, None] / dim_t + pos_x = torch.stack( + (pos_x[:, 0::2].sin(), pos_x[:, 1::2].cos()), dim=2 + ).flatten(1) + pos_y = torch.stack( + (pos_y[:, 0::2].sin(), pos_y[:, 1::2].cos()), dim=2 + ).flatten(1) + return pos_x, pos_y + + @torch.no_grad() + def encode_boxes(self, x, y, w, h): + pos_x, pos_y = self._encode_xy(x, y) + pos = torch.cat((pos_y, pos_x, h[:, None], w[:, None]), dim=1) + return pos + + encode = encode_boxes # Backwards compatibility + + @torch.no_grad() + def encode_points(self, x, y, labels): + (bx, nx), (by, ny), (bl, nl) = x.shape, y.shape, labels.shape + assert bx == by and nx == ny and bx == bl and nx == nl + pos_x, pos_y = self._encode_xy(x.flatten(), y.flatten()) + pos_x, pos_y = pos_x.reshape(bx, nx, -1), pos_y.reshape(by, ny, -1) + pos = torch.cat((pos_y, pos_x, labels[:, :, None]), dim=2) + return pos + + @torch.no_grad() + def _pe(self, B, device, *cache_key): + H, W = cache_key + if cache_key in self.cache: + return self.cache[cache_key].to(device)[None].repeat(B, 1, 1, 1) + + y_embed = ( + torch.arange(1, H + 1, dtype=torch.float32, device=device) + .view(1, -1, 1) + .repeat(B, 1, W) + ) + x_embed = ( + torch.arange(1, W + 1, dtype=torch.float32, device=device) + .view(1, 1, -1) + .repeat(B, H, 1) + ) + + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + self.cache[cache_key] = pos[0] + return pos + + @torch.no_grad() + def forward(self, x: torch.Tensor): + B = x.shape[0] + cache_key = (x.shape[-2], x.shape[-1]) + return self._pe(B, x.device, *cache_key) + + +class PositionEmbeddingRandom(nn.Module): + """ + Positional encoding using random spatial frequencies. + """ + + def __init__(self, num_pos_feats: int = 64, scale: Optional[float] = None) -> None: + super().__init__() + if scale is None or scale <= 0.0: + scale = 1.0 + self.register_buffer( + "positional_encoding_gaussian_matrix", + scale * torch.randn((2, num_pos_feats)), + ) + + def _pe_encoding(self, coords: torch.Tensor) -> torch.Tensor: + """Positionally encode points that are normalized to [0,1].""" + # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape + coords = 2 * coords - 1 + coords = coords @ self.positional_encoding_gaussian_matrix + coords = 2 * np.pi * coords + # outputs d_1 x ... x d_n x C shape + return torch.cat([torch.sin(coords), torch.cos(coords)], dim=-1) + + def forward(self, size: Tuple[int, int]) -> torch.Tensor: + """Generate positional encoding for a grid of the specified size.""" + h, w = size + device: Any = self.positional_encoding_gaussian_matrix.device + grid = torch.ones((h, w), device=device, dtype=torch.float32) + y_embed = grid.cumsum(dim=0) - 0.5 + x_embed = grid.cumsum(dim=1) - 0.5 + y_embed = y_embed / h + x_embed = x_embed / w + + pe = self._pe_encoding(torch.stack([x_embed, y_embed], dim=-1)) + return pe.permute(2, 0, 1) # C x H x W + + def forward_with_coords( + self, coords_input: torch.Tensor, image_size: Tuple[int, int] + ) -> torch.Tensor: + """Positionally encode points that are not normalized to [0,1].""" + coords = coords_input.clone() + coords[:, :, 0] = coords[:, :, 0] / image_size[1] + coords[:, :, 1] = coords[:, :, 1] / image_size[0] + return self._pe_encoding(coords.to(torch.float)) # B x N x C + + +# Rotary Positional Encoding, adapted from: +# 1. https://github.com/meta-llama/codellama/blob/main/llama/model.py +# 2. https://github.com/naver-ai/rope-vit +# 3. https://github.com/lucidrains/rotary-embedding-torch + + +def init_t_xy(end_x: int, end_y: int): + t = torch.arange(end_x * end_y, dtype=torch.float32) + t_x = (t % end_x).float() + t_y = torch.div(t, end_x, rounding_mode="floor").float() + return t_x, t_y + + +def compute_axial_cis(dim: int, end_x: int, end_y: int, theta: float = 10000.0): + freqs_x = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) + freqs_y = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) + + t_x, t_y = init_t_xy(end_x, end_y) + freqs_x = torch.outer(t_x, freqs_x) + freqs_y = torch.outer(t_y, freqs_y) + freqs_cis_x = torch.polar(torch.ones_like(freqs_x), freqs_x) + freqs_cis_y = torch.polar(torch.ones_like(freqs_y), freqs_y) + return torch.cat([freqs_cis_x, freqs_cis_y], dim=-1) + + +def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor): + ndim = x.ndim + assert 0 <= 1 < ndim + assert freqs_cis.shape == (x.shape[-2], x.shape[-1]) + shape = [d if i >= ndim - 2 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def apply_rotary_enc( + xq: torch.Tensor, + xk: torch.Tensor, + freqs_cis: torch.Tensor, + repeat_freqs_k: bool = False, +): + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) + xk_ = ( + torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) + if xk.shape[-2] != 0 + else None + ) + freqs_cis = reshape_for_broadcast(freqs_cis, xq_) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) + if xk_ is None: + # no keys to rotate, due to dropout + return xq_out.type_as(xq).to(xq.device), xk + # repeat freqs along seq_len dim to match k seq_len + if repeat_freqs_k: + r = xk_.shape[-2] // xq_.shape[-2] + if freqs_cis.is_cuda: + freqs_cis = freqs_cis.repeat(*([1] * (freqs_cis.ndim - 2)), r, 1) + else: + # torch.repeat on complex numbers may not be supported on non-CUDA devices + # (freqs_cis has 4 dims and we repeat on dim 2) so we use expand + flatten + freqs_cis = freqs_cis.unsqueeze(2).expand(-1, -1, r, -1, -1).flatten(2, 3) + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) + return xq_out.type_as(xq).to(xq.device), xk_out.type_as(xk).to(xk.device) + + +# Matrix version of rotary enc +# https://github.com/facebookresearch/segment-anything-2/issues/186 + +def get_rotation_matrices(dim, end_x, end_y, theta=10000.0, device=None, dtype=None): + + powers = torch.linspace(0, 1, 1 + (dim // 4), device=device, dtype=dtype)[:-1] + base_angles = torch.pow(theta, -powers) + + end_x, end_y = int(end_x), int(end_y) + x_mults = torch.arange(end_x, device=device, dtype=dtype).repeat(end_y) + y_mults = torch.arange(end_y, device=device, dtype=dtype).repeat_interleave(end_x) + angles_xy = (torch.outer(mults, base_angles) for mults in (x_mults, y_mults)) + + rotmats_list = [] + for angles in angles_xy: + sterm, cterm = torch.sin(-angles), torch.cos(-angles) + rotmat = torch.stack( + [ + torch.stack([cterm, -sterm], dim=-1), + torch.stack([sterm, cterm], dim=-1), + ], + dim=-1, + ) + rotmats_list.append(rotmat) + + return torch.cat(rotmats_list, dim=1).unsqueeze(0).unsqueeze(0) + + +def apply_rotary_matenc(xq, xk, rotmats, repeat_freqs_k=False): + + bq, hq, nq, cq = xq.shape + bk, hk, nk, ck = xk.shape + + q_out = torch.matmul(rotmats, xq.reshape(bq, hq, nq, cq // 2, 2, 1)).flatten(3) + k_rotmat = rotmats.repeat(1, 1, nk // nq, 1, 1, 1) if repeat_freqs_k else rotmats + k_out = torch.matmul(k_rotmat, xk.reshape(bk, hk, nk, ck // 2, 2, 1)).flatten(3) + + return q_out, k_out diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam/__init__.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/__init__.py new file mode 100644 index 00000000..5277f461 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam/mask_decoder.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/mask_decoder.py new file mode 100644 index 00000000..9bebc036 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/mask_decoder.py @@ -0,0 +1,295 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import List, Optional, Tuple, Type + +import torch +from torch import nn + +from sam2.modeling.sam2_utils import LayerNorm2d, MLP + + +class MaskDecoder(nn.Module): + def __init__( + self, + *, + transformer_dim: int, + transformer: nn.Module, + num_multimask_outputs: int = 3, + activation: Type[nn.Module] = nn.GELU, + iou_head_depth: int = 3, + iou_head_hidden_dim: int = 256, + use_high_res_features: bool = False, + iou_prediction_use_sigmoid=False, + dynamic_multimask_via_stability=False, + dynamic_multimask_stability_delta=0.05, + dynamic_multimask_stability_thresh=0.98, + pred_obj_scores: bool = False, + pred_obj_scores_mlp: bool = False, + use_multimask_token_for_obj_ptr: bool = False, + ) -> None: + """ + Predicts masks given an image and prompt embeddings, using a + transformer architecture. + + Arguments: + transformer_dim (int): the channel dimension of the transformer + transformer (nn.Module): the transformer used to predict masks + num_multimask_outputs (int): the number of masks to predict + when disambiguating masks + activation (nn.Module): the type of activation to use when + upscaling masks + iou_head_depth (int): the depth of the MLP used to predict + mask quality + iou_head_hidden_dim (int): the hidden dimension of the MLP + used to predict mask quality + """ + super().__init__() + self.transformer_dim = transformer_dim + self.transformer = transformer + + self.num_multimask_outputs = num_multimask_outputs + + self.iou_token = nn.Embedding(1, transformer_dim) + self.num_mask_tokens = num_multimask_outputs + 1 + self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) + + self.pred_obj_scores = pred_obj_scores + if self.pred_obj_scores: + self.obj_score_token = nn.Embedding(1, transformer_dim) + self.use_multimask_token_for_obj_ptr = use_multimask_token_for_obj_ptr + + self.output_upscaling = nn.Sequential( + nn.ConvTranspose2d( + transformer_dim, transformer_dim // 4, kernel_size=2, stride=2 + ), + LayerNorm2d(transformer_dim // 4), + activation(), + nn.ConvTranspose2d( + transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2 + ), + activation(), + ) + self.use_high_res_features = use_high_res_features + if use_high_res_features: + self.conv_s0 = nn.Conv2d( + transformer_dim, transformer_dim // 8, kernel_size=1, stride=1 + ) + self.conv_s1 = nn.Conv2d( + transformer_dim, transformer_dim // 4, kernel_size=1, stride=1 + ) + + self.output_hypernetworks_mlps = nn.ModuleList( + [ + MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) + for i in range(self.num_mask_tokens) + ] + ) + + self.iou_prediction_head = MLP( + transformer_dim, + iou_head_hidden_dim, + self.num_mask_tokens, + iou_head_depth, + sigmoid_output=iou_prediction_use_sigmoid, + ) + if self.pred_obj_scores: + self.pred_obj_score_head = nn.Linear(transformer_dim, 1) + if pred_obj_scores_mlp: + self.pred_obj_score_head = MLP(transformer_dim, transformer_dim, 1, 3) + + # When outputting a single mask, optionally we can dynamically fall back to the best + # multimask output token if the single mask output token gives low stability scores. + self.dynamic_multimask_via_stability = dynamic_multimask_via_stability + self.dynamic_multimask_stability_delta = dynamic_multimask_stability_delta + self.dynamic_multimask_stability_thresh = dynamic_multimask_stability_thresh + + def forward( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + multimask_output: bool, + repeat_image: bool, + high_res_features: Optional[List[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Predict masks given image and prompt embeddings. + + Arguments: + image_embeddings (torch.Tensor): the embeddings from the image encoder + image_pe (torch.Tensor): positional encoding with the shape of image_embeddings + sparse_prompt_embeddings (torch.Tensor): the embeddings of the points and boxes + dense_prompt_embeddings (torch.Tensor): the embeddings of the mask inputs + multimask_output (bool): Whether to return multiple masks or a single + mask. + + Returns: + torch.Tensor: batched predicted masks + torch.Tensor: batched predictions of mask quality + torch.Tensor: batched SAM token for mask output + """ + masks, iou_pred, mask_tokens_out, object_score_logits = self.predict_masks( + image_embeddings=image_embeddings, + image_pe=image_pe, + sparse_prompt_embeddings=sparse_prompt_embeddings, + dense_prompt_embeddings=dense_prompt_embeddings, + repeat_image=repeat_image, + high_res_features=high_res_features, + ) + + # Select the correct mask or masks for output + if multimask_output: + masks = masks[:, 1:, :, :] + iou_pred = iou_pred[:, 1:] + elif self.dynamic_multimask_via_stability and not self.training: + masks, iou_pred = self._dynamic_multimask_via_stability(masks, iou_pred) + else: + masks = masks[:, 0:1, :, :] + iou_pred = iou_pred[:, 0:1] + + if multimask_output and self.use_multimask_token_for_obj_ptr: + sam_tokens_out = mask_tokens_out[:, 1:] # [b, 3, c] shape + else: + # Take the mask output token. Here we *always* use the token for single mask output. + # At test time, even if we track after 1-click (and using multimask_output=True), + # we still take the single mask token here. The rationale is that we always track + # after multiple clicks during training, so the past tokens seen during training + # are always the single mask token (and we'll let it be the object-memory token). + sam_tokens_out = mask_tokens_out[:, 0:1] # [b, 1, c] shape + + # Prepare output + return masks, iou_pred, sam_tokens_out, object_score_logits + + def predict_masks( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + repeat_image: bool, + high_res_features: Optional[List[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Predicts masks. See 'forward' for more details.""" + # Concatenate output tokens + s = 0 + if self.pred_obj_scores: + output_tokens = torch.cat( + [ + self.obj_score_token.weight, + self.iou_token.weight, + self.mask_tokens.weight, + ], + dim=0, + ) + s = 1 + else: + output_tokens = torch.cat( + [self.iou_token.weight, self.mask_tokens.weight], dim=0 + ) + output_tokens = output_tokens.unsqueeze(0).expand( + sparse_prompt_embeddings.size(0), -1, -1 + ) + tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) + + # Expand per-image data in batch direction to be per-mask + if repeat_image: + src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) + else: + assert image_embeddings.shape[0] == tokens.shape[0] + src = image_embeddings + src = src + dense_prompt_embeddings + assert ( + image_pe.size(0) == 1 + ), "image_pe should have size 1 in batch dim (from `get_dense_pe()`)" + pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) + b, c, h, w = src.shape + + # Run the transformer + hs, src = self.transformer(src, pos_src, tokens) + iou_token_out = hs[:, s, :] + mask_tokens_out = hs[:, s + 1 : (s + 1 + self.num_mask_tokens), :] + + # Upscale mask embeddings and predict masks using the mask tokens + src = src.transpose(1, 2).view(b, c, h, w) + if not self.use_high_res_features: + upscaled_embedding = self.output_upscaling(src) + else: + dc1, ln1, act1, dc2, act2 = self.output_upscaling + feat_s0, feat_s1 = high_res_features + upscaled_embedding = act1(ln1(dc1(src) + feat_s1)) + upscaled_embedding = act2(dc2(upscaled_embedding) + feat_s0) + + hyper_in_list: List[torch.Tensor] = [] + for i in range(self.num_mask_tokens): + hyper_in_list.append( + self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]) + ) + hyper_in = torch.stack(hyper_in_list, dim=1) + b, c, h, w = upscaled_embedding.shape + masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) + + # Generate mask quality predictions + iou_pred = self.iou_prediction_head(iou_token_out) + if self.pred_obj_scores: + assert s == 1 + object_score_logits = self.pred_obj_score_head(hs[:, 0, :]) + else: + # Obj scores logits - default to 10.0, i.e. assuming the object is present, sigmoid(10)=1 + object_score_logits = 10.0 * iou_pred.new_ones(iou_pred.shape[0], 1) + + return masks, iou_pred, mask_tokens_out, object_score_logits + + def _get_stability_scores(self, mask_logits): + """ + Compute stability scores of the mask logits based on the IoU between upper and + lower thresholds. + """ + mask_logits = mask_logits.flatten(-2) + stability_delta = self.dynamic_multimask_stability_delta + area_i = torch.sum(mask_logits > stability_delta, dim=-1).float() + area_u = torch.sum(mask_logits > -stability_delta, dim=-1).float() + stability_scores = torch.where(area_u > 0, area_i / area_u, 1.0) + return stability_scores + + def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores): + """ + When outputting a single mask, if the stability score from the current single-mask + output (based on output token 0) falls below a threshold, we instead select from + multi-mask outputs (based on output token 1~3) the mask with the highest predicted + IoU score. This is intended to ensure a valid mask for both clicking and tracking. + """ + # The best mask from multimask output tokens (1~3) + multimask_logits = all_mask_logits[:, 1:, :, :] + multimask_iou_scores = all_iou_scores[:, 1:] + best_scores_inds = torch.argmax(multimask_iou_scores, dim=-1) + batch_inds = torch.arange( + multimask_iou_scores.size(0), device=all_iou_scores.device + ) + best_multimask_logits = multimask_logits[batch_inds, best_scores_inds] + best_multimask_logits = best_multimask_logits.unsqueeze(1) + best_multimask_iou_scores = multimask_iou_scores[batch_inds, best_scores_inds] + best_multimask_iou_scores = best_multimask_iou_scores.unsqueeze(1) + + # The mask from singlemask output token 0 and its stability score + singlemask_logits = all_mask_logits[:, 0:1, :, :] + singlemask_iou_scores = all_iou_scores[:, 0:1] + stability_scores = self._get_stability_scores(singlemask_logits) + is_stable = stability_scores >= self.dynamic_multimask_stability_thresh + + # Dynamically fall back to best multimask output upon low stability scores. + mask_logits_out = torch.where( + is_stable[..., None, None].expand_as(singlemask_logits), + singlemask_logits, + best_multimask_logits, + ) + iou_scores_out = torch.where( + is_stable.expand_as(singlemask_iou_scores), + singlemask_iou_scores, + best_multimask_iou_scores, + ) + return mask_logits_out, iou_scores_out diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam/prompt_encoder.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/prompt_encoder.py new file mode 100644 index 00000000..c5787626 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/prompt_encoder.py @@ -0,0 +1,202 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional, Tuple, Type + +import torch +from torch import nn + +from sam2.modeling.position_encoding import PositionEmbeddingRandom + +from sam2.modeling.sam2_utils import LayerNorm2d + + +class PromptEncoder(nn.Module): + def __init__( + self, + embed_dim: int, + image_embedding_size: Tuple[int, int], + input_image_size: Tuple[int, int], + mask_in_chans: int, + activation: Type[nn.Module] = nn.GELU, + ) -> None: + """ + Encodes prompts for input to SAM's mask decoder. + + Arguments: + embed_dim (int): The prompts' embedding dimension + image_embedding_size (tuple(int, int)): The spatial size of the + image embedding, as (H, W). + input_image_size (int): The padded size of the image as input + to the image encoder, as (H, W). + mask_in_chans (int): The number of hidden channels used for + encoding input masks. + activation (nn.Module): The activation to use when encoding + input masks. + """ + super().__init__() + self.embed_dim = embed_dim + self.input_image_size = input_image_size + self.image_embedding_size = image_embedding_size + self.pe_layer = PositionEmbeddingRandom(embed_dim // 2) + + self.num_point_embeddings: int = 4 # pos/neg point + 2 box corners + point_embeddings = [ + nn.Embedding(1, embed_dim) for i in range(self.num_point_embeddings) + ] + self.point_embeddings = nn.ModuleList(point_embeddings) + self.not_a_point_embed = nn.Embedding(1, embed_dim) + + self.mask_input_size = ( + 4 * image_embedding_size[0], + 4 * image_embedding_size[1], + ) + self.mask_downscaling = nn.Sequential( + nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans // 4), + activation(), + nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans), + activation(), + nn.Conv2d(mask_in_chans, embed_dim, kernel_size=1), + ) + self.no_mask_embed = nn.Embedding(1, embed_dim) + + def get_dense_pe(self) -> torch.Tensor: + """ + Returns the positional encoding used to encode point prompts, + applied to a dense set of points the shape of the image encoding. + + Returns: + torch.Tensor: Positional encoding with shape + 1x(embed_dim)x(embedding_h)x(embedding_w) + """ + return self.pe_layer(self.image_embedding_size).unsqueeze(0) + + def _embed_points( + self, + points: torch.Tensor, + labels: torch.Tensor, + pad: bool, + ) -> torch.Tensor: + """Embeds point prompts.""" + points = points + 0.5 # Shift to center of pixel + if pad: + padding_point = torch.zeros((points.shape[0], 1, 2), device=points.device) + padding_label = -torch.ones((labels.shape[0], 1), device=labels.device) + points = torch.cat([points, padding_point], dim=1) + labels = torch.cat([labels, padding_label], dim=1) + point_embedding = self.pe_layer.forward_with_coords( + points, self.input_image_size + ) + + point_embedding = torch.where( + (labels == -1).unsqueeze(-1), + torch.zeros_like(point_embedding) + self.not_a_point_embed.weight, + point_embedding, + ) + point_embedding = torch.where( + (labels == 0).unsqueeze(-1), + point_embedding + self.point_embeddings[0].weight, + point_embedding, + ) + point_embedding = torch.where( + (labels == 1).unsqueeze(-1), + point_embedding + self.point_embeddings[1].weight, + point_embedding, + ) + point_embedding = torch.where( + (labels == 2).unsqueeze(-1), + point_embedding + self.point_embeddings[2].weight, + point_embedding, + ) + point_embedding = torch.where( + (labels == 3).unsqueeze(-1), + point_embedding + self.point_embeddings[3].weight, + point_embedding, + ) + return point_embedding + + def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor: + """Embeds box prompts.""" + boxes = boxes + 0.5 # Shift to center of pixel + coords = boxes.reshape(-1, 2, 2) + corner_embedding = self.pe_layer.forward_with_coords( + coords, self.input_image_size + ) + corner_embedding[:, 0, :] += self.point_embeddings[2].weight + corner_embedding[:, 1, :] += self.point_embeddings[3].weight + return corner_embedding + + def _embed_masks(self, masks: torch.Tensor) -> torch.Tensor: + """Embeds mask inputs.""" + mask_embedding = self.mask_downscaling(masks) + return mask_embedding + + def _get_batch_size( + self, + points: Optional[Tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> int: + """ + Gets the batch size of the output given the batch size of the input prompts. + """ + if points is not None: + return points[0].shape[0] + elif boxes is not None: + return boxes.shape[0] + elif masks is not None: + return masks.shape[0] + else: + return 1 + + def _get_device(self) -> torch.device: + return self.point_embeddings[0].weight.device + + def forward( + self, + points: Optional[Tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Embeds different types of prompts, returning both sparse and dense + embeddings. + + Arguments: + points (tuple(torch.Tensor, torch.Tensor) or none): point coordinates + and labels to embed. + boxes (torch.Tensor or none): boxes to embed + masks (torch.Tensor or none): masks to embed + + Returns: + torch.Tensor: sparse embeddings for the points and boxes, with shape + BxNx(embed_dim), where N is determined by the number of input points + and boxes. + torch.Tensor: dense embeddings for the masks, in the shape + Bx(embed_dim)x(embed_H)x(embed_W) + """ + bs = self._get_batch_size(points, boxes, masks) + sparse_embeddings = torch.empty( + (bs, 0, self.embed_dim), device=self._get_device() + ) + if points is not None: + coords, labels = points + point_embeddings = self._embed_points(coords, labels, pad=(boxes is None)) + sparse_embeddings = torch.cat([sparse_embeddings, point_embeddings], dim=1) + if boxes is not None: + box_embeddings = self._embed_boxes(boxes) + sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=1) + + if masks is not None: + dense_embeddings = self._embed_masks(masks) + else: + dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand( + bs, -1, self.image_embedding_size[0], self.image_embedding_size[1] + ) + + return sparse_embeddings, dense_embeddings diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam/transformer.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/transformer.py new file mode 100644 index 00000000..b836c524 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam/transformer.py @@ -0,0 +1,385 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib +import math +import warnings +from functools import partial +from typing import Tuple, Type + +import torch +import torch.nn.functional as F +from torch import nn, Tensor + +from sam2.modeling.position_encoding import apply_rotary_enc, compute_axial_cis +from sam2.modeling.position_encoding import apply_rotary_matenc, get_rotation_matrices +from sam2.modeling.sam2_utils import MLP +from sam2.utils.misc import get_sdpa_settings + +warnings.simplefilter(action="ignore", category=FutureWarning) +# Check whether Flash Attention is available (and use it by default) +OLD_GPU, USE_FLASH_ATTN, MATH_KERNEL_ON = get_sdpa_settings() +# A fallback setting to allow all available kernels if Flash Attention fails +ALLOW_ALL_KERNELS = False + +# Use matrix version of rotrary enc +USE_MAT_ROTARY_ENC = True + +def sdp_kernel_context(dropout_p): + """ + Get the context for the attention scaled dot-product kernel. We use Flash Attention + by default, but fall back to all available kernels if Flash Attention fails. + """ + if ALLOW_ALL_KERNELS: + return contextlib.nullcontext() + + return torch.backends.cuda.sdp_kernel( + enable_flash=USE_FLASH_ATTN, + # if Flash attention kernel is off, then math kernel needs to be enabled + enable_math=(OLD_GPU and dropout_p > 0.0) or MATH_KERNEL_ON, + enable_mem_efficient=OLD_GPU, + ) + + +class TwoWayTransformer(nn.Module): + def __init__( + self, + depth: int, + embedding_dim: int, + num_heads: int, + mlp_dim: int, + activation: Type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + ) -> None: + """ + A transformer decoder that attends to an input image using + queries whose positional embedding is supplied. + + Args: + depth (int): number of layers in the transformer + embedding_dim (int): the channel dimension for the input embeddings + num_heads (int): the number of heads for multihead attention. Must + divide embedding_dim + mlp_dim (int): the channel dimension internal to the MLP block + activation (nn.Module): the activation to use in the MLP block + """ + super().__init__() + self.depth = depth + self.embedding_dim = embedding_dim + self.num_heads = num_heads + self.mlp_dim = mlp_dim + self.layers = nn.ModuleList() + + for i in range(depth): + self.layers.append( + TwoWayAttentionBlock( + embedding_dim=embedding_dim, + num_heads=num_heads, + mlp_dim=mlp_dim, + activation=activation, + attention_downsample_rate=attention_downsample_rate, + skip_first_layer_pe=(i == 0), + ) + ) + + self.final_attn_token_to_image = Attention( + embedding_dim, num_heads, downsample_rate=attention_downsample_rate + ) + self.norm_final_attn = nn.LayerNorm(embedding_dim) + + def forward( + self, + image_embedding: Tensor, + image_pe: Tensor, + point_embedding: Tensor, + ) -> Tuple[Tensor, Tensor]: + """ + Args: + image_embedding (torch.Tensor): image to attend to. Should be shape + B x embedding_dim x h x w for any h and w. + image_pe (torch.Tensor): the positional encoding to add to the image. Must + have the same shape as image_embedding. + point_embedding (torch.Tensor): the embedding to add to the query points. + Must have shape B x N_points x embedding_dim for any N_points. + + Returns: + torch.Tensor: the processed point_embedding + torch.Tensor: the processed image_embedding + """ + # BxCxHxW -> BxHWxC == B x N_image_tokens x C + bs, c, h, w = image_embedding.shape + image_embedding = image_embedding.flatten(2).permute(0, 2, 1) + image_pe = image_pe.flatten(2).permute(0, 2, 1) + + # Prepare queries + queries = point_embedding + keys = image_embedding + + # Apply transformer blocks and final layernorm + for layer in self.layers: + queries, keys = layer( + queries=queries, + keys=keys, + query_pe=point_embedding, + key_pe=image_pe, + ) + + # Apply the final attention layer from the points to the image + q = queries + point_embedding + k = keys + image_pe + attn_out = self.final_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm_final_attn(queries) + + return queries, keys + + +class TwoWayAttentionBlock(nn.Module): + def __init__( + self, + embedding_dim: int, + num_heads: int, + mlp_dim: int = 2048, + activation: Type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + skip_first_layer_pe: bool = False, + ) -> None: + """ + A transformer block with four layers: (1) self-attention of sparse + inputs, (2) cross attention of sparse inputs to dense inputs, (3) mlp + block on sparse inputs, and (4) cross attention of dense inputs to sparse + inputs. + + Arguments: + embedding_dim (int): the channel dimension of the embeddings + num_heads (int): the number of heads in the attention layers + mlp_dim (int): the hidden dimension of the mlp block + activation (nn.Module): the activation of the mlp block + skip_first_layer_pe (bool): skip the PE on the first layer + """ + super().__init__() + self.self_attn = Attention(embedding_dim, num_heads) + self.norm1 = nn.LayerNorm(embedding_dim) + + self.cross_attn_token_to_image = Attention( + embedding_dim, num_heads, downsample_rate=attention_downsample_rate + ) + self.norm2 = nn.LayerNorm(embedding_dim) + + self.mlp = MLP( + embedding_dim, mlp_dim, embedding_dim, num_layers=2, activation=activation + ) + self.norm3 = nn.LayerNorm(embedding_dim) + + self.norm4 = nn.LayerNorm(embedding_dim) + self.cross_attn_image_to_token = Attention( + embedding_dim, num_heads, downsample_rate=attention_downsample_rate + ) + + self.skip_first_layer_pe = skip_first_layer_pe + + def forward( + self, queries: Tensor, keys: Tensor, query_pe: Tensor, key_pe: Tensor + ) -> Tuple[Tensor, Tensor]: + # Self attention block + if self.skip_first_layer_pe: + queries = self.self_attn(q=queries, k=queries, v=queries) + else: + q = queries + query_pe + attn_out = self.self_attn(q=q, k=q, v=queries) + queries = queries + attn_out + queries = self.norm1(queries) + + # Cross attention block, tokens attending to image embedding + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm2(queries) + + # MLP block + mlp_out = self.mlp(queries) + queries = queries + mlp_out + queries = self.norm3(queries) + + # Cross attention block, image embedding attending to tokens + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_image_to_token(q=k, k=q, v=queries) + keys = keys + attn_out + keys = self.norm4(keys) + + return queries, keys + + +class Attention(nn.Module): + """ + An attention layer that allows for downscaling the size of the embedding + after projection to queries, keys, and values. + """ + + def __init__( + self, + embedding_dim: int, + num_heads: int, + downsample_rate: int = 1, + dropout: float = 0.0, + kv_in_dim: int = None, + ) -> None: + super().__init__() + self.embedding_dim = embedding_dim + self.kv_in_dim = kv_in_dim if kv_in_dim is not None else embedding_dim + self.internal_dim = embedding_dim // downsample_rate + self.num_heads = num_heads + assert ( + self.internal_dim % num_heads == 0 + ), "num_heads must divide embedding_dim." + + self.q_proj = nn.Linear(embedding_dim, self.internal_dim) + self.k_proj = nn.Linear(self.kv_in_dim, self.internal_dim) + self.v_proj = nn.Linear(self.kv_in_dim, self.internal_dim) + self.out_proj = nn.Linear(self.internal_dim, embedding_dim) + + self.dropout_p = dropout + + def _separate_heads(self, x: Tensor, num_heads: int) -> Tensor: + b, n, c = x.shape + x = x.reshape(b, n, num_heads, c // num_heads) + return x.transpose(1, 2) # B x N_heads x N_tokens x C_per_head + + def _recombine_heads(self, x: Tensor) -> Tensor: + b, n_heads, n_tokens, c_per_head = x.shape + x = x.transpose(1, 2) + return x.reshape(b, n_tokens, n_heads * c_per_head) # B x N_tokens x C + + def forward(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor: + # Input projections + q = self.q_proj(q) + k = self.k_proj(k) + v = self.v_proj(v) + + # Separate into heads + q = self._separate_heads(q, self.num_heads) + k = self._separate_heads(k, self.num_heads) + v = self._separate_heads(v, self.num_heads) + + dropout_p = self.dropout_p if self.training else 0.0 + # Attention + #try: + # with sdp_kernel_context(dropout_p): + # out = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + #except Exception as e: + if True: + # Fall back to all kernels if the Flash attention kernel fails + #warnings.warn( + # f"Flash Attention kernel failed due to: {e}\nFalling back to all available " + # f"kernels for scaled_dot_product_attention (which may have a slower speed).", + # category=UserWarning, + # stacklevel=2, + #) + global ALLOW_ALL_KERNELS + ALLOW_ALL_KERNELS = True + out = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + + out = self._recombine_heads(out) + out = self.out_proj(out) + + return out + + +class RoPEAttention(Attention): + """Attention with rotary position encoding.""" + + def __init__( + self, + *args, + rope_theta=10000.0, + # whether to repeat q rope to match k length + # this is needed for cross-attention to memories + rope_k_repeat=False, + feat_sizes=(32, 32), # [w, h] for stride 16 feats at 512 resolution + **kwargs, + ): + super().__init__(*args, **kwargs) + + self.compute_cis = partial( + compute_axial_cis, dim=self.internal_dim // self.num_heads, theta=rope_theta + ) + freqs_cis = self.compute_cis(end_x=feat_sizes[0], end_y=feat_sizes[1]) + self.freqs_cis = freqs_cis + self.rope_k_repeat = rope_k_repeat + + if USE_MAT_ROTARY_ENC: + rotmats = get_rotation_matrices(dim=self.internal_dim // self.num_heads, end_x=feat_sizes[0], end_y=feat_sizes[1], theta=rope_theta) + self.rotmats = rotmats + self.rope_theta = rope_theta + + def forward( + self, q: Tensor, k: Tensor, v: Tensor, num_k_exclude_rope: int = 0 + ) -> Tensor: + # Input projections + q = self.q_proj(q) + k = self.k_proj(k) + v = self.v_proj(v) + + # Separate into heads + q = self._separate_heads(q, self.num_heads) + k = self._separate_heads(k, self.num_heads) + v = self._separate_heads(v, self.num_heads) + + # Apply rotary position encoding + w = h = math.sqrt(q.shape[-2]) + + self.freqs_cis = self.freqs_cis.to(q.device) + if self.freqs_cis.shape[0] != q.shape[-2]: + self.freqs_cis = self.compute_cis(end_x=w, end_y=h).to(q.device) + + if USE_MAT_ROTARY_ENC: + self.rotmats = self.rotmats.to(q.device) + if self.rotmats.shape[0] != q.shape[-2]: + self.rotmats = get_rotation_matrices(dim=self.internal_dim // self.num_heads, end_x=w, end_y=h, theta=self.rope_theta) + + if q.shape[-2] != k.shape[-2]: + assert self.rope_k_repeat + + num_k_rope = k.size(-2) - num_k_exclude_rope + if USE_MAT_ROTARY_ENC: + q, k[:, :, :num_k_rope] = apply_rotary_matenc( + q, + k[:, :, :num_k_rope], + rotmats=self.rotmats, + repeat_freqs_k=self.rope_k_repeat, + ) + else: + q, k[:, :, :num_k_rope] = apply_rotary_enc( + q, + k[:, :, :num_k_rope], + freqs_cis=self.freqs_cis, + repeat_freqs_k=self.rope_k_repeat, + ) + + dropout_p = self.dropout_p if self.training else 0.0 + # Attention + #try: + # with sdp_kernel_context(dropout_p): + # out = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + #except Exception as e: + if True: + # Fall back to all kernels if the Flash attention kernel fails + #warnings.warn( + # f"Flash Attention kernel failed due to: {e}\nFalling back to all available " + # f"kernels for scaled_dot_product_attention (which may have a slower speed).", + # category=UserWarning, + # stacklevel=2, + #) + global ALLOW_ALL_KERNELS + ALLOW_ALL_KERNELS = True + out = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + + out = self._recombine_heads(out) + out = self.out_proj(out) + + return out diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_base.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_base.py new file mode 100644 index 00000000..5f3c7ead --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_base.py @@ -0,0 +1,951 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import torch +import torch.distributed +import torch.nn.functional as F + +from torch.nn.init import trunc_normal_ + +from sam2.modeling.sam.mask_decoder import MaskDecoder +from sam2.modeling.sam.prompt_encoder import PromptEncoder +from sam2.modeling.sam.transformer import TwoWayTransformer +from sam2.modeling.sam2_utils import get_1d_sine_pe, MLP, select_closest_cond_frames + +# a large negative value as a placeholder score for missing objects +NO_OBJ_SCORE = -1024.0 + + +class SAM2Base(torch.nn.Module): + def __init__( + self, + image_encoder, + memory_attention, + memory_encoder, + num_maskmem=7, # default 1 input frame + 6 previous frames + image_size=512, + backbone_stride=16, # stride of the image backbone output + sigmoid_scale_for_mem_enc=1.0, # scale factor for mask sigmoid prob + sigmoid_bias_for_mem_enc=0.0, # bias factor for mask sigmoid prob + # During evaluation, whether to binarize the sigmoid mask logits on interacted frames with clicks + binarize_mask_from_pts_for_mem_enc=False, + use_mask_input_as_output_without_sam=False, # on frames with mask input, whether to directly output the input mask without using a SAM prompt encoder + mask decoder + # The maximum number of conditioning frames to participate in the memory attention (-1 means no limit; if there are more conditioning frames than this limit, + # we only cross-attend to the temporally closest `max_cond_frames_in_attn` conditioning frames in the encoder when tracking each frame). This gives the model + # a temporal locality when handling a large number of annotated frames (since closer frames should be more important) and also avoids GPU OOM. + max_cond_frames_in_attn=-1, + # on the first frame, whether to directly add the no-memory embedding to the image feature + # (instead of using the transformer encoder) + directly_add_no_mem_embed=False, + # whether to use high-resolution feature maps in the SAM mask decoder + use_high_res_features_in_sam=False, + # whether to output multiple (3) masks for the first click on initial conditioning frames + multimask_output_in_sam=False, + # the minimum and maximum number of clicks to use multimask_output_in_sam (only relevant when `multimask_output_in_sam=True`; + # default is 1 for both, meaning that only the first click gives multimask output; also note that a box counts as two points) + multimask_min_pt_num=1, + multimask_max_pt_num=1, + # whether to also use multimask output for tracking (not just for the first click on initial conditioning frames; only relevant when `multimask_output_in_sam=True`) + multimask_output_for_tracking=False, + # Whether to use multimask tokens for obj ptr; Only relevant when both + # use_obj_ptrs_in_encoder=True and multimask_output_for_tracking=True + use_multimask_token_for_obj_ptr: bool = False, + # whether to use sigmoid to restrict ious prediction to [0-1] + iou_prediction_use_sigmoid=False, + # The memory bank's temporal stride during evaluation (i.e. the `r` parameter in XMem and Cutie; XMem and Cutie use r=5). + # For r>1, the (self.num_maskmem - 1) non-conditioning memory frames consist of + # (self.num_maskmem - 2) nearest frames from every r-th frames, plus the last frame. + memory_temporal_stride_for_eval=1, + # whether to apply non-overlapping constraints on the object masks in the memory encoder during evaluation (to avoid/alleviate superposing masks) + non_overlap_masks_for_mem_enc=False, + # whether to cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder=False, + # the maximum number of object pointers from other frames in encoder cross attention (only relevant when `use_obj_ptrs_in_encoder=True`) + max_obj_ptrs_in_encoder=16, + # whether to add temporal positional encoding to the object pointers in the encoder (only relevant when `use_obj_ptrs_in_encoder=True`) + add_tpos_enc_to_obj_ptrs=True, + # whether to add an extra linear projection layer for the temporal positional encoding in the object pointers to avoid potential interference + # with spatial positional encoding (only relevant when both `use_obj_ptrs_in_encoder=True` and `add_tpos_enc_to_obj_ptrs=True`) + proj_tpos_enc_in_obj_ptrs=False, + # whether to use signed distance (instead of unsigned absolute distance) in the temporal positional encoding in the object pointers + # (only relevant when both `use_obj_ptrs_in_encoder=True` and `add_tpos_enc_to_obj_ptrs=True`) + use_signed_tpos_enc_to_obj_ptrs=False, + # whether to only attend to object pointers in the past (before the current frame) in the encoder during evaluation + # (only relevant when `use_obj_ptrs_in_encoder=True`; this might avoid pointer information too far in the future to distract the initial tracking) + only_obj_ptrs_in_the_past_for_eval=False, + # Whether to predict if there is an object in the frame + pred_obj_scores: bool = False, + # Whether to use an MLP to predict object scores + pred_obj_scores_mlp: bool = False, + # Only relevant if pred_obj_scores=True and use_obj_ptrs_in_encoder=True; + # Whether to have a fixed no obj pointer when there is no object present + # or to use it as an additive embedding with obj_ptr produced by decoder + fixed_no_obj_ptr: bool = False, + # Soft no object, i.e. mix in no_obj_ptr softly, + # hope to make recovery easier if there is a mistake and mitigate accumulation of errors + soft_no_obj_ptr: bool = False, + use_mlp_for_obj_ptr_proj: bool = False, + # add no obj embedding to spatial frames + no_obj_embed_spatial: bool = False, + # extra arguments used to construct the SAM mask decoder; if not None, it should be a dict of kwargs to be passed into `MaskDecoder` class. + sam_mask_decoder_extra_args=None, + compile_image_encoder: bool = False, + ): + super().__init__() + + # Part 1: the image backbone + self.image_encoder = image_encoder + # Use level 0, 1, 2 for high-res setting, or just level 2 for the default setting + self.use_high_res_features_in_sam = use_high_res_features_in_sam + self.num_feature_levels = 3 if use_high_res_features_in_sam else 1 + self.use_obj_ptrs_in_encoder = use_obj_ptrs_in_encoder + self.max_obj_ptrs_in_encoder = max_obj_ptrs_in_encoder + if use_obj_ptrs_in_encoder: + # A conv layer to downsample the mask prompt to stride 4 (the same stride as + # low-res SAM mask logits) and to change its scales from 0~1 to SAM logit scale, + # so that it can be fed into the SAM mask decoder to generate a pointer. + self.mask_downsample = torch.nn.Conv2d(1, 1, kernel_size=4, stride=4) + self.add_tpos_enc_to_obj_ptrs = add_tpos_enc_to_obj_ptrs + if proj_tpos_enc_in_obj_ptrs: + assert add_tpos_enc_to_obj_ptrs # these options need to be used together + self.proj_tpos_enc_in_obj_ptrs = proj_tpos_enc_in_obj_ptrs + self.use_signed_tpos_enc_to_obj_ptrs = use_signed_tpos_enc_to_obj_ptrs + self.only_obj_ptrs_in_the_past_for_eval = only_obj_ptrs_in_the_past_for_eval + + # Part 2: memory attention to condition current frame's visual features + # with memories (and obj ptrs) from past frames + self.memory_attention = memory_attention + self.hidden_dim = image_encoder.neck.d_model + + # Part 3: memory encoder for the previous frame's outputs + self.memory_encoder = memory_encoder + self.mem_dim = self.hidden_dim + if hasattr(self.memory_encoder, "out_proj") and hasattr( + self.memory_encoder.out_proj, "weight" + ): + # if there is compression of memories along channel dim + self.mem_dim = self.memory_encoder.out_proj.weight.shape[0] + self.num_maskmem = num_maskmem # Number of memories accessible + # Temporal encoding of the memories + self.maskmem_tpos_enc = torch.nn.Parameter( + torch.zeros(num_maskmem, 1, 1, self.mem_dim) + ) + trunc_normal_(self.maskmem_tpos_enc, std=0.02) + # a single token to indicate no memory embedding from previous frames + self.no_mem_embed = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim)) + self.no_mem_pos_enc = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim)) + trunc_normal_(self.no_mem_embed, std=0.02) + trunc_normal_(self.no_mem_pos_enc, std=0.02) + self.directly_add_no_mem_embed = directly_add_no_mem_embed + # Apply sigmoid to the output raw mask logits (to turn them from + # range (-inf, +inf) to range (0, 1)) before feeding them into the memory encoder + self.sigmoid_scale_for_mem_enc = sigmoid_scale_for_mem_enc + self.sigmoid_bias_for_mem_enc = sigmoid_bias_for_mem_enc + self.binarize_mask_from_pts_for_mem_enc = binarize_mask_from_pts_for_mem_enc + self.non_overlap_masks_for_mem_enc = non_overlap_masks_for_mem_enc + self.memory_temporal_stride_for_eval = memory_temporal_stride_for_eval + # On frames with mask input, whether to directly output the input mask without + # using a SAM prompt encoder + mask decoder + self.use_mask_input_as_output_without_sam = use_mask_input_as_output_without_sam + self.multimask_output_in_sam = multimask_output_in_sam + self.multimask_min_pt_num = multimask_min_pt_num + self.multimask_max_pt_num = multimask_max_pt_num + self.multimask_output_for_tracking = multimask_output_for_tracking + self.use_multimask_token_for_obj_ptr = use_multimask_token_for_obj_ptr + self.iou_prediction_use_sigmoid = iou_prediction_use_sigmoid + + # Part 4: SAM-style prompt encoder (for both mask and point inputs) + # and SAM-style mask decoder for the final mask output + self.image_size = image_size + self.backbone_stride = backbone_stride + self.sam_mask_decoder_extra_args = sam_mask_decoder_extra_args + self.pred_obj_scores = pred_obj_scores + self.pred_obj_scores_mlp = pred_obj_scores_mlp + self.fixed_no_obj_ptr = fixed_no_obj_ptr + self.soft_no_obj_ptr = soft_no_obj_ptr + if self.fixed_no_obj_ptr: + assert self.pred_obj_scores + assert self.use_obj_ptrs_in_encoder + if self.pred_obj_scores and self.use_obj_ptrs_in_encoder: + self.no_obj_ptr = torch.nn.Parameter(torch.zeros(1, self.hidden_dim)) + trunc_normal_(self.no_obj_ptr, std=0.02) + self.use_mlp_for_obj_ptr_proj = use_mlp_for_obj_ptr_proj + self.no_obj_embed_spatial = None + if no_obj_embed_spatial: + self.no_obj_embed_spatial = torch.nn.Parameter(torch.zeros(1, self.mem_dim)) + trunc_normal_(self.no_obj_embed_spatial, std=0.02) + + self._build_sam_heads() + self.max_cond_frames_in_attn = max_cond_frames_in_attn + + # Model compilation + if compile_image_encoder: + # Compile the forward function (not the full module) to allow loading checkpoints. + print( + "Image encoder compilation is enabled. First forward pass will be slow." + ) + self.image_encoder.forward = torch.compile( + self.image_encoder.forward, + mode="max-autotune", + fullgraph=True, + dynamic=False, + ) + + assert(self.image_size == 1024) + assert(self.num_feature_levels == 3) + assert(self.hidden_dim == 256) + assert(self.num_maskmem == 7) + assert(self.directly_add_no_mem_embed == True) + # assert(self.training == False) + assert(self.mem_dim == 64) + # assert(self.add_tpos_enc_to_obj_ptrs == False) + assert(self.use_obj_ptrs_in_encoder == True) + assert(self.multimask_output_in_sam == True) + assert(self.multimask_min_pt_num == 0) + assert(self.multimask_max_pt_num == 1) + assert(self.sam_prompt_embed_dim == self.hidden_dim) + assert(self.backbone_stride == 16) + assert(self.sam_image_embedding_size == self.image_size // self.backbone_stride) + assert(self.pred_obj_scores == True) + assert(self.use_obj_ptrs_in_encoder == True) + assert(self.use_mlp_for_obj_ptr_proj == True) + # assert(self.proj_tpos_enc_in_obj_ptrs == False) + assert(self.soft_no_obj_ptr == False) + assert(self.fixed_no_obj_ptr == True) + assert(self.non_overlap_masks_for_mem_enc == False) + assert(self.binarize_mask_from_pts_for_mem_enc == False or self.binarize_mask_from_pts_for_mem_enc == True) # True for video + assert(self.sigmoid_scale_for_mem_enc == 20) + assert(self.sigmoid_bias_for_mem_enc == -10.0) + assert(self.sam_mask_decoder.dynamic_multimask_via_stability == True) + assert(self.sam_mask_decoder.dynamic_multimask_stability_delta == 0.05) + assert(self.sam_mask_decoder.dynamic_multimask_stability_thresh == 0.98) + assert(self.max_cond_frames_in_attn == -1) + assert(self.memory_temporal_stride_for_eval == 1) + assert(self.max_obj_ptrs_in_encoder == 16) + assert(self.only_obj_ptrs_in_the_past_for_eval == True) + assert(self.multimask_output_for_tracking == True) + assert(self.use_multimask_token_for_obj_ptr == True) + + @property + def device(self): + return next(self.parameters()).device + + def forward(self, *args, **kwargs): + raise NotImplementedError( + "Please use the corresponding methods in SAM2VideoPredictor for inference or SAM2Train for training/fine-tuning" + "See notebooks/video_predictor_example.ipynb for an inference example." + ) + + def _build_sam_heads(self): + """Build SAM-style prompt encoder and mask decoder.""" + self.sam_prompt_embed_dim = self.hidden_dim + self.sam_image_embedding_size = self.image_size // self.backbone_stride + + # build PromptEncoder and MaskDecoder from SAM + # (their hyperparameters like `mask_in_chans=16` are from SAM code) + self.sam_prompt_encoder = PromptEncoder( + embed_dim=self.sam_prompt_embed_dim, + image_embedding_size=( + self.sam_image_embedding_size, + self.sam_image_embedding_size, + ), + input_image_size=(self.image_size, self.image_size), + mask_in_chans=16, + ) + self.sam_mask_decoder = MaskDecoder( + num_multimask_outputs=3, + transformer=TwoWayTransformer( + depth=2, + embedding_dim=self.sam_prompt_embed_dim, + mlp_dim=2048, + num_heads=8, + ), + transformer_dim=self.sam_prompt_embed_dim, + iou_head_depth=3, + iou_head_hidden_dim=256, + use_high_res_features=self.use_high_res_features_in_sam, + iou_prediction_use_sigmoid=self.iou_prediction_use_sigmoid, + pred_obj_scores=self.pred_obj_scores, + pred_obj_scores_mlp=self.pred_obj_scores_mlp, + use_multimask_token_for_obj_ptr=self.use_multimask_token_for_obj_ptr, + **(self.sam_mask_decoder_extra_args or {}), + ) + if self.use_obj_ptrs_in_encoder: + # a linear projection on SAM output tokens to turn them into object pointers + self.obj_ptr_proj = torch.nn.Linear(self.hidden_dim, self.hidden_dim) + if self.use_mlp_for_obj_ptr_proj: + self.obj_ptr_proj = MLP( + self.hidden_dim, self.hidden_dim, self.hidden_dim, 3 + ) + else: + self.obj_ptr_proj = torch.nn.Identity() + if self.proj_tpos_enc_in_obj_ptrs: + # a linear projection on temporal positional encoding in object pointers to + # avoid potential interference with spatial positional encoding + self.obj_ptr_tpos_proj = torch.nn.Linear(self.hidden_dim, self.mem_dim) + else: + self.obj_ptr_tpos_proj = torch.nn.Identity() + + def _forward_sam_heads( + self, + backbone_features, + point_inputs=None, + mask_inputs=None, + high_res_features=None, + multimask_output=False, + ): + """ + Forward SAM prompt encoders and mask heads. + + Inputs: + - backbone_features: image features of [B, C, H, W] shape + - point_inputs: a dictionary with "point_coords" and "point_labels", where + 1) "point_coords" has [B, P, 2] shape and float32 dtype and contains the + absolute pixel-unit coordinate in (x, y) format of the P input points + 2) "point_labels" has shape [B, P] and int32 dtype, where 1 means + positive clicks, 0 means negative clicks, and -1 means padding + - mask_inputs: a mask of [B, 1, H*16, W*16] shape, float or bool, with the + same spatial size as the image. + - high_res_features: either 1) None or 2) or a list of length 2 containing + two feature maps of [B, C, 4*H, 4*W] and [B, C, 2*H, 2*W] shapes respectively, + which will be used as high-resolution feature maps for SAM decoder. + - multimask_output: if it's True, we output 3 candidate masks and their 3 + corresponding IoU estimates, and if it's False, we output only 1 mask and + its corresponding IoU estimate. + + Outputs: + - low_res_multimasks: [B, M, H*4, W*4] shape (where M = 3 if + `multimask_output=True` and M = 1 if `multimask_output=False`), the SAM + output mask logits (before sigmoid) for the low-resolution masks, with 4x + the resolution (1/4 stride) of the input backbone_features. + - high_res_multimasks: [B, M, H*16, W*16] shape (where M = 3 + if `multimask_output=True` and M = 1 if `multimask_output=False`), + upsampled from the low-resolution masks, with shape size as the image + (stride is 1 pixel). + - ious, [B, M] shape, where (where M = 3 if `multimask_output=True` and M = 1 + if `multimask_output=False`), the estimated IoU of each output mask. + - low_res_masks: [B, 1, H*4, W*4] shape, the best mask in `low_res_multimasks`. + If `multimask_output=True`, it's the mask with the highest IoU estimate. + If `multimask_output=False`, it's the same as `low_res_multimasks`. + - high_res_masks: [B, 1, H*16, W*16] shape, the best mask in `high_res_multimasks`. + If `multimask_output=True`, it's the mask with the highest IoU estimate. + If `multimask_output=False`, it's the same as `high_res_multimasks`. + - obj_ptr: [B, C] shape, the object pointer vector for the output mask, extracted + based on the output token from the SAM mask decoder. + """ + B = backbone_features.size(0) + device = backbone_features.device + assert backbone_features.size(1) == self.sam_prompt_embed_dim + assert backbone_features.size(2) == self.sam_image_embedding_size + assert backbone_features.size(3) == self.sam_image_embedding_size + + # a) Handle point prompts + if point_inputs is not None: + sam_point_coords = point_inputs["point_coords"] + sam_point_labels = point_inputs["point_labels"] + # assert sam_point_coords.size(0) == B and sam_point_labels.size(0) == B + else: + # If no points are provide, pad with an empty point (with label -1) + sam_point_coords = torch.zeros(B, 1, 2, device=device) + sam_point_labels = -torch.ones(B, 1, dtype=torch.int32, device=device) + + # b) Handle mask prompts + if mask_inputs is not None: + # If mask_inputs is provided, downsize it into low-res mask input if needed + # and feed it as a dense mask prompt into the SAM mask encoder + assert len(mask_inputs.shape) == 4 and mask_inputs.shape[:2] == (B, 1) + if mask_inputs.shape[-2:] != self.sam_prompt_encoder.mask_input_size: + sam_mask_prompt = F.interpolate( + mask_inputs.float(), + size=self.sam_prompt_encoder.mask_input_size, + align_corners=False, + mode="bilinear", + antialias=True, # use antialias for downsampling + ) + else: + sam_mask_prompt = mask_inputs + else: + # Otherwise, simply feed None (and SAM's prompt encoder will add + # a learned `no_mask_embed` to indicate no mask input in this case). + sam_mask_prompt = None + + sparse_embeddings, dense_embeddings = self.sam_prompt_encoder( + points=(sam_point_coords, sam_point_labels), + boxes=None, + masks=sam_mask_prompt, + ) + ( + low_res_multimasks, + ious, + sam_output_tokens, + object_score_logits, + ) = self.sam_mask_decoder( + image_embeddings=backbone_features, + image_pe=self.sam_prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + repeat_image=False, # the image is already batched + high_res_features=high_res_features, + ) + if self.pred_obj_scores: + is_obj_appearing = object_score_logits > 0 + + # Mask used for spatial memories is always a *hard* choice between obj and no obj, + # consistent with the actual mask prediction + low_res_multimasks = torch.where( + is_obj_appearing[:, None, None], + low_res_multimasks, + NO_OBJ_SCORE, + ) + + # convert masks from possibly bfloat16 (or float16) to float32 + # (older PyTorch versions before 2.1 don't support `interpolate` on bf16) + low_res_multimasks = low_res_multimasks.float() + high_res_multimasks = F.interpolate( + low_res_multimasks, + size=(self.image_size, self.image_size), + mode="bilinear", + align_corners=False, + ) + + sam_output_token = sam_output_tokens[:, 0] + if multimask_output: + # take the best mask prediction (with the highest IoU estimation) + best_iou_inds = torch.argmax(ious, dim=-1) + batch_inds = torch.arange(B, device=device) + low_res_masks = low_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1) + high_res_masks = high_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1) + if sam_output_tokens.size(1) > 1: + sam_output_token = sam_output_tokens[batch_inds, best_iou_inds] + else: + low_res_masks, high_res_masks = low_res_multimasks, high_res_multimasks + + # Extract object pointer from the SAM output token (with occlusion handling) + obj_ptr = self.obj_ptr_proj(sam_output_token) + if self.pred_obj_scores: + # Allow *soft* no obj ptr, unlike for masks + if self.soft_no_obj_ptr: + lambda_is_obj_appearing = object_score_logits.sigmoid() + else: + lambda_is_obj_appearing = is_obj_appearing.float() + + if self.fixed_no_obj_ptr: + obj_ptr = lambda_is_obj_appearing * obj_ptr + obj_ptr = obj_ptr + (1 - lambda_is_obj_appearing) * self.no_obj_ptr + + return ( + low_res_multimasks, + high_res_multimasks, + ious, + low_res_masks, + high_res_masks, + obj_ptr, + object_score_logits, + ) + + def _use_mask_as_output(self, backbone_features, high_res_features, mask_inputs): + """ + Directly turn binary `mask_inputs` into a output mask logits without using SAM. + (same input and output shapes as in _forward_sam_heads above). + """ + # Use -10/+10 as logits for neg/pos pixels (very close to 0/1 in prob after sigmoid). + out_scale, out_bias = 20.0, -10.0 # sigmoid(-10.0)=4.5398e-05 + mask_inputs_float = mask_inputs.float() + high_res_masks = mask_inputs_float * out_scale + out_bias + low_res_masks = F.interpolate( + high_res_masks, + size=(high_res_masks.size(-2) // 4, high_res_masks.size(-1) // 4), + align_corners=False, + mode="bilinear", + antialias=True, # use antialias for downsampling + ) + # a dummy IoU prediction of all 1's under mask input + ious = mask_inputs.new_ones(mask_inputs.size(0), 1).float() + if not self.use_obj_ptrs_in_encoder: + # all zeros as a dummy object pointer (of shape [B, C]) + obj_ptr = torch.zeros( + mask_inputs.size(0), self.hidden_dim, device=mask_inputs.device + ) + else: + # produce an object pointer using the SAM decoder from the mask input + _, _, _, _, _, obj_ptr, _ = self._forward_sam_heads( + backbone_features=backbone_features, + mask_inputs=self.mask_downsample(mask_inputs_float), + high_res_features=high_res_features, + ) + # In this method, we are treating mask_input as output, e.g. using it directly to create spatial mem; + # Below, we follow the same design axiom to use mask_input to decide if obj appears or not instead of relying + # on the object_scores from the SAM decoder. + is_obj_appearing = torch.any(mask_inputs.flatten(1).float() > 0.0, dim=1) + is_obj_appearing = is_obj_appearing[..., None] + lambda_is_obj_appearing = is_obj_appearing.float() + object_score_logits = out_scale * lambda_is_obj_appearing + out_bias + if self.pred_obj_scores: + if self.fixed_no_obj_ptr: + obj_ptr = lambda_is_obj_appearing * obj_ptr + obj_ptr = obj_ptr + (1 - lambda_is_obj_appearing) * self.no_obj_ptr + + return ( + low_res_masks, + high_res_masks, + ious, + low_res_masks, + high_res_masks, + obj_ptr, + object_score_logits, + ) + + def forward_image(self, img_batch: torch.Tensor): + """Get the image feature on the input batch.""" + backbone_out = self.image_encoder(img_batch) + if self.use_high_res_features_in_sam: + # precompute projected level 0 and level 1 features in SAM decoder + # to avoid running it again on every SAM click + backbone_out["backbone_fpn"][0] = self.sam_mask_decoder.conv_s0( + backbone_out["backbone_fpn"][0] + ) + backbone_out["backbone_fpn"][1] = self.sam_mask_decoder.conv_s1( + backbone_out["backbone_fpn"][1] + ) + return backbone_out + + def _prepare_backbone_features(self, backbone_out): + """Prepare and flatten visual features.""" + backbone_out = backbone_out.copy() + assert len(backbone_out["backbone_fpn"]) == len(backbone_out["vision_pos_enc"]) + assert len(backbone_out["backbone_fpn"]) >= self.num_feature_levels + + feature_maps = backbone_out["backbone_fpn"][-self.num_feature_levels :] + vision_pos_embeds = backbone_out["vision_pos_enc"][-self.num_feature_levels :] + + feat_sizes = [(x.shape[-2], x.shape[-1]) for x in vision_pos_embeds] + # flatten NxCxHxW to HWxNxC + vision_feats = [x.flatten(2).permute(2, 0, 1) for x in feature_maps] + vision_pos_embeds = [x.flatten(2).permute(2, 0, 1) for x in vision_pos_embeds] + + return backbone_out, vision_feats, vision_pos_embeds, feat_sizes + + def _prepare_memory_conditioned_features( + self, + frame_idx, + is_init_cond_frame, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + output_dict, + num_frames, + track_in_reverse=False, # tracking in reverse time order (for demo usage) + ): + """Fuse the current frame's visual feature map with previous memory.""" + B = current_vision_feats[-1].size(1) # batch size on this frame + C = self.hidden_dim + H, W = feat_sizes[-1] # top-level (lowest-resolution) feature size + device = current_vision_feats[-1].device + # The case of `self.num_maskmem == 0` below is primarily used for reproducing SAM on images. + # In this case, we skip the fusion with any memory. + if self.num_maskmem == 0: # Disable memory and skip fusion + pix_feat = current_vision_feats[-1].permute(1, 2, 0).view(B, C, H, W) + return pix_feat + + num_obj_ptr_tokens = 0 + tpos_sign_mul = -1 if track_in_reverse else 1 + # Step 1: condition the visual features of the current frame on previous memories + if not is_init_cond_frame: + # Retrieve the memories encoded with the maskmem backbone + to_cat_memory, to_cat_memory_pos_embed = [], [] + # Add conditioning frames's output first (all cond frames have t_pos=0 for + # when getting temporal positional embedding below) + assert len(output_dict["cond_frame_outputs"]) > 0 + # Select a maximum number of temporally closest cond frames for cross attention + cond_outputs = output_dict["cond_frame_outputs"] + selected_cond_outputs, unselected_cond_outputs = select_closest_cond_frames( + frame_idx, cond_outputs, self.max_cond_frames_in_attn + ) + t_pos_and_prevs = [(0, out) for out in selected_cond_outputs.values()] + # Add last (self.num_maskmem - 1) frames before current frame for non-conditioning memory + # the earliest one has t_pos=1 and the latest one has t_pos=self.num_maskmem-1 + # We also allow taking the memory frame non-consecutively (with stride>1), in which case + # we take (self.num_maskmem - 2) frames among every stride-th frames plus the last frame. + stride = 1 if self.training else self.memory_temporal_stride_for_eval + for t_pos in range(1, self.num_maskmem): + t_rel = self.num_maskmem - t_pos # how many frames before current frame + if t_rel == 1: + # for t_rel == 1, we take the last frame (regardless of r) + if not track_in_reverse: + # the frame immediately before this frame (i.e. frame_idx - 1) + prev_frame_idx = frame_idx - t_rel + else: + # the frame immediately after this frame (i.e. frame_idx + 1) + prev_frame_idx = frame_idx + t_rel + else: + # for t_rel >= 2, we take the memory frame from every r-th frames + if not track_in_reverse: + # first find the nearest frame among every r-th frames before this frame + # for r=1, this would be (frame_idx - 2) + prev_frame_idx = ((frame_idx - 2) // stride) * stride + # then seek further among every r-th frames + prev_frame_idx = prev_frame_idx - (t_rel - 2) * stride + else: + # first find the nearest frame among every r-th frames after this frame + # for r=1, this would be (frame_idx + 2) + prev_frame_idx = -(-(frame_idx + 2) // stride) * stride + # then seek further among every r-th frames + prev_frame_idx = prev_frame_idx + (t_rel - 2) * stride + out = output_dict["non_cond_frame_outputs"].get(prev_frame_idx, None) + if out is None: + # If an unselected conditioning frame is among the last (self.num_maskmem - 1) + # frames, we still attend to it as if it's a non-conditioning frame. + out = unselected_cond_outputs.get(prev_frame_idx, None) + t_pos_and_prevs.append((t_pos, out)) + + for t_pos, prev in t_pos_and_prevs: + if prev is None: + continue # skip padding frames + # "maskmem_features" might have been offloaded to CPU in demo use cases, + # so we load it back to GPU (it's a no-op if it's already on GPU). + feats = prev["maskmem_features"].to(device, non_blocking=True) + to_cat_memory.append(feats.flatten(2).permute(2, 0, 1)) + # Spatial positional encoding (it might have been offloaded to CPU in eval) + maskmem_enc = prev["maskmem_pos_enc"][-1].to(device) + maskmem_enc = maskmem_enc.flatten(2).permute(2, 0, 1) + # Temporal positional encoding + maskmem_enc = ( + maskmem_enc + self.maskmem_tpos_enc[self.num_maskmem - t_pos - 1] + ) + to_cat_memory_pos_embed.append(maskmem_enc) + + # Construct the list of past object pointers + if self.use_obj_ptrs_in_encoder: + max_obj_ptrs_in_encoder = min(num_frames, self.max_obj_ptrs_in_encoder) + # First add those object pointers from selected conditioning frames + # (optionally, only include object pointers in the past during evaluation) + if not self.training and self.only_obj_ptrs_in_the_past_for_eval: + ptr_cond_outputs = { + t: out + for t, out in selected_cond_outputs.items() + if (t >= frame_idx if track_in_reverse else t <= frame_idx) + } + else: + ptr_cond_outputs = selected_cond_outputs + pos_and_ptrs = [ + # Temporal pos encoding contains how far away each pointer is from current frame + ( + ( + (frame_idx - t) * tpos_sign_mul + if self.use_signed_tpos_enc_to_obj_ptrs + else abs(frame_idx - t) + ), + out["obj_ptr"], + ) + for t, out in ptr_cond_outputs.items() + ] + # Add up to (max_obj_ptrs_in_encoder - 1) non-conditioning frames before current frame + for t_diff in range(1, max_obj_ptrs_in_encoder): + t = frame_idx + t_diff if track_in_reverse else frame_idx - t_diff + if t < 0 or (num_frames is not None and t >= num_frames): + break + out = output_dict["non_cond_frame_outputs"].get( + t, unselected_cond_outputs.get(t, None) + ) + if out is not None: + pos_and_ptrs.append((t_diff, out["obj_ptr"])) + # If we have at least one object pointer, add them to the across attention + if len(pos_and_ptrs) > 0: + pos_list, ptrs_list = zip(*pos_and_ptrs) + # stack object pointers along dim=0 into [ptr_seq_len, B, C] shape + obj_ptrs = torch.stack(ptrs_list, dim=0) + # a temporal positional embedding based on how far each object pointer is from + # the current frame (sine embedding normalized by the max pointer num). + if self.add_tpos_enc_to_obj_ptrs: + t_diff_max = max_obj_ptrs_in_encoder - 1 + tpos_dim = C if self.proj_tpos_enc_in_obj_ptrs else self.mem_dim + obj_pos = torch.tensor(pos_list).to( + device=device, non_blocking=True + ) + obj_pos = get_1d_sine_pe(obj_pos / t_diff_max, dim=tpos_dim) + obj_pos = self.obj_ptr_tpos_proj(obj_pos) + obj_pos = obj_pos.unsqueeze(1).expand(-1, B, self.mem_dim) + else: + obj_pos = obj_ptrs.new_zeros(len(pos_list), B, self.mem_dim) + if self.mem_dim < C: + # split a pointer into (C // self.mem_dim) tokens for self.mem_dim < C + obj_ptrs = obj_ptrs.reshape( + -1, B, C // self.mem_dim, self.mem_dim + ) + obj_ptrs = obj_ptrs.permute(0, 2, 1, 3).flatten(0, 1) + obj_pos = obj_pos.repeat_interleave(C // self.mem_dim, dim=0) + to_cat_memory.append(obj_ptrs) + to_cat_memory_pos_embed.append(obj_pos) + num_obj_ptr_tokens = obj_ptrs.shape[0] + else: + num_obj_ptr_tokens = 0 + else: + # for initial conditioning frames, encode them without using any previous memory + if self.directly_add_no_mem_embed: + # directly add no-mem embedding (instead of using the transformer encoder) + pix_feat_with_mem = current_vision_feats[-1] + self.no_mem_embed + pix_feat_with_mem = pix_feat_with_mem.permute(1, 2, 0).view(B, C, H, W) + return pix_feat_with_mem + + # Use a dummy token on the first frame (to avoid empty memory input to tranformer encoder) + to_cat_memory = [self.no_mem_embed.expand(1, B, self.mem_dim)] + to_cat_memory_pos_embed = [self.no_mem_pos_enc.expand(1, B, self.mem_dim)] + + # Step 2: Concatenate the memories and forward through the transformer encoder + memory = torch.cat(to_cat_memory, dim=0) + memory_pos_embed = torch.cat(to_cat_memory_pos_embed, dim=0) + + pix_feat_with_mem = self.memory_attention( + curr=current_vision_feats, + curr_pos=current_vision_pos_embeds, + memory=memory, + memory_pos=memory_pos_embed, + num_obj_ptr_tokens=num_obj_ptr_tokens, + ) + # reshape the output (HW)BC => BCHW + pix_feat_with_mem = pix_feat_with_mem.permute(1, 2, 0).view(B, C, H, W) + return pix_feat_with_mem + + def _encode_new_memory( + self, + current_vision_feats, + feat_sizes, + pred_masks_high_res, + object_score_logits, + is_mask_from_pts, + ): + """Encode the current image and its prediction into a memory feature.""" + # B = current_vision_feats[-1].size(1) # batch size on this frame + # C = self.hidden_dim + # H, W = feat_sizes[-1] # top-level (lowest-resolution) feature size + # # top-level feature, (HW)BC => BCHW + # pix_feat = current_vision_feats[-1].permute(1, 2, 0).view(B, C, H, W) + + B = current_vision_feats.size(1) + C = self.hidden_dim + H, W = feat_sizes[-1] + pix_feat = current_vision_feats + + + if self.non_overlap_masks_for_mem_enc and not self.training: + # optionally, apply non-overlapping constraints to the masks (it's applied + # in the batch dimension and should only be used during eval, where all + # the objects come from the same video under batch size 1). + pred_masks_high_res = self._apply_non_overlapping_constraints( + pred_masks_high_res + ) + # scale the raw mask logits with a temperature before applying sigmoid + binarize = self.binarize_mask_from_pts_for_mem_enc and is_mask_from_pts + if binarize and not self.training: + mask_for_mem = (pred_masks_high_res > 0).float() + else: + # apply sigmoid on the raw mask logits to turn them into range (0, 1) + mask_for_mem = torch.sigmoid(pred_masks_high_res) + # apply scale and bias terms to the sigmoid probabilities + if self.sigmoid_scale_for_mem_enc != 1.0: + mask_for_mem = mask_for_mem * self.sigmoid_scale_for_mem_enc + if self.sigmoid_bias_for_mem_enc != 0.0: + mask_for_mem = mask_for_mem + self.sigmoid_bias_for_mem_enc + maskmem_out = self.memory_encoder( + pix_feat, mask_for_mem, skip_mask_sigmoid=True # sigmoid already applied + ) + maskmem_features = maskmem_out["vision_features"] + maskmem_pos_enc = maskmem_out["vision_pos_enc"] + # add a no-object embedding to the spatial memory to indicate that the frame + # is predicted to be occluded (i.e. no object is appearing in the frame) + if self.no_obj_embed_spatial is not None: + is_obj_appearing = (object_score_logits > 0).float() + maskmem_features += ( + 1 - is_obj_appearing[..., None, None] + ) * self.no_obj_embed_spatial[..., None, None].expand( + *maskmem_features.shape + ) + + return maskmem_features, maskmem_pos_enc + + def _track_step( + self, + frame_idx, + is_init_cond_frame, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + point_inputs, + mask_inputs, + output_dict, + num_frames, + track_in_reverse, + prev_sam_mask_logits, + ): + current_out = {"point_inputs": point_inputs, "mask_inputs": mask_inputs} + # High-resolution feature maps for the SAM head, reshape (HW)BC => BCHW + if len(current_vision_feats) > 1: + high_res_features = [ + x.permute(1, 2, 0).view(x.size(1), x.size(2), *s) + for x, s in zip(current_vision_feats[:-1], feat_sizes[:-1]) + ] + else: + high_res_features = None + if mask_inputs is not None and self.use_mask_input_as_output_without_sam: + # When use_mask_input_as_output_without_sam=True, we directly output the mask input + # (see it as a GT mask) without using a SAM prompt encoder + mask decoder. + pix_feat = current_vision_feats[-1].permute(1, 2, 0) + pix_feat = pix_feat.view(-1, self.hidden_dim, *feat_sizes[-1]) + sam_outputs = self._use_mask_as_output( + pix_feat, high_res_features, mask_inputs + ) + else: + # fused the visual feature with previous memory features in the memory bank + pix_feat = self._prepare_memory_conditioned_features( + frame_idx=frame_idx, + is_init_cond_frame=is_init_cond_frame, + current_vision_feats=current_vision_feats[-1:], + current_vision_pos_embeds=current_vision_pos_embeds[-1:], + feat_sizes=feat_sizes[-1:], + output_dict=output_dict, + num_frames=num_frames, + track_in_reverse=track_in_reverse, + ) + # apply SAM-style segmentation head + # here we might feed previously predicted low-res SAM mask logits into the SAM mask decoder, + # e.g. in demo where such logits come from earlier interaction instead of correction sampling + # (in this case, any `mask_inputs` shouldn't reach here as they are sent to _use_mask_as_output instead) + if prev_sam_mask_logits is not None: + assert point_inputs is not None and mask_inputs is None + mask_inputs = prev_sam_mask_logits + multimask_output = self._use_multimask(is_init_cond_frame, point_inputs) + sam_outputs = self._forward_sam_heads( + backbone_features=pix_feat, + point_inputs=point_inputs, + mask_inputs=mask_inputs, + high_res_features=high_res_features, + multimask_output=multimask_output, + ) + + return current_out, sam_outputs, high_res_features, pix_feat + + def _encode_memory_in_output( + self, + current_vision_feats, + feat_sizes, + point_inputs, + run_mem_encoder, + high_res_masks, + object_score_logits, + current_out, + ): + if run_mem_encoder and self.num_maskmem > 0: + high_res_masks_for_mem_enc = high_res_masks + maskmem_features, maskmem_pos_enc = self._encode_new_memory( + current_vision_feats=current_vision_feats, + feat_sizes=feat_sizes, + pred_masks_high_res=high_res_masks_for_mem_enc, + object_score_logits=object_score_logits, + is_mask_from_pts=(point_inputs is not None), + ) + current_out["maskmem_features"] = maskmem_features + current_out["maskmem_pos_enc"] = maskmem_pos_enc + else: + current_out["maskmem_features"] = None + current_out["maskmem_pos_enc"] = None + + def track_step( + self, + frame_idx, + is_init_cond_frame, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + point_inputs, + mask_inputs, + output_dict, + num_frames, + track_in_reverse=False, # tracking in reverse time order (for demo usage) + # Whether to run the memory encoder on the predicted masks. Sometimes we might want + # to skip the memory encoder with `run_mem_encoder=False`. For example, + # in demo we might call `track_step` multiple times for each user click, + # and only encode the memory when the user finalizes their clicks. And in ablation + # settings like SAM training on static images, we don't need the memory encoder. + run_mem_encoder=True, + # The previously predicted SAM mask logits (which can be fed together with new clicks in demo). + prev_sam_mask_logits=None, + ): + current_out, sam_outputs, _, _ = self._track_step( + frame_idx, + is_init_cond_frame, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + point_inputs, + mask_inputs, + output_dict, + num_frames, + track_in_reverse, + prev_sam_mask_logits, + ) + + ( + _, + _, + _, + low_res_masks, + high_res_masks, + obj_ptr, + object_score_logits, + ) = sam_outputs + + current_out["pred_masks"] = low_res_masks + current_out["pred_masks_high_res"] = high_res_masks + current_out["obj_ptr"] = obj_ptr + if not self.training: + # Only add this in inference (to avoid unused param in activation checkpointing; + # it's mainly used in the demo to encode spatial memories w/ consolidated masks) + current_out["object_score_logits"] = object_score_logits + + # Finally run the memory encoder on the predicted mask to encode + # it into a new memory feature (that can be used in future frames) + self._encode_memory_in_output( + current_vision_feats, + feat_sizes, + point_inputs, + run_mem_encoder, + high_res_masks, + object_score_logits, + current_out, + ) + + return current_out + + def _use_multimask(self, is_init_cond_frame, point_inputs): + """Whether to use multimask output in the SAM head.""" + num_pts = 0 if point_inputs is None else point_inputs["point_labels"].size(1) + multimask_output = ( + self.multimask_output_in_sam + and (is_init_cond_frame or self.multimask_output_for_tracking) + and (self.multimask_min_pt_num <= num_pts <= self.multimask_max_pt_num) + ) + return multimask_output + + def _apply_non_overlapping_constraints(self, pred_masks): + """ + Apply non-overlapping constraints to the object scores in pred_masks. Here we + keep only the highest scoring object at each spatial location in pred_masks. + """ + batch_size = pred_masks.size(0) + if batch_size == 1: + return pred_masks + + device = pred_masks.device + # "max_obj_inds": object index of the object with the highest score at each location + max_obj_inds = torch.argmax(pred_masks, dim=0, keepdim=True) + # "batch_obj_inds": object index of each object slice (along dim 0) in `pred_masks` + batch_obj_inds = torch.arange(batch_size, device=device)[:, None, None, None] + keep = max_obj_inds == batch_obj_inds + # suppress overlapping regions' scores below -10.0 so that the foreground regions + # don't overlap (here sigmoid(-10.0)=4.5398e-05) + pred_masks = torch.where(keep, pred_masks, torch.clamp(pred_masks, max=-10.0)) + return pred_masks diff --git a/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_utils.py b/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_utils.py new file mode 100644 index 00000000..e16caae3 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/modeling/sam2_utils.py @@ -0,0 +1,323 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +import copy +from typing import Tuple + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sam2.utils.misc import mask_to_box + + +def select_closest_cond_frames(frame_idx, cond_frame_outputs, max_cond_frame_num): + """ + Select up to `max_cond_frame_num` conditioning frames from `cond_frame_outputs` + that are temporally closest to the current frame at `frame_idx`. Here, we take + - a) the closest conditioning frame before `frame_idx` (if any); + - b) the closest conditioning frame after `frame_idx` (if any); + - c) any other temporally closest conditioning frames until reaching a total + of `max_cond_frame_num` conditioning frames. + + Outputs: + - selected_outputs: selected items (keys & values) from `cond_frame_outputs`. + - unselected_outputs: items (keys & values) not selected in `cond_frame_outputs`. + """ + if max_cond_frame_num == -1 or len(cond_frame_outputs) <= max_cond_frame_num: + selected_outputs = cond_frame_outputs + unselected_outputs = {} + else: + assert max_cond_frame_num >= 2, "we should allow using 2+ conditioning frames" + selected_outputs = {} + + # the closest conditioning frame before `frame_idx` (if any) + idx_before = max((t for t in cond_frame_outputs if t < frame_idx), default=None) + if idx_before is not None: + selected_outputs[idx_before] = cond_frame_outputs[idx_before] + + # the closest conditioning frame after `frame_idx` (if any) + idx_after = min((t for t in cond_frame_outputs if t >= frame_idx), default=None) + if idx_after is not None: + selected_outputs[idx_after] = cond_frame_outputs[idx_after] + + # add other temporally closest conditioning frames until reaching a total + # of `max_cond_frame_num` conditioning frames. + num_remain = max_cond_frame_num - len(selected_outputs) + inds_remain = sorted( + (t for t in cond_frame_outputs if t not in selected_outputs), + key=lambda x: abs(x - frame_idx), + )[:num_remain] + selected_outputs.update((t, cond_frame_outputs[t]) for t in inds_remain) + unselected_outputs = { + t: v for t, v in cond_frame_outputs.items() if t not in selected_outputs + } + + return selected_outputs, unselected_outputs + + +def get_1d_sine_pe(pos_inds, dim, temperature=10000): + """ + Get 1D sine positional embedding as in the original Transformer paper. + """ + pe_dim = dim // 2 + dim_t = torch.arange(pe_dim, dtype=torch.float32, device=pos_inds.device) + dim_t = temperature ** (2 * (dim_t // 2) / pe_dim) + + pos_embed = pos_inds.unsqueeze(-1) / dim_t + pos_embed = torch.cat([pos_embed.sin(), pos_embed.cos()], dim=-1) + return pos_embed + + +def get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(f"activation should be relu/gelu, not {activation}.") + + +def get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class DropPath(nn.Module): + # adapted from https://github.com/huggingface/pytorch-image-models/blob/main/timm/layers/drop.py + def __init__(self, drop_prob=0.0, scale_by_keep=True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + if self.drop_prob == 0.0 or not self.training: + return x + keep_prob = 1 - self.drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and self.scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +# Lightly adapted from +# https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/transformer/transformer_predictor.py # noqa +class MLP(nn.Module): + def __init__( + self, + input_dim: int, + hidden_dim: int, + output_dim: int, + num_layers: int, + activation: nn.Module = nn.ReLU, + sigmoid_output: bool = False, + ) -> None: + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList( + nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]) + ) + self.sigmoid_output = sigmoid_output + self.act = activation() + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = self.act(layer(x)) if i < self.num_layers - 1 else layer(x) + if self.sigmoid_output: + x = F.sigmoid(x) + return x + + +# From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa +# Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa +class LayerNorm2d(nn.Module): + def __init__(self, num_channels: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(num_channels)) + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + + +def sample_box_points( + masks: torch.Tensor, + noise: float = 0.1, # SAM default + noise_bound: int = 20, # SAM default + top_left_label: int = 2, + bottom_right_label: int = 3, +) -> Tuple[np.array, np.array]: + """ + Sample a noised version of the top left and bottom right corners of a given `bbox` + + Inputs: + - masks: [B, 1, H,W] boxes, dtype=torch.Tensor + - noise: noise as a fraction of box width and height, dtype=float + - noise_bound: maximum amount of noise (in pure pixesl), dtype=int + + Returns: + - box_coords: [B, num_pt, 2], contains (x, y) coordinates of top left and bottom right box corners, dtype=torch.float + - box_labels: [B, num_pt], label 2 is reserverd for top left and 3 for bottom right corners, dtype=torch.int32 + """ + device = masks.device + box_coords = mask_to_box(masks) + B, _, H, W = masks.shape + box_labels = torch.tensor( + [top_left_label, bottom_right_label], dtype=torch.int, device=device + ).repeat(B) + if noise > 0.0: + if not isinstance(noise_bound, torch.Tensor): + noise_bound = torch.tensor(noise_bound, device=device) + bbox_w = box_coords[..., 2] - box_coords[..., 0] + bbox_h = box_coords[..., 3] - box_coords[..., 1] + max_dx = torch.min(bbox_w * noise, noise_bound) + max_dy = torch.min(bbox_h * noise, noise_bound) + box_noise = 2 * torch.rand(B, 1, 4, device=device) - 1 + box_noise = box_noise * torch.stack((max_dx, max_dy, max_dx, max_dy), dim=-1) + + box_coords = box_coords + box_noise + img_bounds = ( + torch.tensor([W, H, W, H], device=device) - 1 + ) # uncentered pixel coords + box_coords.clamp_(torch.zeros_like(img_bounds), img_bounds) # In place clamping + + box_coords = box_coords.reshape(-1, 2, 2) # always 2 points + box_labels = box_labels.reshape(-1, 2) + return box_coords, box_labels + + +def sample_random_points_from_errors(gt_masks, pred_masks, num_pt=1): + """ + Sample `num_pt` random points (along with their labels) independently from the error regions. + + Inputs: + - gt_masks: [B, 1, H_im, W_im] masks, dtype=torch.bool + - pred_masks: [B, 1, H_im, W_im] masks, dtype=torch.bool or None + - num_pt: int, number of points to sample independently for each of the B error maps + + Outputs: + - points: [B, num_pt, 2], dtype=torch.float, contains (x, y) coordinates of each sampled point + - labels: [B, num_pt], dtype=torch.int32, where 1 means positive clicks and 0 means + negative clicks + """ + if pred_masks is None: # if pred_masks is not provided, treat it as empty + pred_masks = torch.zeros_like(gt_masks) + assert gt_masks.dtype == torch.bool and gt_masks.size(1) == 1 + assert pred_masks.dtype == torch.bool and pred_masks.shape == gt_masks.shape + assert num_pt >= 0 + + B, _, H_im, W_im = gt_masks.shape + device = gt_masks.device + + # false positive region, a new point sampled in this region should have + # negative label to correct the FP error + fp_masks = ~gt_masks & pred_masks + # false negative region, a new point sampled in this region should have + # positive label to correct the FN error + fn_masks = gt_masks & ~pred_masks + # whether the prediction completely match the ground-truth on each mask + all_correct = torch.all((gt_masks == pred_masks).flatten(2), dim=2) + all_correct = all_correct[..., None, None] + + # channel 0 is FP map, while channel 1 is FN map + pts_noise = torch.rand(B, num_pt, H_im, W_im, 2, device=device) + # sample a negative new click from FP region or a positive new click + # from FN region, depend on where the maximum falls, + # and in case the predictions are all correct (no FP or FN), we just + # sample a negative click from the background region + pts_noise[..., 0] *= fp_masks | (all_correct & ~gt_masks) + pts_noise[..., 1] *= fn_masks + pts_idx = pts_noise.flatten(2).argmax(dim=2) + labels = (pts_idx % 2).to(torch.int32) + pts_idx = pts_idx // 2 + pts_x = pts_idx % W_im + pts_y = pts_idx // W_im + points = torch.stack([pts_x, pts_y], dim=2).to(torch.float) + return points, labels + + +def sample_one_point_from_error_center(gt_masks, pred_masks, padding=True): + """ + Sample 1 random point (along with its label) from the center of each error region, + that is, the point with the largest distance to the boundary of each error region. + This is the RITM sampling method from https://github.com/saic-vul/ritm_interactive_segmentation/blob/master/isegm/inference/clicker.py + + Inputs: + - gt_masks: [B, 1, H_im, W_im] masks, dtype=torch.bool + - pred_masks: [B, 1, H_im, W_im] masks, dtype=torch.bool or None + - padding: if True, pad with boundary of 1 px for distance transform + + Outputs: + - points: [B, 1, 2], dtype=torch.float, contains (x, y) coordinates of each sampled point + - labels: [B, 1], dtype=torch.int32, where 1 means positive clicks and 0 means negative clicks + """ + import cv2 + + if pred_masks is None: + pred_masks = torch.zeros_like(gt_masks) + assert gt_masks.dtype == torch.bool and gt_masks.size(1) == 1 + assert pred_masks.dtype == torch.bool and pred_masks.shape == gt_masks.shape + + B, _, _, W_im = gt_masks.shape + device = gt_masks.device + + # false positive region, a new point sampled in this region should have + # negative label to correct the FP error + fp_masks = ~gt_masks & pred_masks + # false negative region, a new point sampled in this region should have + # positive label to correct the FN error + fn_masks = gt_masks & ~pred_masks + + fp_masks = fp_masks.cpu().numpy() + fn_masks = fn_masks.cpu().numpy() + points = torch.zeros(B, 1, 2, dtype=torch.float) + labels = torch.ones(B, 1, dtype=torch.int32) + for b in range(B): + fn_mask = fn_masks[b, 0] + fp_mask = fp_masks[b, 0] + if padding: + fn_mask = np.pad(fn_mask, ((1, 1), (1, 1)), "constant") + fp_mask = np.pad(fp_mask, ((1, 1), (1, 1)), "constant") + # compute the distance of each point in FN/FP region to its boundary + fn_mask_dt = cv2.distanceTransform(fn_mask.astype(np.uint8), cv2.DIST_L2, 0) + fp_mask_dt = cv2.distanceTransform(fp_mask.astype(np.uint8), cv2.DIST_L2, 0) + if padding: + fn_mask_dt = fn_mask_dt[1:-1, 1:-1] + fp_mask_dt = fp_mask_dt[1:-1, 1:-1] + + # take the point in FN/FP region with the largest distance to its boundary + fn_mask_dt_flat = fn_mask_dt.reshape(-1) + fp_mask_dt_flat = fp_mask_dt.reshape(-1) + fn_argmax = np.argmax(fn_mask_dt_flat) + fp_argmax = np.argmax(fp_mask_dt_flat) + is_positive = fn_mask_dt_flat[fn_argmax] > fp_mask_dt_flat[fp_argmax] + pt_idx = fn_argmax if is_positive else fp_argmax + points[b, 0, 0] = pt_idx % W_im # x + points[b, 0, 1] = pt_idx // W_im # y + labels[b, 0] = int(is_positive) + + points = points.to(device) + labels = labels.to(device) + return points, labels + + +def get_next_point(gt_masks, pred_masks, method): + if method == "uniform": + return sample_random_points_from_errors(gt_masks, pred_masks) + elif method == "center": + return sample_one_point_from_error_center(gt_masks, pred_masks) + else: + raise ValueError(f"unknown sampling method {method}") diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2.1_hiera_large.yaml b/tools/sam2-onnx-tensorrt/sam2/sam2.1_hiera_large.yaml new file mode 100644 index 00000000..23073ea7 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2.1_hiera_large.yaml @@ -0,0 +1,120 @@ +# @package _global_ + +# Model +model: + _target_: sam2.modeling.sam2_base.SAM2Base + image_encoder: + _target_: sam2.modeling.backbones.image_encoder.ImageEncoder + scalp: 1 + trunk: + _target_: sam2.modeling.backbones.hieradet.Hiera + embed_dim: 144 + num_heads: 2 + stages: [2, 6, 36, 4] + global_att_blocks: [23, 33, 43] + window_pos_embed_bkg_spatial_size: [7, 7] + window_spec: [8, 4, 16, 8] + neck: + _target_: sam2.modeling.backbones.image_encoder.FpnNeck + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 256 + normalize: true + scale: null + temperature: 10000 + d_model: 256 + backbone_channel_list: [1152, 576, 288, 144] + fpn_top_down_levels: [2, 3] # output level 0 and 1 directly use the backbone features + fpn_interp_model: nearest + + memory_attention: + _target_: sam2.modeling.memory_attention.MemoryAttention + d_model: 256 + pos_enc_at_input: true + layer: + _target_: sam2.modeling.memory_attention.MemoryAttentionLayer + activation: relu + dim_feedforward: 2048 + dropout: 0.1 + pos_enc_at_attn: false + self_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + d_model: 256 + pos_enc_at_cross_attn_keys: true + pos_enc_at_cross_attn_queries: false + cross_attention: + _target_: sam2.modeling.sam.transformer.RoPEAttention + rope_theta: 10000.0 + feat_sizes: [64, 64] + rope_k_repeat: True + embedding_dim: 256 + num_heads: 1 + downsample_rate: 1 + dropout: 0.1 + kv_in_dim: 64 + num_layers: 4 + + memory_encoder: + _target_: sam2.modeling.memory_encoder.MemoryEncoder + out_dim: 64 + position_encoding: + _target_: sam2.modeling.position_encoding.PositionEmbeddingSine + num_pos_feats: 64 + normalize: true + scale: null + temperature: 10000 + mask_downsampler: + _target_: sam2.modeling.memory_encoder.MaskDownSampler + kernel_size: 3 + stride: 2 + padding: 1 + fuser: + _target_: sam2.modeling.memory_encoder.Fuser + layer: + _target_: sam2.modeling.memory_encoder.CXBlock + dim: 256 + kernel_size: 7 + padding: 3 + layer_scale_init_value: 1e-6 + use_dwconv: True # depth-wise convs + num_layers: 2 + + num_maskmem: 7 + image_size: 1024 + # apply scaled sigmoid on mask logits for memory encoder, and directly feed input mask as output mask + sigmoid_scale_for_mem_enc: 20.0 + sigmoid_bias_for_mem_enc: -10.0 + use_mask_input_as_output_without_sam: true + # Memory + directly_add_no_mem_embed: true + no_obj_embed_spatial: true + # use high-resolution feature map in the SAM mask decoder + use_high_res_features_in_sam: true + # output 3 masks on the first click on initial conditioning frames + multimask_output_in_sam: true + # SAM heads + iou_prediction_use_sigmoid: True + # cross-attend to object pointers from other frames (based on SAM output tokens) in the encoder + use_obj_ptrs_in_encoder: true + add_tpos_enc_to_obj_ptrs: true + proj_tpos_enc_in_obj_ptrs: true + use_signed_tpos_enc_to_obj_ptrs: true + only_obj_ptrs_in_the_past_for_eval: true + # object occlusion prediction + pred_obj_scores: true + pred_obj_scores_mlp: true + fixed_no_obj_ptr: true + # multimask tracking settings + multimask_output_for_tracking: true + use_multimask_token_for_obj_ptr: true + multimask_min_pt_num: 0 + multimask_max_pt_num: 1 + use_mlp_for_obj_ptr_proj: true + # Compilation flag + compile_image_encoder: False diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_b+.yaml b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_b+.yaml new file mode 120000 index 00000000..998d9c98 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_b+.yaml @@ -0,0 +1 @@ +configs/sam2/sam2_hiera_b+.yaml \ No newline at end of file diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_l.yaml b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_l.yaml new file mode 120000 index 00000000..c0e7e58e --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_l.yaml @@ -0,0 +1 @@ +configs/sam2/sam2_hiera_l.yaml \ No newline at end of file diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_s.yaml b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_s.yaml new file mode 120000 index 00000000..41896a26 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_s.yaml @@ -0,0 +1 @@ +configs/sam2/sam2_hiera_s.yaml \ No newline at end of file diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_t.yaml b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_t.yaml new file mode 120000 index 00000000..71ff3abb --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_hiera_t.yaml @@ -0,0 +1 @@ +configs/sam2/sam2_hiera_t.yaml \ No newline at end of file diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_image_predictor.py b/tools/sam2-onnx-tensorrt/sam2/sam2_image_predictor.py new file mode 100644 index 00000000..41ce53af --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_image_predictor.py @@ -0,0 +1,466 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import logging + +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from PIL.Image import Image + +from sam2.modeling.sam2_base import SAM2Base + +from sam2.utils.transforms import SAM2Transforms + + +class SAM2ImagePredictor: + def __init__( + self, + sam_model: SAM2Base, + mask_threshold=0.0, + max_hole_area=0.0, + max_sprinkle_area=0.0, + **kwargs, + ) -> None: + """ + Uses SAM-2 to calculate the image embedding for an image, and then + allow repeated, efficient mask prediction given prompts. + + Arguments: + sam_model (Sam-2): The model to use for mask prediction. + mask_threshold (float): The threshold to use when converting mask logits + to binary masks. Masks are thresholded at 0 by default. + max_hole_area (int): If max_hole_area > 0, we fill small holes in up to + the maximum area of max_hole_area in low_res_masks. + max_sprinkle_area (int): If max_sprinkle_area > 0, we remove small sprinkles up to + the maximum area of max_sprinkle_area in low_res_masks. + """ + super().__init__() + self.model = sam_model + self._transforms = SAM2Transforms( + resolution=self.model.image_size, + mask_threshold=mask_threshold, + max_hole_area=max_hole_area, + max_sprinkle_area=max_sprinkle_area, + ) + + # Predictor state + self._is_image_set = False + self._features = None + self._orig_hw = None + # Whether the predictor is set for single image or a batch of images + self._is_batch = False + + # Predictor config + self.mask_threshold = mask_threshold + + # Spatial dim for backbone feature maps + self._bb_feat_sizes = [ + (256, 256), + (128, 128), + (64, 64), + ] + + @classmethod + def from_pretrained(cls, model_id: str, **kwargs) -> "SAM2ImagePredictor": + """ + Load a pretrained model from the Hugging Face hub. + + Arguments: + model_id (str): The Hugging Face repository ID. + **kwargs: Additional arguments to pass to the model constructor. + + Returns: + (SAM2ImagePredictor): The loaded model. + """ + from sam2.build_sam import build_sam2_hf + + sam_model = build_sam2_hf(model_id, **kwargs) + return cls(sam_model, **kwargs) + + @torch.no_grad() + def set_image( + self, + image: Union[np.ndarray, Image], + ) -> None: + """ + Calculates the image embeddings for the provided image, allowing + masks to be predicted with the 'predict' method. + + Arguments: + image (np.ndarray or PIL Image): The input image to embed in RGB format. The image should be in HWC format if np.ndarray, or WHC format if PIL Image + with pixel values in [0, 255]. + image_format (str): The color format of the image, in ['RGB', 'BGR']. + """ + self.reset_predictor() + # Transform the image to the form expected by the model + if isinstance(image, np.ndarray): + logging.info("For numpy array image, we assume (HxWxC) format") + self._orig_hw = [image.shape[:2]] + elif isinstance(image, Image): + w, h = image.size + self._orig_hw = [(h, w)] + else: + raise NotImplementedError("Image format not supported") + + input_image = self._transforms(image) + input_image = input_image[None, ...].to(self.device) + + assert ( + len(input_image.shape) == 4 and input_image.shape[1] == 3 + ), f"input_image must be of size 1x3xHxW, got {input_image.shape}" + logging.info("Computing image embeddings for the provided image...") + backbone_out = self.model.forward_image(input_image) + _, vision_feats, _, _ = self.model._prepare_backbone_features(backbone_out) + # Add no_mem_embed, which is added to the lowest rest feat. map during training on videos + if self.model.directly_add_no_mem_embed: + vision_feats[-1] = vision_feats[-1] + self.model.no_mem_embed + + feats = [ + feat.permute(1, 2, 0).view(1, -1, *feat_size) + for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) + ][::-1] + self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} + self._is_image_set = True + logging.info("Image embeddings computed.") + + @torch.no_grad() + def set_image_batch( + self, + image_list: List[Union[np.ndarray]], + ) -> None: + """ + Calculates the image embeddings for the provided image batch, allowing + masks to be predicted with the 'predict_batch' method. + + Arguments: + image_list (List[np.ndarray]): The input images to embed in RGB format. The image should be in HWC format if np.ndarray + with pixel values in [0, 255]. + """ + self.reset_predictor() + assert isinstance(image_list, list) + self._orig_hw = [] + for image in image_list: + assert isinstance( + image, np.ndarray + ), "Images are expected to be an np.ndarray in RGB format, and of shape HWC" + self._orig_hw.append(image.shape[:2]) + # Transform the image to the form expected by the model + img_batch = self._transforms.forward_batch(image_list) + img_batch = img_batch.to(self.device) + batch_size = img_batch.shape[0] + assert ( + len(img_batch.shape) == 4 and img_batch.shape[1] == 3 + ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}" + logging.info("Computing image embeddings for the provided images...") + backbone_out = self.model.forward_image(img_batch) + _, vision_feats, _, _ = self.model._prepare_backbone_features(backbone_out) + # Add no_mem_embed, which is added to the lowest rest feat. map during training on videos + if self.model.directly_add_no_mem_embed: + vision_feats[-1] = vision_feats[-1] + self.model.no_mem_embed + + feats = [ + feat.permute(1, 2, 0).view(batch_size, -1, *feat_size) + for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) + ][::-1] + self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} + self._is_image_set = True + self._is_batch = True + logging.info("Image embeddings computed.") + + def predict_batch( + self, + point_coords_batch: List[np.ndarray] = None, + point_labels_batch: List[np.ndarray] = None, + box_batch: List[np.ndarray] = None, + mask_input_batch: List[np.ndarray] = None, + multimask_output: bool = True, + return_logits: bool = False, + normalize_coords=True, + ) -> Tuple[List[np.ndarray], List[np.ndarray], List[np.ndarray]]: + """This function is very similar to predict(...), however it is used for batched mode, when the model is expected to generate predictions on multiple images. + It returns a tuple of lists of masks, ious, and low_res_masks_logits. + """ + assert self._is_batch, "This function should only be used when in batched mode" + if not self._is_image_set: + raise RuntimeError( + "An image must be set with .set_image_batch(...) before mask prediction." + ) + num_images = len(self._features["image_embed"]) + all_masks = [] + all_ious = [] + all_low_res_masks = [] + for img_idx in range(num_images): + # Transform input prompts + point_coords = ( + point_coords_batch[img_idx] if point_coords_batch is not None else None + ) + point_labels = ( + point_labels_batch[img_idx] if point_labels_batch is not None else None + ) + box = box_batch[img_idx] if box_batch is not None else None + mask_input = ( + mask_input_batch[img_idx] if mask_input_batch is not None else None + ) + mask_input, unnorm_coords, labels, unnorm_box = self._prep_prompts( + point_coords, + point_labels, + box, + mask_input, + normalize_coords, + img_idx=img_idx, + ) + masks, iou_predictions, low_res_masks = self._predict( + unnorm_coords, + labels, + unnorm_box, + mask_input, + multimask_output, + return_logits=return_logits, + img_idx=img_idx, + ) + masks_np = masks.squeeze(0).float().detach().cpu().numpy() + iou_predictions_np = ( + iou_predictions.squeeze(0).float().detach().cpu().numpy() + ) + low_res_masks_np = low_res_masks.squeeze(0).float().detach().cpu().numpy() + all_masks.append(masks_np) + all_ious.append(iou_predictions_np) + all_low_res_masks.append(low_res_masks_np) + + return all_masks, all_ious, all_low_res_masks + + def predict( + self, + point_coords: Optional[np.ndarray] = None, + point_labels: Optional[np.ndarray] = None, + box: Optional[np.ndarray] = None, + mask_input: Optional[np.ndarray] = None, + multimask_output: bool = True, + return_logits: bool = False, + normalize_coords=True, + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Predict masks for the given input prompts, using the currently set image. + + Arguments: + point_coords (np.ndarray or None): A Nx2 array of point prompts to the + model. Each point is in (X,Y) in pixels. + point_labels (np.ndarray or None): A length N array of labels for the + point prompts. 1 indicates a foreground point and 0 indicates a + background point. + box (np.ndarray or None): A length 4 array given a box prompt to the + model, in XYXY format. + mask_input (np.ndarray): A low resolution mask input to the model, typically + coming from a previous prediction iteration. Has form 1xHxW, where + for SAM, H=W=256. + multimask_output (bool): If true, the model will return three masks. + For ambiguous input prompts (such as a single click), this will often + produce better masks than a single prediction. If only a single + mask is needed, the model's predicted quality score can be used + to select the best mask. For non-ambiguous prompts, such as multiple + input prompts, multimask_output=False can give better results. + return_logits (bool): If true, returns un-thresholded masks logits + instead of a binary mask. + normalize_coords (bool): If true, the point coordinates will be normalized to the range [0,1] and point_coords is expected to be wrt. image dimensions. + + Returns: + (np.ndarray): The output masks in CxHxW format, where C is the + number of masks, and (H, W) is the original image size. + (np.ndarray): An array of length C containing the model's + predictions for the quality of each mask. + (np.ndarray): An array of shape CxHxW, where C is the number + of masks and H=W=256. These low resolution logits can be passed to + a subsequent iteration as mask input. + """ + if not self._is_image_set: + raise RuntimeError( + "An image must be set with .set_image(...) before mask prediction." + ) + + # Transform input prompts + + mask_input, unnorm_coords, labels, unnorm_box = self._prep_prompts( + point_coords, point_labels, box, mask_input, normalize_coords + ) + + masks, iou_predictions, low_res_masks = self._predict( + unnorm_coords, + labels, + unnorm_box, + mask_input, + multimask_output, + return_logits=return_logits, + ) + + masks_np = masks.squeeze(0).float().detach().cpu().numpy() + iou_predictions_np = iou_predictions.squeeze(0).float().detach().cpu().numpy() + low_res_masks_np = low_res_masks.squeeze(0).float().detach().cpu().numpy() + return masks_np, iou_predictions_np, low_res_masks_np + + def _prep_prompts( + self, point_coords, point_labels, box, mask_logits, normalize_coords, img_idx=-1 + ): + + unnorm_coords, labels, unnorm_box, mask_input = None, None, None, None + if point_coords is not None: + assert ( + point_labels is not None + ), "point_labels must be supplied if point_coords is supplied." + point_coords = torch.as_tensor( + point_coords, dtype=torch.float, device=self.device + ) + unnorm_coords = self._transforms.transform_coords( + point_coords, normalize=normalize_coords, orig_hw=self._orig_hw[img_idx] + ) + labels = torch.as_tensor(point_labels, dtype=torch.int, device=self.device) + if len(unnorm_coords.shape) == 2: + unnorm_coords, labels = unnorm_coords[None, ...], labels[None, ...] + if box is not None: + box = torch.as_tensor(box, dtype=torch.float, device=self.device) + unnorm_box = self._transforms.transform_boxes( + box, normalize=normalize_coords, orig_hw=self._orig_hw[img_idx] + ) # Bx2x2 + if mask_logits is not None: + mask_input = torch.as_tensor( + mask_logits, dtype=torch.float, device=self.device + ) + if len(mask_input.shape) == 3: + mask_input = mask_input[None, :, :, :] + return mask_input, unnorm_coords, labels, unnorm_box + + @torch.no_grad() + def _predict( + self, + point_coords: Optional[torch.Tensor], + point_labels: Optional[torch.Tensor], + boxes: Optional[torch.Tensor] = None, + mask_input: Optional[torch.Tensor] = None, + multimask_output: bool = True, + return_logits: bool = False, + img_idx: int = -1, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Predict masks for the given input prompts, using the currently set image. + Input prompts are batched torch tensors and are expected to already be + transformed to the input frame using SAM2Transforms. + + Arguments: + point_coords (torch.Tensor or None): A BxNx2 array of point prompts to the + model. Each point is in (X,Y) in pixels. + point_labels (torch.Tensor or None): A BxN array of labels for the + point prompts. 1 indicates a foreground point and 0 indicates a + background point. + boxes (np.ndarray or None): A Bx4 array given a box prompt to the + model, in XYXY format. + mask_input (np.ndarray): A low resolution mask input to the model, typically + coming from a previous prediction iteration. Has form Bx1xHxW, where + for SAM, H=W=256. Masks returned by a previous iteration of the + predict method do not need further transformation. + multimask_output (bool): If true, the model will return three masks. + For ambiguous input prompts (such as a single click), this will often + produce better masks than a single prediction. If only a single + mask is needed, the model's predicted quality score can be used + to select the best mask. For non-ambiguous prompts, such as multiple + input prompts, multimask_output=False can give better results. + return_logits (bool): If true, returns un-thresholded masks logits + instead of a binary mask. + + Returns: + (torch.Tensor): The output masks in BxCxHxW format, where C is the + number of masks, and (H, W) is the original image size. + (torch.Tensor): An array of shape BxC containing the model's + predictions for the quality of each mask. + (torch.Tensor): An array of shape BxCxHxW, where C is the number + of masks and H=W=256. These low res logits can be passed to + a subsequent iteration as mask input. + """ + if not self._is_image_set: + raise RuntimeError( + "An image must be set with .set_image(...) before mask prediction." + ) + + if point_coords is not None: + concat_points = (point_coords, point_labels) + else: + concat_points = None + + # Embed prompts + if boxes is not None: + box_coords = boxes.reshape(-1, 2, 2) + box_labels = torch.tensor([[2, 3]], dtype=torch.int, device=boxes.device) + box_labels = box_labels.repeat(boxes.size(0), 1) + # we merge "boxes" and "points" into a single "concat_points" input (where + # boxes are added at the beginning) to sam_prompt_encoder + if concat_points is not None: + concat_coords = torch.cat([box_coords, concat_points[0]], dim=1) + concat_labels = torch.cat([box_labels, concat_points[1]], dim=1) + concat_points = (concat_coords, concat_labels) + else: + concat_points = (box_coords, box_labels) + + sparse_embeddings, dense_embeddings = self.model.sam_prompt_encoder( + points=concat_points, + boxes=None, + masks=mask_input, + ) + + # Predict masks + batched_mode = ( + concat_points is not None and concat_points[0].shape[0] > 1 + ) # multi object prediction + high_res_features = [ + feat_level[img_idx].unsqueeze(0) + for feat_level in self._features["high_res_feats"] + ] + low_res_masks, iou_predictions, _, _ = self.model.sam_mask_decoder( + image_embeddings=self._features["image_embed"][img_idx].unsqueeze(0), + image_pe=self.model.sam_prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + repeat_image=batched_mode, + high_res_features=high_res_features, + ) + + # Upscale the masks to the original image resolution + masks = self._transforms.postprocess_masks( + low_res_masks, self._orig_hw[img_idx] + ) + low_res_masks = torch.clamp(low_res_masks, -32.0, 32.0) + if not return_logits: + masks = masks > self.mask_threshold + + return masks, iou_predictions, low_res_masks + + def get_image_embedding(self) -> torch.Tensor: + """ + Returns the image embeddings for the currently set image, with + shape 1xCxHxW, where C is the embedding dimension and (H,W) are + the embedding spatial dimension of SAM (typically C=256, H=W=64). + """ + if not self._is_image_set: + raise RuntimeError( + "An image must be set with .set_image(...) to generate an embedding." + ) + assert ( + self._features is not None + ), "Features must exist if an image has been set." + return self._features["image_embed"] + + @property + def device(self) -> torch.device: + return self.model.device + + def reset_predictor(self) -> None: + """ + Resets the image embeddings and other state variables. + """ + self._is_image_set = False + self._features = None + self._orig_hw = None + self._is_batch = False diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor.py b/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor.py new file mode 100644 index 00000000..5a7e1a01 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor.py @@ -0,0 +1,1223 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import warnings +from collections import OrderedDict + +import torch +import torch.nn.functional as F + +from tqdm import tqdm + +from sam2.modeling.sam2_base import NO_OBJ_SCORE, SAM2Base +from sam2.utils.misc import concat_points, fill_holes_in_mask_scores, load_video_frames + + +class SAM2VideoPredictor(SAM2Base): + """The predictor class to handle user interactions and manage inference states.""" + + def __init__( + self, + fill_hole_area=0, + # whether to apply non-overlapping constraints on the output object masks + non_overlap_masks=False, + # whether to clear non-conditioning memory of the surrounding frames (which may contain outdated information) after adding correction clicks; + # note that this would only apply to *single-object tracking* unless `clear_non_cond_mem_for_multi_obj` is also set to True) + clear_non_cond_mem_around_input=False, + # if `add_all_frames_to_correct_as_cond` is True, we also append to the conditioning frame list any frame that receives a later correction click + # if `add_all_frames_to_correct_as_cond` is False, we conditioning frame list to only use those initial conditioning frames + add_all_frames_to_correct_as_cond=False, + **kwargs, + ): + super().__init__(**kwargs) + self.fill_hole_area = fill_hole_area + self.non_overlap_masks = non_overlap_masks + self.clear_non_cond_mem_around_input = clear_non_cond_mem_around_input + self.add_all_frames_to_correct_as_cond = add_all_frames_to_correct_as_cond + + @torch.inference_mode() + def init_state( + self, + video_path, + offload_video_to_cpu=False, + offload_state_to_cpu=False, + async_loading_frames=False, + ): + """Initialize an inference state.""" + compute_device = self.device # device of the model + images, video_height, video_width = load_video_frames( + video_path=video_path, + image_size=self.image_size, + offload_video_to_cpu=offload_video_to_cpu, + async_loading_frames=async_loading_frames, + compute_device=compute_device, + ) + inference_state = {} + inference_state["images"] = images + inference_state["num_frames"] = len(images) + # whether to offload the video frames to CPU memory + # turning on this option saves the GPU memory with only a very small overhead + inference_state["offload_video_to_cpu"] = offload_video_to_cpu + # whether to offload the inference state to CPU memory + # turning on this option saves the GPU memory at the cost of a lower tracking fps + # (e.g. in a test case of 768x768 model, fps dropped from 27 to 24 when tracking one object + # and from 24 to 21 when tracking two objects) + inference_state["offload_state_to_cpu"] = offload_state_to_cpu + # the original video height and width, used for resizing final output scores + inference_state["video_height"] = video_height + inference_state["video_width"] = video_width + inference_state["device"] = compute_device + if offload_state_to_cpu: + inference_state["storage_device"] = torch.device("cpu") + else: + inference_state["storage_device"] = compute_device + # inputs on each frame + inference_state["point_inputs_per_obj"] = {} + inference_state["mask_inputs_per_obj"] = {} + # visual features on a small number of recently visited frames for quick interactions + inference_state["cached_features"] = {} + # values that don't change across frames (so we only need to hold one copy of them) + inference_state["constants"] = {} + # mapping between client-side object id and model-side object index + inference_state["obj_id_to_idx"] = OrderedDict() + inference_state["obj_idx_to_id"] = OrderedDict() + inference_state["obj_ids"] = [] + # Slice (view) of each object tracking results, sharing the same memory with "output_dict" + inference_state["output_dict_per_obj"] = {} + # A temporary storage to hold new outputs when user interact with a frame + # to add clicks or mask (it's merged into "output_dict" before propagation starts) + inference_state["temp_output_dict_per_obj"] = {} + # Frames that already holds consolidated outputs from click or mask inputs + # (we directly use their consolidated outputs during tracking) + # metadata for each tracking frame (e.g. which direction it's tracked) + inference_state["frames_tracked_per_obj"] = {} + # Warm up the visual backbone and cache the image feature on frame 0 + self._get_image_feature(inference_state, frame_idx=0, batch_size=1) + return inference_state + + @classmethod + def from_pretrained(cls, model_id: str, **kwargs) -> "SAM2VideoPredictor": + """ + Load a pretrained model from the Hugging Face hub. + + Arguments: + model_id (str): The Hugging Face repository ID. + **kwargs: Additional arguments to pass to the model constructor. + + Returns: + (SAM2VideoPredictor): The loaded model. + """ + from sam2.build_sam import build_sam2_video_predictor_hf + + sam_model = build_sam2_video_predictor_hf(model_id, **kwargs) + return sam_model + + def _obj_id_to_idx(self, inference_state, obj_id): + """Map client-side object id to model-side object index.""" + obj_idx = inference_state["obj_id_to_idx"].get(obj_id, None) + if obj_idx is not None: + return obj_idx + + # We always allow adding new objects (including after tracking starts). + allow_new_object = True + if allow_new_object: + # get the next object slot + obj_idx = len(inference_state["obj_id_to_idx"]) + inference_state["obj_id_to_idx"][obj_id] = obj_idx + inference_state["obj_idx_to_id"][obj_idx] = obj_id + inference_state["obj_ids"] = list(inference_state["obj_id_to_idx"]) + # set up input and output structures for this object + inference_state["point_inputs_per_obj"][obj_idx] = {} + inference_state["mask_inputs_per_obj"][obj_idx] = {} + inference_state["output_dict_per_obj"][obj_idx] = { + "cond_frame_outputs": {}, # dict containing {frame_idx: } + "non_cond_frame_outputs": {}, # dict containing {frame_idx: } + } + inference_state["temp_output_dict_per_obj"][obj_idx] = { + "cond_frame_outputs": {}, # dict containing {frame_idx: } + "non_cond_frame_outputs": {}, # dict containing {frame_idx: } + } + inference_state["frames_tracked_per_obj"][obj_idx] = {} + return obj_idx + else: + raise RuntimeError( + f"Cannot add new object id {obj_id} after tracking starts. " + f"All existing object ids: {inference_state['obj_ids']}. " + f"Please call 'reset_state' to restart from scratch." + ) + + def _obj_idx_to_id(self, inference_state, obj_idx): + """Map model-side object index to client-side object id.""" + return inference_state["obj_idx_to_id"][obj_idx] + + def _get_obj_num(self, inference_state): + """Get the total number of unique object ids received so far in this session.""" + return len(inference_state["obj_idx_to_id"]) + + @torch.inference_mode() + def add_new_points_or_box( + self, + inference_state, + frame_idx, + obj_id, + points=None, + labels=None, + clear_old_points=True, + normalize_coords=True, + box=None, + ): + """Add new points to a frame.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + point_inputs_per_frame = inference_state["point_inputs_per_obj"][obj_idx] + mask_inputs_per_frame = inference_state["mask_inputs_per_obj"][obj_idx] + + if (points is not None) != (labels is not None): + raise ValueError("points and labels must be provided together") + if points is None and box is None: + raise ValueError("at least one of points or box must be provided as input") + + if points is None: + points = torch.zeros(0, 2, dtype=torch.float32) + elif not isinstance(points, torch.Tensor): + points = torch.tensor(points, dtype=torch.float32) + if labels is None: + labels = torch.zeros(0, dtype=torch.int32) + elif not isinstance(labels, torch.Tensor): + labels = torch.tensor(labels, dtype=torch.int32) + if points.dim() == 2: + points = points.unsqueeze(0) # add batch dimension + if labels.dim() == 1: + labels = labels.unsqueeze(0) # add batch dimension + + # If `box` is provided, we add it as the first two points with labels 2 and 3 + # along with the user-provided points (consistent with how SAM 2 is trained). + if box is not None: + if not clear_old_points: + raise ValueError( + "cannot add box without clearing old points, since " + "box prompt must be provided before any point prompt " + "(please use clear_old_points=True instead)" + ) + if not isinstance(box, torch.Tensor): + box = torch.tensor(box, dtype=torch.float32, device=points.device) + box_coords = box.reshape(1, 2, 2) + box_labels = torch.tensor([2, 3], dtype=torch.int32, device=labels.device) + box_labels = box_labels.reshape(1, 2) + points = torch.cat([box_coords, points], dim=1) + labels = torch.cat([box_labels, labels], dim=1) + + if normalize_coords: + video_H = inference_state["video_height"] + video_W = inference_state["video_width"] + points = points / torch.tensor([video_W, video_H]).to(points.device) + # scale the (normalized) coordinates by the model's internal image size + points = points * self.image_size + points = points.to(inference_state["device"]) + labels = labels.to(inference_state["device"]) + + if not clear_old_points: + point_inputs = point_inputs_per_frame.get(frame_idx, None) + else: + point_inputs = None + point_inputs = concat_points(point_inputs, points, labels) + + point_inputs_per_frame[frame_idx] = point_inputs + mask_inputs_per_frame.pop(frame_idx, None) + # If this frame hasn't been tracked before, we treat it as an initial conditioning + # frame, meaning that the inputs points are to generate segments on this frame without + # using any memory from other frames, like in SAM. Otherwise (if it has been tracked), + # the input points will be used to correct the already tracked masks. + obj_frames_tracked = inference_state["frames_tracked_per_obj"][obj_idx] + is_init_cond_frame = frame_idx not in obj_frames_tracked + # whether to track in reverse time order + if is_init_cond_frame: + reverse = False + else: + reverse = obj_frames_tracked[frame_idx]["reverse"] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + # Add a frame to conditioning output if it's an initial conditioning frame or + # if the model sees all frames receiving clicks/mask as conditioning frames. + is_cond = is_init_cond_frame or self.add_all_frames_to_correct_as_cond + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + + # Get any previously predicted mask logits on this object and feed it along with + # the new clicks into the SAM mask decoder. + prev_sam_mask_logits = None + # lookup temporary output dict first, which contains the most recent output + # (if not found, then lookup conditioning and non-conditioning frame output) + prev_out = obj_temp_output_dict[storage_key].get(frame_idx) + if prev_out is None: + prev_out = obj_output_dict["cond_frame_outputs"].get(frame_idx) + if prev_out is None: + prev_out = obj_output_dict["non_cond_frame_outputs"].get(frame_idx) + + if prev_out is not None and prev_out["pred_masks"] is not None: + device = inference_state["device"] + prev_sam_mask_logits = prev_out["pred_masks"].to(device, non_blocking=True) + # Clamp the scale of prev_sam_mask_logits to avoid rare numerical issues. + prev_sam_mask_logits = torch.clamp(prev_sam_mask_logits, -32.0, 32.0) + current_out, _ = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=obj_output_dict, # run on the slice of a single object + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + is_init_cond_frame=is_init_cond_frame, + point_inputs=point_inputs, + mask_inputs=None, + reverse=reverse, + # Skip the memory encoder when adding clicks or mask. We execute the memory encoder + # at the beginning of `propagate_in_video` (after user finalize their clicks). This + # allows us to enforce non-overlapping constraints on all objects before encoding + # them into memory. + run_mem_encoder=False, + prev_sam_mask_logits=prev_sam_mask_logits, + ) + # Add the output to the output dict (to be used as future memory) + obj_temp_output_dict[storage_key][frame_idx] = current_out + + # Resize the output mask to the original video resolution + obj_ids = inference_state["obj_ids"] + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + def add_new_points(self, *args, **kwargs): + """Deprecated method. Please use `add_new_points_or_box` instead.""" + return self.add_new_points_or_box(*args, **kwargs) + + @torch.inference_mode() + def add_new_mask( + self, + inference_state, + frame_idx, + obj_id, + mask, + ): + """Add new mask to a frame.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + point_inputs_per_frame = inference_state["point_inputs_per_obj"][obj_idx] + mask_inputs_per_frame = inference_state["mask_inputs_per_obj"][obj_idx] + + if not isinstance(mask, torch.Tensor): + mask = torch.tensor(mask, dtype=torch.bool) + assert mask.dim() == 2 + mask_H, mask_W = mask.shape + mask_inputs_orig = mask[None, None] # add batch and channel dimension + mask_inputs_orig = mask_inputs_orig.float().to(inference_state["device"]) + + # resize the mask if it doesn't match the model's image size + if mask_H != self.image_size or mask_W != self.image_size: + mask_inputs = torch.nn.functional.interpolate( + mask_inputs_orig, + size=(self.image_size, self.image_size), + align_corners=False, + mode="bilinear", + antialias=True, # use antialias for downsampling + ) + mask_inputs = (mask_inputs >= 0.5).float() + else: + mask_inputs = mask_inputs_orig + + mask_inputs_per_frame[frame_idx] = mask_inputs + point_inputs_per_frame.pop(frame_idx, None) + # If this frame hasn't been tracked before, we treat it as an initial conditioning + # frame, meaning that the inputs points are to generate segments on this frame without + # using any memory from other frames, like in SAM. Otherwise (if it has been tracked), + # the input points will be used to correct the already tracked masks. + obj_frames_tracked = inference_state["frames_tracked_per_obj"][obj_idx] + is_init_cond_frame = frame_idx not in obj_frames_tracked + # whether to track in reverse time order + if is_init_cond_frame: + reverse = False + else: + reverse = obj_frames_tracked[frame_idx]["reverse"] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + # Add a frame to conditioning output if it's an initial conditioning frame or + # if the model sees all frames receiving clicks/mask as conditioning frames. + is_cond = is_init_cond_frame or self.add_all_frames_to_correct_as_cond + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + + current_out, _ = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=obj_output_dict, # run on the slice of a single object + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + is_init_cond_frame=is_init_cond_frame, + point_inputs=None, + mask_inputs=mask_inputs, + reverse=reverse, + # Skip the memory encoder when adding clicks or mask. We execute the memory encoder + # at the beginning of `propagate_in_video` (after user finalize their clicks). This + # allows us to enforce non-overlapping constraints on all objects before encoding + # them into memory. + run_mem_encoder=False, + ) + # Add the output to the output dict (to be used as future memory) + obj_temp_output_dict[storage_key][frame_idx] = current_out + + # Resize the output mask to the original video resolution + obj_ids = inference_state["obj_ids"] + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + def _get_orig_video_res_output(self, inference_state, any_res_masks): + """ + Resize the object scores to the original video resolution (video_res_masks) + and apply non-overlapping constraints for final output. + """ + device = inference_state["device"] + video_H = inference_state["video_height"] + video_W = inference_state["video_width"] + any_res_masks = any_res_masks.to(device, non_blocking=True) + if any_res_masks.shape[-2:] == (video_H, video_W): + video_res_masks = any_res_masks + else: + video_res_masks = torch.nn.functional.interpolate( + any_res_masks, + size=(video_H, video_W), + mode="bilinear", + align_corners=False, + ) + if self.non_overlap_masks: + video_res_masks = self._apply_non_overlapping_constraints(video_res_masks) + return any_res_masks, video_res_masks + + def _consolidate_temp_output_across_obj( + self, + inference_state, + frame_idx, + is_cond, + consolidate_at_video_res=False, + ): + """ + Consolidate the per-object temporary outputs in `temp_output_dict_per_obj` on + a frame into a single output for all objects, including + 1) fill any missing objects either from `output_dict_per_obj` (if they exist in + `output_dict_per_obj` for this frame) or leave them as placeholder values + (if they don't exist in `output_dict_per_obj` for this frame); + 2) if specified, rerun memory encoder after apply non-overlapping constraints + on the object scores. + """ + batch_size = self._get_obj_num(inference_state) + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + # Optionally, we allow consolidating the temporary outputs at the original + # video resolution (to provide a better editing experience for mask prompts). + if consolidate_at_video_res: + consolidated_H = inference_state["video_height"] + consolidated_W = inference_state["video_width"] + consolidated_mask_key = "pred_masks_video_res" + else: + consolidated_H = consolidated_W = self.image_size // 4 + consolidated_mask_key = "pred_masks" + + # Initialize `consolidated_out`. Its "maskmem_features" and "maskmem_pos_enc" + # will be added when rerunning the memory encoder after applying non-overlapping + # constraints to object scores. Its "pred_masks" are prefilled with a large + # negative value (NO_OBJ_SCORE) to represent missing objects. + consolidated_out = { + consolidated_mask_key: torch.full( + size=(batch_size, 1, consolidated_H, consolidated_W), + fill_value=NO_OBJ_SCORE, + dtype=torch.float32, + device=inference_state["storage_device"], + ), + } + for obj_idx in range(batch_size): + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + out = obj_temp_output_dict[storage_key].get(frame_idx, None) + # If the object doesn't appear in "temp_output_dict_per_obj" on this frame, + # we fall back and look up its previous output in "output_dict_per_obj". + # We look up both "cond_frame_outputs" and "non_cond_frame_outputs" in + # "output_dict_per_obj" to find a previous output for this object. + if out is None: + out = obj_output_dict["cond_frame_outputs"].get(frame_idx, None) + if out is None: + out = obj_output_dict["non_cond_frame_outputs"].get(frame_idx, None) + # If the object doesn't appear in "output_dict_per_obj" either, we skip it + # and leave its mask scores to the default scores (i.e. the NO_OBJ_SCORE + # placeholder above) and set its object pointer to be a dummy pointer. + if out is None: + continue + # Add the temporary object output mask to consolidated output mask + obj_mask = out["pred_masks"] + consolidated_pred_masks = consolidated_out[consolidated_mask_key] + if obj_mask.shape[-2:] == consolidated_pred_masks.shape[-2:]: + consolidated_pred_masks[obj_idx : obj_idx + 1] = obj_mask + else: + # Resize first if temporary object mask has a different resolution + resized_obj_mask = torch.nn.functional.interpolate( + obj_mask, + size=consolidated_pred_masks.shape[-2:], + mode="bilinear", + align_corners=False, + ) + consolidated_pred_masks[obj_idx : obj_idx + 1] = resized_obj_mask + + return consolidated_out + + @torch.inference_mode() + def propagate_in_video_preflight(self, inference_state): + """Prepare inference_state and consolidate temporary outputs before tracking.""" + # Check and make sure that every object has received input points or masks. + batch_size = self._get_obj_num(inference_state) + if batch_size == 0: + raise RuntimeError( + "No input points or masks are provided for any object; please add inputs first." + ) + + # Consolidate per-object temporary outputs in "temp_output_dict_per_obj" and + # add them into "output_dict". + for obj_idx in range(batch_size): + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + for is_cond in [False, True]: + # Separately consolidate conditioning and non-conditioning temp outputs + storage_key = ( + "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + ) + # Find all the frames that contain temporary outputs for any objects + # (these should be the frames that have just received clicks for mask inputs + # via `add_new_points_or_box` or `add_new_mask`) + for frame_idx, out in obj_temp_output_dict[storage_key].items(): + # Run memory encoder on the temporary outputs (if the memory feature is missing) + if out["maskmem_features"] is None: + high_res_masks = torch.nn.functional.interpolate( + out["pred_masks"].to(inference_state["device"]), + size=(self.image_size, self.image_size), + mode="bilinear", + align_corners=False, + ) + maskmem_features, maskmem_pos_enc = self._run_memory_encoder( + inference_state=inference_state, + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + high_res_masks=high_res_masks, + object_score_logits=out["object_score_logits"], + # these frames are what the user interacted with + is_mask_from_pts=True, + ) + out["maskmem_features"] = maskmem_features + out["maskmem_pos_enc"] = maskmem_pos_enc + + obj_output_dict[storage_key][frame_idx] = out + if self.clear_non_cond_mem_around_input: + # clear non-conditioning memory of the surrounding frames + self._clear_obj_non_cond_mem_around_input( + inference_state, frame_idx, obj_idx + ) + + # clear temporary outputs in `temp_output_dict_per_obj` + obj_temp_output_dict[storage_key].clear() + + # check and make sure that every object has received input points or masks + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + if len(obj_output_dict["cond_frame_outputs"]) == 0: + obj_id = self._obj_idx_to_id(inference_state, obj_idx) + raise RuntimeError( + f"No input points or masks are provided for object id {obj_id}; please add inputs first." + ) + # edge case: if an output is added to "cond_frame_outputs", we remove any prior + # output on the same frame in "non_cond_frame_outputs" + for frame_idx in obj_output_dict["cond_frame_outputs"]: + obj_output_dict["non_cond_frame_outputs"].pop(frame_idx, None) + + @torch.inference_mode() + def propagate_in_video( + self, + inference_state, + start_frame_idx=None, + max_frame_num_to_track=None, + reverse=False, + ): + """Propagate the input points across frames to track in the entire video.""" + self.propagate_in_video_preflight(inference_state) + + obj_ids = inference_state["obj_ids"] + num_frames = inference_state["num_frames"] + batch_size = self._get_obj_num(inference_state) + + # set start index, end index, and processing order + if start_frame_idx is None: + # default: start from the earliest frame with input points + start_frame_idx = min( + t + for obj_output_dict in inference_state["output_dict_per_obj"].values() + for t in obj_output_dict["cond_frame_outputs"] + ) + if max_frame_num_to_track is None: + # default: track all the frames in the video + max_frame_num_to_track = num_frames + if reverse: + end_frame_idx = max(start_frame_idx - max_frame_num_to_track, 0) + if start_frame_idx > 0: + processing_order = range(start_frame_idx, end_frame_idx - 1, -1) + else: + processing_order = [] # skip reverse tracking if starting from frame 0 + else: + end_frame_idx = min( + start_frame_idx + max_frame_num_to_track, num_frames - 1 + ) + processing_order = range(start_frame_idx, end_frame_idx + 1) + + for frame_idx in tqdm(processing_order, desc="propagate in video"): + pred_masks_per_obj = [None] * batch_size + for obj_idx in range(batch_size): + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + # We skip those frames already in consolidated outputs (these are frames + # that received input clicks or mask). Note that we cannot directly run + # batched forward on them via `_run_single_frame_inference` because the + # number of clicks on each object might be different. + if frame_idx in obj_output_dict["cond_frame_outputs"]: + storage_key = "cond_frame_outputs" + current_out = obj_output_dict[storage_key][frame_idx] + device = inference_state["device"] + pred_masks = current_out["pred_masks"].to(device, non_blocking=True) + if self.clear_non_cond_mem_around_input: + # clear non-conditioning memory of the surrounding frames + self._clear_obj_non_cond_mem_around_input( + inference_state, frame_idx, obj_idx + ) + else: + storage_key = "non_cond_frame_outputs" + current_out, pred_masks = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=obj_output_dict, + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + is_init_cond_frame=False, + point_inputs=None, + mask_inputs=None, + reverse=reverse, + run_mem_encoder=True, + ) + obj_output_dict[storage_key][frame_idx] = current_out + + inference_state["frames_tracked_per_obj"][obj_idx][frame_idx] = { + "reverse": reverse + } + pred_masks_per_obj[obj_idx] = pred_masks + + # Resize the output mask to the original video resolution (we directly use + # the mask scores on GPU for output to avoid any CPU conversion in between) + if len(pred_masks_per_obj) > 1: + all_pred_masks = torch.cat(pred_masks_per_obj, dim=0) + else: + all_pred_masks = pred_masks_per_obj[0] + _, video_res_masks = self._get_orig_video_res_output( + inference_state, all_pred_masks + ) + yield frame_idx, obj_ids, video_res_masks + + @torch.inference_mode() + def clear_all_prompts_in_frame( + self, inference_state, frame_idx, obj_id, need_output=True + ): + """Remove all input points or mask in a specific frame for a given object.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + + # Clear the conditioning information on the given frame + inference_state["point_inputs_per_obj"][obj_idx].pop(frame_idx, None) + inference_state["mask_inputs_per_obj"][obj_idx].pop(frame_idx, None) + + temp_output_dict_per_obj = inference_state["temp_output_dict_per_obj"] + temp_output_dict_per_obj[obj_idx]["cond_frame_outputs"].pop(frame_idx, None) + temp_output_dict_per_obj[obj_idx]["non_cond_frame_outputs"].pop(frame_idx, None) + + # Remove the frame's conditioning output (possibly downgrading it to non-conditioning) + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + out = obj_output_dict["cond_frame_outputs"].pop(frame_idx, None) + if out is not None: + # The frame is not a conditioning frame anymore since it's not receiving inputs, + # so we "downgrade" its output (if exists) to a non-conditioning frame output. + obj_output_dict["non_cond_frame_outputs"][frame_idx] = out + inference_state["frames_tracked_per_obj"][obj_idx].pop(frame_idx, None) + + if not need_output: + return + # Finally, output updated masks per object (after removing the inputs above) + obj_ids = inference_state["obj_ids"] + is_cond = any( + frame_idx in obj_temp_output_dict["cond_frame_outputs"] + for obj_temp_output_dict in temp_output_dict_per_obj.values() + ) + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + @torch.inference_mode() + def reset_state(self, inference_state): + """Remove all input points or mask in all frames throughout the video.""" + self._reset_tracking_results(inference_state) + # Remove all object ids + inference_state["obj_id_to_idx"].clear() + inference_state["obj_idx_to_id"].clear() + inference_state["obj_ids"].clear() + inference_state["point_inputs_per_obj"].clear() + inference_state["mask_inputs_per_obj"].clear() + inference_state["output_dict_per_obj"].clear() + inference_state["temp_output_dict_per_obj"].clear() + inference_state["frames_tracked_per_obj"].clear() + + def _reset_tracking_results(self, inference_state): + """Reset all tracking inputs and results across the videos.""" + for v in inference_state["point_inputs_per_obj"].values(): + v.clear() + for v in inference_state["mask_inputs_per_obj"].values(): + v.clear() + for v in inference_state["output_dict_per_obj"].values(): + v["cond_frame_outputs"].clear() + v["non_cond_frame_outputs"].clear() + for v in inference_state["temp_output_dict_per_obj"].values(): + v["cond_frame_outputs"].clear() + v["non_cond_frame_outputs"].clear() + for v in inference_state["frames_tracked_per_obj"].values(): + v.clear() + + def _get_image_feature(self, inference_state, frame_idx, batch_size): + """Compute the image features on a given frame.""" + # Look up in the cache first + image, backbone_out = inference_state["cached_features"].get( + frame_idx, (None, None) + ) + if backbone_out is None: + # Cache miss -- we will run inference on a single image + device = inference_state["device"] + image = inference_state["images"][frame_idx].to(device).float().unsqueeze(0) + backbone_out = self.forward_image(image) + # Cache the most recent frame's feature (for repeated interactions with + # a frame; we can use an LRU cache for more frames in the future). + inference_state["cached_features"] = {frame_idx: (image, backbone_out)} + + # expand the features to have the same dimension as the number of objects + expanded_image = image.expand(batch_size, -1, -1, -1) + expanded_backbone_out = { + "backbone_fpn": backbone_out["backbone_fpn"].copy(), + "vision_pos_enc": backbone_out["vision_pos_enc"].copy(), + } + for i, feat in enumerate(expanded_backbone_out["backbone_fpn"]): + expanded_backbone_out["backbone_fpn"][i] = feat.expand( + batch_size, -1, -1, -1 + ) + for i, pos in enumerate(expanded_backbone_out["vision_pos_enc"]): + pos = pos.expand(batch_size, -1, -1, -1) + expanded_backbone_out["vision_pos_enc"][i] = pos + + features = self._prepare_backbone_features(expanded_backbone_out) + features = (expanded_image,) + features + return features + + def _run_single_frame_inference( + self, + inference_state, + output_dict, + frame_idx, + batch_size, + is_init_cond_frame, + point_inputs, + mask_inputs, + reverse, + run_mem_encoder, + prev_sam_mask_logits=None, + ): + """Run tracking on a single frame based on current inputs and previous memory.""" + # Retrieve correct image features + ( + _, + _, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + ) = self._get_image_feature(inference_state, frame_idx, batch_size) + + # point and mask should not appear as input simultaneously on the same frame + assert point_inputs is None or mask_inputs is None + current_out = self.track_step( + frame_idx=frame_idx, + is_init_cond_frame=is_init_cond_frame, + current_vision_feats=current_vision_feats, + current_vision_pos_embeds=current_vision_pos_embeds, + feat_sizes=feat_sizes, + point_inputs=point_inputs, + mask_inputs=mask_inputs, + output_dict=output_dict, + num_frames=inference_state["num_frames"], + track_in_reverse=reverse, + run_mem_encoder=run_mem_encoder, + prev_sam_mask_logits=prev_sam_mask_logits, + ) + + # optionally offload the output to CPU memory to save GPU space + storage_device = inference_state["storage_device"] + maskmem_features = current_out["maskmem_features"] + if maskmem_features is not None: + maskmem_features = maskmem_features.to(torch.bfloat16) + maskmem_features = maskmem_features.to(storage_device, non_blocking=True) + pred_masks_gpu = current_out["pred_masks"] + # potentially fill holes in the predicted masks + if self.fill_hole_area > 0: + pred_masks_gpu = fill_holes_in_mask_scores( + pred_masks_gpu, self.fill_hole_area + ) + pred_masks = pred_masks_gpu.to(storage_device, non_blocking=True) + # "maskmem_pos_enc" is the same across frames, so we only need to store one copy of it + maskmem_pos_enc = self._get_maskmem_pos_enc(inference_state, current_out) + # object pointer is a small tensor, so we always keep it on GPU memory for fast access + obj_ptr = current_out["obj_ptr"] + object_score_logits = current_out["object_score_logits"] + # make a compact version of this frame's output to reduce the state size + compact_current_out = { + "maskmem_features": maskmem_features, + "maskmem_pos_enc": maskmem_pos_enc, + "pred_masks": pred_masks, + "obj_ptr": obj_ptr, + "object_score_logits": object_score_logits, + } + return compact_current_out, pred_masks_gpu + + def _run_memory_encoder( + self, + inference_state, + frame_idx, + batch_size, + high_res_masks, + object_score_logits, + is_mask_from_pts, + ): + """ + Run the memory encoder on `high_res_masks`. This is usually after applying + non-overlapping constraints to object scores. Since their scores changed, their + memory also need to be computed again with the memory encoder. + """ + # Retrieve correct image features + _, _, current_vision_feats, _, feat_sizes = self._get_image_feature( + inference_state, frame_idx, batch_size + ) + maskmem_features, maskmem_pos_enc = self._encode_new_memory( + current_vision_feats=current_vision_feats, + feat_sizes=feat_sizes, + pred_masks_high_res=high_res_masks, + object_score_logits=object_score_logits, + is_mask_from_pts=is_mask_from_pts, + ) + + # optionally offload the output to CPU memory to save GPU space + storage_device = inference_state["storage_device"] + maskmem_features = maskmem_features.to(torch.bfloat16) + maskmem_features = maskmem_features.to(storage_device, non_blocking=True) + # "maskmem_pos_enc" is the same across frames, so we only need to store one copy of it + maskmem_pos_enc = self._get_maskmem_pos_enc( + inference_state, {"maskmem_pos_enc": maskmem_pos_enc} + ) + return maskmem_features, maskmem_pos_enc + + def _get_maskmem_pos_enc(self, inference_state, current_out): + """ + `maskmem_pos_enc` is the same across frames and objects, so we cache it as + a constant in the inference session to reduce session storage size. + """ + model_constants = inference_state["constants"] + # "out_maskmem_pos_enc" should be either a list of tensors or None + out_maskmem_pos_enc = current_out["maskmem_pos_enc"] + if out_maskmem_pos_enc is not None: + if "maskmem_pos_enc" not in model_constants: + assert isinstance(out_maskmem_pos_enc, list) + # only take the slice for one object, since it's same across objects + maskmem_pos_enc = [x[0:1].clone() for x in out_maskmem_pos_enc] + model_constants["maskmem_pos_enc"] = maskmem_pos_enc + else: + maskmem_pos_enc = model_constants["maskmem_pos_enc"] + # expand the cached maskmem_pos_enc to the actual batch size + batch_size = out_maskmem_pos_enc[0].size(0) + expanded_maskmem_pos_enc = [ + x.expand(batch_size, -1, -1, -1) for x in maskmem_pos_enc + ] + else: + expanded_maskmem_pos_enc = None + return expanded_maskmem_pos_enc + + @torch.inference_mode() + def remove_object(self, inference_state, obj_id, strict=False, need_output=True): + """ + Remove an object id from the tracking state. If strict is True, we check whether + the object id actually exists and raise an error if it doesn't exist. + """ + old_obj_idx_to_rm = inference_state["obj_id_to_idx"].get(obj_id, None) + updated_frames = [] + # Check whether this object_id to remove actually exists and possibly raise an error. + if old_obj_idx_to_rm is None: + if not strict: + return inference_state["obj_ids"], updated_frames + raise RuntimeError( + f"Cannot remove object id {obj_id} as it doesn't exist. " + f"All existing object ids: {inference_state['obj_ids']}." + ) + + # If this is the only remaining object id, we simply reset the state. + if len(inference_state["obj_id_to_idx"]) == 1: + self.reset_state(inference_state) + return inference_state["obj_ids"], updated_frames + + # There are still remaining objects after removing this object id. In this case, + # we need to delete the object storage from inference state tensors. + # Step 0: clear the input on those frames where this object id has point or mask input + # (note that this step is required as it might downgrade conditioning frames to + # non-conditioning ones) + obj_input_frames_inds = set() + obj_input_frames_inds.update( + inference_state["point_inputs_per_obj"][old_obj_idx_to_rm] + ) + obj_input_frames_inds.update( + inference_state["mask_inputs_per_obj"][old_obj_idx_to_rm] + ) + for frame_idx in obj_input_frames_inds: + self.clear_all_prompts_in_frame( + inference_state, frame_idx, obj_id, need_output=False + ) + + # Step 1: Update the object id mapping (note that it must be done after Step 0, + # since Step 0 still requires the old object id mappings in inference_state) + old_obj_ids = inference_state["obj_ids"] + old_obj_inds = list(range(len(old_obj_ids))) + remain_old_obj_inds = old_obj_inds.copy() + remain_old_obj_inds.remove(old_obj_idx_to_rm) + new_obj_ids = [old_obj_ids[old_idx] for old_idx in remain_old_obj_inds] + new_obj_inds = list(range(len(new_obj_ids))) + # build new mappings + old_idx_to_new_idx = dict(zip(remain_old_obj_inds, new_obj_inds)) + inference_state["obj_id_to_idx"] = dict(zip(new_obj_ids, new_obj_inds)) + inference_state["obj_idx_to_id"] = dict(zip(new_obj_inds, new_obj_ids)) + inference_state["obj_ids"] = new_obj_ids + + # Step 2: For per-object tensor storage, we shift their obj_idx in the dict keys. + def _map_keys(container): + new_kvs = [] + for k in old_obj_inds: + v = container.pop(k) + if k in old_idx_to_new_idx: + new_kvs.append((old_idx_to_new_idx[k], v)) + container.update(new_kvs) + + _map_keys(inference_state["point_inputs_per_obj"]) + _map_keys(inference_state["mask_inputs_per_obj"]) + _map_keys(inference_state["output_dict_per_obj"]) + _map_keys(inference_state["temp_output_dict_per_obj"]) + _map_keys(inference_state["frames_tracked_per_obj"]) + + # Step 3: Further collect the outputs on those frames in `obj_input_frames_inds`, which + # could show an updated mask for objects previously occluded by the object being removed + if need_output: + temp_output_dict_per_obj = inference_state["temp_output_dict_per_obj"] + for frame_idx in obj_input_frames_inds: + is_cond = any( + frame_idx in obj_temp_output_dict["cond_frame_outputs"] + for obj_temp_output_dict in temp_output_dict_per_obj.values() + ) + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + updated_frames.append((frame_idx, video_res_masks)) + + return inference_state["obj_ids"], updated_frames + + def _clear_non_cond_mem_around_input(self, inference_state, frame_idx): + """ + Remove the non-conditioning memory around the input frame. When users provide + correction clicks, the surrounding frames' non-conditioning memories can still + contain outdated object appearance information and could confuse the model. + + This method clears those non-conditioning memories surrounding the interacted + frame to avoid giving the model both old and new information about the object. + """ + r = self.memory_temporal_stride_for_eval + frame_idx_begin = frame_idx - r * self.num_maskmem + frame_idx_end = frame_idx + r * self.num_maskmem + batch_size = self._get_obj_num(inference_state) + for obj_idx in range(batch_size): + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + non_cond_frame_outputs = obj_output_dict["non_cond_frame_outputs"] + for t in range(frame_idx_begin, frame_idx_end + 1): + non_cond_frame_outputs.pop(t, None) + + +class SAM2VideoPredictorVOS(SAM2VideoPredictor): + """Optimized for the VOS setting""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._compile_all_components() + + def _compile_all_components(self): + print("Compiling all components for VOS setting. First time may be very slow.") + self.memory_encoder.forward = torch.compile( + self.memory_encoder.forward, + mode="max-autotune", + fullgraph=True, + dynamic=False, + ) + + self.memory_attention.forward = torch.compile( + self.memory_attention.forward, + mode="max-autotune", + fullgraph=True, + dynamic=True, # Num. of memories varies + ) + + self.sam_prompt_encoder.forward = torch.compile( + self.sam_prompt_encoder.forward, + mode="max-autotune", + fullgraph=True, + dynamic=False, # Accuracy regression on True + ) + + self.sam_mask_decoder.forward = torch.compile( + self.sam_mask_decoder.forward, + mode="max-autotune", + fullgraph=True, + dynamic=False, # Accuracy regression on True + ) + + def forward_image(self, img_batch: torch.Tensor): + """ + Identical to the corresponding method in the parent (SAM2VideoPredictor), but + cloning the backbone features and pos encoding to enable compilation. + """ + backbone_out = self.image_encoder(img_batch) + if self.use_high_res_features_in_sam: + # precompute projected level 0 and level 1 features in SAM decoder + # to avoid running it again on every SAM click + backbone_out["backbone_fpn"][0] = self.sam_mask_decoder.conv_s0( + backbone_out["backbone_fpn"][0] + ) + backbone_out["backbone_fpn"][1] = self.sam_mask_decoder.conv_s1( + backbone_out["backbone_fpn"][1] + ) + # Clone to help torch.compile + for i in range(len(backbone_out["backbone_fpn"])): + backbone_out["backbone_fpn"][i] = backbone_out["backbone_fpn"][i].clone() + backbone_out["vision_pos_enc"][i] = backbone_out["vision_pos_enc"][ + i + ].clone() + return backbone_out + + def _forward_sam_heads( + self, + backbone_features, + point_inputs=None, + mask_inputs=None, + high_res_features=None, + multimask_output=False, + ): + """ + Identical to the corresponding method in the parent (SAM2VideoPredictor), but + cloning the outputs of prompt_encoder and mask_decoder to enable compilation. + """ + B = backbone_features.size(0) + device = backbone_features.device + assert backbone_features.size(1) == self.sam_prompt_embed_dim + assert backbone_features.size(2) == self.sam_image_embedding_size + assert backbone_features.size(3) == self.sam_image_embedding_size + + # a) Handle point prompts + if point_inputs is not None: + sam_point_coords = point_inputs["point_coords"] + sam_point_labels = point_inputs["point_labels"] + assert sam_point_coords.size(0) == B and sam_point_labels.size(0) == B + else: + # If no points are provide, pad with an empty point (with label -1) + sam_point_coords = torch.zeros(B, 1, 2, device=device) + sam_point_labels = -torch.ones(B, 1, dtype=torch.int32, device=device) + + # b) Handle mask prompts + if mask_inputs is not None: + # If mask_inputs is provided, downsize it into low-res mask input if needed + # and feed it as a dense mask prompt into the SAM mask encoder + assert len(mask_inputs.shape) == 4 and mask_inputs.shape[:2] == (B, 1) + if mask_inputs.shape[-2:] != self.sam_prompt_encoder.mask_input_size: + sam_mask_prompt = F.interpolate( + mask_inputs.float(), + size=self.sam_prompt_encoder.mask_input_size, + align_corners=False, + mode="bilinear", + antialias=True, # use antialias for downsampling + ) + else: + sam_mask_prompt = mask_inputs + else: + # Otherwise, simply feed None (and SAM's prompt encoder will add + # a learned `no_mask_embed` to indicate no mask input in this case). + sam_mask_prompt = None + + sparse_embeddings, dense_embeddings = self.sam_prompt_encoder( + points=(sam_point_coords, sam_point_labels), + boxes=None, + masks=sam_mask_prompt, + ) + # Clone image_pe and the outputs of sam_prompt_encoder + # to enable compilation + sparse_embeddings = sparse_embeddings.clone() + dense_embeddings = dense_embeddings.clone() + image_pe = self.sam_prompt_encoder.get_dense_pe().clone() + ( + low_res_multimasks, + ious, + sam_output_tokens, + object_score_logits, + ) = self.sam_mask_decoder( + image_embeddings=backbone_features, + image_pe=image_pe, + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + repeat_image=False, # the image is already batched + high_res_features=high_res_features, + ) + # Clone the output of sam_mask_decoder + # to enable compilation + low_res_multimasks = low_res_multimasks.clone() + ious = ious.clone() + sam_output_tokens = sam_output_tokens.clone() + object_score_logits = object_score_logits.clone() + + if self.pred_obj_scores: + is_obj_appearing = object_score_logits > 0 + + # Mask used for spatial memories is always a *hard* choice between obj and no obj, + # consistent with the actual mask prediction + low_res_multimasks = torch.where( + is_obj_appearing[:, None, None], + low_res_multimasks, + NO_OBJ_SCORE, + ) + + # convert masks from possibly bfloat16 (or float16) to float32 + # (older PyTorch versions before 2.1 don't support `interpolate` on bf16) + low_res_multimasks = low_res_multimasks.float() + high_res_multimasks = F.interpolate( + low_res_multimasks, + size=(self.image_size, self.image_size), + mode="bilinear", + align_corners=False, + ) + + sam_output_token = sam_output_tokens[:, 0] + if multimask_output: + # take the best mask prediction (with the highest IoU estimation) + best_iou_inds = torch.argmax(ious, dim=-1) + batch_inds = torch.arange(B, device=device) + low_res_masks = low_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1) + high_res_masks = high_res_multimasks[batch_inds, best_iou_inds].unsqueeze(1) + if sam_output_tokens.size(1) > 1: + sam_output_token = sam_output_tokens[batch_inds, best_iou_inds] + else: + low_res_masks, high_res_masks = low_res_multimasks, high_res_multimasks + + # Extract object pointer from the SAM output token (with occlusion handling) + obj_ptr = self.obj_ptr_proj(sam_output_token) + if self.pred_obj_scores: + # Allow *soft* no obj ptr, unlike for masks + if self.soft_no_obj_ptr: + lambda_is_obj_appearing = object_score_logits.sigmoid() + else: + lambda_is_obj_appearing = is_obj_appearing.float() + + if self.fixed_no_obj_ptr: + obj_ptr = lambda_is_obj_appearing * obj_ptr + obj_ptr = obj_ptr + (1 - lambda_is_obj_appearing) * self.no_obj_ptr + + return ( + low_res_multimasks, + high_res_multimasks, + ious, + low_res_masks, + high_res_masks, + obj_ptr, + object_score_logits, + ) + + def _encode_new_memory( + self, + current_vision_feats, + feat_sizes, + pred_masks_high_res, + object_score_logits, + is_mask_from_pts, + ): + """ + Identical to the corresponding method in the parent (SAM2VideoPredictor), but + cloning the memories and their pos enc to enable compilation. + """ + B = current_vision_feats[-1].size(1) # batch size on this frame + C = self.hidden_dim + H, W = feat_sizes[-1] # top-level (lowest-resolution) feature size + # top-level feature, (HW)BC => BCHW + pix_feat = current_vision_feats[-1].permute(1, 2, 0).view(B, C, H, W) + if self.non_overlap_masks_for_mem_enc and not self.training: + # optionally, apply non-overlapping constraints to the masks (it's applied + # in the batch dimension and should only be used during eval, where all + # the objects come from the same video under batch size 1). + pred_masks_high_res = self._apply_non_overlapping_constraints( + pred_masks_high_res + ) + # scale the raw mask logits with a temperature before applying sigmoid + binarize = self.binarize_mask_from_pts_for_mem_enc and is_mask_from_pts + if binarize and not self.training: + mask_for_mem = (pred_masks_high_res > 0).float() + else: + # apply sigmoid on the raw mask logits to turn them into range (0, 1) + mask_for_mem = torch.sigmoid(pred_masks_high_res) + # apply scale and bias terms to the sigmoid probabilities + if self.sigmoid_scale_for_mem_enc != 1.0: + mask_for_mem = mask_for_mem * self.sigmoid_scale_for_mem_enc + if self.sigmoid_bias_for_mem_enc != 0.0: + mask_for_mem = mask_for_mem + self.sigmoid_bias_for_mem_enc + maskmem_out = self.memory_encoder( + pix_feat, mask_for_mem, skip_mask_sigmoid=True # sigmoid already applied + ) + # Clone the feats and pos_enc to enable compilation + maskmem_features = maskmem_out["vision_features"].clone() + maskmem_pos_enc = [m.clone() for m in maskmem_out["vision_pos_enc"]] + # add a no-object embedding to the spatial memory to indicate that the frame + # is predicted to be occluded (i.e. no object is appearing in the frame) + if self.no_obj_embed_spatial is not None: + is_obj_appearing = (object_score_logits > 0).float() + maskmem_features += ( + 1 - is_obj_appearing[..., None, None] + ) * self.no_obj_embed_spatial[..., None, None].expand( + *maskmem_features.shape + ) + + return maskmem_features, maskmem_pos_enc diff --git a/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor_legacy.py b/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor_legacy.py new file mode 100644 index 00000000..c7e01ccf --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/sam2_video_predictor_legacy.py @@ -0,0 +1,1172 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import warnings +from collections import OrderedDict + +import torch + +from tqdm import tqdm + +from sam2.modeling.sam2_base import NO_OBJ_SCORE, SAM2Base +from sam2.utils.misc import concat_points, fill_holes_in_mask_scores, load_video_frames + + +class SAM2VideoPredictor(SAM2Base): + """The predictor class to handle user interactions and manage inference states.""" + + def __init__( + self, + fill_hole_area=0, + # whether to apply non-overlapping constraints on the output object masks + non_overlap_masks=False, + # whether to clear non-conditioning memory of the surrounding frames (which may contain outdated information) after adding correction clicks; + # note that this would only apply to *single-object tracking* unless `clear_non_cond_mem_for_multi_obj` is also set to True) + clear_non_cond_mem_around_input=False, + # whether to also clear non-conditioning memory of the surrounding frames (only effective when `clear_non_cond_mem_around_input` is True). + clear_non_cond_mem_for_multi_obj=False, + # if `add_all_frames_to_correct_as_cond` is True, we also append to the conditioning frame list any frame that receives a later correction click + # if `add_all_frames_to_correct_as_cond` is False, we conditioning frame list to only use those initial conditioning frames + add_all_frames_to_correct_as_cond=False, + **kwargs, + ): + super().__init__(**kwargs) + self.fill_hole_area = fill_hole_area + self.non_overlap_masks = non_overlap_masks + self.clear_non_cond_mem_around_input = clear_non_cond_mem_around_input + self.clear_non_cond_mem_for_multi_obj = clear_non_cond_mem_for_multi_obj + self.add_all_frames_to_correct_as_cond = add_all_frames_to_correct_as_cond + + @torch.inference_mode() + def init_state( + self, + video_path, + offload_video_to_cpu=False, + offload_state_to_cpu=False, + async_loading_frames=False, + ): + """Initialize an inference state.""" + compute_device = self.device # device of the model + images, video_height, video_width = load_video_frames( + video_path=video_path, + image_size=self.image_size, + offload_video_to_cpu=offload_video_to_cpu, + async_loading_frames=async_loading_frames, + compute_device=compute_device, + ) + inference_state = {} + inference_state["images"] = images + inference_state["num_frames"] = len(images) + # whether to offload the video frames to CPU memory + # turning on this option saves the GPU memory with only a very small overhead + inference_state["offload_video_to_cpu"] = offload_video_to_cpu + # whether to offload the inference state to CPU memory + # turning on this option saves the GPU memory at the cost of a lower tracking fps + # (e.g. in a test case of 768x768 model, fps dropped from 27 to 24 when tracking one object + # and from 24 to 21 when tracking two objects) + inference_state["offload_state_to_cpu"] = offload_state_to_cpu + # the original video height and width, used for resizing final output scores + inference_state["video_height"] = video_height + inference_state["video_width"] = video_width + inference_state["device"] = compute_device + if offload_state_to_cpu: + inference_state["storage_device"] = torch.device("cpu") + else: + inference_state["storage_device"] = compute_device + # inputs on each frame + inference_state["point_inputs_per_obj"] = {} + inference_state["mask_inputs_per_obj"] = {} + # visual features on a small number of recently visited frames for quick interactions + inference_state["cached_features"] = {} + # values that don't change across frames (so we only need to hold one copy of them) + inference_state["constants"] = {} + # mapping between client-side object id and model-side object index + inference_state["obj_id_to_idx"] = OrderedDict() + inference_state["obj_idx_to_id"] = OrderedDict() + inference_state["obj_ids"] = [] + # A storage to hold the model's tracking results and states on each frame + inference_state["output_dict"] = { + "cond_frame_outputs": {}, # dict containing {frame_idx: } + "non_cond_frame_outputs": {}, # dict containing {frame_idx: } + } + # Slice (view) of each object tracking results, sharing the same memory with "output_dict" + inference_state["output_dict_per_obj"] = {} + # A temporary storage to hold new outputs when user interact with a frame + # to add clicks or mask (it's merged into "output_dict" before propagation starts) + inference_state["temp_output_dict_per_obj"] = {} + # Frames that already holds consolidated outputs from click or mask inputs + # (we directly use their consolidated outputs during tracking) + inference_state["consolidated_frame_inds"] = { + "cond_frame_outputs": set(), # set containing frame indices + "non_cond_frame_outputs": set(), # set containing frame indices + } + # metadata for each tracking frame (e.g. which direction it's tracked) + inference_state["tracking_has_started"] = False + inference_state["frames_already_tracked"] = {} + # Warm up the visual backbone and cache the image feature on frame 0 + self._get_image_feature(inference_state, frame_idx=0, batch_size=1) + return inference_state + + @classmethod + def from_pretrained(cls, model_id: str, **kwargs) -> "SAM2VideoPredictor": + """ + Load a pretrained model from the Hugging Face hub. + + Arguments: + model_id (str): The Hugging Face repository ID. + **kwargs: Additional arguments to pass to the model constructor. + + Returns: + (SAM2VideoPredictor): The loaded model. + """ + from sam2.build_sam import build_sam2_video_predictor_hf + + sam_model = build_sam2_video_predictor_hf(model_id, **kwargs) + return sam_model + + def _obj_id_to_idx(self, inference_state, obj_id): + """Map client-side object id to model-side object index.""" + obj_idx = inference_state["obj_id_to_idx"].get(obj_id, None) + if obj_idx is not None: + return obj_idx + + # This is a new object id not sent to the server before. We only allow adding + # new objects *before* the tracking starts. + allow_new_object = not inference_state["tracking_has_started"] + if allow_new_object: + # get the next object slot + obj_idx = len(inference_state["obj_id_to_idx"]) + inference_state["obj_id_to_idx"][obj_id] = obj_idx + inference_state["obj_idx_to_id"][obj_idx] = obj_id + inference_state["obj_ids"] = list(inference_state["obj_id_to_idx"]) + # set up input and output structures for this object + inference_state["point_inputs_per_obj"][obj_idx] = {} + inference_state["mask_inputs_per_obj"][obj_idx] = {} + inference_state["output_dict_per_obj"][obj_idx] = { + "cond_frame_outputs": {}, # dict containing {frame_idx: } + "non_cond_frame_outputs": {}, # dict containing {frame_idx: } + } + inference_state["temp_output_dict_per_obj"][obj_idx] = { + "cond_frame_outputs": {}, # dict containing {frame_idx: } + "non_cond_frame_outputs": {}, # dict containing {frame_idx: } + } + return obj_idx + else: + raise RuntimeError( + f"Cannot add new object id {obj_id} after tracking starts. " + f"All existing object ids: {inference_state['obj_ids']}. " + f"Please call 'reset_state' to restart from scratch." + ) + + def _obj_idx_to_id(self, inference_state, obj_idx): + """Map model-side object index to client-side object id.""" + return inference_state["obj_idx_to_id"][obj_idx] + + def _get_obj_num(self, inference_state): + """Get the total number of unique object ids received so far in this session.""" + return len(inference_state["obj_idx_to_id"]) + + @torch.inference_mode() + def add_new_points_or_box( + self, + inference_state, + frame_idx, + obj_id, + points=None, + labels=None, + clear_old_points=True, + normalize_coords=True, + box=None, + ): + """Add new points to a frame.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + point_inputs_per_frame = inference_state["point_inputs_per_obj"][obj_idx] + mask_inputs_per_frame = inference_state["mask_inputs_per_obj"][obj_idx] + + if (points is not None) != (labels is not None): + raise ValueError("points and labels must be provided together") + if points is None and box is None: + raise ValueError("at least one of points or box must be provided as input") + + if points is None: + points = torch.zeros(0, 2, dtype=torch.float32) + elif not isinstance(points, torch.Tensor): + points = torch.tensor(points, dtype=torch.float32) + if labels is None: + labels = torch.zeros(0, dtype=torch.int32) + elif not isinstance(labels, torch.Tensor): + labels = torch.tensor(labels, dtype=torch.int32) + if points.dim() == 2: + points = points.unsqueeze(0) # add batch dimension + if labels.dim() == 1: + labels = labels.unsqueeze(0) # add batch dimension + + # If `box` is provided, we add it as the first two points with labels 2 and 3 + # along with the user-provided points (consistent with how SAM 2 is trained). + if box is not None: + if not clear_old_points: + raise ValueError( + "cannot add box without clearing old points, since " + "box prompt must be provided before any point prompt " + "(please use clear_old_points=True instead)" + ) + if inference_state["tracking_has_started"]: + warnings.warn( + "You are adding a box after tracking starts. SAM 2 may not always be " + "able to incorporate a box prompt for *refinement*. If you intend to " + "use box prompt as an *initial* input before tracking, please call " + "'reset_state' on the inference state to restart from scratch.", + category=UserWarning, + stacklevel=2, + ) + if not isinstance(box, torch.Tensor): + box = torch.tensor(box, dtype=torch.float32, device=points.device) + box_coords = box.reshape(1, 2, 2) + box_labels = torch.tensor([2, 3], dtype=torch.int32, device=labels.device) + box_labels = box_labels.reshape(1, 2) + points = torch.cat([box_coords, points], dim=1) + labels = torch.cat([box_labels, labels], dim=1) + + if normalize_coords: + video_H = inference_state["video_height"] + video_W = inference_state["video_width"] + points = points / torch.tensor([video_W, video_H]).to(points.device) + # scale the (normalized) coordinates by the model's internal image size + points = points * self.image_size + points = points.to(inference_state["device"]) + labels = labels.to(inference_state["device"]) + + if not clear_old_points: + point_inputs = point_inputs_per_frame.get(frame_idx, None) + else: + point_inputs = None + point_inputs = concat_points(point_inputs, points, labels) + + point_inputs_per_frame[frame_idx] = point_inputs + mask_inputs_per_frame.pop(frame_idx, None) + # If this frame hasn't been tracked before, we treat it as an initial conditioning + # frame, meaning that the inputs points are to generate segments on this frame without + # using any memory from other frames, like in SAM. Otherwise (if it has been tracked), + # the input points will be used to correct the already tracked masks. + is_init_cond_frame = frame_idx not in inference_state["frames_already_tracked"] + # whether to track in reverse time order + if is_init_cond_frame: + reverse = False + else: + reverse = inference_state["frames_already_tracked"][frame_idx]["reverse"] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + # Add a frame to conditioning output if it's an initial conditioning frame or + # if the model sees all frames receiving clicks/mask as conditioning frames. + is_cond = is_init_cond_frame or self.add_all_frames_to_correct_as_cond + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + + # Get any previously predicted mask logits on this object and feed it along with + # the new clicks into the SAM mask decoder. + prev_sam_mask_logits = None + # lookup temporary output dict first, which contains the most recent output + # (if not found, then lookup conditioning and non-conditioning frame output) + prev_out = obj_temp_output_dict[storage_key].get(frame_idx) + if prev_out is None: + prev_out = obj_output_dict["cond_frame_outputs"].get(frame_idx) + if prev_out is None: + prev_out = obj_output_dict["non_cond_frame_outputs"].get(frame_idx) + + if prev_out is not None and prev_out["pred_masks"] is not None: + device = inference_state["device"] + prev_sam_mask_logits = prev_out["pred_masks"].to(device, non_blocking=True) + # Clamp the scale of prev_sam_mask_logits to avoid rare numerical issues. + prev_sam_mask_logits = torch.clamp(prev_sam_mask_logits, -32.0, 32.0) + current_out, _ = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=obj_output_dict, # run on the slice of a single object + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + is_init_cond_frame=is_init_cond_frame, + point_inputs=point_inputs, + mask_inputs=None, + reverse=reverse, + # Skip the memory encoder when adding clicks or mask. We execute the memory encoder + # at the beginning of `propagate_in_video` (after user finalize their clicks). This + # allows us to enforce non-overlapping constraints on all objects before encoding + # them into memory. + run_mem_encoder=False, + prev_sam_mask_logits=prev_sam_mask_logits, + ) + # Add the output to the output dict (to be used as future memory) + obj_temp_output_dict[storage_key][frame_idx] = current_out + + # Resize the output mask to the original video resolution + obj_ids = inference_state["obj_ids"] + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + run_mem_encoder=False, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + def add_new_points(self, *args, **kwargs): + """Deprecated method. Please use `add_new_points_or_box` instead.""" + return self.add_new_points_or_box(*args, **kwargs) + + @torch.inference_mode() + def add_new_mask( + self, + inference_state, + frame_idx, + obj_id, + mask, + ): + """Add new mask to a frame.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + point_inputs_per_frame = inference_state["point_inputs_per_obj"][obj_idx] + mask_inputs_per_frame = inference_state["mask_inputs_per_obj"][obj_idx] + + if not isinstance(mask, torch.Tensor): + mask = torch.tensor(mask, dtype=torch.bool) + assert mask.dim() == 2 + mask_H, mask_W = mask.shape + mask_inputs_orig = mask[None, None] # add batch and channel dimension + mask_inputs_orig = mask_inputs_orig.float().to(inference_state["device"]) + + # resize the mask if it doesn't match the model's image size + if mask_H != self.image_size or mask_W != self.image_size: + mask_inputs = torch.nn.functional.interpolate( + mask_inputs_orig, + size=(self.image_size, self.image_size), + align_corners=False, + mode="bilinear", + antialias=True, # use antialias for downsampling + ) + mask_inputs = (mask_inputs >= 0.5).float() + else: + mask_inputs = mask_inputs_orig + + mask_inputs_per_frame[frame_idx] = mask_inputs + point_inputs_per_frame.pop(frame_idx, None) + # If this frame hasn't been tracked before, we treat it as an initial conditioning + # frame, meaning that the inputs points are to generate segments on this frame without + # using any memory from other frames, like in SAM. Otherwise (if it has been tracked), + # the input points will be used to correct the already tracked masks. + is_init_cond_frame = frame_idx not in inference_state["frames_already_tracked"] + # whether to track in reverse time order + if is_init_cond_frame: + reverse = False + else: + reverse = inference_state["frames_already_tracked"][frame_idx]["reverse"] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + # Add a frame to conditioning output if it's an initial conditioning frame or + # if the model sees all frames receiving clicks/mask as conditioning frames. + is_cond = is_init_cond_frame or self.add_all_frames_to_correct_as_cond + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + + current_out, _ = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=obj_output_dict, # run on the slice of a single object + frame_idx=frame_idx, + batch_size=1, # run on the slice of a single object + is_init_cond_frame=is_init_cond_frame, + point_inputs=None, + mask_inputs=mask_inputs, + reverse=reverse, + # Skip the memory encoder when adding clicks or mask. We execute the memory encoder + # at the beginning of `propagate_in_video` (after user finalize their clicks). This + # allows us to enforce non-overlapping constraints on all objects before encoding + # them into memory. + run_mem_encoder=False, + ) + # Add the output to the output dict (to be used as future memory) + obj_temp_output_dict[storage_key][frame_idx] = current_out + + # Resize the output mask to the original video resolution + obj_ids = inference_state["obj_ids"] + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + run_mem_encoder=False, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + def _get_orig_video_res_output(self, inference_state, any_res_masks): + """ + Resize the object scores to the original video resolution (video_res_masks) + and apply non-overlapping constraints for final output. + """ + device = inference_state["device"] + video_H = inference_state["video_height"] + video_W = inference_state["video_width"] + any_res_masks = any_res_masks.to(device, non_blocking=True) + if any_res_masks.shape[-2:] == (video_H, video_W): + video_res_masks = any_res_masks + else: + video_res_masks = torch.nn.functional.interpolate( + any_res_masks, + size=(video_H, video_W), + mode="bilinear", + align_corners=False, + ) + if self.non_overlap_masks: + video_res_masks = self._apply_non_overlapping_constraints(video_res_masks) + return any_res_masks, video_res_masks + + def _consolidate_temp_output_across_obj( + self, + inference_state, + frame_idx, + is_cond, + run_mem_encoder, + consolidate_at_video_res=False, + ): + """ + Consolidate the per-object temporary outputs in `temp_output_dict_per_obj` on + a frame into a single output for all objects, including + 1) fill any missing objects either from `output_dict_per_obj` (if they exist in + `output_dict_per_obj` for this frame) or leave them as placeholder values + (if they don't exist in `output_dict_per_obj` for this frame); + 2) if specified, rerun memory encoder after apply non-overlapping constraints + on the object scores. + """ + batch_size = self._get_obj_num(inference_state) + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + # Optionally, we allow consolidating the temporary outputs at the original + # video resolution (to provide a better editing experience for mask prompts). + if consolidate_at_video_res: + assert not run_mem_encoder, "memory encoder cannot run at video resolution" + consolidated_H = inference_state["video_height"] + consolidated_W = inference_state["video_width"] + consolidated_mask_key = "pred_masks_video_res" + else: + consolidated_H = consolidated_W = self.image_size // 4 + consolidated_mask_key = "pred_masks" + + # Initialize `consolidated_out`. Its "maskmem_features" and "maskmem_pos_enc" + # will be added when rerunning the memory encoder after applying non-overlapping + # constraints to object scores. Its "pred_masks" are prefilled with a large + # negative value (NO_OBJ_SCORE) to represent missing objects. + consolidated_out = { + "maskmem_features": None, + "maskmem_pos_enc": None, + consolidated_mask_key: torch.full( + size=(batch_size, 1, consolidated_H, consolidated_W), + fill_value=NO_OBJ_SCORE, + dtype=torch.float32, + device=inference_state["storage_device"], + ), + "obj_ptr": torch.full( + size=(batch_size, self.hidden_dim), + fill_value=NO_OBJ_SCORE, + dtype=torch.float32, + device=inference_state["device"], + ), + "object_score_logits": torch.full( + size=(batch_size, 1), + # default to 10.0 for object_score_logits, i.e. assuming the object is + # present as sigmoid(10)=1, same as in `predict_masks` of `MaskDecoder` + fill_value=10.0, + dtype=torch.float32, + device=inference_state["device"], + ), + } + empty_mask_ptr = None + for obj_idx in range(batch_size): + obj_temp_output_dict = inference_state["temp_output_dict_per_obj"][obj_idx] + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx] + out = obj_temp_output_dict[storage_key].get(frame_idx, None) + # If the object doesn't appear in "temp_output_dict_per_obj" on this frame, + # we fall back and look up its previous output in "output_dict_per_obj". + # We look up both "cond_frame_outputs" and "non_cond_frame_outputs" in + # "output_dict_per_obj" to find a previous output for this object. + if out is None: + out = obj_output_dict["cond_frame_outputs"].get(frame_idx, None) + if out is None: + out = obj_output_dict["non_cond_frame_outputs"].get(frame_idx, None) + # If the object doesn't appear in "output_dict_per_obj" either, we skip it + # and leave its mask scores to the default scores (i.e. the NO_OBJ_SCORE + # placeholder above) and set its object pointer to be a dummy pointer. + if out is None: + # Fill in dummy object pointers for those objects without any inputs or + # tracking outcomes on this frame (only do it under `run_mem_encoder=True`, + # i.e. when we need to build the memory for tracking). + if run_mem_encoder: + if empty_mask_ptr is None: + empty_mask_ptr = self._get_empty_mask_ptr( + inference_state, frame_idx + ) + # fill object pointer with a dummy pointer (based on an empty mask) + consolidated_out["obj_ptr"][obj_idx : obj_idx + 1] = empty_mask_ptr + continue + # Add the temporary object output mask to consolidated output mask + obj_mask = out["pred_masks"] + consolidated_pred_masks = consolidated_out[consolidated_mask_key] + if obj_mask.shape[-2:] == consolidated_pred_masks.shape[-2:]: + consolidated_pred_masks[obj_idx : obj_idx + 1] = obj_mask + else: + # Resize first if temporary object mask has a different resolution + resized_obj_mask = torch.nn.functional.interpolate( + obj_mask, + size=consolidated_pred_masks.shape[-2:], + mode="bilinear", + align_corners=False, + ) + consolidated_pred_masks[obj_idx : obj_idx + 1] = resized_obj_mask + consolidated_out["obj_ptr"][obj_idx : obj_idx + 1] = out["obj_ptr"] + consolidated_out["object_score_logits"][obj_idx : obj_idx + 1] = out[ + "object_score_logits" + ] + + # Optionally, apply non-overlapping constraints on the consolidated scores + # and rerun the memory encoder + if run_mem_encoder: + device = inference_state["device"] + high_res_masks = torch.nn.functional.interpolate( + consolidated_out["pred_masks"].to(device, non_blocking=True), + size=(self.image_size, self.image_size), + mode="bilinear", + align_corners=False, + ) + if self.non_overlap_masks_for_mem_enc: + high_res_masks = self._apply_non_overlapping_constraints(high_res_masks) + maskmem_features, maskmem_pos_enc = self._run_memory_encoder( + inference_state=inference_state, + frame_idx=frame_idx, + batch_size=batch_size, + high_res_masks=high_res_masks, + object_score_logits=consolidated_out["object_score_logits"], + is_mask_from_pts=True, # these frames are what the user interacted with + ) + consolidated_out["maskmem_features"] = maskmem_features + consolidated_out["maskmem_pos_enc"] = maskmem_pos_enc + + return consolidated_out + + def _get_empty_mask_ptr(self, inference_state, frame_idx): + """Get a dummy object pointer based on an empty mask on the current frame.""" + # A dummy (empty) mask with a single object + batch_size = 1 + mask_inputs = torch.zeros( + (batch_size, 1, self.image_size, self.image_size), + dtype=torch.float32, + device=inference_state["device"], + ) + + # Retrieve correct image features + ( + _, + _, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + ) = self._get_image_feature(inference_state, frame_idx, batch_size) + + # Feed the empty mask and image feature above to get a dummy object pointer + current_out = self.track_step( + frame_idx=frame_idx, + is_init_cond_frame=True, + current_vision_feats=current_vision_feats, + current_vision_pos_embeds=current_vision_pos_embeds, + feat_sizes=feat_sizes, + point_inputs=None, + mask_inputs=mask_inputs, + output_dict={}, + num_frames=inference_state["num_frames"], + track_in_reverse=False, + run_mem_encoder=False, + prev_sam_mask_logits=None, + ) + return current_out["obj_ptr"] + + @torch.inference_mode() + def propagate_in_video_preflight(self, inference_state): + """Prepare inference_state and consolidate temporary outputs before tracking.""" + # Tracking has started and we don't allow adding new objects until session is reset. + inference_state["tracking_has_started"] = True + batch_size = self._get_obj_num(inference_state) + + # Consolidate per-object temporary outputs in "temp_output_dict_per_obj" and + # add them into "output_dict". + temp_output_dict_per_obj = inference_state["temp_output_dict_per_obj"] + output_dict = inference_state["output_dict"] + # "consolidated_frame_inds" contains indices of those frames where consolidated + # temporary outputs have been added (either in this call or any previous calls + # to `propagate_in_video_preflight`). + consolidated_frame_inds = inference_state["consolidated_frame_inds"] + for is_cond in [False, True]: + # Separately consolidate conditioning and non-conditioning temp outputs + storage_key = "cond_frame_outputs" if is_cond else "non_cond_frame_outputs" + # Find all the frames that contain temporary outputs for any objects + # (these should be the frames that have just received clicks for mask inputs + # via `add_new_points_or_box` or `add_new_mask`) + temp_frame_inds = set() + for obj_temp_output_dict in temp_output_dict_per_obj.values(): + temp_frame_inds.update(obj_temp_output_dict[storage_key].keys()) + consolidated_frame_inds[storage_key].update(temp_frame_inds) + # consolidate the temporary output across all objects on this frame + for frame_idx in temp_frame_inds: + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, frame_idx, is_cond=is_cond, run_mem_encoder=True + ) + # merge them into "output_dict" and also create per-object slices + output_dict[storage_key][frame_idx] = consolidated_out + self._add_output_per_object( + inference_state, frame_idx, consolidated_out, storage_key + ) + clear_non_cond_mem = self.clear_non_cond_mem_around_input and ( + self.clear_non_cond_mem_for_multi_obj or batch_size <= 1 + ) + if clear_non_cond_mem: + # clear non-conditioning memory of the surrounding frames + self._clear_non_cond_mem_around_input(inference_state, frame_idx) + + # clear temporary outputs in `temp_output_dict_per_obj` + for obj_temp_output_dict in temp_output_dict_per_obj.values(): + obj_temp_output_dict[storage_key].clear() + + # edge case: if an output is added to "cond_frame_outputs", we remove any prior + # output on the same frame in "non_cond_frame_outputs" + for frame_idx in output_dict["cond_frame_outputs"]: + output_dict["non_cond_frame_outputs"].pop(frame_idx, None) + for obj_output_dict in inference_state["output_dict_per_obj"].values(): + for frame_idx in obj_output_dict["cond_frame_outputs"]: + obj_output_dict["non_cond_frame_outputs"].pop(frame_idx, None) + for frame_idx in consolidated_frame_inds["cond_frame_outputs"]: + assert frame_idx in output_dict["cond_frame_outputs"] + consolidated_frame_inds["non_cond_frame_outputs"].discard(frame_idx) + + # Make sure that the frame indices in "consolidated_frame_inds" are exactly those frames + # with either points or mask inputs (which should be true under a correct workflow). + all_consolidated_frame_inds = ( + consolidated_frame_inds["cond_frame_outputs"] + | consolidated_frame_inds["non_cond_frame_outputs"] + ) + input_frames_inds = set() + for point_inputs_per_frame in inference_state["point_inputs_per_obj"].values(): + input_frames_inds.update(point_inputs_per_frame.keys()) + for mask_inputs_per_frame in inference_state["mask_inputs_per_obj"].values(): + input_frames_inds.update(mask_inputs_per_frame.keys()) + assert all_consolidated_frame_inds == input_frames_inds + + @torch.inference_mode() + def propagate_in_video( + self, + inference_state, + start_frame_idx=None, + max_frame_num_to_track=None, + reverse=False, + ): + """Propagate the input points across frames to track in the entire video.""" + self.propagate_in_video_preflight(inference_state) + + output_dict = inference_state["output_dict"] + consolidated_frame_inds = inference_state["consolidated_frame_inds"] + obj_ids = inference_state["obj_ids"] + num_frames = inference_state["num_frames"] + batch_size = self._get_obj_num(inference_state) + if len(output_dict["cond_frame_outputs"]) == 0: + raise RuntimeError("No points are provided; please add points first") + clear_non_cond_mem = self.clear_non_cond_mem_around_input and ( + self.clear_non_cond_mem_for_multi_obj or batch_size <= 1 + ) + + # set start index, end index, and processing order + if start_frame_idx is None: + # default: start from the earliest frame with input points + start_frame_idx = min(output_dict["cond_frame_outputs"]) + if max_frame_num_to_track is None: + # default: track all the frames in the video + max_frame_num_to_track = num_frames + if reverse: + end_frame_idx = max(start_frame_idx - max_frame_num_to_track, 0) + if start_frame_idx > 0: + processing_order = range(start_frame_idx, end_frame_idx - 1, -1) + else: + processing_order = [] # skip reverse tracking if starting from frame 0 + else: + end_frame_idx = min( + start_frame_idx + max_frame_num_to_track, num_frames - 1 + ) + processing_order = range(start_frame_idx, end_frame_idx + 1) + + for frame_idx in tqdm(processing_order, desc="propagate in video"): + # We skip those frames already in consolidated outputs (these are frames + # that received input clicks or mask). Note that we cannot directly run + # batched forward on them via `_run_single_frame_inference` because the + # number of clicks on each object might be different. + if frame_idx in consolidated_frame_inds["cond_frame_outputs"]: + storage_key = "cond_frame_outputs" + current_out = output_dict[storage_key][frame_idx] + pred_masks = current_out["pred_masks"] + if clear_non_cond_mem: + # clear non-conditioning memory of the surrounding frames + self._clear_non_cond_mem_around_input(inference_state, frame_idx) + elif frame_idx in consolidated_frame_inds["non_cond_frame_outputs"]: + storage_key = "non_cond_frame_outputs" + current_out = output_dict[storage_key][frame_idx] + pred_masks = current_out["pred_masks"] + else: + storage_key = "non_cond_frame_outputs" + current_out, pred_masks = self._run_single_frame_inference( + inference_state=inference_state, + output_dict=output_dict, + frame_idx=frame_idx, + batch_size=batch_size, + is_init_cond_frame=False, + point_inputs=None, + mask_inputs=None, + reverse=reverse, + run_mem_encoder=True, + ) + output_dict[storage_key][frame_idx] = current_out + # Create slices of per-object outputs for subsequent interaction with each + # individual object after tracking. + self._add_output_per_object( + inference_state, frame_idx, current_out, storage_key + ) + inference_state["frames_already_tracked"][frame_idx] = {"reverse": reverse} + + # Resize the output mask to the original video resolution (we directly use + # the mask scores on GPU for output to avoid any CPU conversion in between) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, pred_masks + ) + yield frame_idx, obj_ids, video_res_masks + + def _add_output_per_object( + self, inference_state, frame_idx, current_out, storage_key + ): + """ + Split a multi-object output into per-object output slices and add them into + `output_dict_per_obj`. The resulting slices share the same tensor storage. + """ + maskmem_features = current_out["maskmem_features"] + assert maskmem_features is None or isinstance(maskmem_features, torch.Tensor) + + maskmem_pos_enc = current_out["maskmem_pos_enc"] + assert maskmem_pos_enc is None or isinstance(maskmem_pos_enc, list) + + output_dict_per_obj = inference_state["output_dict_per_obj"] + for obj_idx, obj_output_dict in output_dict_per_obj.items(): + obj_slice = slice(obj_idx, obj_idx + 1) + obj_out = { + "maskmem_features": None, + "maskmem_pos_enc": None, + "pred_masks": current_out["pred_masks"][obj_slice], + "obj_ptr": current_out["obj_ptr"][obj_slice], + "object_score_logits": current_out["object_score_logits"][obj_slice], + } + if maskmem_features is not None: + obj_out["maskmem_features"] = maskmem_features[obj_slice] + if maskmem_pos_enc is not None: + obj_out["maskmem_pos_enc"] = [x[obj_slice] for x in maskmem_pos_enc] + obj_output_dict[storage_key][frame_idx] = obj_out + + @torch.inference_mode() + def clear_all_prompts_in_frame( + self, inference_state, frame_idx, obj_id, need_output=True + ): + """Remove all input points or mask in a specific frame for a given object.""" + obj_idx = self._obj_id_to_idx(inference_state, obj_id) + + # Clear the conditioning information on the given frame + inference_state["point_inputs_per_obj"][obj_idx].pop(frame_idx, None) + inference_state["mask_inputs_per_obj"][obj_idx].pop(frame_idx, None) + + temp_output_dict_per_obj = inference_state["temp_output_dict_per_obj"] + temp_output_dict_per_obj[obj_idx]["cond_frame_outputs"].pop(frame_idx, None) + temp_output_dict_per_obj[obj_idx]["non_cond_frame_outputs"].pop(frame_idx, None) + + # Check and see if there are still any inputs left on this frame + batch_size = self._get_obj_num(inference_state) + frame_has_input = False + for obj_idx2 in range(batch_size): + if frame_idx in inference_state["point_inputs_per_obj"][obj_idx2]: + frame_has_input = True + break + if frame_idx in inference_state["mask_inputs_per_obj"][obj_idx2]: + frame_has_input = True + break + + # If this frame has no remaining inputs for any objects, we further clear its + # conditioning frame status + if not frame_has_input: + output_dict = inference_state["output_dict"] + consolidated_frame_inds = inference_state["consolidated_frame_inds"] + consolidated_frame_inds["cond_frame_outputs"].discard(frame_idx) + consolidated_frame_inds["non_cond_frame_outputs"].discard(frame_idx) + # Remove the frame's conditioning output (possibly downgrading it to non-conditioning) + out = output_dict["cond_frame_outputs"].pop(frame_idx, None) + if out is not None: + # The frame is not a conditioning frame anymore since it's not receiving inputs, + # so we "downgrade" its output (if exists) to a non-conditioning frame output. + output_dict["non_cond_frame_outputs"][frame_idx] = out + inference_state["frames_already_tracked"].pop(frame_idx, None) + # Similarly, do it for the sliced output on each object. + for obj_idx2 in range(batch_size): + obj_output_dict = inference_state["output_dict_per_obj"][obj_idx2] + obj_out = obj_output_dict["cond_frame_outputs"].pop(frame_idx, None) + if obj_out is not None: + obj_output_dict["non_cond_frame_outputs"][frame_idx] = obj_out + + # If all the conditioning frames have been removed, we also clear the tracking outputs + if len(output_dict["cond_frame_outputs"]) == 0: + self._reset_tracking_results(inference_state) + + if not need_output: + return + # Finally, output updated masks per object (after removing the inputs above) + obj_ids = inference_state["obj_ids"] + is_cond = any( + frame_idx in obj_temp_output_dict["cond_frame_outputs"] + for obj_temp_output_dict in temp_output_dict_per_obj.values() + ) + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + run_mem_encoder=False, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + return frame_idx, obj_ids, video_res_masks + + @torch.inference_mode() + def reset_state(self, inference_state): + """Remove all input points or mask in all frames throughout the video.""" + self._reset_tracking_results(inference_state) + # Remove all object ids + inference_state["obj_id_to_idx"].clear() + inference_state["obj_idx_to_id"].clear() + inference_state["obj_ids"].clear() + inference_state["point_inputs_per_obj"].clear() + inference_state["mask_inputs_per_obj"].clear() + inference_state["output_dict_per_obj"].clear() + inference_state["temp_output_dict_per_obj"].clear() + + def _reset_tracking_results(self, inference_state): + """Reset all tracking inputs and results across the videos.""" + for v in inference_state["point_inputs_per_obj"].values(): + v.clear() + for v in inference_state["mask_inputs_per_obj"].values(): + v.clear() + for v in inference_state["output_dict_per_obj"].values(): + v["cond_frame_outputs"].clear() + v["non_cond_frame_outputs"].clear() + for v in inference_state["temp_output_dict_per_obj"].values(): + v["cond_frame_outputs"].clear() + v["non_cond_frame_outputs"].clear() + inference_state["output_dict"]["cond_frame_outputs"].clear() + inference_state["output_dict"]["non_cond_frame_outputs"].clear() + inference_state["consolidated_frame_inds"]["cond_frame_outputs"].clear() + inference_state["consolidated_frame_inds"]["non_cond_frame_outputs"].clear() + inference_state["tracking_has_started"] = False + inference_state["frames_already_tracked"].clear() + + def _get_image_feature(self, inference_state, frame_idx, batch_size): + """Compute the image features on a given frame.""" + # Look up in the cache first + image, backbone_out = inference_state["cached_features"].get( + frame_idx, (None, None) + ) + if backbone_out is None: + # Cache miss -- we will run inference on a single image + device = inference_state["device"] + image = inference_state["images"][frame_idx].to(device).float().unsqueeze(0) + backbone_out = self.forward_image(image) + # Cache the most recent frame's feature (for repeated interactions with + # a frame; we can use an LRU cache for more frames in the future). + inference_state["cached_features"] = {frame_idx: (image, backbone_out)} + + # expand the features to have the same dimension as the number of objects + expanded_image = image.expand(batch_size, -1, -1, -1) + expanded_backbone_out = { + "backbone_fpn": backbone_out["backbone_fpn"].copy(), + "vision_pos_enc": backbone_out["vision_pos_enc"].copy(), + } + for i, feat in enumerate(expanded_backbone_out["backbone_fpn"]): + expanded_backbone_out["backbone_fpn"][i] = feat.expand( + batch_size, -1, -1, -1 + ) + for i, pos in enumerate(expanded_backbone_out["vision_pos_enc"]): + pos = pos.expand(batch_size, -1, -1, -1) + expanded_backbone_out["vision_pos_enc"][i] = pos + + features = self._prepare_backbone_features(expanded_backbone_out) + features = (expanded_image,) + features + return features + + def _run_single_frame_inference( + self, + inference_state, + output_dict, + frame_idx, + batch_size, + is_init_cond_frame, + point_inputs, + mask_inputs, + reverse, + run_mem_encoder, + prev_sam_mask_logits=None, + ): + """Run tracking on a single frame based on current inputs and previous memory.""" + # Retrieve correct image features + ( + _, + _, + current_vision_feats, + current_vision_pos_embeds, + feat_sizes, + ) = self._get_image_feature(inference_state, frame_idx, batch_size) + + # point and mask should not appear as input simultaneously on the same frame + assert point_inputs is None or mask_inputs is None + current_out = self.track_step( + frame_idx=frame_idx, + is_init_cond_frame=is_init_cond_frame, + current_vision_feats=current_vision_feats, + current_vision_pos_embeds=current_vision_pos_embeds, + feat_sizes=feat_sizes, + point_inputs=point_inputs, + mask_inputs=mask_inputs, + output_dict=output_dict, + num_frames=inference_state["num_frames"], + track_in_reverse=reverse, + run_mem_encoder=run_mem_encoder, + prev_sam_mask_logits=prev_sam_mask_logits, + ) + + # optionally offload the output to CPU memory to save GPU space + storage_device = inference_state["storage_device"] + maskmem_features = current_out["maskmem_features"] + if maskmem_features is not None: + maskmem_features = maskmem_features.to(torch.bfloat16) + maskmem_features = maskmem_features.to(storage_device, non_blocking=True) + pred_masks_gpu = current_out["pred_masks"] + # potentially fill holes in the predicted masks + if self.fill_hole_area > 0: + pred_masks_gpu = fill_holes_in_mask_scores( + pred_masks_gpu, self.fill_hole_area + ) + pred_masks = pred_masks_gpu.to(storage_device, non_blocking=True) + # "maskmem_pos_enc" is the same across frames, so we only need to store one copy of it + maskmem_pos_enc = self._get_maskmem_pos_enc(inference_state, current_out) + # object pointer is a small tensor, so we always keep it on GPU memory for fast access + obj_ptr = current_out["obj_ptr"] + object_score_logits = current_out["object_score_logits"] + # make a compact version of this frame's output to reduce the state size + compact_current_out = { + "maskmem_features": maskmem_features, + "maskmem_pos_enc": maskmem_pos_enc, + "pred_masks": pred_masks, + "obj_ptr": obj_ptr, + "object_score_logits": object_score_logits, + } + return compact_current_out, pred_masks_gpu + + def _run_memory_encoder( + self, + inference_state, + frame_idx, + batch_size, + high_res_masks, + object_score_logits, + is_mask_from_pts, + ): + """ + Run the memory encoder on `high_res_masks`. This is usually after applying + non-overlapping constraints to object scores. Since their scores changed, their + memory also need to be computed again with the memory encoder. + """ + # Retrieve correct image features + _, _, current_vision_feats, _, feat_sizes = self._get_image_feature( + inference_state, frame_idx, batch_size + ) + maskmem_features, maskmem_pos_enc = self._encode_new_memory( + current_vision_feats=current_vision_feats, + feat_sizes=feat_sizes, + pred_masks_high_res=high_res_masks, + object_score_logits=object_score_logits, + is_mask_from_pts=is_mask_from_pts, + ) + + # optionally offload the output to CPU memory to save GPU space + storage_device = inference_state["storage_device"] + maskmem_features = maskmem_features.to(torch.bfloat16) + maskmem_features = maskmem_features.to(storage_device, non_blocking=True) + # "maskmem_pos_enc" is the same across frames, so we only need to store one copy of it + maskmem_pos_enc = self._get_maskmem_pos_enc( + inference_state, {"maskmem_pos_enc": maskmem_pos_enc} + ) + return maskmem_features, maskmem_pos_enc + + def _get_maskmem_pos_enc(self, inference_state, current_out): + """ + `maskmem_pos_enc` is the same across frames and objects, so we cache it as + a constant in the inference session to reduce session storage size. + """ + model_constants = inference_state["constants"] + # "out_maskmem_pos_enc" should be either a list of tensors or None + out_maskmem_pos_enc = current_out["maskmem_pos_enc"] + if out_maskmem_pos_enc is not None: + if "maskmem_pos_enc" not in model_constants: + assert isinstance(out_maskmem_pos_enc, list) + # only take the slice for one object, since it's same across objects + maskmem_pos_enc = [x[0:1].clone() for x in out_maskmem_pos_enc] + model_constants["maskmem_pos_enc"] = maskmem_pos_enc + else: + maskmem_pos_enc = model_constants["maskmem_pos_enc"] + # expand the cached maskmem_pos_enc to the actual batch size + batch_size = out_maskmem_pos_enc[0].size(0) + expanded_maskmem_pos_enc = [ + x.expand(batch_size, -1, -1, -1) for x in maskmem_pos_enc + ] + else: + expanded_maskmem_pos_enc = None + return expanded_maskmem_pos_enc + + @torch.inference_mode() + def remove_object(self, inference_state, obj_id, strict=False, need_output=True): + """ + Remove an object id from the tracking state. If strict is True, we check whether + the object id actually exists and raise an error if it doesn't exist. + """ + old_obj_idx_to_rm = inference_state["obj_id_to_idx"].get(obj_id, None) + updated_frames = [] + # Check whether this object_id to remove actually exists and possibly raise an error. + if old_obj_idx_to_rm is None: + if not strict: + return inference_state["obj_ids"], updated_frames + raise RuntimeError( + f"Cannot remove object id {obj_id} as it doesn't exist. " + f"All existing object ids: {inference_state['obj_ids']}." + ) + + # If this is the only remaining object id, we simply reset the state. + if len(inference_state["obj_id_to_idx"]) == 1: + self.reset_state(inference_state) + return inference_state["obj_ids"], updated_frames + + # There are still remaining objects after removing this object id. In this case, + # we need to delete the object storage from inference state tensors. + # Step 0: clear the input on those frames where this object id has point or mask input + # (note that this step is required as it might downgrade conditioning frames to + # non-conditioning ones) + obj_input_frames_inds = set() + obj_input_frames_inds.update( + inference_state["point_inputs_per_obj"][old_obj_idx_to_rm] + ) + obj_input_frames_inds.update( + inference_state["mask_inputs_per_obj"][old_obj_idx_to_rm] + ) + for frame_idx in obj_input_frames_inds: + self.clear_all_prompts_in_frame( + inference_state, frame_idx, obj_id, need_output=False + ) + + # Step 1: Update the object id mapping (note that it must be done after Step 0, + # since Step 0 still requires the old object id mappings in inference_state) + old_obj_ids = inference_state["obj_ids"] + old_obj_inds = list(range(len(old_obj_ids))) + remain_old_obj_inds = old_obj_inds.copy() + remain_old_obj_inds.remove(old_obj_idx_to_rm) + new_obj_ids = [old_obj_ids[old_idx] for old_idx in remain_old_obj_inds] + new_obj_inds = list(range(len(new_obj_ids))) + # build new mappings + old_idx_to_new_idx = dict(zip(remain_old_obj_inds, new_obj_inds)) + inference_state["obj_id_to_idx"] = dict(zip(new_obj_ids, new_obj_inds)) + inference_state["obj_idx_to_id"] = dict(zip(new_obj_inds, new_obj_ids)) + inference_state["obj_ids"] = new_obj_ids + + # Step 2: For per-object tensor storage, we shift their obj_idx in the dict keys. + # (note that "consolidated_frame_inds" doesn't need to be updated in this step as + # it's already handled in Step 0) + def _map_keys(container): + new_kvs = [] + for k in old_obj_inds: + v = container.pop(k) + if k in old_idx_to_new_idx: + new_kvs.append((old_idx_to_new_idx[k], v)) + container.update(new_kvs) + + _map_keys(inference_state["point_inputs_per_obj"]) + _map_keys(inference_state["mask_inputs_per_obj"]) + _map_keys(inference_state["output_dict_per_obj"]) + _map_keys(inference_state["temp_output_dict_per_obj"]) + + # Step 3: For packed tensor storage, we index the remaining ids and rebuild the per-object slices. + def _slice_state(output_dict, storage_key): + for frame_idx, out in output_dict[storage_key].items(): + out["maskmem_features"] = out["maskmem_features"][remain_old_obj_inds] + out["maskmem_pos_enc"] = [ + x[remain_old_obj_inds] for x in out["maskmem_pos_enc"] + ] + # "maskmem_pos_enc" is the same across frames, so we only need to store one copy of it + out["maskmem_pos_enc"] = self._get_maskmem_pos_enc(inference_state, out) + out["pred_masks"] = out["pred_masks"][remain_old_obj_inds] + out["obj_ptr"] = out["obj_ptr"][remain_old_obj_inds] + out["object_score_logits"] = out["object_score_logits"][ + remain_old_obj_inds + ] + # also update the per-object slices + self._add_output_per_object( + inference_state, frame_idx, out, storage_key + ) + + _slice_state(inference_state["output_dict"], "cond_frame_outputs") + _slice_state(inference_state["output_dict"], "non_cond_frame_outputs") + + # Step 4: Further collect the outputs on those frames in `obj_input_frames_inds`, which + # could show an updated mask for objects previously occluded by the object being removed + if need_output: + temp_output_dict_per_obj = inference_state["temp_output_dict_per_obj"] + for frame_idx in obj_input_frames_inds: + is_cond = any( + frame_idx in obj_temp_output_dict["cond_frame_outputs"] + for obj_temp_output_dict in temp_output_dict_per_obj.values() + ) + consolidated_out = self._consolidate_temp_output_across_obj( + inference_state, + frame_idx, + is_cond=is_cond, + run_mem_encoder=False, + consolidate_at_video_res=True, + ) + _, video_res_masks = self._get_orig_video_res_output( + inference_state, consolidated_out["pred_masks_video_res"] + ) + updated_frames.append((frame_idx, video_res_masks)) + + return inference_state["obj_ids"], updated_frames + + def _clear_non_cond_mem_around_input(self, inference_state, frame_idx): + """ + Remove the non-conditioning memory around the input frame. When users provide + correction clicks, the surrounding frames' non-conditioning memories can still + contain outdated object appearance information and could confuse the model. + + This method clears those non-conditioning memories surrounding the interacted + frame to avoid giving the model both old and new information about the object. + """ + r = self.memory_temporal_stride_for_eval + frame_idx_begin = frame_idx - r * self.num_maskmem + frame_idx_end = frame_idx + r * self.num_maskmem + output_dict = inference_state["output_dict"] + non_cond_frame_outputs = output_dict["non_cond_frame_outputs"] + for t in range(frame_idx_begin, frame_idx_end + 1): + non_cond_frame_outputs.pop(t, None) + for obj_output_dict in inference_state["output_dict_per_obj"].values(): + obj_output_dict["non_cond_frame_outputs"].pop(t, None) diff --git a/tools/sam2-onnx-tensorrt/sam2/utils/__init__.py b/tools/sam2-onnx-tensorrt/sam2/utils/__init__.py new file mode 100644 index 00000000..5277f461 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/utils/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. diff --git a/tools/sam2-onnx-tensorrt/sam2/utils/amg.py b/tools/sam2-onnx-tensorrt/sam2/utils/amg.py new file mode 100644 index 00000000..98684296 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/utils/amg.py @@ -0,0 +1,348 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import math +from copy import deepcopy +from itertools import product +from typing import Any, Dict, Generator, ItemsView, List, Tuple + +import numpy as np +import torch + +# Very lightly adapted from https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/utils/amg.py + + +class MaskData: + """ + A structure for storing masks and their related data in batched format. + Implements basic filtering and concatenation. + """ + + def __init__(self, **kwargs) -> None: + for v in kwargs.values(): + assert isinstance( + v, (list, np.ndarray, torch.Tensor) + ), "MaskData only supports list, numpy arrays, and torch tensors." + self._stats = dict(**kwargs) + + def __setitem__(self, key: str, item: Any) -> None: + assert isinstance( + item, (list, np.ndarray, torch.Tensor) + ), "MaskData only supports list, numpy arrays, and torch tensors." + self._stats[key] = item + + def __delitem__(self, key: str) -> None: + del self._stats[key] + + def __getitem__(self, key: str) -> Any: + return self._stats[key] + + def items(self) -> ItemsView[str, Any]: + return self._stats.items() + + def filter(self, keep: torch.Tensor) -> None: + for k, v in self._stats.items(): + if v is None: + self._stats[k] = None + elif isinstance(v, torch.Tensor): + self._stats[k] = v[torch.as_tensor(keep, device=v.device)] + elif isinstance(v, np.ndarray): + self._stats[k] = v[keep.detach().cpu().numpy()] + elif isinstance(v, list) and keep.dtype == torch.bool: + self._stats[k] = [a for i, a in enumerate(v) if keep[i]] + elif isinstance(v, list): + self._stats[k] = [v[i] for i in keep] + else: + raise TypeError(f"MaskData key {k} has an unsupported type {type(v)}.") + + def cat(self, new_stats: "MaskData") -> None: + for k, v in new_stats.items(): + if k not in self._stats or self._stats[k] is None: + self._stats[k] = deepcopy(v) + elif isinstance(v, torch.Tensor): + self._stats[k] = torch.cat([self._stats[k], v], dim=0) + elif isinstance(v, np.ndarray): + self._stats[k] = np.concatenate([self._stats[k], v], axis=0) + elif isinstance(v, list): + self._stats[k] = self._stats[k] + deepcopy(v) + else: + raise TypeError(f"MaskData key {k} has an unsupported type {type(v)}.") + + def to_numpy(self) -> None: + for k, v in self._stats.items(): + if isinstance(v, torch.Tensor): + self._stats[k] = v.float().detach().cpu().numpy() + + +def is_box_near_crop_edge( + boxes: torch.Tensor, crop_box: List[int], orig_box: List[int], atol: float = 20.0 +) -> torch.Tensor: + """Filter masks at the edge of a crop, but not at the edge of the original image.""" + crop_box_torch = torch.as_tensor(crop_box, dtype=torch.float, device=boxes.device) + orig_box_torch = torch.as_tensor(orig_box, dtype=torch.float, device=boxes.device) + boxes = uncrop_boxes_xyxy(boxes, crop_box).float() + near_crop_edge = torch.isclose(boxes, crop_box_torch[None, :], atol=atol, rtol=0) + near_image_edge = torch.isclose(boxes, orig_box_torch[None, :], atol=atol, rtol=0) + near_crop_edge = torch.logical_and(near_crop_edge, ~near_image_edge) + return torch.any(near_crop_edge, dim=1) + + +def box_xyxy_to_xywh(box_xyxy: torch.Tensor) -> torch.Tensor: + box_xywh = deepcopy(box_xyxy) + box_xywh[2] = box_xywh[2] - box_xywh[0] + box_xywh[3] = box_xywh[3] - box_xywh[1] + return box_xywh + + +def batch_iterator(batch_size: int, *args) -> Generator[List[Any], None, None]: + assert len(args) > 0 and all( + len(a) == len(args[0]) for a in args + ), "Batched iteration must have inputs of all the same size." + n_batches = len(args[0]) // batch_size + int(len(args[0]) % batch_size != 0) + for b in range(n_batches): + yield [arg[b * batch_size : (b + 1) * batch_size] for arg in args] + + +def mask_to_rle_pytorch(tensor: torch.Tensor) -> List[Dict[str, Any]]: + """ + Encodes masks to an uncompressed RLE, in the format expected by + pycoco tools. + """ + # Put in fortran order and flatten h,w + b, h, w = tensor.shape + tensor = tensor.permute(0, 2, 1).flatten(1) + + # Compute change indices + diff = tensor[:, 1:] ^ tensor[:, :-1] + change_indices = diff.nonzero() + + # Encode run length + out = [] + for i in range(b): + cur_idxs = change_indices[change_indices[:, 0] == i, 1] + cur_idxs = torch.cat( + [ + torch.tensor([0], dtype=cur_idxs.dtype, device=cur_idxs.device), + cur_idxs + 1, + torch.tensor([h * w], dtype=cur_idxs.dtype, device=cur_idxs.device), + ] + ) + btw_idxs = cur_idxs[1:] - cur_idxs[:-1] + counts = [] if tensor[i, 0] == 0 else [0] + counts.extend(btw_idxs.detach().cpu().tolist()) + out.append({"size": [h, w], "counts": counts}) + return out + + +def rle_to_mask(rle: Dict[str, Any]) -> np.ndarray: + """Compute a binary mask from an uncompressed RLE.""" + h, w = rle["size"] + mask = np.empty(h * w, dtype=bool) + idx = 0 + parity = False + for count in rle["counts"]: + mask[idx : idx + count] = parity + idx += count + parity ^= True + mask = mask.reshape(w, h) + return mask.transpose() # Put in C order + + +def area_from_rle(rle: Dict[str, Any]) -> int: + return sum(rle["counts"][1::2]) + + +def calculate_stability_score( + masks: torch.Tensor, mask_threshold: float, threshold_offset: float +) -> torch.Tensor: + """ + Computes the stability score for a batch of masks. The stability + score is the IoU between the binary masks obtained by thresholding + the predicted mask logits at high and low values. + """ + # One mask is always contained inside the other. + # Save memory by preventing unnecessary cast to torch.int64 + intersections = ( + (masks > (mask_threshold + threshold_offset)) + .sum(-1, dtype=torch.int16) + .sum(-1, dtype=torch.int32) + ) + unions = ( + (masks > (mask_threshold - threshold_offset)) + .sum(-1, dtype=torch.int16) + .sum(-1, dtype=torch.int32) + ) + return intersections / unions + + +def build_point_grid(n_per_side: int) -> np.ndarray: + """Generates a 2D grid of points evenly spaced in [0,1]x[0,1].""" + offset = 1 / (2 * n_per_side) + points_one_side = np.linspace(offset, 1 - offset, n_per_side) + points_x = np.tile(points_one_side[None, :], (n_per_side, 1)) + points_y = np.tile(points_one_side[:, None], (1, n_per_side)) + points = np.stack([points_x, points_y], axis=-1).reshape(-1, 2) + return points + + +def build_all_layer_point_grids( + n_per_side: int, n_layers: int, scale_per_layer: int +) -> List[np.ndarray]: + """Generates point grids for all crop layers.""" + points_by_layer = [] + for i in range(n_layers + 1): + n_points = int(n_per_side / (scale_per_layer**i)) + points_by_layer.append(build_point_grid(n_points)) + return points_by_layer + + +def generate_crop_boxes( + im_size: Tuple[int, ...], n_layers: int, overlap_ratio: float +) -> Tuple[List[List[int]], List[int]]: + """ + Generates a list of crop boxes of different sizes. Each layer + has (2**i)**2 boxes for the ith layer. + """ + crop_boxes, layer_idxs = [], [] + im_h, im_w = im_size + short_side = min(im_h, im_w) + + # Original image + crop_boxes.append([0, 0, im_w, im_h]) + layer_idxs.append(0) + + def crop_len(orig_len, n_crops, overlap): + return int(math.ceil((overlap * (n_crops - 1) + orig_len) / n_crops)) + + for i_layer in range(n_layers): + n_crops_per_side = 2 ** (i_layer + 1) + overlap = int(overlap_ratio * short_side * (2 / n_crops_per_side)) + + crop_w = crop_len(im_w, n_crops_per_side, overlap) + crop_h = crop_len(im_h, n_crops_per_side, overlap) + + crop_box_x0 = [int((crop_w - overlap) * i) for i in range(n_crops_per_side)] + crop_box_y0 = [int((crop_h - overlap) * i) for i in range(n_crops_per_side)] + + # Crops in XYWH format + for x0, y0 in product(crop_box_x0, crop_box_y0): + box = [x0, y0, min(x0 + crop_w, im_w), min(y0 + crop_h, im_h)] + crop_boxes.append(box) + layer_idxs.append(i_layer + 1) + + return crop_boxes, layer_idxs + + +def uncrop_boxes_xyxy(boxes: torch.Tensor, crop_box: List[int]) -> torch.Tensor: + x0, y0, _, _ = crop_box + offset = torch.tensor([[x0, y0, x0, y0]], device=boxes.device) + # Check if boxes has a channel dimension + if len(boxes.shape) == 3: + offset = offset.unsqueeze(1) + return boxes + offset + + +def uncrop_points(points: torch.Tensor, crop_box: List[int]) -> torch.Tensor: + x0, y0, _, _ = crop_box + offset = torch.tensor([[x0, y0]], device=points.device) + # Check if points has a channel dimension + if len(points.shape) == 3: + offset = offset.unsqueeze(1) + return points + offset + + +def uncrop_masks( + masks: torch.Tensor, crop_box: List[int], orig_h: int, orig_w: int +) -> torch.Tensor: + x0, y0, x1, y1 = crop_box + if x0 == 0 and y0 == 0 and x1 == orig_w and y1 == orig_h: + return masks + # Coordinate transform masks + pad_x, pad_y = orig_w - (x1 - x0), orig_h - (y1 - y0) + pad = (x0, pad_x - x0, y0, pad_y - y0) + return torch.nn.functional.pad(masks, pad, value=0) + + +def remove_small_regions( + mask: np.ndarray, area_thresh: float, mode: str +) -> Tuple[np.ndarray, bool]: + """ + Removes small disconnected regions and holes in a mask. Returns the + mask and an indicator of if the mask has been modified. + """ + import cv2 # type: ignore + + assert mode in ["holes", "islands"] + correct_holes = mode == "holes" + working_mask = (correct_holes ^ mask).astype(np.uint8) + n_labels, regions, stats, _ = cv2.connectedComponentsWithStats(working_mask, 8) + sizes = stats[:, -1][1:] # Row 0 is background label + small_regions = [i + 1 for i, s in enumerate(sizes) if s < area_thresh] + if len(small_regions) == 0: + return mask, False + fill_labels = [0] + small_regions + if not correct_holes: + fill_labels = [i for i in range(n_labels) if i not in fill_labels] + # If every region is below threshold, keep largest + if len(fill_labels) == 0: + fill_labels = [int(np.argmax(sizes)) + 1] + mask = np.isin(regions, fill_labels) + return mask, True + + +def coco_encode_rle(uncompressed_rle: Dict[str, Any]) -> Dict[str, Any]: + from pycocotools import mask as mask_utils # type: ignore + + h, w = uncompressed_rle["size"] + rle = mask_utils.frPyObjects(uncompressed_rle, h, w) + rle["counts"] = rle["counts"].decode("utf-8") # Necessary to serialize with json + return rle + + +def batched_mask_to_box(masks: torch.Tensor) -> torch.Tensor: + """ + Calculates boxes in XYXY format around masks. Return [0,0,0,0] for + an empty mask. For input shape C1xC2x...xHxW, the output shape is C1xC2x...x4. + """ + # torch.max below raises an error on empty inputs, just skip in this case + if torch.numel(masks) == 0: + return torch.zeros(*masks.shape[:-2], 4, device=masks.device) + + # Normalize shape to CxHxW + shape = masks.shape + h, w = shape[-2:] + if len(shape) > 2: + masks = masks.flatten(0, -3) + else: + masks = masks.unsqueeze(0) + + # Get top and bottom edges + in_height, _ = torch.max(masks, dim=-1) + in_height_coords = in_height * torch.arange(h, device=in_height.device)[None, :] + bottom_edges, _ = torch.max(in_height_coords, dim=-1) + in_height_coords = in_height_coords + h * (~in_height) + top_edges, _ = torch.min(in_height_coords, dim=-1) + + # Get left and right edges + in_width, _ = torch.max(masks, dim=-2) + in_width_coords = in_width * torch.arange(w, device=in_width.device)[None, :] + right_edges, _ = torch.max(in_width_coords, dim=-1) + in_width_coords = in_width_coords + w * (~in_width) + left_edges, _ = torch.min(in_width_coords, dim=-1) + + # If the mask is empty the right edge will be to the left of the left edge. + # Replace these boxes with [0, 0, 0, 0] + empty_filter = (right_edges < left_edges) | (bottom_edges < top_edges) + out = torch.stack([left_edges, top_edges, right_edges, bottom_edges], dim=-1) + out = out * (~empty_filter).unsqueeze(-1) + + # Return to original shape + if len(shape) > 2: + out = out.reshape(*shape[:-2], 4) + else: + out = out[0] + + return out diff --git a/tools/sam2-onnx-tensorrt/sam2/utils/misc.py b/tools/sam2-onnx-tensorrt/sam2/utils/misc.py new file mode 100644 index 00000000..b65ee825 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/utils/misc.py @@ -0,0 +1,349 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import os +import warnings +from threading import Thread + +import numpy as np +import torch +from PIL import Image +from tqdm import tqdm + + +def get_sdpa_settings(): + if torch.cuda.is_available(): + old_gpu = torch.cuda.get_device_properties(0).major < 7 + # only use Flash Attention on Ampere (8.0) or newer GPUs + use_flash_attn = torch.cuda.get_device_properties(0).major >= 8 + if not use_flash_attn: + warnings.warn( + "Flash Attention is disabled as it requires a GPU with Ampere (8.0) CUDA capability.", + category=UserWarning, + stacklevel=2, + ) + # keep math kernel for PyTorch versions before 2.2 (Flash Attention v2 is only + # available on PyTorch 2.2+, while Flash Attention v1 cannot handle all cases) + pytorch_version = tuple(int(v) for v in torch.__version__.split(".")[:2]) + if pytorch_version < (2, 2): + warnings.warn( + f"You are using PyTorch {torch.__version__} without Flash Attention v2 support. " + "Consider upgrading to PyTorch 2.2+ for Flash Attention v2 (which could be faster).", + category=UserWarning, + stacklevel=2, + ) + math_kernel_on = pytorch_version < (2, 2) or not use_flash_attn + else: + old_gpu = True + use_flash_attn = False + math_kernel_on = True + + return old_gpu, use_flash_attn, math_kernel_on + + +def get_connected_components(mask): + """ + Get the connected components (8-connectivity) of binary masks of shape (N, 1, H, W). + + Inputs: + - mask: A binary mask tensor of shape (N, 1, H, W), where 1 is foreground and 0 is + background. + + Outputs: + - labels: A tensor of shape (N, 1, H, W) containing the connected component labels + for foreground pixels and 0 for background pixels. + - counts: A tensor of shape (N, 1, H, W) containing the area of the connected + components for foreground pixels and 0 for background pixels. + """ + from sam2 import _C + + return _C.get_connected_componnets(mask.to(torch.uint8).contiguous()) + + +def mask_to_box(masks: torch.Tensor): + """ + compute bounding box given an input mask + + Inputs: + - masks: [B, 1, H, W] masks, dtype=torch.Tensor + + Returns: + - box_coords: [B, 1, 4], contains (x, y) coordinates of top left and bottom right box corners, dtype=torch.Tensor + """ + B, _, h, w = masks.shape + device = masks.device + xs = torch.arange(w, device=device, dtype=torch.int32) + ys = torch.arange(h, device=device, dtype=torch.int32) + grid_xs, grid_ys = torch.meshgrid(xs, ys, indexing="xy") + grid_xs = grid_xs[None, None, ...].expand(B, 1, h, w) + grid_ys = grid_ys[None, None, ...].expand(B, 1, h, w) + min_xs, _ = torch.min(torch.where(masks, grid_xs, w).flatten(-2), dim=-1) + max_xs, _ = torch.max(torch.where(masks, grid_xs, -1).flatten(-2), dim=-1) + min_ys, _ = torch.min(torch.where(masks, grid_ys, h).flatten(-2), dim=-1) + max_ys, _ = torch.max(torch.where(masks, grid_ys, -1).flatten(-2), dim=-1) + bbox_coords = torch.stack((min_xs, min_ys, max_xs, max_ys), dim=-1) + + return bbox_coords + + +def _load_img_as_tensor(img_path, image_size): + img_pil = Image.open(img_path) + img_np = np.array(img_pil.convert("RGB").resize((image_size, image_size))) + if img_np.dtype == np.uint8: # np.uint8 is expected for JPEG images + img_np = img_np / 255.0 + else: + raise RuntimeError(f"Unknown image dtype: {img_np.dtype} on {img_path}") + img = torch.from_numpy(img_np).permute(2, 0, 1) + video_width, video_height = img_pil.size # the original video size + return img, video_height, video_width + + +class AsyncVideoFrameLoader: + """ + A list of video frames to be load asynchronously without blocking session start. + """ + + def __init__( + self, + img_paths, + image_size, + offload_video_to_cpu, + img_mean, + img_std, + compute_device, + ): + self.img_paths = img_paths + self.image_size = image_size + self.offload_video_to_cpu = offload_video_to_cpu + self.img_mean = img_mean + self.img_std = img_std + # items in `self.images` will be loaded asynchronously + self.images = [None] * len(img_paths) + # catch and raise any exceptions in the async loading thread + self.exception = None + # video_height and video_width be filled when loading the first image + self.video_height = None + self.video_width = None + self.compute_device = compute_device + + # load the first frame to fill video_height and video_width and also + # to cache it (since it's most likely where the user will click) + self.__getitem__(0) + + # load the rest of frames asynchronously without blocking the session start + def _load_frames(): + try: + for n in tqdm(range(len(self.images)), desc="frame loading (JPEG)"): + self.__getitem__(n) + except Exception as e: + self.exception = e + + self.thread = Thread(target=_load_frames, daemon=True) + self.thread.start() + + def __getitem__(self, index): + if self.exception is not None: + raise RuntimeError("Failure in frame loading thread") from self.exception + + img = self.images[index] + if img is not None: + return img + + img, video_height, video_width = _load_img_as_tensor( + self.img_paths[index], self.image_size + ) + self.video_height = video_height + self.video_width = video_width + # normalize by mean and std + img -= self.img_mean + img /= self.img_std + if not self.offload_video_to_cpu: + img = img.to(self.compute_device, non_blocking=True) + self.images[index] = img + return img + + def __len__(self): + return len(self.images) + + +def load_video_frames( + video_path, + image_size, + offload_video_to_cpu, + img_mean=(0.485, 0.456, 0.406), + img_std=(0.229, 0.224, 0.225), + async_loading_frames=False, + compute_device=torch.device("cuda"), +): + """ + Load the video frames from video_path. The frames are resized to image_size as in + the model and are loaded to GPU if offload_video_to_cpu=False. This is used by the demo. + """ + is_bytes = isinstance(video_path, bytes) + is_str = isinstance(video_path, str) + is_mp4_path = is_str and os.path.splitext(video_path)[-1] in [".mp4", ".MP4"] + if is_bytes or is_mp4_path: + return load_video_frames_from_video_file( + video_path=video_path, + image_size=image_size, + offload_video_to_cpu=offload_video_to_cpu, + img_mean=img_mean, + img_std=img_std, + compute_device=compute_device, + ) + elif is_str and os.path.isdir(video_path): + return load_video_frames_from_jpg_images( + video_path=video_path, + image_size=image_size, + offload_video_to_cpu=offload_video_to_cpu, + img_mean=img_mean, + img_std=img_std, + async_loading_frames=async_loading_frames, + compute_device=compute_device, + ) + else: + raise NotImplementedError( + "Only MP4 video and JPEG folder are supported at this moment" + ) + + +def load_video_frames_from_jpg_images( + video_path, + image_size, + offload_video_to_cpu, + img_mean=(0.485, 0.456, 0.406), + img_std=(0.229, 0.224, 0.225), + async_loading_frames=False, + compute_device=torch.device("cuda"), +): + """ + Load the video frames from a directory of JPEG files (".jpg" format). + + The frames are resized to image_size x image_size and are loaded to GPU if + `offload_video_to_cpu` is `False` and to CPU if `offload_video_to_cpu` is `True`. + + You can load a frame asynchronously by setting `async_loading_frames` to `True`. + """ + if isinstance(video_path, str) and os.path.isdir(video_path): + jpg_folder = video_path + else: + raise NotImplementedError( + "Only JPEG frames are supported at this moment. For video files, you may use " + "ffmpeg (https://ffmpeg.org/) to extract frames into a folder of JPEG files, such as \n" + "```\n" + "ffmpeg -i .mp4 -q:v 2 -start_number 0 /'%05d.jpg'\n" + "```\n" + "where `-q:v` generates high-quality JPEG frames and `-start_number 0` asks " + "ffmpeg to start the JPEG file from 00000.jpg." + ) + + frame_names = [ + p + for p in os.listdir(jpg_folder) + if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"] + ] + frame_names.sort(key=lambda p: int(os.path.splitext(p)[0])) + num_frames = len(frame_names) + if num_frames == 0: + raise RuntimeError(f"no images found in {jpg_folder}") + img_paths = [os.path.join(jpg_folder, frame_name) for frame_name in frame_names] + img_mean = torch.tensor(img_mean, dtype=torch.float32)[:, None, None] + img_std = torch.tensor(img_std, dtype=torch.float32)[:, None, None] + + if async_loading_frames: + lazy_images = AsyncVideoFrameLoader( + img_paths, + image_size, + offload_video_to_cpu, + img_mean, + img_std, + compute_device, + ) + return lazy_images, lazy_images.video_height, lazy_images.video_width + + images = torch.zeros(num_frames, 3, image_size, image_size, dtype=torch.float32) + for n, img_path in enumerate(tqdm(img_paths, desc="frame loading (JPEG)")): + images[n], video_height, video_width = _load_img_as_tensor(img_path, image_size) + if not offload_video_to_cpu: + images = images.to(compute_device) + img_mean = img_mean.to(compute_device) + img_std = img_std.to(compute_device) + # normalize by mean and std + images -= img_mean + images /= img_std + return images, video_height, video_width + + +def load_video_frames_from_video_file( + video_path, + image_size, + offload_video_to_cpu, + img_mean=(0.485, 0.456, 0.406), + img_std=(0.229, 0.224, 0.225), + compute_device=torch.device("cuda"), +): + """Load the video frames from a video file.""" + import decord + + img_mean = torch.tensor(img_mean, dtype=torch.float32)[:, None, None] + img_std = torch.tensor(img_std, dtype=torch.float32)[:, None, None] + # Get the original video height and width + decord.bridge.set_bridge("torch") + video_height, video_width, _ = decord.VideoReader(video_path).next().shape + # Iterate over all frames in the video + images = [] + for frame in decord.VideoReader(video_path, width=image_size, height=image_size): + images.append(frame.permute(2, 0, 1)) + + images = torch.stack(images, dim=0).float() / 255.0 + if not offload_video_to_cpu: + images = images.to(compute_device) + img_mean = img_mean.to(compute_device) + img_std = img_std.to(compute_device) + # normalize by mean and std + images -= img_mean + images /= img_std + return images, video_height, video_width + + +def fill_holes_in_mask_scores(mask, max_area): + """ + A post processor to fill small holes in mask scores with area under `max_area`. + """ + # Holes are those connected components in background with area <= self.max_area + # (background regions are those with mask scores <= 0) + assert max_area > 0, "max_area must be positive" + + input_mask = mask + try: + labels, areas = get_connected_components(mask <= 0) + is_hole = (labels > 0) & (areas <= max_area) + # We fill holes with a small positive mask score (0.1) to change them to foreground. + mask = torch.where(is_hole, 0.1, mask) + except Exception as e: + # Skip the post-processing step on removing small holes if the CUDA kernel fails + warnings.warn( + f"{e}\n\nSkipping the post-processing step due to the error above. You can " + "still use SAM 2 and it's OK to ignore the error above, although some post-processing " + "functionality may be limited (which doesn't affect the results in most cases; see " + "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/facebookresearch/sam2/blob/main/INSTALL.md).", + category=UserWarning, + stacklevel=2, + ) + mask = input_mask + + return mask + + +def concat_points(old_point_inputs, new_points, new_labels): + """Add new points and labels to previous point inputs (add at the end).""" + if old_point_inputs is None: + points, labels = new_points, new_labels + else: + points = torch.cat([old_point_inputs["point_coords"], new_points], dim=1) + labels = torch.cat([old_point_inputs["point_labels"], new_labels], dim=1) + + return {"point_coords": points, "point_labels": labels} diff --git a/tools/sam2-onnx-tensorrt/sam2/utils/transforms.py b/tools/sam2-onnx-tensorrt/sam2/utils/transforms.py new file mode 100644 index 00000000..cc17bebf --- /dev/null +++ b/tools/sam2-onnx-tensorrt/sam2/utils/transforms.py @@ -0,0 +1,118 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.transforms import Normalize, Resize, ToTensor + + +class SAM2Transforms(nn.Module): + def __init__( + self, resolution, mask_threshold, max_hole_area=0.0, max_sprinkle_area=0.0 + ): + """ + Transforms for SAM2. + """ + super().__init__() + self.resolution = resolution + self.mask_threshold = mask_threshold + self.max_hole_area = max_hole_area + self.max_sprinkle_area = max_sprinkle_area + self.mean = [0.485, 0.456, 0.406] + self.std = [0.229, 0.224, 0.225] + self.to_tensor = ToTensor() + self.transforms = torch.jit.script( + nn.Sequential( + Resize((self.resolution, self.resolution)), + Normalize(self.mean, self.std), + ) + ) + + def __call__(self, x): + x = self.to_tensor(x) + return self.transforms(x) + + def forward_batch(self, img_list): + img_batch = [self.transforms(self.to_tensor(img)) for img in img_list] + img_batch = torch.stack(img_batch, dim=0) + return img_batch + + def transform_coords( + self, coords: torch.Tensor, normalize=False, orig_hw=None + ) -> torch.Tensor: + """ + Expects a torch tensor with length 2 in the last dimension. The coordinates can be in absolute image or normalized coordinates, + If the coords are in absolute image coordinates, normalize should be set to True and original image size is required. + + Returns + Un-normalized coordinates in the range of [0, 1] which is expected by the SAM2 model. + """ + if normalize: + assert orig_hw is not None + h, w = orig_hw + coords = coords.clone() + coords[..., 0] = coords[..., 0] / w + coords[..., 1] = coords[..., 1] / h + + coords = coords * self.resolution # unnormalize coords + return coords + + def transform_boxes( + self, boxes: torch.Tensor, normalize=False, orig_hw=None + ) -> torch.Tensor: + """ + Expects a tensor of shape Bx4. The coordinates can be in absolute image or normalized coordinates, + if the coords are in absolute image coordinates, normalize should be set to True and original image size is required. + """ + boxes = self.transform_coords(boxes.reshape(-1, 2, 2), normalize, orig_hw) + return boxes + + def postprocess_masks(self, masks: torch.Tensor, orig_hw) -> torch.Tensor: + """ + Perform PostProcessing on output masks. + """ + from sam2.utils.misc import get_connected_components + + masks = masks.float() + input_masks = masks + mask_flat = masks.flatten(0, 1).unsqueeze(1) # flatten as 1-channel image + try: + if self.max_hole_area > 0: + # Holes are those connected components in background with area <= self.fill_hole_area + # (background regions are those with mask scores <= self.mask_threshold) + labels, areas = get_connected_components( + mask_flat <= self.mask_threshold + ) + is_hole = (labels > 0) & (areas <= self.max_hole_area) + is_hole = is_hole.reshape_as(masks) + # We fill holes with a small positive mask score (10.0) to change them to foreground. + masks = torch.where(is_hole, self.mask_threshold + 10.0, masks) + + if self.max_sprinkle_area > 0: + labels, areas = get_connected_components( + mask_flat > self.mask_threshold + ) + is_hole = (labels > 0) & (areas <= self.max_sprinkle_area) + is_hole = is_hole.reshape_as(masks) + # We fill holes with negative mask score (-10.0) to change them to background. + masks = torch.where(is_hole, self.mask_threshold - 10.0, masks) + except Exception as e: + # Skip the post-processing step if the CUDA kernel fails + warnings.warn( + f"{e}\n\nSkipping the post-processing step due to the error above. You can " + "still use SAM 2 and it's OK to ignore the error above, although some post-processing " + "functionality may be limited (which doesn't affect the results in most cases; see " + "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/facebookresearch/sam2/blob/main/INSTALL.md).", + category=UserWarning, + stacklevel=2, + ) + masks = input_masks + + masks = F.interpolate(masks, orig_hw, mode="bilinear", align_corners=False) + return masks diff --git a/tools/sam2-onnx-tensorrt/setup.py b/tools/sam2-onnx-tensorrt/setup.py new file mode 100644 index 00000000..a8646799 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/setup.py @@ -0,0 +1,177 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +import os + +from setuptools import find_packages, setup + +# Package metadata +NAME = "SAM-2" +VERSION = "1.0" +DESCRIPTION = "SAM 2: Segment Anything in Images and Videos" +URL = "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/facebookresearch/sam2" +AUTHOR = "Meta AI" +AUTHOR_EMAIL = "segment-anything@meta.com" +LICENSE = "Apache 2.0" + +# Read the contents of README file +with open("README.md", "r", encoding="utf-8") as f: + LONG_DESCRIPTION = f.read() + +# Required dependencies +REQUIRED_PACKAGES = [ + "torch>=2.5.1", + "torch<=2.8.1", + "torchvision>=0.20.1", + "numpy>=1.24.4", + "tqdm>=4.66.1", + "hydra-core>=1.3.2", + "iopath>=0.1.10", + "pillow>=9.4.0", + "onnx>=1.18.0", + "onnxscript==0.4.0", +] + +EXTRA_PACKAGES = { + "notebooks": [ + "matplotlib>=3.9.1", + "jupyter>=1.0.0", + "opencv-python>=4.7.0", + "eva-decord>=0.6.1", + ], + "interactive-demo": [ + "Flask>=3.0.3", + "Flask-Cors>=5.0.0", + "av>=13.0.0", + "dataclasses-json>=0.6.7", + "eva-decord>=0.6.1", + "gunicorn>=23.0.0", + "imagesize>=1.4.1", + "pycocotools>=2.0.8", + "strawberry-graphql>=0.243.0", + ], + "dev": [ + "black==24.2.0", + "usort==1.0.2", + "ufmt==2.0.0b2", + "fvcore>=0.1.5.post20221221", + "pandas>=2.2.2", + "scikit-image>=0.24.0", + "tensorboard>=2.17.0", + "pycocotools>=2.0.8", + "tensordict>=0.6.0", + "opencv-python>=4.7.0", + "submitit>=1.5.1", + ], +} + +# By default, we also build the SAM 2 CUDA extension. +# You may turn off CUDA build with `export SAM2_BUILD_CUDA=0`. +BUILD_CUDA = os.getenv("SAM2_BUILD_CUDA", "1") == "1" +# By default, we allow SAM 2 installation to proceed even with build errors. +# You may force stopping on errors with `export SAM2_BUILD_ALLOW_ERRORS=0`. +BUILD_ALLOW_ERRORS = os.getenv("SAM2_BUILD_ALLOW_ERRORS", "1") == "1" + +# Catch and skip errors during extension building and print a warning message +# (note that this message only shows up under verbose build mode +# "pip install -v -e ." or "python setup.py build_ext -v") +CUDA_ERROR_MSG = ( + "{}\n\n" + "Failed to build the SAM 2 CUDA extension due to the error above. " + "You can still use SAM 2 and it's OK to ignore the error above, although some " + "post-processing functionality may be limited (which doesn't affect the results in most cases; " + "(see https://github.com/facebookresearch/sam2/blob/main/INSTALL.md).\n" +) + + +def get_extensions(): + if not BUILD_CUDA: + return [] + + try: + from torch.utils.cpp_extension import CUDAExtension + + srcs = ["sam2/csrc/connected_components.cu"] + compile_args = { + "cxx": [], + "nvcc": [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ], + } + ext_modules = [CUDAExtension("sam2._C", srcs, extra_compile_args=compile_args)] + except Exception as e: + if BUILD_ALLOW_ERRORS: + print(CUDA_ERROR_MSG.format(e)) + ext_modules = [] + else: + raise e + + return ext_modules + + +try: + from torch.utils.cpp_extension import BuildExtension + + class BuildExtensionIgnoreErrors(BuildExtension): + + def finalize_options(self): + try: + super().finalize_options() + except Exception as e: + print(CUDA_ERROR_MSG.format(e)) + self.extensions = [] + + def build_extensions(self): + try: + super().build_extensions() + except Exception as e: + print(CUDA_ERROR_MSG.format(e)) + self.extensions = [] + + def get_ext_filename(self, ext_name): + try: + return super().get_ext_filename(ext_name) + except Exception as e: + print(CUDA_ERROR_MSG.format(e)) + self.extensions = [] + return "_C.so" + + cmdclass = { + "build_ext": ( + BuildExtensionIgnoreErrors.with_options(no_python_abi_suffix=True) + if BUILD_ALLOW_ERRORS + else BuildExtension.with_options(no_python_abi_suffix=True) + ) + } +except Exception as e: + cmdclass = {} + if BUILD_ALLOW_ERRORS: + print(CUDA_ERROR_MSG.format(e)) + else: + raise e + + +# Setup configuration +setup( + name=NAME, + version=VERSION, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + url=URL, + author=AUTHOR, + author_email=AUTHOR_EMAIL, + license=LICENSE, + packages=find_packages(exclude="notebooks"), + include_package_data=True, + install_requires=REQUIRED_PACKAGES, + extras_require=EXTRA_PACKAGES, + python_requires=">=3.10.0", + ext_modules=get_extensions(), + cmdclass=cmdclass, +) diff --git a/tools/sam2-onnx-tensorrt/src/Module.py b/tools/sam2-onnx-tensorrt/src/Module.py new file mode 100644 index 00000000..3b34eca3 --- /dev/null +++ b/tools/sam2-onnx-tensorrt/src/Module.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import time +from torch import nn +from typing import Any +from sam2.build_sam import build_sam2 +from sam2.modeling.sam2_base import SAM2Base +from sam2.modeling.sam2_utils import get_1d_sine_pe +from sam2.utils.misc import fill_holes_in_mask_scores + +class ImageEncoder(nn.Module): + def __init__(self, sam_model: SAM2Base) -> None: + super().__init__() + self.model = sam_model + self.no_mem_embed = sam_model.no_mem_embed # [1,1,256] + self.image_encoder = sam_model.image_encoder + self.num_feature_levels = sam_model.num_feature_levels + self.prepare_backbone_features = sam_model._prepare_backbone_features + + @torch.no_grad() + def forward(self, image: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + start_time = time.time() + backbone_out = self.image_encoder(image) # {"vision_features","vision_pos_enc","backbone_fpn"} + backbone_out["backbone_fpn"][0] = self.model.sam_mask_decoder.conv_s0(backbone_out["backbone_fpn"][0]) + backbone_out["backbone_fpn"][1] = self.model.sam_mask_decoder.conv_s1(backbone_out["backbone_fpn"][1]) + + vision_pos_enc = backbone_out["vision_pos_enc"] # three tensors + backbone_fpn = backbone_out["backbone_fpn"] # three tensor + pix_feat = backbone_out["vision_features"] # one tensor + + expanded_backbone_out = { + "backbone_fpn": backbone_fpn, + "vision_pos_enc": vision_pos_enc, + } + for i, feat in enumerate(expanded_backbone_out["backbone_fpn"]): + expanded_backbone_out["backbone_fpn"][i] = feat.expand(1, -1, -1, -1) + for i, pos in enumerate(expanded_backbone_out["vision_pos_enc"]): + expanded_backbone_out["vision_pos_enc"][i] = pos.expand(1, -1, -1, -1) + + (_, current_vision_feats, current_vision_pos_embeds, _) = self.prepare_backbone_features(expanded_backbone_out) + + current_vision_feat = current_vision_feats[-1] + self.no_mem_embed + current_vision_feat2 = current_vision_feat.reshape(64, 64, 1, 256).permute(2, 3, 0, 1) # [1,256,64,64] + + # flatten HWxNxC -> NxCxHxW + high_res_features_0 = current_vision_feats[0].reshape(256, 256, 1, 32).permute(2, 3, 0, 1) # [1, 32, 256, 256] + high_res_features_1 = current_vision_feats[1].reshape(128, 128, 1, 64).permute(2, 3, 0, 1) # [1, 64, 128, 128] + + # pix_feat [1, 256, 64, 64] + # high_res_features_0 [1, 32, 256, 256] + # high_res_features_1 [1, 64, 128, 128] + # current_vision_feat [1, 256, 64, 64] + # current_vision_pos_embed2 [4096, 1, 256] + end_time = time.time() + + return pix_feat, high_res_features_0, high_res_features_1, current_vision_feat2, current_vision_pos_embeds[-1] + +class MemAttention(nn.Module): + def __init__(self, sam_model: SAM2Base) -> None: + super().__init__() + self.model = sam_model + self.no_mem_embed = sam_model.no_mem_embed + self.memory_attention = sam_model.memory_attention + self.obj_ptr_tpos_proj = sam_model.obj_ptr_tpos_proj + + # @torch.no_grad() + def forward( + self, + current_vision_feat: torch.Tensor, # [1, 256, 64, 64] + current_vision_pos_embed: torch.Tensor, # [4096, 1, 256] + memory_0: torch.Tensor, # [batch,num_obj_ptr,256]->[batch,num_obj_ptr,4,64]->[4*num_obj_ptr,batch,64] + memory_1: torch.Tensor, # [batch,num_masks,64,64,64]->[batch,num_masks,64,4096]->[4096*num_masks,batch,64] + memory_pos_embed: torch.Tensor, # [y*4096,1,64] + cond_frame_id_diff: torch.Tensor, # single float, current frame id - first cond frame id + ) -> tuple[Any]: + start_time = time.time() + num_obj_ptr_tokens = memory_0.shape[1] * 4 # old: shape[0] + current_vision_feat = current_vision_feat.permute(2, 3, 0, 1).reshape(4096, 1, 256) + current_vision_feat = current_vision_feat - self.no_mem_embed + + batch_size = memory_0.size()[0] + num_obj_ptr = memory_0.size()[1] + num_masks = memory_1.size()[1] + + current_vision_feat = current_vision_feat.repeat(1, batch_size, 1) + current_vision_pos_embed = current_vision_pos_embed.repeat(1, batch_size, 1) + + # [20,16,256] -> [20,16,4,64] -> [16,4,20,64] -> [64,20,64] + memory_0 = memory_0.reshape(batch_size, -1, 4, 64) + memory_0 = memory_0.permute(1, 2, 0, 3).flatten(0, 1) + + # old [16,256] -> [16,1,4,64] -> [16,4,1,64] -> [64,1,64] + # memory_0 = memory_0.reshape(-1,1,4,64) + # memory_0 = memory_0.permute(0, 2, 1, 3).flatten(0, 1) + + # [20,7,64,64,64] -> [20,7,64,64*64] -> [7,64*64,20,64] -> [7*64*64,20, 64] + memory_1 = memory_1.view(batch_size, -1, 64, 64*64).permute(1, 3, 0, 2) + memory_1 = memory_1.reshape(-1, batch_size, 64) + + # old [7,64,64,64] -> [7,64,64*64] -> [7,64*64,64] -> [7*64*64,1, 64] + # memory_1 = memory_1.view(-1, 64, 64*64).permute(0,2,1) + # memory_1 = memory_1.reshape(-1,1,64) + + # [20,7*4096+64,64] -> [7*4096+64,20,64] + memory_pos_embed = memory_pos_embed.permute(1, 0, 2) + + if True: + # generate obj_pos as add_tpos_enc_to_obj_ptrs is used in SAM 2.1 + # add it to memory_pos_embed + + # obj_pos frame id is like [cond_frame_id_diff, 1, 2, 3,...,15] + obj_pos = torch.cat([cond_frame_id_diff.unsqueeze(0), + torch.arange(1, num_obj_ptr, dtype=torch.float32)]) + t_diff_max = 15.0 + obj_pos = get_1d_sine_pe(obj_pos / t_diff_max, dim=256) # 256 channels # [num_obj_ptr,256] + obj_pos = self.obj_ptr_tpos_proj(obj_pos) + obj_pos = obj_pos.unsqueeze(1).expand(-1, batch_size, 64) # [num_obj_ptr,batch_size,64] + obj_pos = obj_pos.repeat_interleave(4, dim=0) # [4*num_obj_ptr,batch_size,64] + memory_pos_embed[num_masks*4096 : num_masks*4096+4*num_obj_ptr] = obj_pos + + + memory = torch.cat((memory_1, memory_0), dim=0) + pix_feat_with_mem = self.memory_attention( + curr=current_vision_feat, + curr_pos=current_vision_pos_embed, + memory=memory, + memory_pos=memory_pos_embed, + num_obj_ptr_tokens=num_obj_ptr_tokens, + ) + # reshape the output (HW)xBxC => BxCxHxW + image_embed = pix_feat_with_mem.permute(1, 2, 0).view(batch_size, 256, 64, 64) # [1,256,64,64] + end_time = time.time() + + return image_embed # [1,256,64,64] + +class MemEncoder(nn.Module): + def __init__(self, sam_model: SAM2Base) -> None: + super().__init__() + self.model = sam_model + self.maskmem_tpos_enc = sam_model.maskmem_tpos_enc + self.feat_sizes = [(256, 256), (128, 128), (64, 64)] + @torch.no_grad() + def forward( + self, + mask_for_mem: torch.Tensor, # [1,1,1024,1024] + pix_feat: torch.Tensor, # [1,256,64,64] + occ_logit: torch.Tensor, # [1,1] + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + start_time = time.time() + + batch_size = mask_for_mem.shape[0] + pix_feat = pix_feat.repeat(batch_size, 1, 1, 1) + + maskmem_features, maskmem_pos_enc = self.model._encode_new_memory( + current_vision_feats=pix_feat, + feat_sizes=self.feat_sizes, + pred_masks_high_res=mask_for_mem, + is_mask_from_pts=True, + object_score_logits=occ_logit, + ) + # maskmem_features = maskmem_features.view(1, 64, 64*64) # .permute(2, 0, 1) + maskmem_pos_enc = maskmem_pos_enc[0].view(batch_size, 64, 64*64).permute(0, 2, 1) # .permute(2, 0, 1) + + + end_time = time.time() + + + return maskmem_features, maskmem_pos_enc, self.maskmem_tpos_enc + +class MaskDecoder(nn.Module): + def __init__(self, sam_model: SAM2Base) -> None: + super().__init__() + self.model = sam_model + self.sigmoid_scale_for_mem_enc = sam_model.sigmoid_scale_for_mem_enc + self.sigmoid_bias_for_mem_enc = sam_model.sigmoid_bias_for_mem_enc + @torch.no_grad() + def forward( + self, + point_coords: torch.Tensor, # [num_labels,num_points,2] + point_labels: torch.Tensor, # [num_labels,num_points] + # frame_size: torch.Tensor, # [2] + image_embed: torch.Tensor, # [1,256,64,64] + high_res_feats_0: torch.Tensor, # [1, 32, 256, 256] + high_res_feats_1: torch.Tensor, # [1, 64, 128, 128] + ): + start_time = time.time() + frame_size = [256, 256] + point_inputs = {"point_coords": point_coords, "point_labels": point_labels} + + batch_size = point_coords.size()[0] + if True: + # image_embed = image_embed.repeat(batch_size, 1, 1, 1) image embedding can be different for each object when using memory bank + high_res_feats_0 = high_res_feats_0.repeat(batch_size, 1, 1, 1) + high_res_feats_1 = high_res_feats_1.repeat(batch_size, 1, 1, 1) + high_res_feats = [high_res_feats_0, high_res_feats_1] + + sam_outputs = self.model._forward_sam_heads( + backbone_features=image_embed, + point_inputs=point_inputs, + mask_inputs=None, + high_res_features=high_res_feats, + multimask_output=True + ) + ( + _, + _, + ious, # [1,3] + low_res_masks, # [1,1,256,256] + high_res_masks, # [1,1,1024,1024] + obj_ptr, # [1,256] + occ_logit, # [1,1] + ) = sam_outputs + # high resolution mask + mask_for_mem = torch.sigmoid(high_res_masks) + mask_for_mem = mask_for_mem * self.sigmoid_scale_for_mem_enc + mask_for_mem = mask_for_mem + self.sigmoid_bias_for_mem_enc + # fill holes + low_res_masks = fill_holes_in_mask_scores(low_res_masks, 8) + # rescaling + pred_mask = torch.nn.functional.interpolate( + low_res_masks, + size=(frame_size[0], frame_size[1]), + mode="bilinear", + align_corners=False, + ) + # Pick highest IOU + iou = torch.max(ious, dim=-1, keepdim=True)[0] + + + end_time = time.time() + + return obj_ptr, mask_for_mem, pred_mask, iou, occ_logit diff --git a/tools/yolo_deepstream/.gitkeep b/tools/yolo_deepstream/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/yolo_deepstream/CLA.md b/tools/yolo_deepstream/CLA.md new file mode 100644 index 00000000..389a4746 --- /dev/null +++ b/tools/yolo_deepstream/CLA.md @@ -0,0 +1,58 @@ +## Individual Contributor License Agreement (CLA) + +**Thank you for submitting your contributions to this project.** + +By signing this CLA, you agree that the following terms apply to all of your past, present and future contributions +to the project. + +### License. + +You hereby represent that all present, past and future contributions are governed by the +[MIT License](https://opensource.org/licenses/MIT) +copyright statement. + +This entails that to the extent possible under law, you transfer all copyright and related or neighboring rights +of the code or documents you contribute to the project itself or its maintainers. +Furthermore you also represent that you have the authority to perform the above waiver +with respect to the entirety of you contributions. + +### Moral Rights. + +To the fullest extent permitted under applicable law, you hereby waive, and agree not to +assert, all of your “moral rights” in or relating to your contributions for the benefit of the project. + +### Third Party Content. + +If your Contribution includes or is based on any source code, object code, bug fixes, configuration changes, tools, +specifications, documentation, data, materials, feedback, information or other works of authorship that were not +authored by you (“Third Party Content”) or if you are aware of any third party intellectual property or proprietary +rights associated with your Contribution (“Third Party Rights”), +then you agree to include with the submission of your Contribution full details respecting such Third Party +Content and Third Party Rights, including, without limitation, identification of which aspects of your +Contribution contain Third Party Content or are associated with Third Party Rights, the owner/author of the +Third Party Content and Third Party Rights, where you obtained the Third Party Content, and any applicable +third party license terms or restrictions respecting the Third Party Content and Third Party Rights. For greater +certainty, the foregoing obligations respecting the identification of Third Party Content and Third Party Rights +do not apply to any portion of a Project that is incorporated into your Contribution to that same Project. + +### Representations. + +You represent that, other than the Third Party Content and Third Party Rights identified by +you in accordance with this Agreement, you are the sole author of your Contributions and are legally entitled +to grant the foregoing licenses and waivers in respect of your Contributions. If your Contributions were +created in the course of your employment with your past or present employer(s), you represent that such +employer(s) has authorized you to make your Contributions on behalf of such employer(s) or such employer +(s) has waived all of their right, title or interest in or to your Contributions. + +### Disclaimer. + +To the fullest extent permitted under applicable law, your Contributions are provided on an "as is" +basis, without any warranties or conditions, express or implied, including, without limitation, any implied +warranties or conditions of non-infringement, merchantability or fitness for a particular purpose. You are not +required to provide support for your Contributions, except to the extent you desire to provide support. + +### No Obligation. + +You acknowledge that the maintainers of this project are under no obligation to use or incorporate your contributions +into the project. The decision to use or incorporate your contributions into the project will be made at the +sole discretion of the maintainers or their authorized delegates. diff --git a/tools/yolo_deepstream/LICENSE.md b/tools/yolo_deepstream/LICENSE.md new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/tools/yolo_deepstream/LICENSE.md @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/yolo_deepstream/README.md b/tools/yolo_deepstream/README.md new file mode 100644 index 00000000..e78c6920 --- /dev/null +++ b/tools/yolo_deepstream/README.md @@ -0,0 +1,167 @@ +# Yolo DeepStream + +## Description + +This repo have 4 parts: +### 1) yolov7_qat +In [yolov7_qat](yolov7_qat), We use [TensorRT's pytorch quntization tool](https://github.com/NVIDIA/TensorRT/tree/main/tools/pytorch-quantization) to Finetune training QAT yolov7 from the pre-trained weight. +Finally we get the same performance of PTQ in TensorRT on Jetson OrinX. And the accuracy(mAP) of the model only dropped a little. + +### 2) tensorrt_yolo +In [tensorrt_yolo](tensorrt_yolo), We provide a standalone c++ yolov7-app sample here. You can use trtexec to convert FP32 onnx models or QAT-int8 models exported from repo [yolov7_qat](yolov7_qat) to trt-engines. And set the trt-engine as yolov7-app's input. It can do detections on images/videos. Or test mAP on COCO dataset. It also support [yolov8](#onnx-model-list) and [yolov9](#onnx-model-list) models. + +### 3) deepstream_yolo +In [deepstream_yolo](deepstream_yolo), This sample shows how to integrate YOLO models with customized output layer parsing for detected objects with DeepStreamSDK. + +## Performance + +Below table shows the end-to-end performance of processing 1080p videos with this sample application. +- Testing Device : + + 1. Jetson AGX Orin 64GB(PowerMode:MAXN + GPU-freq:1.3GHz + CPU:12-core-2.2GHz) + + 2. Tesla T4 + +- Testing Models + 1. Yolov7 models and calibration file can be found in sub dir: [yolov7_qat](./yolov7_qat/README.md#description) + 2. Yolov8s model and calibration files can be found here: [model list](#onnx-model-list) + 3. Yolov9s model can be generated by the method in [yolov9 model generation](deepstream_yolo/#yolov9) + 2. Yolov11s model can be generated by the method in: [yolov11s model generation](deepstream_yolo/#yolov11s) + + + #### Performance in Jetson Thor + + The data in the following table are tested with scaling-compute-hw=1. + + |Model |precision | Batch Size | Number
of streams | trtexec FPS| deepstream-app FPS
with cuda-post-process | mAPval
0.5:0.95 | + |--------------|----------- |----------- |----------------- | -----------|----------- | --------- | + |yolov4 | FP16 | 1 | 1 | 278 | 52 | 51.13 | + |yolov7 | FP16 | 1 | 1 | 600 | 60 | 51.13 | + |yolov8s | FP16 | 1 | 1 | 1097 | 64 |44.5| + |yolov9s| FP16 | 1 | 1 | 582 | 59 |46.8| + |yolov11s| FP16 | 1 | 1 | 977 | 63 | 46.5| + |yolov4 | FP16 | 16 | 16 | 391 | 384 |51.13| + |yolov7 | FP16 | 16 | 16 | 740 | 656 |51.13| + |yolov8s | FP16 | 16 | 16 | 1061 | 944 |44.5| + |yolov9s| FP16 | 16 | 16 | 788 | 656 |46.8| + |yolov11s| FP16 | 16 | 16 | 1197 | 768 |46.5| + + - Note: Jetson Thor does not support DLA + + The data in the following table are tested with scaling-compute-hw=2. + + |Model |precision | Batch Size | Number
of streams | trtexec FPS| deepstream-app FPS
with cuda-post-process | mAPval
0.5:0.95 | + |--------------|----------- |----------- |----------------- | -----------|----------- | --------- | + |yolov4 | FP16 | 1 | 1 | 278 | 52 | 51.13 | + |yolov7 | FP16 | 1 | 1 | 600 | 61 | 51.13 | + |yolov8s | FP16 | 1 | 1 | 1097 | 63 |44.5| + |yolov9s| FP16 | 1 | 1 | 582 | 60 |46.8| + |yolov11s| FP16 | 1 | 1 | 977 | 61 | 46.5| + |yolov4 | FP16 | 16 | 16 | 391 | 368 |51.13| + |yolov7 | FP16 | 16 | 16 | 740 | 608 |51.13| + |yolov8s | FP16 | 16 | 16 | 1061 | 944 |44.5| + |yolov9s| FP16 | 16 | 16 | 788 | 608 |46.8| + |yolov11s| FP16 | 16 | 16 | 1197 | 768 |46.5| + + #### Performance in Tegra T4 + + The data in the following table are tested with scaling-compute-hw=1. + + |Model |precision | Batch Size | Number
of streams | trtexec FPS| deepstream-app FPS
with cuda-post-process | mAPval
0.5:0.95 | + |--------------|-----------|-----------|----------------- | -----------|----------- | ---------| + |yolov7| FP16 | 1 | 1 | 208 | 58 | 51.13 | + |yolov8s| FP16 | 1 | 1 | 1064 | 70 | 44.5 | + |yolov9s| FP16 | 1 | 1 | 326 | 58 | 46.8 | + |yolov11s| FP16 | 1 | 1 | 1132 | 64 | 46.5 | + |yolov7| FP16 | 16 | 16 | 305 | 304 | 51.13 | + |yolov8s| FP16 | 16 | 16 | 1590 | 752 | 44.5 | + |yolov9s| FP16 | 16 | 16 | 688 | 368 | 46.8 | + |yolov11s| FP16 | 16 | 16 | 1532 | 688 | 46.5 | + + + - Note: trtexec cudaGraph not enabled as deepstream not support cudaGraph + + +## Onnx model list + +we provide the following onnx models + + +| model name | calibrationFile |Hardware | resolution | precision | mAPval
0.5:0.95 | +|----------- |----------- |----------------- | ------ | ------ |------ | +| [yolov7_ptq_640.onnx](https://nvidia.box.com/shared/static/rlv3buq7sei2log2d3beyg1jhjyw59hn) | explict quant model with QDQ nodes | gpu | batch x 3 x 640 x 640 | int8 |51.00 | +| [yolov7.onnx](https://nvidia.box.com/shared/static/rmh8rttesg4cgrysb2qm12udpvd95as1) | - | gpu | batch x 3 x 640 x 640 | fp16 | 51.24| +| [yolov7_qat_640.onnx](https://nvidia.box.com/shared/static/v1ze885p35hfjl96xtw8s0xbcpv64tfr) | explict quant model with QDQ | gpu | batch x 3 x 640 x 640 | int8 | 51.13 | +| [yolov8s_DAT_640_noqdq.onnx](https://nvidia.box.com/shared/static/ownxazhmtpnlo3jvbkx4r62ffccm8hu5) | [yolov8s_DAT_precision_config_calib.cache](https://nvidia.box.com/shared/static/6bua0bo57cb6s44048os9qq9i1xjw5u1) | dla | 1 x 3 x 640 x 640 | int8 |44.6| +| [yolov8s_640_dynamic.onnx](https://nvidia.box.com/shared/static/yie26fuadn2wdm21bqqagbjat68ih38p) | [yolov8s_gpu_precision_config_calib.cache](https://nvidia.box.com/shared/static/041fltrp4i0u8nv37fy3oj31453bjbp3) | gpu | batch x 3 x 640 x 640 | int8/fp16 |44.5/44.9 | +| [yolov9-s-converted.sim.trans.onnx](https://nvidia.box.com/shared/static/dzch7bx0xlap4hoc5nk9huy72w33wbc9) | - | gpu | 1 x 3 x 640 x 640 | fp16 |46.8| +| [yolov11s_qat_int8_672_dynamic.onnx](https://nvidia.box.com/shared/static/87pt9tlgx588l9j9a9wfdk2yjljjrffn) | explict quant model with QDQ | gpu | batch x 3 x 672 x 672 | int8 | 46.5 | + + +## Code structure +```bash +├── CLA.md +├── deepstream_yolo +│   ├── append_transpose_yolov8_v9.py +│   ├── build_DLA_engine.sh +│   ├── config_infer_primary_yoloV4.txt # config file for yolov4 model +│   ├── config_infer_primary_yoloV7.txt # config file for yolov7 model +│   ├── config_infer_primary_yoloV8_dla.txt # config file for yolov8 for Jetson DLA model +│   ├── config_infer_primary_yoloV8.txt # config file for yolov8 model +│   ├── config_infer_primary_yoloV9.txt # config file for yolov9 model +│ ├── config_infer_primary_yoloV11.txt # config file for yolov11 model +│ ├── config_infer_primary_yoloV11_obb.txt # config file for yolov11 OBB model +│   ├── deepstream_app_config_yolo.txt +│   ├── labels.txt # labels for coco detection +│   ├── labels_obb.txt # labels for dotav1 detection +│   ├── nvdsinfer_custom_impl_Yolo # output layer parsing functions for detected objects for the Yolo model. +│   │   ├── Makefile +│   │   ├── nvdsparsebbox_Yolo.cpp +│   │   └── nvdsparsebbox_Yolo_cuda.cu +│   └── README.md +├── LICENSE.md +├── README.md +├── tensorrt_yolo +│   ├── CMakeLists.txt +│   ├── imgs # the demo images +│   │   ├── horses.jpg +│   │   └── zidane.jpg +│   ├── README.md +│   ├── samples +│   │   ├── detect.cpp # detection app for images detection +│   │   ├── validate_coco.cpp # validate coco dataset app +│   │   └── video_detect.cpp # detection app for video detection +│   ├── src +│   │   ├── argsParser.cpp # argsParser helper class for commandline parsing +│   │   ├── argsParser.h # argsParser helper class for commandline parsing +│   │   ├── tools.h # helper function for yolo class +│   │   ├── Yolo.cpp # Class Yolo +│   │   └── Yolo.h # Class Yolo +│   └── test_coco_map.py # tool for test coco map with json file +└── yolov7_qat + ├── doc + │   ├── Guidance_of_QAT_performance_optimization.md # guidance for Q&DQ insert and placement for pytorch-quantization tool + │   └── imgs # the demo images + │   ├── int8_q_recommended_procedure.png + │   ├── monkey-patch-qat-conv-fp16-issue_ptqonnx.png + │   ├── monkey-patch-qat-conv-fp16-issue_ptq.png + │   ├── monkey-patch-qat-conv-fp16-issue_qatonnx_edit.png + │   ├── monkey-patch-qat-conv-fp16-issue_qatonnx.png + │   ├── monkey-patch-qat-conv-fp16-issue_qat.png + │   ├── monkey-patch-qat-maxpooling-qat.png + │   ├── QATConv.png + │   └── QATFlow.png + ├── quantization + │   ├── quantize.py # helper class for quantize yolov7 model + │   └── rules.py # rules for Q&DQ nodes insert and restrictions + ├── README.md + └── scripts + ├── detect-trt.py # detect a image with tensorrt engine + ├── draw-engine.py # draw tensorrt engine to graph + ├── eval-trt.py # the script for evalating tensorrt mAP + ├── eval-trt.sh # the command lne script for evaluating tensorrt mAP + ├── qat.py # main function for QAT and PTQ + ├── qat-yolov5.py + ├── quantize_utils.py + └── trt-int8.py # tensorrt build-in calibration +``` diff --git a/tools/yolo_deepstream/deepstream_yolo/README.md b/tools/yolo_deepstream/deepstream_yolo/README.md new file mode 100644 index 00000000..bccf7a4e --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/README.md @@ -0,0 +1,244 @@ +# Deploy YOLO Models With DeepStream # + +**This sample shows how to integrate YOLO models with customized output layer parsing for detected objects with DeepStreamSDK.** + +## 1. Sample contents: ## +- `deepstream_app_config_yolo.txt`: DeepStream reference app configuration file for using YOLO models as the primary detector. +- `config_infer_primary_yoloV4.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV4 detector model. +- `config_infer_primary_yoloV7.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV7 detector model. +- `config_infer_primary_yoloV8.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV8 detector model. +- `config_infer_primary_yoloV8_dla.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV8 detector model running with DLA on Jetson. +- `config_infer_primary_yoloV9.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV9 detector model. +- `config_infer_primary_yoloV11.txt`: Configuration file for the GStreamer nvinfer plugin for the YoloV11 detector model. +- `config_infer_primary_yoloV11_obb.txt`: Configuration file for the GStreamer nvinfer plugin for the YOLOv11 OBB (Oriented Bounding Box) detector model. +- `nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp`: Output layer parsing function for detected objects for the Yolo models. +- `nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo_cuda.cu`: Output layer parsing function for detected objects for the Yolo models by CUDA. + +## 2. Download and Build ## + +```sh + $ cd ~/ + $ git clone https://github.com/NVIDIA/deepstream.git + $ cd ~/deepstream/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo + $ make + $ cd .. +``` +NOTE: To improve performance on specific GPUs, please add "-gencode=arch=compute_xx,code=sm_xx" in Makefile. Computing capability can be found in this link https://developer.nvidia.com/zh-cn/cuda-gpus#compute. + +## 3. Prepare model and Run with deepstream-app ## + +### TIPS: DeepStream 7.1+ is required ### + +#### Yolov4 + +- Go to this pytorch repository where you can convert YOLOv4 Pytorch model into **ONNX** +- Other famous YOLOv4 pytorch repositories as references: + - + - + - + - +- Or you can download reference ONNX model directly from here ([link](https://nvidia.box.com/s/achcifjwl1ac99tdvfwtfmxgec5d0pro)). + +- Run + ```bash + $ deepstream-app -c deepstream_app_config_yolo.txt + ``` + The output result will output to `yolo.mp4` + +#### YOLOv7 +- Follow the guide https://github.com/WongKinYiu/yolov7#export, export a dynamic-batch-1-output onnx-model + +```bash +$ python export.py --weights ./yolov7.pt --grid --simplify --topk-all 100 --iou-thres 0.65 --conf-thres 0.35 --img-size 640 640 --dynamic-batch +``` + +- Or use the qat model exported from [yolov7_qat](../yolov7_qat/README.md) + +- Generate the TensorRT engine with the following command: + +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7_qat_640.onnx --int8 --fp16 --minShapes=images:1x3x640x640 --optShapes=images:8x3x640x640 --maxShapes=images:16x3x640x640 --saveEngine=yolov7_qat_640_gpu_b16.engine +``` + +- Run +```bash +$ deepstream-app -c deepstream_app_config_yolo.txt -t +``` +The output result will output to `yolo.mp4` + +#### YOLOv8 +##### 1. Run on GPU with int8 precision with calibrated model(GPU Only) +- Convert model with trtexec + + The model can be found in [model-list](../README.md#model-list) + + ```bash + # batchsize = 16 + $ /usr/src/tensorrt/bin/trtexec --onnx=yolov8s_640_dynamic.onnx --fp16 --int8 --verbose --calib=yolov8s_gpu_precision_config_calib.cache --saveEngine=yolov8s_ptq_640_gpu_b16.engine --minShapes=x.1:16x3x640x640 --optShapes=x.1:16x3x640x640 --maxShapes=x.1:16x3x640x640 --precisionConstraints=obey --layerPrecisions=Split_36:fp16,Reshape_37:fp16,Transpose_38:fp16,Softmax_39:fp16,Conv_41:fp16,Sub_64:fp16,Concat_65:fp16,Mul_67:fp16,Sigmoid_68:fp16,Concat_69:fp16 + ``` + +- Enable the configure on config files, edit [deepstream_app_config_yolo.txt](./deepstream_app_config_yolo.txt) + ```ini + # batch-size = 16 + config-file=config_infer_primary_yoloV8.txt + ``` + +##### 2. Run on DLA with our DLA-spec-finetuned model(DLA Only) +- Download sample ONNX models and convert the ONNX model to TensorRT engine + + Download model and coresponding calibration file from [model-list](../README.md#model-list) + + Choose one of the following methods + + * Convert model with trtexec + + ```bash + $ /usr/src/tensorrt/bin/trtexec --onnx=yolov8s_DAT_640_noqdq.onnx --fp16 --int8 --verbose --calib=yolov8s_DAT_precision_config_calib.cache --precisionConstraints=obey --layerPrecisions=Split_36:fp16,Reshape_37:fp16,Transpose_38:fp16,Softmax_39:fp16,Conv_41:fp16,Sub_64:fp16,Concat_65:fp16,Mul_67:fp16,Sigmoid_68:fp16,Concat_69:fp16 --saveEngine=yolov8s_DAT_640_noqdq_DLA.engine --useDLACore=0 --allowGPUFallback + ``` + + * Run the download and build script + + ```bash + ./build_DLA_engine.sh + ``` + +- Enable the configuration for yolov8 in config files, edit [deepstream_app_config_yolo.txt](./deepstream_app_config_yolo.txt) + + ```ini + ... + num-sources=1 + ... + config-file=config_infer_primary_yoloV8_dla.txt + [streammux] + ... + batch-size=1 + ... + + [primary-gie] + ... + batch-size=1 + ... + ``` + + Edit [config_infer_primary_yoloV8_dla.txt](./config_infer_primary_yoloV8_dla.txt) + ``` + model-engine-file=yolov8s_DAT_640_noqdq_DLA.engine + ... + enable-dla=1 + use-dla-core=1 # or 0 + ``` + +##### 3. Run + ```bash + $ deepstream-app -c deepstream_app_config_yolo.txt + ``` + The output result will output to `yolo.mp4` + +#### YOLOv9 +- Get ONNX model by one of the following methods + * Follow the guide https://github.com/WongKinYiu/yolov9/issues/2#issuecomment-1960519506, export a dynamic-batch-1-output onnx-model + ```bash + $ python export.py --weights yolov9-s-converted.pt --dynamic --include onnx + ``` + * Download the ONNX model directly from [yolov9-s onnx model](https://nvidia.box.com/s/dzch7bx0xlap4hoc5nk9huy72w33wbc9) +- Put the ONNX model file under yolo_deepstream/deepstream_yolo directory +- Modify the [deepstream_app_config_yolo.txt](./deepstream_app_config_yolo.txt) file + ```ini + config-file=config_infer_primary_yoloV9.txt + ``` +- Run + ```bash + deepstream-app -c deepstream_app_config_yolo.txt -t + ``` + The output result will output to `yolo.mp4` + +#### YOLOv8/YOLOv9 TIPS + + - How to export yolov8 models with official repo + Follow the guide https://github.com/ultralytics/ultralytics/tree/v8.2.103, export a dynamic-batch-1-output onnx-model + + ```python + from ultralytics import YOLO + # Load a model + model = YOLO("yolov8s.pt") # load a pretrained model (recommended for training) + path = model.export(format="onnx") # export the model to ONNX format + ``` + + - For yolov8 and yolov9 models user must add a transpose node at the end of the network through [onnx-graphsurgeon](https://github.com/NVIDIA/TensorRT/tree/release/9.2/tools/onnx-graphsurgeon) : + ```bash + $ python append_transpose_yolov8_v9.py + ``` + +#### YOLOv11s + +- Use the following command to download the yolov11s onnx-model + +```bash +wget https://nvidia.box.com/shared/static/87pt9tlgx588l9j9a9wfdk2yjljjrffn -O yolov11s_qat_int8_672_dynamic.onnx +``` + +- Convert model with trtexec +```bash +# batch size 16 +/usr/src/tensorrt/bin/trtexec --minShapes=images:1x3x672x672 --optShapes=images:16x3x672x672 --maxShapes=images:16x3x672x672 --onnx=yolov11s_qat_int8_672_dynamic.onnx --saveEngine=yolov11s_qat_int8_672_dynamic.onnx_b16_gpu0_int8.engine --int8 --fp16 --dumpLayerInfo --profilingVerbosity=detailed --exportLayerInfo=yolov11s_qat_int8_672_dynamic_layer.json --exportProfile=yolov11s_qat_int8_672_dynamic_profile.json +``` + +- Modify the [deepstream_app_config_yolo.txt](./deepstream_app_config_yolo.txt) file +```ini +config-file=config_infer_primary_yoloV11.txt +``` + +- Run +```bash +deepstream-app -c deepstream_app_config_yolo.txt -t +``` +The output result will output to `yolo.mp4` + +#### YOLOv11 OBB (Oriented Bounding Box) + +- Export the YOLOv11 OBB model to ONNX (e.g. with Ultralytics). Requires DeepStream SDK for YOLOv11 OBB custom parser. + +```bash +python3 -c " +from ultralytics import YOLO +model = YOLO('yolo11n-obb.pt') +model.export(format='onnx', dynamic=False, simplify=True) +" +``` + +- Put `yolo11n-obb.onnx` and `labels_obb.txt` under this directory. nvinfer will build the engine on first run. Update `custom-lib-path` in [config_infer_primary_yoloV11_obb.txt](./config_infer_primary_yoloV11_obb.txt) if your DeepStream custom parser is installed elsewhere. + +- Modify the [deepstream_app_config_yolo.txt](./deepstream_app_config_yolo.txt) file + * Set `uri` under `[source0]` to your own aerial video (e.g. DOTA-style) to test with this model. + * Under `[primary-gie]`, set `config-file` and `labelfile-path`. + + ```ini + # [source0] + uri=file:///path/to/your/aerial_video.mp4 + + + # [primary-gie] + config-file=config_infer_primary_yoloV11_obb.txt + labelfile-path=labels_obb.txt + ``` + +- Run +```bash +deepstream-app -c deepstream_app_config_yolo.txt -t +``` +The output result will output to `yolo.mp4` + +## 4. CUDA Post Processing + +this sample provide two ways of yolov7/yolov8/yolov9/yolov11 post-processing(decode yolo result, not include NMS), CPU version and GPU version +- CPU implement can be found in: [nvdsparsebbox_Yolo.cpp](nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp) +- CUDA implement can be found in: [nvdsparsebbox_Yolo_cuda.cu](nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo_cuda.cu) + +Default will use CUDA-post processing. To enable CPU post-processing: +in [config_infer_primary_yoloV7.txt](config_infer_primary_yoloV7.txt), [config_infer_primary_yoloV8.txt](config_infer_primary_yoloV8.txt), [config_infer_primary_yoloV9.txt](config_infer_primary_yoloV9.txt) or [config_infer_primary_yoloV11.txt](config_infer_primary_yoloV11.txt) + +- `parse-bbox-func-name=NvDsInferParseCustomYoloV7_cuda` -> `parse-bbox-func-name=NvDsInferParseCustomYoloV7` +- `disable-output-host-copy=1` -> `disable-output-host-copy=0` + +The performance of the CPU-post-processing and CUDA-post-processing result can be found in [Performance](../#Performance) + diff --git a/tools/yolo_deepstream/deepstream_yolo/append_transpose_yolov8_v9.py b/tools/yolo_deepstream/deepstream_yolo/append_transpose_yolov8_v9.py new file mode 100644 index 00000000..07645544 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/append_transpose_yolov8_v9.py @@ -0,0 +1,14 @@ +import onnx_graphsurgeon as gs +import numpy as np +import onnx + +graph = gs.import_onnx(onnx.load("yolov9-t-converted.onnx")) +# graph = gs.import_onnx(onnx.load("yolov8-s.onnx")) +ori_output = graph.outputs[0] +trans_out = gs.Variable(name="trans_out", dtype=np.float32, shape=(-1, 8400, 84)) +trans_node = gs.Node(op="Transpose",name="transpose_output_node", attrs={"perm":np.array([0,2,1])}, inputs=[ori_output], outputs=[trans_out]) +graph.nodes.append(trans_node) +graph.outputs = [trans_out] +graph.cleanup(remove_unused_graph_inputs=True).toposort() +model = onnx.shape_inference.infer_shapes(gs.export_onnx(graph)) +onnx.save(model, "yolov9-t-converted-trans-dynamic_batch_640.onnx") \ No newline at end of file diff --git a/tools/yolo_deepstream/deepstream_yolo/build_DLA_engine.sh b/tools/yolo_deepstream/deepstream_yolo/build_DLA_engine.sh new file mode 100644 index 00000000..85f14663 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/build_DLA_engine.sh @@ -0,0 +1,4 @@ +#! /bin/bash +wget https://nvidia.box.com/shared/static/ownxazhmtpnlo3jvbkx4r62ffccm8hu5 -O yolov8s_DAT_640_noqdq.onnx && \ +wget https://nvidia.box.com/shared/static/6bua0bo57cb6s44048os9qq9i1xjw5u1 -O yolov8s_DAT_precision_config_calib.cache && \ +/usr/src/tensorrt/bin/trtexec --onnx=yolov8s_DAT_640_noqdq.onnx --fp16 --int8 --verbose --calib=yolov8s_DAT_precision_config_calib.cache --precisionConstraints=obey --layerPrecisions=Split_36:fp16,Reshape_37:fp16,Transpose_38:fp16,Softmax_39:fp16,Conv_41:fp16,Sub_64:fp16,Concat_65:fp16,Mul_67:fp16,Sigmoid_68:fp16,Concat_69:fp16 --saveEngine=yolov8s_DAT_640_noqdq_DLA.engine --useDLACore=0 --allowGPUFallback diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11.txt new file mode 100644 index 00000000..487ee759 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11.txt @@ -0,0 +1,100 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=yolov11s_qat_int8_672_dynamic.onnx +infer-dims=3;672;672 +batch-size=16 +model-engine-file=yolov11s_qat_int8_672_dynamic.onnx_b16_gpu0_best.engine +labelfile-path=labels.txt +## 0=FP32, 1=INT8, 2=FP16, 3=BEST mode +network-mode=3 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +symmetric-padding=1 +## Bilinear Interpolation +scaling-filter=1 +# parse-bbox-func-name=NvDsInferParseCustomYoloV8 +parse-bbox-func-name=NvDsInferParseCustomYoloV11_cuda + +# disable-output-host-copy=0 +disable-output-host-copy=1 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-compute-hw=0 +## start from DS6.2 +crop-objects-to-roi-boundary=1 + + +[class-attrs-all] +#nms-iou-threshold=0.3 +#threshold=0.7 +nms-iou-threshold=0.65 +pre-cluster-threshold=0.25 +topk=300 + diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11_obb.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11_obb.txt new file mode 100644 index 00000000..aaa7b32a --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV11_obb.txt @@ -0,0 +1,59 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# YOLO11 OBB (Oriented Bounding Box) primary inference config. +# Requires DeepStream SDK with YOLO11 OBB custom parser (NvDsInferParseCustomYoloV11OBB). +# Update onnx-file, model-engine-file, labelfile-path and custom-lib-path for your setup. + +[property] +gpu-id=0 +net-scale-factor=0.003921569790137 +model-color-format=0 +labelfile-path=labels_obb.txt +model-engine-file=yolo11n-obb.onnx_b1_gpu0_fp32.engine +onnx-file=yolo11n-obb.onnx + +# Network precision mode: 0=FP32, 1=INT8, 2=FP16 +network-mode=0 +workspace-size=2048 +infer-dims=3;1024;1024 +uff-input-blob-name=input_1 +batch-size=1 +process-mode=1 +gie-unique-id=1 + +# 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3=DBSCAN+NMS Hybrid, 4=None (custom parser does postprocessing including NMS) +cluster-mode=4 +maintain-aspect-ratio=0 +symmetric-padding=0 + +# YOLO11 OBB custom parser (from DeepStream SDK). Update path if your parser is installed elsewhere. +custom-lib-path=/opt/nvidia/deepstream/deepstream/lib/libnvds_infercustomparser.so +parse-bbox-func-name=NvDsInferParseCustomYoloV11OBB +num-detected-classes=80 + +[class-attrs-all] +# When cluster-mode=4, nms-iou-threshold and topk are not used; NMS is handled inside the custom parser. +pre-cluster-threshold=0.5 +border-color=1;0;0;1 +bg-color=1;0;0;0.5 diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV4.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV4.txt new file mode 100644 index 00000000..3d76ab51 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV4.txt @@ -0,0 +1,86 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=yolov4_-1_3_416_416_dynamic.onnx.nms.onnx +model-engine-file=yolov4_-1_3_416_416_dynamic.onnx.nms.onnx_b16_gpu0_fp16.engine +labelfile-path=labels.txt +batch-size=16 +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 0=Group Rectangles, 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +parse-bbox-func-name=NvDsInferParseCustomYoloV4 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-filter=0 +#scaling-compute-hw=0 + +[class-attrs-all] +nms-iou-threshold=0.6 +pre-cluster-threshold=0.4 diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV7.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV7.txt new file mode 100644 index 00000000..69c57751 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV7.txt @@ -0,0 +1,97 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +#onnx-file=yolov7.onnx +model-engine-file=yolov7_qat_640_gpu_b16.engine +labelfile-path=labels.txt +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +symmetric-padding=1 +## Bilinear Interpolation +scaling-filter=1 +# parse-bbox-func-name=NvDsInferParseCustomYoloV7 +parse-bbox-func-name=NvDsInferParseCustomYoloV7_cuda +# disable-output-host-copy=0 +disable-output-host-copy=1 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-compute-hw=0 +## start from DS6.2 +crop-objects-to-roi-boundary=1 + + +[class-attrs-all] +#nms-iou-threshold=0.3 +#threshold=0.7 +nms-iou-threshold=0.65 +pre-cluster-threshold=0.25 +topk=300 + diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8.txt new file mode 100644 index 00000000..e9ca9908 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8.txt @@ -0,0 +1,101 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=yolov8s_640_dynamic.onnx +int8-calib-file=yolov8s_gpu_precision_config_calib.cache +layer-device-precision=Split_36:fp16:gpu;Reshape_37:fp16:gpu;Transpose_38:fp16:gpu;Softmax_39:fp16:gpu;Conv_41:fp16:gpu;Sub_64:fp16:gpu;Concat_65:fp16:gpu;Mul_67:fp16:gpu;Sigmoid_68:fp16:gpu;Concat_69:fp16:gpu +infer-dims=3;640;640 +batch-size=16 +model-engine-file=yolov8s_ptq_640_gpu_b16.engine +labelfile-path=labels.txt +## 0=FP32, 1=INT8, 2=FP16, 3=BEST mode +network-mode=3 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +symmetric-padding=1 +## Bilinear Interpolation +scaling-filter=1 +# parse-bbox-func-name=NvDsInferParseCustomYoloV8 +parse-bbox-func-name=NvDsInferParseCustomYoloV8_cuda + +# disable-output-host-copy=0 +disable-output-host-copy=1 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-compute-hw=0 +## start from DS6.2 +crop-objects-to-roi-boundary=1 + +[class-attrs-all] +#nms-iou-threshold=0.3 +#threshold=0.7 +nms-iou-threshold=0.65 +pre-cluster-threshold=0.25 +topk=300 + diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8_dla.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8_dla.txt new file mode 100644 index 00000000..ad9b3181 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV8_dla.txt @@ -0,0 +1,102 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=yolov8s_DAT_640_noqdq.onnx +int8-calib-file=yolov8s_DAT_precision_config_calib.cache +layer-device-precision=Split_36:fp16:dla;Reshape_37:fp16:dla;Transpose_38:fp16:dla;Softmax_39:fp16:gpu;Conv_41:fp16:gpu;Sub_64:fp16:gpu;Concat_65:fp16:gpu;Mul_67:fp16:gpu;Sigmoid_68:fp16:gpu;Concat_69:fp16:gpu +batch-size=1 +model-engine-file=yolov8s_DAT_640_noqdq_DLA.engine +labelfile-path=labels.txt +## 0=FP32, 1=INT8, 2=FP16, 3=BEST mode +network-mode=3 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +symmetric-padding=1 +## Bilinear Interpolation +scaling-filter=1 +# parse-bbox-func-name=NvDsInferParseCustomYoloV8 +parse-bbox-func-name=NvDsInferParseCustomYoloV8_cuda + +# disable-output-host-copy=0 +disable-output-host-copy=1 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-compute-hw=0 +## start from DS6.2 +crop-objects-to-roi-boundary=1 +enable-dla=1 +use-dla-core=0 # 0 or 1 + +[class-attrs-all] +#nms-iou-threshold=0.3 +#threshold=0.7 +nms-iou-threshold=0.65 +pre-cluster-threshold=0.45 +topk=100 + diff --git a/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV9.txt b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV9.txt new file mode 100644 index 00000000..fa607138 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/config_infer_primary_yoloV9.txt @@ -0,0 +1,100 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +# Following properties are mandatory when engine files are not specified: +# int8-calib-file(Only in INT8), model-file-format +# Caffemodel mandatory properties: model-file, proto-file, output-blob-names +# UFF: uff-file, input-dims, uff-input-blob-name, output-blob-names +# ONNX: onnx-file +# +# Mandatory properties for detectors: +# num-detected-classes +# +# Optional properties for detectors: +# cluster-mode(Default=Group Rectangles), interval(Primary mode only, Default=0) +# custom-lib-path +# parse-bbox-func-name +# +# Mandatory properties for classifiers: +# classifier-threshold, is-classifier +# +# Optional properties for classifiers: +# classifier-async-mode(Secondary mode only, Default=false) +# +# Optional properties in secondary mode: +# operate-on-gie-id(Default=0), operate-on-class-ids(Defaults to all classes), +# input-object-min-width, input-object-min-height, input-object-max-width, +# input-object-max-height +# +# Following properties are always recommended: +# batch-size(Default=1) +# +# Other optional properties: +# net-scale-factor(Default=1), network-mode(Default=0 i.e FP32), +# model-color-format(Default=0 i.e. RGB) model-engine-file, labelfile-path, +# mean-file, gie-unique-id(Default=0), offsets, process-mode (Default=1 i.e. primary), +# custom-lib-path, network-mode(Default=0 i.e FP32) +# +# The values in the config file are overridden by values set through GObject +# properties. + +[property] +gpu-id=0 +net-scale-factor=0.0039215697906911373 +#0=RGB, 1=BGR +model-color-format=0 +onnx-file=yolov9-s-converted.sim.trans.onnx +infer-dims=3;640;640 +batch-size=16 +model-engine-file=yolov9-s-converted.sim.trans.onnx_b16_gpu0_best.engine +labelfile-path=labels.txt +## 0=FP32, 1=INT8, 2=FP16 mode +network-mode=2 +num-detected-classes=80 +gie-unique-id=1 +network-type=0 +is-classifier=0 +## 1=DBSCAN, 2=NMS, 3= DBSCAN+NMS Hybrid, 4 = None(No clustering) +cluster-mode=2 +maintain-aspect-ratio=1 +symmetric-padding=1 +## Bilinear Interpolation +scaling-filter=1 +# parse-bbox-func-name=NvDsInferParseCustomYoloV8 +parse-bbox-func-name=NvDsInferParseCustomYoloV8_cuda + +# disable-output-host-copy=0 +disable-output-host-copy=1 +custom-lib-path=nvdsinfer_custom_impl_Yolo/libnvdsinfer_custom_impl_Yolo.so +#scaling-compute-hw=0 +## start from DS6.2 +crop-objects-to-roi-boundary=1 + + +[class-attrs-all] +#nms-iou-threshold=0.3 +#threshold=0.7 +nms-iou-threshold=0.65 +pre-cluster-threshold=0.25 +topk=300 + diff --git a/tools/yolo_deepstream/deepstream_yolo/deepstream_app_config_yolo.txt b/tools/yolo_deepstream/deepstream_yolo/deepstream_app_config_yolo.txt new file mode 100644 index 00000000..40496ccb --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/deepstream_app_config_yolo.txt @@ -0,0 +1,141 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +[application] +enable-perf-measurement=1 +perf-measurement-interval-sec=5 +#gie-kitti-output-dir=streamscl + +[tiled-display] +enable=1 +rows=4 +columns=4 +width=1280 +height=720 +gpu-id=0 +#(0): nvbuf-mem-default - Default memory allocated, specific to particular platform +#(1): nvbuf-mem-cuda-pinned - Allocate Pinned/Host cuda memory, applicable for Tesla +#(2): nvbuf-mem-cuda-device - Allocate Device cuda memory, applicable for Tesla +#(3): nvbuf-mem-cuda-unified - Allocate Unified cuda memory, applicable for Tesla +#(4): nvbuf-mem-surface-array - Allocate Surface Array memory, applicable for Jetson +nvbuf-memory-type=0 + +[source0] +enable=1 +#Type - 1=CameraV4L2 2=URI 3=MultiURI +type=3 +uri=file:///opt/nvidia/deepstream/deepstream/samples/streams/sample_1080p_h265.mp4 +num-sources=16 +gpu-id=0 +# (0): memtype_device - Memory type Device +# (1): memtype_pinned - Memory type Host Pinned +# (2): memtype_unified - Memory type Unified +cudadec-memtype=0 + +[sink0] +enable=1 +#Type - 1=FakeSink 2=EglSink 3=File +type=3 +sync=0 +source-id=0 +gpu-id=0 +nvbuf-memory-type=0 +#1=mp4 2=mkv +container=1 +#1=h264 2=h265 +codec=1 +output-file=yolo.mp4 + +[osd] +enable=1 +gpu-id=0 +border-width=1 +text-size=12 +text-color=1;1;1;1; +text-bg-color=0.3;0.3;0.3;1 +font=Serif +show-clock=0 +clock-x-offset=800 +clock-y-offset=820 +clock-text-size=12 +clock-color=1;0;0;0 +nvbuf-memory-type=0 + +[streammux] +gpu-id=0 +##Boolean property to inform muxer that sources are live +live-source=0 +batch-size=16 +##time out in usec, to wait after the first buffer is available +##to push the batch even if the complete batch is not formed +batched-push-timeout=40000 +## Set muxer output width and height +width=1280 +height=720 +##Enable to maintain aspect ratio wrt source, and allow black borders, works +##along with width, height properties +enable-padding=0 +nvbuf-memory-type=0 + +# config-file property is mandatory for any gie section. +# Other properties are optional and if set will override the properties set in +# the infer config file. +[primary-gie] +enable=1 +gpu-id=0 +labelfile-path=labels.txt +#batch-size=1 +#Required by the app for OSD, not a plugin property +bbox-border-color0=1;0;0;1 +bbox-border-color1=0;1;1;1 +bbox-border-color2=0;0;1;1 +bbox-border-color3=0;1;0;1 +interval=0 +gie-unique-id=1 +nvbuf-memory-type=0 +#config-file=config_infer_primary_yoloV4.txt +config-file=config_infer_primary_yoloV7.txt +#config-file=config_infer_primary_yoloV8.txt +#config-file=config_infer_primary_yoloV8_dla.txt +#config-file=config_infer_primary_yoloV9.txt +#config-file=config_infer_primary_yoloV11.txt +#config-file=config_infer_primary_yoloV11_obb.txt + +[tracker] +enable=0 +# For NvDCF and DeepSORT tracker, tracker-width and tracker-height must be a multiple of 32, respectively +tracker-width=640 +tracker-height=384 +ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so +# ll-config-file required to set different tracker types +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_IOU.yml +ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_accuracy.yml +# ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_DeepSORT.yml +gpu-id=0 +enable-batch-process=1 +enable-past-frame=1 +display-tracking-id=1 + +[tests] +file-loop=0 diff --git a/tools/yolo_deepstream/deepstream_yolo/labels.txt b/tools/yolo_deepstream/deepstream_yolo/labels.txt new file mode 100644 index 00000000..ca76c80b --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/labels.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/tools/yolo_deepstream/deepstream_yolo/labels_obb.txt b/tools/yolo_deepstream/deepstream_yolo/labels_obb.txt new file mode 100644 index 00000000..d2f04630 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/labels_obb.txt @@ -0,0 +1,15 @@ +plane +ship +storage-tank +baseball-diamond +tennis-court +basketball-court +ground-track-field +harbor +bridge +large-vehicle +small-vehicle +helicopter +roundabout +soccer-ball-field +swimming-pool diff --git a/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/Makefile b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/Makefile new file mode 100644 index 00000000..978b64e6 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/Makefile @@ -0,0 +1,67 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -) +CC:= g++ +NVCC:=/usr/local/cuda/bin/nvcc + +CFLAGS:= -Wall -std=c++11 -shared -fPIC -Wno-error=deprecated-declarations +CFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes/ -I/usr/local/cuda/include + +CUFLAGS:= -std=c++17 -shared +CUFLAGS+= -I/opt/nvidia/deepstream/deepstream/sources/includes/ -I/usr/local/cuda/include +LIBS:= -lnvinfer_plugin -lnvinfer -L/usr/local/cuda/lib64 -lcudart -lcublas -lstdc++fs +LFLAGS:= -shared -Wl,--start-group $(LIBS) -Wl,--end-group + +INCS:= $(wildcard *.h) +SRCFILES:= nvdsparsebbox_Yolo.cpp\ + nvdsparsebbox_Yolo_cuda.cu + +TARGET_LIB:= libnvdsinfer_custom_impl_Yolo.so + +# Gencode arguments +SMS ?= 75 80 86 89 90 100 120 +$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm))) +ifeq ($(TARGET_DEVICE),aarch64) + GENCODE_FLAGS := +else + GENCODE_FLAGS := $(GENCODE_FLAGS) +endif + + +TARGET_OBJS:= $(SRCFILES:.cpp=.o) +TARGET_OBJS:= $(TARGET_OBJS:.cu=.o) + +all: $(TARGET_LIB) + +%.o: %.cpp $(INCS) Makefile + $(CC) -c -o $@ $(CFLAGS) $< + +%.o: %.cu $(INCS) Makefile + $(NVCC) -c -o $@ $(GENCODE_FLAGS) --compiler-options '-fPIC' $(CUFLAGS) $< + +$(TARGET_LIB) : $(TARGET_OBJS) + $(CC) -o $@ $(TARGET_OBJS) $(LFLAGS) + +clean: + rm -rf $(TARGET_LIB) *.o diff --git a/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp new file mode 100644 index 00000000..68ff8166 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo.cpp @@ -0,0 +1,356 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include "nvdsinfer_custom_impl.h" + +static const int NUM_CLASSES_YOLO = 80; + +float clamp(const float val, const float minVal, const float maxVal) +{ + assert(minVal <= maxVal); + return std::min(maxVal, std::max(minVal, val)); +} + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList); + +extern "C" bool NvDsInferParseCustomYoloV7( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList); + +/* YOLOv4 implementations */ +static NvDsInferParseObjectInfo convertBBoxYoloV4(const float& bx1, const float& by1, const float& bx2, + const float& by2, const uint& netW, const uint& netH) +{ + NvDsInferParseObjectInfo b; + // Restore coordinates to network input resolution + + float x1 = bx1 * netW; + float y1 = by1 * netH; + float x2 = bx2 * netW; + float y2 = by2 * netH; + + x1 = clamp(x1, 0, netW); + y1 = clamp(y1, 0, netH); + x2 = clamp(x2, 0, netW); + y2 = clamp(y2, 0, netH); + + b.left = x1; + b.width = clamp(x2 - x1, 0, netW); + b.top = y1; + b.height = clamp(y2 - y1, 0, netH); + + return b; +} + +static void addBBoxProposalYoloV4(const float bx, const float by, const float bw, const float bh, + const uint& netW, const uint& netH, const int maxIndex, + const float maxProb, std::vector& binfo) +{ + NvDsInferParseObjectInfo bbi = convertBBoxYoloV4(bx, by, bw, bh, netW, netH); + if (bbi.width < 1 || bbi.height < 1) return; + + bbi.detectionConfidence = maxProb; + bbi.classId = maxIndex; + binfo.push_back(bbi); +} + +static std::vector +decodeYoloV4Tensor( + const float* boxes, const float* scores, const float* classes, + const uint num_bboxes, NvDsInferParseDetectionParams const& detectionParams, + const uint& netW, const uint& netH) +{ + std::vector binfo; + + uint bbox_location = 0; + for (uint b = 0; b < num_bboxes; ++b) + { + float bx1 = boxes[bbox_location]; + float by1 = boxes[bbox_location + 1]; + float bx2 = boxes[bbox_location + 2]; + float by2 = boxes[bbox_location + 3]; + + int class_id = (int)classes[b]; + + if (class_id < detectionParams.numClassesConfigured && class_id >=0) + { + float prob = scores[b]; + if (prob > detectionParams.perClassPreclusterThreshold[class_id]) + { + addBBoxProposalYoloV4(bx1, by1, bx2, by2, netW, netH, class_id, prob, binfo); + } + + } + + + bbox_location += 4; + } + + return binfo; +} + +extern "C" bool NvDsInferParseCustomYoloV4( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) + { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + const NvDsInferLayerInfo *boxes = NULL; + const NvDsInferLayerInfo *scores = NULL; + const NvDsInferLayerInfo *num = NULL; + const NvDsInferLayerInfo *classes_layer = NULL; + int outputLayernum = outputLayersInfo.size(); + for(int l=0; lbuffer; + + std::vector objects; + + uint num_bboxes = num_layer[0]; + // 2 dimensional: [num_bboxes, 4] + assert(boxes->inferDims.numDims == 2); + // Single dimensional: [num_bboxes] + assert(scores->inferDims.numDims == 1); + + // std::cout << "Network Info: " << networkInfo.height << " " << networkInfo.width << std::endl; + + std::vector outObjs = + decodeYoloV4Tensor( + (const float*)(boxes->buffer), (const float*)(scores->buffer), (const float*)(classes_layer->buffer), num_bboxes, detectionParams, + networkInfo.width, networkInfo.height); + + objects.insert(objects.end(), outObjs.begin(), outObjs.end()); + + objectList = objects; + + return true; +} +/* YOLOv4 implementations end*/ + +/*Yolov7 bbox parser*/ +static NvDsInferParseObjectInfo convertBBoxYoloV7(const float& bx, const float& by, const float& bw, + const float& bh, const int& stride, const uint& netW, + const uint& netH) +{ + NvDsInferParseObjectInfo b; + // Restore coordinates to network input resolution + float xCenter = bx * stride; + float yCenter = by * stride; + float x0 = xCenter - bw / 2; + float y0 = yCenter - bh / 2; + float x1 = x0 + bw; + float y1 = y0 + bh; + + x0 = clamp(x0, 0, netW); + y0 = clamp(y0, 0, netH); + x1 = clamp(x1, 0, netW); + y1 = clamp(y1, 0, netH); + + b.left = x0; + b.width = clamp(x1 - x0, 0, netW); + b.top = y0; + b.height = clamp(y1 - y0, 0, netH); + + return b; +} + +static void addBBoxProposalYoloV7(const float bx, const float by, const float bw, const float bh, + const uint stride, const uint& netW, const uint& netH, const int maxIndex, + const float maxProb, std::vector& binfo) +{ + NvDsInferParseObjectInfo bbi = convertBBoxYoloV7(bx, by, bw, bh, stride, netW, netH); + if (bbi.width < 1 || bbi.height < 1) return; + + bbi.detectionConfidence = maxProb; + bbi.classId = maxIndex; + binfo.push_back(bbi); +} + +static bool NvDsInferParseYoloV7( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + + + if (outputLayersInfo.empty()) { + std::cerr << "Could not find output layer in bbox parsing" << std::endl;; + return false; + } + const NvDsInferLayerInfo &layer = outputLayersInfo[0]; + + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) + { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + std::vector objects; + + float* data = (float*)layer.buffer; + const int dimensions = layer.inferDims.d[1]; + int rows = layer.inferDims.numElements / layer.inferDims.d[1]; + + for (int i = 0; i < rows; ++i) { + //85 = x, y, w, h, maxProb, score0......score79 + float bx = data[ 0]; + float by = data[ 1]; + float bw = data[ 2]; + float bh = data[ 3]; + float maxProb = data[ 4]; + int maxIndex; + float * classes_scores = data + 5; + + float maxScore = 0; + int index = 0; + for (int j = 0 ;j < NUM_CLASSES_YOLO; j++){ + if(*classes_scores > maxScore){ + index = j; + maxScore = *classes_scores; + } + classes_scores++; + } + + maxIndex = index; + data += dimensions; + + addBBoxProposalYoloV7(bx, by, bw, bh, 1, networkInfo.width, networkInfo.height, maxIndex, maxProb, objects); + } + objectList = objects; + return true; +} + +extern "C" bool NvDsInferParseCustomYoloV7( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + return NvDsInferParseYoloV7 ( + outputLayersInfo, networkInfo, detectionParams, objectList); +} + + +static bool NvDsInferParseYoloV8( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + if (outputLayersInfo.empty()) { + std::cerr << "Could not find output layer in bbox parsing" << std::endl;; + return false; + } + const NvDsInferLayerInfo &layer = outputLayersInfo[0]; + + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) + { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + std::vector objects; + + float* data = (float*)layer.buffer; + const int dimensions = layer.inferDims.d[1]; + int rows = layer.inferDims.numElements / layer.inferDims.d[1]; + + + for (int i = 0; i < rows; ++i) { + //85 = x, y, w, h, score0......score79 + float bx = data[ 0]; + float by = data[ 1]; + float bw = data[ 2]; + float bh = data[ 3]; + float * classes_scores = data + 4; + + float maxScore = 0; + int index = 0; + for (int j = 0 ;j < NUM_CLASSES_YOLO; j++){ + if(*classes_scores > maxScore){ + index = j; + maxScore = *classes_scores; + } + classes_scores++; + } + + int maxIndex = index; + data += dimensions; + float maxProb = 1.0; + // share the same addBBoxProposal function + addBBoxProposalYoloV7(bx, by, bw, bh, 1, networkInfo.width, networkInfo.height, maxIndex, maxScore, objects); + + } + objectList = objects; + return true; +} + +extern "C" bool NvDsInferParseCustomYoloV8( + std::vector const& outputLayersInfo, + NvDsInferNetworkInfo const& networkInfo, + NvDsInferParseDetectionParams const& detectionParams, + std::vector& objectList) +{ + return NvDsInferParseYoloV8 ( + outputLayersInfo, networkInfo, detectionParams, objectList); +} +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV4); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV7); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV8); diff --git a/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo_cuda.cu b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo_cuda.cu new file mode 100644 index 00000000..d741c6c5 --- /dev/null +++ b/tools/yolo_deepstream/deepstream_yolo/nvdsinfer_custom_impl_Yolo/nvdsparsebbox_Yolo_cuda.cu @@ -0,0 +1,181 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "nvdsinfer_custom_impl.h" +#include "nvtx3/nvToolsExt.h" + +#include +#include +#include +#include +#include +#include + +static const int NUM_CLASSES_YOLO = 80; +#define OBJECTLISTSIZEV7 25200 +#define OBJECTLISTSIZEV8 8400 +#define OBJECTLISTSIZEV11 9261 + +#define BLOCKSIZE 1024 +thrust::device_vector objects_v7(OBJECTLISTSIZEV7); +thrust::device_vector objects_v8(OBJECTLISTSIZEV8); +thrust::device_vector objects_v11(OBJECTLISTSIZEV11); + +template +__global__ void decodeYoloTensor_cuda(NvDsInferParseObjectInfo *binfo, + float *data, int dimensions, int rows, + int netW, int netH, float Threshold) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx < rows) { + data = data + idx * dimensions; + + float maxProb = isYoloV7 ? data[4] : 1.0f; + if (isYoloV7 && maxProb < Threshold) { + binfo[idx].detectionConfidence = 0.0; + return; + } + + float bx = data[0]; + float by = data[1]; + float bw = data[2]; + float bh = data[3]; + + float *classes_scores = (float *)(data + (isYoloV7 ? 5 : 4)); + float maxScore = 0; + int maxIndex = 0; + +#pragma unroll + for (int j = 0; j < NUM_CLASSES_YOLO; j++) { + if (*classes_scores > maxScore) { + maxIndex = j; + maxScore = *classes_scores; + } + classes_scores++; + } + + float finalScore = isYoloV7 ? (maxProb * maxScore) : maxScore; + if (finalScore < Threshold) { + binfo[idx].detectionConfidence = 0.0; + return; + } + + float xCenter = bx; + float yCenter = by; + float x0 = xCenter - bw / 2.0f; + float y0 = yCenter - bh / 2.0f; + float x1 = x0 + bw; + float y1 = y0 + bh; + + binfo[idx].left = fminf(float(netW), fmaxf(0.0f, x0)); + binfo[idx].top = fminf(float(netH), fmaxf(0.0f, y0)); + binfo[idx].width = fminf(float(netW), fmaxf(0.0f, x1 - x0)); + binfo[idx].height = fminf(float(netH), fmaxf(0.0f, y1 - y0)); + binfo[idx].detectionConfidence = finalScore; + binfo[idx].classId = maxIndex; + } +} + +template +static bool NvDsInferParseYolo_cuda( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList, + thrust::device_vector &objects) { + + if (outputLayersInfo.empty()) { + std::cerr << "Could not find output layer in bbox parsing" << std::endl; + return false; + } + const NvDsInferLayerInfo &layer = outputLayersInfo[0]; + + if (NUM_CLASSES_YOLO != detectionParams.numClassesConfigured) { + std::cerr << "WARNING: Num classes mismatch. Configured:" + << detectionParams.numClassesConfigured + << ", detected by network: " << NUM_CLASSES_YOLO << std::endl; + } + + float *data = (float *)layer.buffer; + const int dimensions = layer.inferDims.d[1]; + int rows = layer.inferDims.numElements / layer.inferDims.d[1]; + + int GRIDSIZE = ((OBJECTLISTSIZE - 1) / BLOCKSIZE) + 1; + float min_PreclusterThreshold = + *(std::min_element(detectionParams.perClassPreclusterThreshold.begin(), + detectionParams.perClassPreclusterThreshold.end())); + + decodeYoloTensor_cuda<<>>( + thrust::raw_pointer_cast(objects.data()), data, dimensions, rows, + networkInfo.width, networkInfo.height, min_PreclusterThreshold); + + objectList.resize(OBJECTLISTSIZE); + thrust::copy(objects.begin(), objects.end(), objectList.begin()); + + return true; +} + +extern "C" { + +bool NvDsInferParseCustomYoloV7_cuda( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + nvtxRangePush("NvDsInferParseYoloV7"); + bool ret = NvDsInferParseYolo_cuda( + outputLayersInfo, networkInfo, detectionParams, objectList, objects_v7); + + nvtxRangePop(); + return ret; +} + +bool NvDsInferParseCustomYoloV8_cuda( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + nvtxRangePush("NvDsInferParseYoloV8"); + bool ret = NvDsInferParseYolo_cuda( + outputLayersInfo, networkInfo, detectionParams, objectList, objects_v8); + nvtxRangePop(); + + return ret; +} + +bool NvDsInferParseCustomYoloV11_cuda( + std::vector const &outputLayersInfo, + NvDsInferNetworkInfo const &networkInfo, + NvDsInferParseDetectionParams const &detectionParams, + std::vector &objectList) { + nvtxRangePush("NvDsInferParseYoloV11"); + bool ret = NvDsInferParseYolo_cuda( + outputLayersInfo, networkInfo, detectionParams, objectList, objects_v11); + nvtxRangePop(); + return ret; +} +} + +/* Check that the custom function has been defined correctly */ +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV7_cuda); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV8_cuda); +CHECK_CUSTOM_PARSE_FUNC_PROTOTYPE(NvDsInferParseCustomYoloV11_cuda); diff --git a/tools/yolo_deepstream/tensorrt_yolo/CMakeLists.txt b/tools/yolo_deepstream/tensorrt_yolo/CMakeLists.txt new file mode 100644 index 00000000..5861fc07 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/CMakeLists.txt @@ -0,0 +1,81 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +cmake_minimum_required( VERSION 3.0 ) + +project( YOLOV7 ) +enable_language( CUDA ) +find_package(CUDA) +set( CMAKE_C_STANDARD 99 ) +set( CMAKE_CXX_STANDARD 11 ) +set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fpic -fpie -fpermissive -std=c++11 -pthread" ) + +find_package(OpenCV REQUIRED) +include_directories( ${OpenCV_INCLUDE_DIRS}) +find_package(jsoncpp CONFIG REQUIRED) + + + +# global include_directories +include_directories( /usr/local/cuda/include ) +#add judgement about system: + +MESSAGE(STATUS "CMAKE_HOST_SYSTEM_PROCESSOR is ${CMAKE_HOST_SYSTEM_PROCESSOR}") + +if (${CMAKE_HOST_SYSTEM_PROCESSOR} EQUAL aarch64) + include_directories( /usr/include/aarch64-linux-gnu/ ) # for jetson +elseif(${CMAKE_HOST_SYSTEM_PROCESSOR} EQUAL x86_64) + include_directories( /usr/lib/x86_64-linux-gnu/ ) +endif() + +include_directories( "${CMAKE_SOURCE_DIR}/src/" ) +include_directories( "/usr/include/jsoncpp/") +# global definitions +add_definitions( -w) + +# global library path +if (${CMAKE_HOST_SYSTEM_PROCESSOR} EQUAL aarch64) + link_directories( "/usr/lib/aarch64-linux-gnu/" ) +elseif(${CMAKE_HOST_SYSTEM_PROCESSOR} EQUAL x86_64) + link_directories( "/usr/lib/x86_64-linux-gnu/" ) +endif() + +link_directories( "/usr/lib/" ) +link_directories( "/usr/local/lib/") +link_directories( "/usr/local/cuda/lib64/" ) + +FILE(GLOB_RECURSE YOLO_SRC src/*.cpp ) +add_library( yolo SHARED ${YOLO_SRC} ) +target_link_libraries(yolo PRIVATE nvinfer) +target_link_libraries(yolo PRIVATE nvinfer_plugin) +# target_link_libraries(yolo PRIVATE nvparsers) +target_link_libraries(yolo PRIVATE nvonnxparser cudart ${OpenCV_LIBS}) + +add_executable(detect samples/detect.cpp ) +target_link_libraries(detect yolo cudart ${OpenCV_LIBS} ) + +add_executable(video_detect samples/video_detect.cpp ) +target_link_libraries(video_detect yolo cudart ${OpenCV_LIBS} ) + +add_executable(validate_coco samples/validate_coco.cpp ) +target_link_libraries(validate_coco yolo cudart ${OpenCV_LIBS} ) +target_link_libraries(validate_coco jsoncpp) diff --git a/tools/yolo_deepstream/tensorrt_yolo/README.md b/tools/yolo_deepstream/tensorrt_yolo/README.md new file mode 100644 index 00000000..1c27b995 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/README.md @@ -0,0 +1,104 @@ +# Yolo TensorRT cpp + +## Description +This is a yolo TensorRT cpp app. Fisrt, using trtexec to convert onnx model to FP32 or FP16 TensorRT engine ,or INT8 TensorRT engine from the QAT model finetuned from [yolov7_qat](../yolov7_qat) and [yolov8](../README.md#onnx-model-list). +Then you can use the `detect/video_detect` app to detect a list of images(images number must smaller than the batchsize of the model)/video. or use `validate_coco` app to test mAP of the TensorRT engine. +## Prerequisites +#### Install opencv +- Note: There are OpenCV4 dependencies in this program. +Follow README and documents of this repository https://github.com/opencv/opencv to install OpenCV. +And, if you want use detect_video app, please install opencv with `ffmpeg` enabled + +#### Install jsoncpp libs +jsoncpp lib is used to write coco-dataset-validate-result to json file. +```bash +$ sudo apt-get install libjsoncpp-dev +``` +## Build and Run yolo-TensorRT-app +### Build +```bash +$ mkdir build && cd build +$ cmake .. +$ make -j4 +``` + +### Prepare TensorRT engines +model can be found in [model-list](../README.md#model-list) +convert onnx model to tensorrt-engine +```bash +# fp32 model +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7.onnx --saveEngine=yolov7fp32.engine +# fp16 model +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7.onnx --saveEngine=yolov7fp16.engine --fp16 +# int8 QAT model, the onnx model with Q&DQ nodes +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7qat.onnx --saveEngine=yolov7QAT.engine --fp16 --int8 +# int8 implicit quant yolov8s model running on gpu +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov8s_640_dynamic.onnx --fp16 --int8 --verbose --calib=yolov8s_ptq_precision_config_calib.cache --saveEngine=yolov8s_ptq_640_gpu_b1.engine --minShapes=x.1:1x3x640x640 --optShapes=x.1:1x3x640x640 --maxShapes=x.1:1x3x640x640 --precisionConstraints=obey --layerPrecisions=Split_36:fp16,Reshape_37:fp16,Transpose_38:fp16,Softmax_39:fp16,Conv_41:fp16,Sub_64:fp16,Concat_65:fp16,Mul_67:fp16,Sigmoid_68:fp16,Concat_69:fp16 +# int8 implicit quant DLA-spec-finetuned yolov8s model running on DLA +/usr/src/tensorrt/bin/trtexec --onnx=yolov8s_DAT_640_noqdq.onnx --fp16 --int8 --verbose --calib=yolov8s_DAT_precision_config_calib.cache --precisionConstraints=obey --layerPrecisions=Split_36:fp16,Reshape_37:fp16,Transpose_38:fp16,Softmax_39:fp16,Conv_41:fp16,Sub_64:fp16,Concat_65:fp16,Mul_67:fp16,Sigmoid_68:fp16,Concat_69:fp16 --saveEngine=yolov8s_DAT_noqdq_640_DLA.engine --useDLACore=0 --allowGPUFallback +``` +### Detection & Validate +- detect with image: + - Run on yolov7 + ```bash + $ ./build/detect --engine=yolov7db4fp32.engine --img=./imgs/horses.jpg,./imgs/zidane.jpg --version=v7 + ``` + - Run on yolov8 gpu + ```bash + $ ./build/detect --engine=yolov8s_ptq_640_gpu_b1.engine --img=./imgs/horses.jpg,./imgs/zidane.jpg --version=v8 + ``` + - Run on yolov8 dla + ```bash + $ ./build/detect --engine=yolov8s_DAT_noqdq_640_DLA.engine --img=./imgs/horses.jpg,./imgs/zidane.jpg --version=v8 + ``` + +- detect with video: + + note: only support batchsize = 1 now. + - Run on yolov7 + ```bash + $ ./build/video_detect --engine=./yolov7fp32.engine --video=YOUR_VIDEO_PATH.mp4 --version=v7 + ``` + - Run on yolov8 gpu + ```bash + $ ./build/video_detect --engine=yolov8s_ptq_640_gpu_b1.engine --video=YOUR_VIDEO_PATH.mp4 --version=v8 + ``` + - Run on yolov8 dla + ```bash + $ ./build/video_detect --engine=yolov8s_DAT_noqdq_640_DLA.engine --video=YOUR_VIDEO_PATH.mp4 --version=v8 + ``` + +- validate mAP on dataset + - note: validate_coco only support model inputsize `[batchsize, 3, 672, 672]` or `[batchsize, 3, 640, 640]` + + - Run on yolov7 + ```bash + $ ./build/validate_coco --engine=./yolov7fp32.engine --coco=/YOUR/COCO/DATA/PATH/ --version=v7 + ``` + - Run on yolov8 on gpu + ```bash + $ ./build/validate_coco --engine=./yolov8s_DAT_noqdq_672_gpu.engine --coco=/YOUR/COCO/DATA/PATH/ --version=v8 + ``` + + - Run on yolov8 on dla + ```bash + $ ./build/validate_coco --engine=./yolov8s_DAT_noqdq_672_DLA.engine --coco=/YOUR/COCO/DATA/PATH/ --version=v8 + ``` + + output: + ``` + -------------------------------------------------------- + Yolov7 initialized from: yolov7672.engine + input : images , shape : [ 1,3,672,672,] + output : output , shape : [ 1,27783,85,] + -------------------------------------------------------- + 5000 / 5000 + predict result has been written to ./predict.json + ``` + validate output with `test_coco_map.py`: + ``` + $ python test_coco_map.py --predict ./predict.json --coco /YOUR/COCO/DATA/PATH/ + ... + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.51005 + ... + ``` diff --git a/tools/yolo_deepstream/tensorrt_yolo/imgs/horses.jpg b/tools/yolo_deepstream/tensorrt_yolo/imgs/horses.jpg new file mode 100644 index 00000000..3a761f46 Binary files /dev/null and b/tools/yolo_deepstream/tensorrt_yolo/imgs/horses.jpg differ diff --git a/tools/yolo_deepstream/tensorrt_yolo/imgs/zidane.jpg b/tools/yolo_deepstream/tensorrt_yolo/imgs/zidane.jpg new file mode 100644 index 00000000..92d72ea1 Binary files /dev/null and b/tools/yolo_deepstream/tensorrt_yolo/imgs/zidane.jpg differ diff --git a/tools/yolo_deepstream/tensorrt_yolo/samples/detect.cpp b/tools/yolo_deepstream/tensorrt_yolo/samples/detect.cpp new file mode 100644 index 00000000..23c440b6 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/samples/detect.cpp @@ -0,0 +1,118 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +std::vector parse_img_paths(argsParser& cmdLine) { + return cmdLine.ParseStringList("img"); +} + +std::string parse_model_path(argsParser& cmdLine) { + const char* engine_path_str = cmdLine.ParseString("engine"); + std::string engine_path; + if (engine_path_str) engine_path = std::string(engine_path_str); + return engine_path; +} + +std::string parse_yolov_version(argsParser& cmdLine) { + const char* version_str = cmdLine.ParseString("version"); + std::string version; + if (version_str) version = std::string(version_str); + return version; +} + +bool print_help() { + printf("--------------------------------------------------------------------------------------------------------\n"); + printf("---------------------------- yolo images detector ---------------------------------------------\n"); + printf(" '--help': print help information \n"); + printf(" '--engine=yolo.engine' Load yolo trt-engine \n"); + printf(" '--version=v7' Run yolov7/v8/v9, default v7 \n"); + printf(" '--img=img1,jpg,img2.jpg,img3.jpg' specify the path of the images, split by `,`\n"); + return true; +} + + +int main(int argc, char** argv){ + + argsParser cmdLine(argc, argv); + //! parse device_flag, see parse_device_flag + if(cmdLine.ParseFlag("help")) { print_help(); return 0; } + + std::string engine_path = parse_model_path(cmdLine); + std::vector img_paths = parse_img_paths(cmdLine); + std::string yolo_version = parse_yolov_version(cmdLine); + bool isYolov7 = true; + if(yolo_version == "v8" || yolo_version == "v9"){ + isYolov7 = false; + } + else{ + isYolov7 = true; + } + // print img paths + std::cout<<"input "< bgr_imgs; + for(int i = 0; i< img_paths.size();i++){ + bgr_imgs.push_back(cv::imread(img_paths[i])); + } + + std::cout<<"preprocess start"<>> nmsresults; + if(isYolov7) // yolov7 + nmsresults = yolo.PostProcess(0.45f, 0.25f, isYolov7); + else + nmsresults = yolo.PostProcess(0.75f, 0.25f, isYolov7); + for(int j =0; j < nmsresults.size();j++){ + Yolo::DrawBoxesonGraph(bgr_imgs[j],nmsresults[j]); + std::string output_path = img_paths[j] + "detect" + std::to_string(j)+".jpg"; + cv::imwrite(output_path, bgr_imgs[j]); + std::cout<<"detectec image written to: "< +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +std::string parse_model_path(argsParser& cmdLine) { + const char* engine_path_str = cmdLine.ParseString("engine"); + std::string engine_path; + if (engine_path_str) engine_path = std::string(engine_path_str); + return engine_path; +} + +std::string parse_coco_path(argsParser& cmdLine) { + const char* coco_path_str = cmdLine.ParseString("coco"); + std::string coco_path; + if (coco_path_str) coco_path = std::string(coco_path_str); + return coco_path; +} +std::string parse_yolov_version(argsParser& cmdLine) { + const char* version_str = cmdLine.ParseString("version"); + std::string version; + if (version_str) version = std::string(version_str); + return version; +} + +bool print_help() { + printf("--------------------------------------------------------------------------------------------------------\n"); + printf("---------------------------- yolo coco validate tool ---------------------------------------------\n"); + printf(" '--help': print help information \n"); + printf(" '--engine=yolo.engine' Load yolo trt-engine \n"); + printf(" '--coco=./data/coco/' specify the path of the coco dataset\n"); + printf(" '--version=v7' Run yolov7/v8/v9, default v7 \n"); + return true; +} + +int coco80_to_coco91_class(int id) { + //# converts 80-index (val2014) to 91-index (paper) + // # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ + // # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') + // # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') + // # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco + // # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet + std::vector x = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90}; + return x[id]; +} +std::vector xyxy2xywh(float x0, float x1, float x2, float x3){ + // # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right + // y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + std::vector y; + y.resize(4); + y[0] = (x0 + x2) / 2;// # x center + y[1] = (x1 + x3) / 2;// # y center + y[2] = x2 - x0;// # width + y[3] = x3 - x1;// # height + y[0] -= y[2]/2; + y[1] -= y[3]/2; + // box[:, :2] -= box[:, 2:] / 2 + + return y; +} + +int number_classes = 80; + +std::vector readCocoPaths(std::string coco_file_path) { + std::vector result; + std::ifstream coco_test_file(coco_file_path); + std::string line; + std::string folder_path = coco_file_path.substr(0, coco_file_path.find_last_of("/")+1); + if(coco_test_file) { + while(getline(coco_test_file, line)){ + + result.push_back(folder_path+line); + // std::cout<<"folder_path+line:"< bgr_imgs; + std::vector imgPathList = readCocoPaths(coco_path);; + std::vector>> batchNmsResult; + int maxBatchsize = yolo.getInputDim().d[0]; + int resolution = yolo.getInputDim().d[2]; + + + Json::Value root; + Json::FastWriter writer; + + for(int i = 0 ; i < imgPathList.size(); ){ + //infer with a batch + for(int j = 0; j < maxBatchsize && i nchwMats; + if(640 == resolution){ + nchwMats= yolo.preProcess(bgr_imgs); + } + else if (672 == resolution) + { + nchwMats = yolo.preProcess4Validate(bgr_imgs); + } + else{ + std::err << "the resolution must be 672 or 640, but got :"<< resolution < point = xyxy2xywh(batchNmsResult[j][k][0],batchNmsResult[j][k][1],batchNmsResult[j][k][2],batchNmsResult[j][k][3]); + bboxObj.append(point[0]); + bboxObj.append(point[1]); + bboxObj.append(point[2]); + bboxObj.append(point[3]); + OneResult["bbox"] = bboxObj; + root.append(OneResult); + } + } + bgr_imgs.clear(); + } + + std::string json_file = writer.write(root); + std::ofstream out("./predict.json"); + out << json_file; + std::cout< +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +std::string parse_video_path(argsParser& cmdLine) { + const char* video_path_str = cmdLine.ParseString("video"); + std::string video_path; + if (video_path_str) video_path = std::string(video_path_str); + return video_path; +} + +std::string parse_model_path(argsParser& cmdLine) { + const char* engine_path_str = cmdLine.ParseString("engine"); + std::string engine_path; + if (engine_path_str) engine_path = std::string(engine_path_str); + return engine_path; +} +std::string parse_yolov_version(argsParser& cmdLine) { + const char* version_str = cmdLine.ParseString("version"); + std::string version; + if (version_str) version = std::string(version_str); + return version; +} + +bool print_help() { + printf("--------------------------------------------------------------------------------------------------------\n"); + printf("---------------------------- yolo images detector ---------------------------------------------\n"); + printf(" '--help': print help information \n"); + printf(" '--engine=yolo.engine' Load yolo trt-engine \n"); + printf(" '--video=video.mp4' specify the path of the video \n"); + printf(" '--version=v7' Run yolov7/v8/v9, default v7 \n"); + return true; +} + + +int main(int argc, char** argv){ + + argsParser cmdLine(argc, argv); + //! parse device_flag, see parse_device_flag + if(cmdLine.ParseFlag("help")) { print_help(); return 0; } + + std::string engine_path = parse_model_path(cmdLine); + std::string video_path = parse_video_path(cmdLine); + std::string yolo_version = parse_yolov_version(cmdLine); + bool isYolov7 = true; + if(yolo_version == "v8" || yolo_version == "v9"){ + isYolov7 = false; + } + else{ + isYolov7 = true; + } + Yolo yolo(engine_path); + + cv::VideoCapture capture; + cv::Mat frame; + frame= capture.open(video_path); + if(!capture.isOpened()) + { + printf("can not open ... please check whether your opencv has installed with ffmpeg..\n"); + return -1; + } + cv::Size size = cv::Size(capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)); + cv::VideoWriter writer; + writer.open(std::string(video_path+".detect.mp4"), cv::VideoWriter::fourcc('M', 'J', 'P', 'G'), 10, size, true); + std::vector framev; + std::vector>> nmsresults; + int total_frame_count = capture.get(cv::CAP_PROP_FRAME_COUNT); + int i = 0; + while (capture.read(frame)){ + framev.push_back(frame); + yolo.preProcess(framev); + yolo.infer(); + nmsresults = yolo.PostProcess(0.45f, 0.25f, isYolov7); + Yolo::DrawBoxesonGraph(frame,nmsresults[0]); + writer.write(frame); + framev.clear(); + i++; + printf("\r%d / %d", i, total_frame_count); + fflush(stdout); + } + capture.release(); + std::cout<<"Done..."<mTotal_inference_time = 0; + this->mInference_count = 0; + + this->mStream = makeCudaStream(cudaEventDefault,0); + this->mEvent = makeCudaEvent(cudaEventDefault); + + this->mEnginePath = engine_path; + Logger mLoggern; + initLibNvInferPlugins(&mLoggern, ""); + this->mRuntime = std::unique_ptr>{nvinfer1::createInferRuntime(mLoggern)}; + // this->mCudaGraphEnabled = enableCudaGraph; + + std::ifstream fin(engine_path, std::ios::binary); + std::vector inBuffer((std::istreambuf_iterator(fin)), std::istreambuf_iterator()); + fin.close(); + mEngine.reset(mRuntime->deserializeCudaEngine(inBuffer.data(), inBuffer.size())); + mContext.reset(mEngine->createExecutionContext()); + mImgPushed = 0; + /* + malloc cuda memory for binding + */ + const int nbBindings = this->mEngine->getNbIOTensors(); + + this->mDynamicBatch = this->mEngine->getTensorShape(mEngine->getIOTensorName(0)).d[0] == -1 ? true:false; + for (int i = 0; i < nbBindings; i++) { + const auto dataType = this->mEngine->getTensorDataType(mEngine->getIOTensorName(i)); + const int elemSize = [&]() -> int { + switch (dataType) { + case nvinfer1::DataType::kFLOAT: + return 4; + case nvinfer1::DataType::kHALF: + return 2; + default: + throw std::runtime_error("invalid data type"); + } + }(); + + nvinfer1::Dims dims; + + //input + if (mEngine->getTensorIOMode(mEngine->getIOTensorName(i)) == nvinfer1::TensorIOMode::kINPUT) { + if(this->mDynamicBatch) + dims = mEngine->getProfileShape(mEngine->getIOTensorName(i), 0, nvinfer1::OptProfileSelector::kMAX); + else + dims = mEngine->getTensorShape(mEngine->getIOTensorName(i)); + this->mInputDim = dims; + } + else{ // output + dims = mEngine->getTensorShape(mEngine->getIOTensorName(i)); + //if dynamic batch, change dim[0] to max-batchsize of input + if(this->mDynamicBatch) + dims.d[0] = mEngine->getProfileShape(mEngine->getIOTensorName(i), 0, nvinfer1::OptProfileSelector::kMAX).d[0]; + + this->mOutputDim = dims; + } + const int bindingSize = elemSize * std::accumulate(dims.d, &dims.d[dims.nbDims], 1, std::multiplies{}); + if (mEngine->getTensorIOMode(mEngine->getIOTensorName(i)) == nvinfer1::TensorIOMode::kINPUT) //intput + mImgBufferSize = bindingSize / dims.d[0]; + else //output + mHostOutputBuffer.resize(bindingSize / elemSize); + + this->mBindings.emplace_back(mallocCudaMem(bindingSize)); + this->mBindingArray.emplace_back(mBindings.back().get()); + if(!mContext->setTensorAddress(mEngine->getIOTensorName(i), this->mBindingArray[i])){ + printf("Failed to set tensor address for tensor %s\n", mEngine->getIOTensorName(i)); + return; + } + } + mMaxBatchSize = mInputDim.d[0]; + + this->ReportArgs(); + + if(this->mDynamicBatch) + mContext->setOptimizationProfileAsync(0, mStream.get()); + + return; +} + +void Yolo::ReportArgs() { + std::cout << "--------------------------------------------------------" << std::endl; + std::cout << "Yolo initialized from: " << mEnginePath << std::endl; + const int nbBindings = mEngine->getNbIOTensors(); + for (int i = 0; i < nbBindings; i++) { + const auto dims = mEngine->getTensorShape(mEngine->getIOTensorName(i)); + if (mEngine->getTensorIOMode(mEngine->getIOTensorName(i)) == nvinfer1::TensorIOMode::kINPUT) + std::cout << "input : " << mEngine->getIOTensorName(i); + else + std::cout << "output : " << mEngine->getIOTensorName(i); + std::cout << " , shape : [ "; + for (int j = 0; j < dims.nbDims; j++) std::cout << dims.d[j] << ","; + std::cout << "]" << std::endl; + } + std::cout << "--------------------------------------------------------" << std::endl; +} + +static void hwc_to_chw(cv::InputArray src, cv::OutputArray dst) { + std::vector channels; + cv::split(src, channels); + // Stretch one-channel images to vector + for (auto &img : channels) { + img = img.reshape(1, 1); + } + // Concatenate three vectors to one + cv::hconcat( channels, dst ); +} + +std::vector Yolo::preProcess(std::vector &cv_img) { + if(cv_img.size() > mInputDim.d[0] || cv_img.size() <=0) { + std::cerr<<"error cv_img.size() in "<<__FUNCTION__< nchwMats; + + for(int i = 0; i< cv_img.size();i++){ + float scale_x = mInputDim.d[3] / (float)cv_img[i].cols; + float scale_y = mInputDim.d[2] / (float)cv_img[i].rows; + float scale = std::min(scale_x, scale_y); + float i2d[6], d2i[6]; + + // resize the image, the src img and the dst img have the same center + i2d[0] = scale; i2d[1] = 0; i2d[2] = (-scale * cv_img[i].cols + mInputDim.d[3] + scale - 1) * 0.5; + i2d[3] = 0; i2d[4] = scale; i2d[5] = (-scale * cv_img[i].rows + mInputDim.d[2] + scale - 1) * 0.5; + + cv::Mat m2x3_i2d(2, 3, CV_32F, i2d); // image to dst(network), 2x3 matrix + cv::Mat m2x3_d2i(2, 3, CV_32F, d2i); // dst to image, 2x3 matrix + cv::invertAffineTransform(m2x3_i2d, m2x3_d2i); + std::vector d2i_1{d2i[0],d2i[1],d2i[2],d2i[3],d2i[4],d2i[5]}; + this->md2i.push_back(d2i_1); + cv::Mat input_image; + cv::cvtColor(cv_img[i], input_image, cv::COLOR_BGR2RGB); + cv::warpAffine(input_image, input_image, m2x3_i2d, cv::Size(mInputDim.d[3], mInputDim.d[2]), cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar::all(114)); + input_image.convertTo(input_image, CV_32FC3, 1.0f/255.0f, 0); + cv::Mat nchwMat; + hwc_to_chw(input_image, nchwMat); + nchwMats.push_back(nchwMat); + this->pushImg(nchwMat.data, 1 ,true); + } + return nchwMats; +} +std::vector Yolo::preProcess4Validate(std::vector &cv_img) { + std::vector nchwMats; + if(cv_img.size() > mInputDim.d[0] || cv_img.size() <=0) { + std::cerr<<"error cv_img.size() in "<<__FUNCTION__< from_{cv_img[i].cols,cv_img[i].rows}; + std::vector to_{640, 640}; + float scale = to_[0]/from_[0] < to_[1]/from_[1]? to_[0]/from_[0]:to_[1]/from_[1]; + std::vector M_{scale, 0, -scale * from_[0] * 0.5 + to_[0] * 0.5 + scale * 0.5 - 0.5 + 16, + 0, scale, -scale * from_[1] * 0.5 + to_[1] * 0.5 + scale * 0.5 - 0.5 + 16}; + + + cv::Mat M(2,3,CV_32FC1, M_.data()); + float d2i[6]; + cv::Mat m2x3_d2i(2, 3, CV_32F, d2i); // dst to image, 2x3 matrix + cv::invertAffineTransform(M, m2x3_d2i); + std::vector d2i_1{d2i[0],d2i[1],d2i[2],d2i[3],d2i[4],d2i[5]}; + md2i.push_back(d2i_1); + + cv::Mat input_image; + cv::Mat nchwMat; + + cv::Scalar scalar = cv::Scalar::all(114); + cv::cvtColor(cv_img[i], input_image, cv::COLOR_BGR2RGB); + cv::warpAffine(input_image, input_image, M,cv::Size(672,672),cv::INTER_LINEAR,cv::BORDER_CONSTANT, scalar); + + input_image.convertTo(input_image, CV_32FC3, 1.0f/255.0f, 0); + + hwc_to_chw(input_image, nchwMat); + nchwMats.push_back(nchwMat); + this->pushImg(nchwMat.data, 1 ,true); + } + return nchwMats; + +} +int Yolo::pushImg(void *imgBuffer, int numImg, bool fromCPU) { + if(mImgPushed + numImg > mMaxBatchSize) { + std::cerr <<" error: mImgPushed = "<< mImgPushed <<" numImg = "< mMaxBatchSize "<mBindings[0].get() + mImgPushed*mImgBufferSize, imgBuffer, mImgBufferSize * numImg, cudaMemcpyHostToDevice)); + } + else { + checkCudaErrors(cudaMemcpy(this->mBindings[0].get() + mImgPushed*mImgBufferSize, imgBuffer, mImgBufferSize * numImg, cudaMemcpyDeviceToDevice)); + } + mImgPushed += numImg; + + return 0; +} + +int Yolo::infer() { + if(mImgPushed == 0){ + std::cerr <<" error: mImgPushed = "<< mImgPushed <<" ,mImgPushed == 0!"<mContext->setInputShape(mEngine->getIOTensorName(0), inferDims); + } + + if (!mContext->enqueueV3(mStream.get())) { + std::cout << "failed to enqueue TensorRT context on device "<< std::endl; + return -1; + } + + if (cudaSuccess != cudaStreamSynchronize(mStream.get())) { + std::cout << "Stream Sync failed "<< std::endl; + return -1; + } + mImgPushed = 0; + mInference_count++; + return 0; +} + + +nvinfer1::Dims Yolo::getInputDim() { + return mInputDim; +} + +nvinfer1::Dims Yolo::getOutputDim() { + return mOutputDim; +} +std::vector>> Yolo::decode_yolov7_result(float conf_thres) { + // for now, copy all buffer to host + std::vector>> all_bboxes; + if(cudaSuccess != cudaMemcpyAsync((void*)(mHostOutputBuffer.data()),this->mBindings[1].get() , sizeof(float) * mHostOutputBuffer.size(), cudaMemcpyDeviceToHost)){ + std::cerr<<"error cv_img.size() in "<<__FUNCTION__<> bboxes; + float *h_one_output; + for(int j = 0; j < md2i.size();j++){ + bboxes.clear(); + h_one_output = mHostOutputBuffer.data() + j * std::accumulate(&mOutputDim.d[1], &mOutputDim.d[mOutputDim.nbDims], 1, std::multiplies{}); + // float conf_thres = 0.4; + + int output_numbox = mOutputDim.d[1]; + int output_numprob = mOutputDim.d[2]; + int num_classes = output_numprob - 5; + + for(int i = 0; i < output_numbox; ++i){ + float* ptr = h_one_output + i * output_numprob; + float objness = ptr[4]; + if(objness < conf_thres) + continue; + + float* pclass = ptr + 5; + int label = std::max_element(pclass, pclass + num_classes) - pclass; + float prob = pclass[label]; + float confidence = prob * objness; + if(confidence < conf_thres) + continue; + + // center point, width, height + float cx = ptr[0]; + float cy = ptr[1]; + float width = ptr[2]; + float height = ptr[3]; + + // predict box + float left = cx - width * 0.5; + float top = cy - height * 0.5; + float right = cx + width * 0.5; + float bottom = cy + height * 0.5; + + // the position on the picture + float image_base_left = md2i[j][0] * left + md2i[j][2]; + float image_base_right = md2i[j][0] * right + md2i[j][2]; + float image_base_top = md2i[j][0] * top + md2i[j][5]; + float image_base_bottom = md2i[j][0] * bottom + md2i[j][5]; + bboxes.push_back({image_base_left, image_base_top, image_base_right, image_base_bottom, (float)label, confidence}); + } + all_bboxes.push_back(bboxes); + } + md2i.clear(); + return all_bboxes; +} + +std::vector>> Yolo::decode_yolov8v9_result(float conf_thres) { + // for now, copy all buffer to host + std::vector>> all_bboxes; + if(cudaSuccess != cudaMemcpyAsync((void*)(mHostOutputBuffer.data()),this->mBindings[1].get() , sizeof(float) * mHostOutputBuffer.size(), cudaMemcpyDeviceToHost)){ + std::cerr<<"error cv_img.size() in "<<__FUNCTION__<> bboxes; + float *h_one_output; + for(int j = 0; j < md2i.size();j++){ + bboxes.clear(); + h_one_output = mHostOutputBuffer.data() + j * std::accumulate(&mOutputDim.d[1], &mOutputDim.d[mOutputDim.nbDims], 1, std::multiplies{}); + // float conf_thres = 0.4; + + int output_numbox = mOutputDim.d[1]; + int output_numprob = mOutputDim.d[2]; + int num_classes = output_numprob - 4; + + for(int i = 0; i < output_numbox; ++i){ + float* ptr = h_one_output + i * output_numprob; + + float* pclass = ptr + 4; + int label = std::max_element(pclass, pclass + num_classes) - pclass; + float prob = pclass[label]; + float confidence = prob; + if(confidence < conf_thres) + continue; + // center point, width, height + float cx = ptr[0]; + float cy = ptr[1]; + float width = ptr[2]; + float height = ptr[3]; + + // predict box + float left = cx - width * 0.5; + float top = cy - height * 0.5; + float right = cx + width * 0.5; + float bottom = cy + height * 0.5; + + // the position on the picture + float image_base_left = md2i[j][0] * left + md2i[j][2]; + float image_base_right = md2i[j][0] * right + md2i[j][2]; + float image_base_top = md2i[j][0] * top + md2i[j][5]; + float image_base_bottom = md2i[j][0] * bottom + md2i[j][5]; + bboxes.push_back({image_base_left, image_base_top, image_base_right, image_base_bottom, (float)label, confidence}); + } + all_bboxes.push_back(bboxes); + } + md2i.clear(); + return all_bboxes; +} + +std::vector> Yolo::nms(std::vector> &bboxes, float iou_thres) { + std::sort(bboxes.begin(), bboxes.end(), [](std::vector& a, std::vector& b){return a[5] > b[5];}); + std::vector remove_flags(bboxes.size()); + std::vector> box_result; + box_result.reserve(bboxes.size()); + + auto iou = [](const std::vector& a, const std::vector& b){ + float cross_left = std::max(a[0], b[0]); + float cross_top = std::max(a[1], b[1]); + float cross_right = std::min(a[2], b[2]); + float cross_bottom = std::min(a[3], b[3]); + + float cross_area = std::max(0.0f, cross_right - cross_left) * std::max(0.0f, cross_bottom - cross_top); + float union_area = std::max(0.0f, a[2] - a[0]) * std::max(0.0f, a[3] - a[1]) + + std::max(0.0f, b[2] - b[0]) * std::max(0.0f, b[3] - b[1]) - cross_area; + if(cross_area == 0 || union_area == 0) return 0.0f; + return cross_area / union_area; + }; + + for(int i = 0; i < bboxes.size(); ++i){ + if(remove_flags[i]) continue; + + auto& ibox = bboxes[i]; + box_result.emplace_back(ibox); + for(int j = i + 1; j < bboxes.size(); ++j){ + if(remove_flags[j]) continue; + + auto& jbox = bboxes[j]; + if(ibox[4] == jbox[4]){ + // class matched + if(iou(ibox, jbox) >= iou_thres) + remove_flags[j] = true; + } + } + } + return box_result; +} +std::vector>> Yolo::yolo_nms(std::vector>> &bboxes, float iou_thres) { + std::vector>> nms_result; + for(int i = 0;i < bboxes.size();i++) { + nms_result.push_back(this->nms(bboxes[i], iou_thres)); + } + return nms_result; +} + +std::vector>> Yolo::PostProcess(float iou_thres, float conf_thres, bool isYolov7){ + std::vector>> PostProcessingResult; + //decode & nms + std::vector>> decode_result; + if(isYolov7) + decode_result = this->decode_yolov7_result(conf_thres); + else + decode_result = this->decode_yolov8v9_result(conf_thres); + PostProcessingResult = this->yolo_nms(decode_result, iou_thres); + return PostProcessingResult; +} + +//help functions for drawing boxes on cv::Mat +static std::tuple hsv2bgr(float h, float s, float v){ + const int h_i = static_cast(h * 6); + const float f = h * 6 - h_i; + const float p = v * (1 - s); + const float q = v * (1 - f*s); + const float t = v * (1 - (1 - f) * s); + float r, g, b; + switch (h_i) { + case 0:r = v; g = t; b = p;break; + case 1:r = q; g = v; b = p;break; + case 2:r = p; g = v; b = t;break; + case 3:r = p; g = q; b = v;break; + case 4:r = t; g = p; b = v;break; + case 5:r = v; g = p; b = q;break; + default:r = 1; g = 1; b = 1;break;} + return std::make_tuple(static_cast(b * 255), static_cast(g * 255), static_cast(r * 255)); +} +static std::tuple random_color(int id){ + float h_plane = ((((unsigned int)id << 2) ^ 0x937151) % 100) / 100.0f;; + float s_plane = ((((unsigned int)id << 3) ^ 0x315793) % 100) / 100.0f; + return hsv2bgr(h_plane, s_plane, 1); +} + +static int Yolo::DrawBoxesonGraph(cv::Mat &bgr_img, std::vector> nmsresult){ + for(int i = 0; i < nmsresult.size(); ++i){ + auto& ibox = nmsresult[i]; + float left = ibox[0]; + float top = ibox[1]; + float right = ibox[2]; + float bottom = ibox[3]; + int class_label = ibox[4]; + float confidence = ibox[5]; + cv::Scalar color; + std::tie(color[0], color[1], color[2]) = random_color(class_label); + cv::rectangle(bgr_img, cv::Point(left, top), cv::Point(right, bottom), color, 3); + + auto name = cocolabels[class_label]; + auto caption = cv::format("%s %.2f", name, confidence); + int text_width = cv::getTextSize(caption, 0, 1, 2, nullptr).width + 10; + cv::rectangle(bgr_img, cv::Point(left-3, top-33), cv::Point(left + text_width, top), color, -1); + cv::putText(bgr_img, caption, cv::Point(left, top-5), 0, 1, cv::Scalar::all(0), 2, 16); + } + return 0; +} diff --git a/tools/yolo_deepstream/tensorrt_yolo/src/Yolo.h b/tools/yolo_deepstream/tensorrt_yolo/src/Yolo.h new file mode 100644 index 00000000..a7bc3e7f --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/src/Yolo.h @@ -0,0 +1,174 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +//! +//! This file contains the implementation of the yolov7 sample. +//! +#pragma once + +#include +#include +#include +#include +#include "NvInfer.h" +#include +#include +#include +#include +#include +#include +//opencv for preprocessing & postprocessing +#include +#include +#include +#include + +class Yolo { +public: + //! + //! \brief init Yolo class object + //! + //! \param engine_path The path of trt engine file + //! + Yolo(std::string engine_path); + + //! + //! \brief preprocess a list of image, the image will remembered inside the class by Yolo object + //! + //! \param cv_img input images with BGR-UInt8, the size of the vector must smmaller than the maxBatchsize of the model + //! + std::vector preProcess(std::vector &cv_img);// + + //! + //! \brief run tensorRT inference with the data preProcessed + //! + int infer(); + + //! + //! \brief PostProcess, will decode and nms the batch inference result of yolov7 + //! + //! \param cv_img + //! \return return all the nms result of Yolo + //! + std::vector>> PostProcess(float iou_thres = 0.45f, float conf_thres = 0.25f, bool isYolov7 = true); + //! + //! \brief Get the input dimenssion of the model + //! + //! \return return Dims of input + //! + nvinfer1::Dims getInputDim(); + + //! + //! \brief Get the output dimenssion of the model + //! + //! \return return the Dims of output + //! + nvinfer1::Dims getOutputDim(); + + //! + //! \brief Draw boxes on bgr image + //! \param bgr_img The images need to be drawed with boxes + //! \param nmsresult nms result get from PostProcess function + //! + static int Yolo::DrawBoxesonGraph(cv::Mat &bgr_img, std::vector> nmsresult); + + //! + //! \brief preprocess a list of image for validate mAP on coco dataset! the model must have a [batchsize, 3, 672, 672] input + //! + //! \param cv_img input images with BGR-UInt8, the size of the vector must smmaller than the maxBatchsize of the model + //! + std::vector preProcess4Validate(std::vector &cv_img); + + //! + //! \brief PostProcess for validate mAP on coco dataset!, will decode the batch inference result of yolov7 + //! + //! \param cv_img + //! \return return all the nms result of Yolo + //! + std::vector>> PostProcess4Validate(float iou_thres = 0.45f, float conf_thres = 0.25f); +private: + + int pushImg(void *imgBuffer, int numImg, bool fromCPU = true); + + std::vector>> decode_yolov7_result(float conf_thres); + std::vector>> decode_yolov8v9_result(float conf_thres); + std::vector>> yolo_nms(std::vector>> &bboxes, float iou_thres); + std::vector> nms(std::vector> &bboxes, float iou_thres); + + //TODO: to be imp + void CudaGraphEndCapture(cudaStream_t stream); + + void CudaGraphBeginCapture(cudaStream_t stream); + + bool CudaGraphLaunch(cudaStream_t stream); + + bool enableCudaGraph(); + + void ReportArgs(); + +private: + + int mImgPushed; + int mMaxBatchSize; + bool mDynamicBatch; + + //stream and event + std::unique_ptr mStream; + std::unique_ptr mEvent; + + // trt objects + std::unique_ptr> mRuntime; + std::unique_ptr> mEngine; + std::unique_ptr> mContext; + std::vector>> mBindings; + + std::vector mBindingArray; + std::vector mHostOutputBuffer; + std::vector mHostNMSBuffer; + + std::string mEnginePath; + nvinfer1::Dims mInputDim; //maxB,3,640,640 + nvinfer1::Dims mOutputDim; + int mImgBufferSize;//sizeof(float)x3x640x640 + + //cuda graph objects + cudaGraph_t mGraph{}; + cudaGraphExec_t mGraphExec{}; + + std::vector> md2i; + + bool mCudaGraphEnabled; + +//TODOs + //! + //! get how many imgs has been totally processed + //! + // caculate fps real time + unsigned long long mLast_inference_time; + unsigned long long mTotal_inference_time; + int mInference_count; +public: + int imgProcessed() { return mInference_count; }; +}; diff --git a/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.cpp b/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.cpp new file mode 100644 index 00000000..35f00082 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.cpp @@ -0,0 +1,88 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#include "argsParser.h" + +// constructor +argsParser::argsParser(const int pArgc, char** pArgv) { + argc = pArgc; + argv = pArgv; +} +// ParseFlag +bool argsParser::ParseFlag(std::string string_ref) const { + if (argc < 1) return false; + + for (int i = 0; i < argc; i++) { + const int string_start = std::string(argv[i]).find_last_of('-') + 1; + if (string_start == 0) continue; + + const char* string_argv = &argv[i][string_start]; + + const char* equal_pos = strchr(string_argv, '='); + + const int argv_length = (int)(equal_pos == 0 ? strlen(string_argv) : equal_pos - string_argv); + const int length = (int)(string_ref.size()); + + if (length == argv_length && !strncasecmp(string_argv, string_ref.c_str(), length)) return true; + } + return false; +} + +// ParseString +const char* argsParser::ParseString(std::string string_ref) const { + if (argc < 1) return NULL; + + for (int i = 0; i < argc; i++) { + const int string_start = std::string(argv[i]).find_last_of('-') + 1; + + if (string_start == 0) continue; + + char* string_argv = (char*)&argv[i][string_start]; + const int length = (int)(string_ref.size()); + + if (!strncasecmp(string_argv, string_ref.c_str(), length)) return (string_argv + length + 1); + //*string_retval = &string_argv[length+1]; + } + return NULL; +} + + +// ParseStringList eg. img1,img2,img3 +std::vector argsParser::ParseStringList(std::string argName, const char delimiter) const{ + const char* ListStr = ParseString(argName); + std::vector result; + if (ListStr == NULL) return result; + int string_start = 0; + int string_end = 0; + + int strLen = (int)strlen(ListStr); + while(string_end < strLen){ + while (delimiter != ListStr[string_end] && string_end < strLen) string_end++; + result.push_back(std::string(ListStr).substr(string_start,string_end-string_start)); + string_end++; + string_start = string_end; + } + return result; +} diff --git a/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.h b/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.h new file mode 100644 index 00000000..436db667 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/src/argsParser.h @@ -0,0 +1,67 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef __COMMAND_LINE_H_ +#define __COMMAND_LINE_H_ + +#include +#include +#include +#include + +#include +#include + +/** + * args line parser + */ +class argsParser { +public: + argsParser(const int argc, char** argv); + + /** + * Parse Flag + */ + bool ParseFlag(const std::string argName) const; + + /** + * Parse String + */ + const char* ParseString(const std::string argName) const; + // const char* ParseString2(const std::string argName, const char* defaultValue = NULL, bool allowOtherDelimiters = true) const; + + /** + * Parse String list delimited by "," + */ + std::vector ParseStringList(std::string argName, const char delimiter = ',') const; + + /** + * The argument count that the object was created with from main() + */ + int argc; + char** argv; +}; + +#endif diff --git a/tools/yolo_deepstream/tensorrt_yolo/src/tools.h b/tools/yolo_deepstream/tensorrt_yolo/src/tools.h new file mode 100644 index 00000000..3cfe95b3 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/src/tools.h @@ -0,0 +1,109 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __TOOLS_H__ +#define __TOOLS_H__ + +#include +#include +#include +#include +#include "NvInfer.h" + +void checkCudaErrors(cudaError_t err) { + if (err != cudaSuccess) throw std::runtime_error(cudaGetErrorName(err)); +} + +// Logger for TensorRT info/warning/errors +class Logger : public nvinfer1::ILogger { +public: + Logger(Severity severity = Severity::kWARNING) : reportableSeverity(severity) {} + + void log(Severity severity, const char* msg) noexcept override { + // suppress messages with severity enum value greater than the reportable + if (severity > reportableSeverity) return; + + switch (severity) { + case Severity::kINTERNAL_ERROR: + std::cerr << "INTERNAL_ERROR: "; + break; + case Severity::kERROR: + std::cerr << "ERROR: "; + break; + case Severity::kWARNING: + std::cerr << "WARNING: "; + break; + case Severity::kINFO: + std::cerr << "INFO: "; + break; + default: + std::cerr << "UNKNOWN: "; + break; + } + std::cerr << msg << std::endl; + } + + Severity reportableSeverity; +}; +template +struct TrtDeleter { + void operator()(T* p) noexcept { + if (p != nullptr) delete p; + } +}; + +template +struct CuMemDeleter { + void operator()(T* p) noexcept { checkCudaErrors(cudaFree(p)); } +}; + +template +std::unique_ptr> mallocCudaMem(size_t nbElems) { + T* ptr = nullptr; + checkCudaErrors(cudaMalloc((void**)&ptr, sizeof(T) * nbElems)); + return std::unique_ptr>{ptr}; +} + +struct EventDeleter { + void operator()(CUevent_st* event) noexcept { checkCudaErrors(cudaEventDestroy(event)); } +}; +struct StreamDeleter { + void operator()(CUstream_st* stream) noexcept { checkCudaErrors(cudaStreamDestroy(stream)); } +}; + +std::unique_ptr makeCudaEvent(int flags) { + cudaEvent_t event; + checkCudaErrors(cudaEventCreateWithFlags(&event, flags)); + return std::unique_ptr{event}; +} + +std::unique_ptr makeCudaStream(int flags, int priority) { + cudaStream_t stream; + checkCudaErrors(cudaStreamCreateWithPriority(&stream, flags, priority)); + return std::unique_ptr{stream}; +} + + + +#endif diff --git a/tools/yolo_deepstream/tensorrt_yolo/test_coco_map.py b/tools/yolo_deepstream/tensorrt_yolo/test_coco_map.py new file mode 100644 index 00000000..42b4dc37 --- /dev/null +++ b/tools/yolo_deepstream/tensorrt_yolo/test_coco_map.py @@ -0,0 +1,46 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import json +import os +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog='test.py') + parser.add_argument('--predict', type=str, default='./predict.json', help='model.pt path(s)') + parser.add_argument('--coco', type=str, default='./coco/', help='*.data path') + opt = parser.parse_args() + print('\nEvaluating pycocotools mAP... saving %s...' % opt.predict) + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + anno = COCO(opt.coco+"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/annotations/instances_val2017.json") # init annotations api + pred = anno.loadRes(opt.predict) # init predictions api + eval = COCOeval(anno, pred, 'bbox') + # if is_coco: + # eval.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] # image IDs to evaluate + eval.evaluate() + eval.accumulate() + eval.summarize() + map, map50 = eval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) + except Exception as e: + print(f'pycocotools unable to run: {e}') diff --git a/tools/yolo_deepstream/yolov7_qat/README.md b/tools/yolo_deepstream/yolov7_qat/README.md new file mode 100644 index 00000000..55d269e1 --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/README.md @@ -0,0 +1,80 @@ +# YoloV7 Quantization Aware Training +## Description + We use [TensorRT's pytorch quntization tool](https://github.com/NVIDIA/TensorRT/tree/main/tools/pytorch-quantization) to finetune training QAT yolov7 from the pre-trained weight, then export the model to onnx and deploy it with TensorRT. The accuray and performance can be found in below table. + +| Method | Calibration method | mAPval
0.5|mAPval
0.5:0.95 |batch-1 fps
Jetson Orin-X |batch-16 fps
Jetson Orin-X |weight| +| ---- | ---- |---- |---- |----|----|-| +| pytorch FP16 | - | 0.6972 | 0.5120 |-|-|[yolov7.pt](https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7.pt)| +| pytorch PTQ-INT8 | Histogram(MSE) | 0.6957 | 0.5100 |-|-|[yolov7_ptq.pt](https://drive.google.com/file/d/1AMymKjKMDmhuNSI3jzL6dv_Pc3rdDDj1/view?usp=sharing) [yolov7_ptq_640.onnx](https://drive.google.com/file/d/1kvCV8PxV6RCidehN4Wp78M116oZ_mSTX/view?usp=sharing)| +| pytorch QAT-INT8 | Histogram(MSE) | 0.6961 | 0.5111 |-|-|[yolov7_qat.pt](https://drive.google.com/file/d/16Ylot5AfkjKeCyVlX3ECsuT6VmHULkd-/view?usp=sharing)| +| TensorRT FP16| - | 0.6973 | 0.5124 |140 |168|[yolov7.onnx](https://drive.google.com/file/d/1R5muSJWVC_BQKml4s4wQQewUXdmQl0Mm/view?usp=sharing) | +| TensorRT PTQ-INT8 | TensorRT built in EntropyCalibratorV2 | 0.6317 | 0.4573 |207|264|-| +| TensorRT QAT-INT8 | Histogram(MSE) | 0.6962 | 0.5113 |207|266|[yolov7_qat_640.onnx](https://drive.google.com/file/d/1qn-p4N3GZojIOvvxkzmPGCQKR6q4ov73/view?usp=sharing)| + - network input resolution: 3x640x640 + - note: trtexec cudaGraph is enabled + +## How To QAT Training +### 1.Setup + +Suggest to use docker environment. +```bash +$ docker pull nvcr.io/nvidia/pytorch:22.09-py3 +``` + +1. Clone and apply patch +```bash +# use this YoloV7 as a sample base +git clone https://github.com/WongKinYiu/yolov7.git +cp -r yolov_deepstream/yolov7_qat/* yolov7/ +``` + +2. Install dependencies +```bash +$ pip install pytorch-quantization --extra-index-url https://pypi.ngc.nvidia.com +``` + +3. Download dataset and pretrained model +```bash +$ bash scripts/get_coco.sh +$ wget https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7.pt +``` + +### 2. Start QAT training + ```bash + $ python scripts/qat.py quantize yolov7.pt --ptq=ptq.pt --qat=qat.pt --eval-ptq --eval-origin + ``` + This script includes steps below: + - Insert Q&DQ nodes to get fake-quant pytorch model
+ [Pytorch quntization tool](https://github.com/NVIDIA/TensorRT/tree/main/tools/pytorch-quantization) provides automatic insertion of QDQ function. But for yolov7 model, it can not get the same performance as PTQ, because in Explicit mode(QAT mode), TensorRT will henceforth refer Q/DQ nodes' placement to restrict the precision of the model. Some of the automatic added Q&DQ nodes can not be fused with other layers which will cause some extra useless precision convertion. In our script, We find Some rules and restrictions for yolov7, QDQ nodes are automatically analyzed and configured in a rule-based manner, ensuring that they are optimal under TensorRT. Ensuring that all nodes are running INT8(confirmed with tool:[trt-engine-explorer](https://github.com/NVIDIA/TensorRT/tree/main/tools/experimental/trt-engine-explorer), see [scripts/draw-engine.py](./scripts/draw-engine.py)). for details of this part, please refer [quantization/rules.py](./quantization/rules.py), About the guidance of Q&DQ insert, please refer [Guidance_of_QAT_performance_optimization](./doc/Guidance_of_QAT_performance_optimization.md) + + - PTQ calibration
+ After inserting Q&DQ nodes, we recommend to run PTQ-Calibration first. Per experiments, `Histogram(MSE)` is the best PTQ calibration method for yolov7. + Note: if you are satisfied with PTQ result, you could also skip QAT. + + - QAT training
+ After QAT, need to finetune traning our model. after getting the accuracy we are satisfied, Saving the weights to files + +### 3. Export onnx + ```bash + $ python scripts/qat.py export qat.pt --size=640 --save=qat.onnx --dynamic + ``` + +### 4. Evaluate model accuracy on coco + ```bash + $ bash scripts/eval-trt.sh qat.pt + ``` + +### 5. Benchmark + ```bash + $ /usr/src/tensorrt/bin/trtexec --onnx=qat.onnx --int8 --fp16 --workspace=1024000 --minShapes=images:4x3x640x640 --optShapes=images:4x3x640x640 --maxShapes=images:4x3x640x640 + ``` + + +## Quantization Yolov7-Tiny +```bash +$ python scripts/qat.py quantize yolov7-tiny.pt --qat=qat.pt --ptq=ptq.pt --ignore-policy="model\.77\.m\.(.*)|model\.0\.(.*)" --supervision-stride=1 --eval-ptq --eval-origin +``` + +## Note +- For YoloV5, please use the script `scripts/qat-yolov5.py`. This adds QAT support for `Add operator`, making it more performant. +- Please refer to the `quantize.replace_bottleneck_forward` function to handle the `Add operator`. \ No newline at end of file diff --git a/tools/yolo_deepstream/yolov7_qat/doc/Guidance_of_QAT_performance_optimization.md b/tools/yolo_deepstream/yolov7_qat/doc/Guidance_of_QAT_performance_optimization.md new file mode 100644 index 00000000..b06dd56e --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/doc/Guidance_of_QAT_performance_optimization.md @@ -0,0 +1,221 @@ + +# Get QAT models' best performance on TensorRT + +## 1. Description +This guidance will show how to get the best performance QAT model on yolov7. + +There are two workflows for quantizing networks in TensorRT, one is Post-training quantization (PTQ).(ref:[tensorrt-developer-guide/intro-quantization](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#intro-quantization)). The other is QAT.(ref:[tensorrt-developer-guide/work-with-qat-networks](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#work-with-qat-networks). In PTQ mode, TensorRT will have the best performance, as it always choose the best layer fusion tactics and fastest kernels to make the global optimal network enqueue graph. +In QAT modes, the enqueue graph is designed by user. Which depends on the QDQ placement, The accuracy conversion and layer fusion strategies in the network are selected strictly according to the QDQ placement.(About the Q&DQ processing of TensorRT, please refer :[TensorRT-developer-guide: Processing of Q/DQ Networks](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#tensorrt-process-qdq)). That is, If we want to get the best performance of QAT, The Q&DQ nodes must make sure: +1. All the computationally intensive layers will run with INT8. +2. Q&DQ can not break down the layer fusion of QAT model. +3. Do not have unnecessary data conversion between INT8 and FLOAT + +One effective way to get best performance of QAT is comparing the enqueue graph of QAT-TensorRT model with PTQ, and ensure they are the same. + +## 2. Workflow +Our solution is: verbosing the QAT-Graph and compare with the PTQ-Graph. And back to fineTune the Q&DQ nodes placement. The procedure can be summaried as below. +1. Insert QDQ in the model and export it to onnx +2. Convert PTQ-Onnx and QAT-onnx to TensorRT model and draw the TensorRT-model-graph +3. Compare the TensorRT-enqueue-Graph and performance between QAT and PTQ +4. If the QAT Graph is different from PTQ Graph and the performance also wrose. modify the QDQ placement. Back to Step 1. Else, to Step 5 +5. Run PTQ benchmark and QAT benchmark to verify + +QATFlow + +For the layer-fusion rules: We can refer: [TensorRT-developer-guide: Types of Fusions](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#fusion-types) +For the tools for verbosing the TensorRT-model graph：[github-TensorRT: trt-engine-explorer](https://github.com/NVIDIA/TensorRT/tree/main/tools/experimental/trt-engine-explorer)(ref: [blog:exploring-tensorrt-engines-with-trex](https://developer.nvidia.com/blog/exploring-tensorrt-engines-with-trex/)) + + +## 3. Step by step guidance of QAT optimization on yolov7 + +Now we will step by step optimizing a QAT model performance, We only care about the performance rather than accuracy at this time as we had not starting finetune the accuracy with training. +we use pytorch-quantization tool [pytorch-quantization](https://github.com/NVIDIA/TensorRT/blob/main/tools/pytorch-quantization) to quantize our pytorch model. And export onnx model with Q&DQ nodes. +This package provides a number of quantized layer modules, which contain quantizers for inputs and weights. e.g. `quant_nn.QuantLinear`, which can be used in place of `nn.Linear. ` These quantized layers can be substituted automatically, via monkey-patching, or by manually modifying the model definition. +Automatic layer substitution is done with `quant_modules`. This should be called before model creation.[ref: [pytorch-quantization-toolkit-tutorials](https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/tutorials/quant_resnet50.html#quantizing-resnet50)] + +### 1) Insert QDQ to model with monkey-patch quantization + +with `quant_modules.initialize()` and `quant_modules.deactivate()`. The tool will automatic insert Q&DQ nodes in the network. + +```python +quant_modules.initialize() +# Load PyTorch model +device = select_device(opt.device) +model = Model(opt.cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) +labels = model.names +quant_modules.deactivate() +``` +calibrate the onnx model to get the scale of Q&DQ nodes. +```python +def calibrate_model(model, model_name, data_loader, num_calib_batch, calibrator,hist_percentile, out_dir, device): + """ + Feed data to the network and calibrate. + Arguments: + model: classification model + model_name: name to use when creating state files + data_loader: calibration data set + num_calib_batch: amount of calibration passes to perform + calibrator: type of calibration to use (max/histogram) + hist_percentile: percentiles to be used for historgram calibration + out_dir: dir to save state files in + """ + if num_calib_batch > 0: + print("Calibrating model") + with torch.no_grad(): + collect_stats(model, data_loader, num_calib_batch, device) + if not calibrator == "histogram": + compute_amax(model, method="max") + calib_output = os.path.join( + out_dir, + F"{model_name}-max-{num_calib_batch*data_loader.batch_size}.pth") + ckpt = {'model': deepcopy(model)} + torch.save(ckpt, calib_output) + else: + for percentile in hist_percentile: + print(F"{percentile} percentile calibration") + compute_amax(model, method="percentile") + calib_output = os.path.join( + out_dir, + F"{model_name}-percentile-{percentile}-{num_calib_batch*data_loader.batch_size}.pth") + ckpt = {'model': deepcopy(model)} + torch.save(ckpt, calib_output) + for method in ["mse", "entropy"]: + print(F"{method} calibration") + compute_amax(model, method=method) + calib_output = os.path.join( + out_dir, + F"{model_name}-{method}-{num_calib_batch*data_loader.batch_size}.pth") + ckpt = {'model': deepcopy(model)} + torch.save(ckpt, calib_output) +``` +### 2) export the calibrated-pytorch model to onnx +```python +quant_nn.TensorQuantizer.use_fb_fake_quant = True +torch.onnx.export(model, img, f, verbose=False, opset_version=13, input_names['images'], + output_names=output_names, + dynamic_axes=dynamic_axes) +quant_nn.TensorQuantizer.use_fb_fake_quant = False +``` +***Now we got a onnx model with Q&DQ layers. TensorRT will process the onnx model with QDQ nodes as QAT models, With this way. Calibration is no longer needed as TensorRT will automatically performs INT8 quantization based on scales of Q and DQ nodes.*** + +TIPS: We calibrate the pytorch model with fake-quant, the exported onnx will have Q&DQ nodes. In the eye of pytorch, it is a ptq-model as we only did a calibration but no finetune training. But in the eye of TensorRT, as long as there are Q&DQ nodes inside the onnx， TensorRT will regard it as a QAT model. + +### 3) Run TensorRT benchmark and export layers information to json +we can export the TensorRT-engine-graph and profile information with flag `--exportLayerInfo=layer.json --profilingVerbosity=detailed --exportProfile=profile.json`. +first we export fp32 onnx model +```bash +$ python export.py --weights ./yolov7.pt --grid --simplify --topk-all 100 --iou-thres 0.65 --conf-thres 0.35 --img-size 640 640 +``` +Then we copy the onnx to target device, Here we use Jetson OrinX as our target device, TensorRT has different behavior on different GPUs. So the test must run on your final target device + +Run PTQ benchmark +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7.onnx --fp16 --int8 --verbose --saveEngine=yolov7_ptq.engine --workspace=1024000 --warmUp=500 --duration=10 --useCudaGraph --useSpinWait --noDataTransfers --exportLayerInfo=yolov7_ptq_layer.json --profilingVerbosity=detailed --exportProfile=yolov7_ptq_profile.json +``` +Run fp16 benchmark +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7.onnx --fp16 --verbose --saveEngine=yolov7_fp16.engine --workspace=1024000 --warmUp=500 --duration=10 --useCudaGraph --useSpinWait --noDataTransfers --exportLayerInfo=yolov7_fp16_layer.json --profilingVerbosity=detailed --exportProfile=yolov7_fp16_profile.json +``` +Run QAT benchmark +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7_qat.onnx --fp16 --int8 --verbose --saveEngine=yolov7_qat.engine --workspace=1024000 --warmUp=500 --duration=10 --useCudaGraph --useSpinWait --noDataTransfers --exportLayerInfo=yolov7_qat_layer.json --profilingVerbosity=detailed --exportProfile=yolov7_qat_profile.json +``` + +Run QAT_mask detect benchmark +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7_qat_maskdet.onnx --fp16 --int8 --verbose --saveEngine=yolov7_qat_maskdet.engine --workspace=1024000 --warmUp=500 --duration=10 --useCudaGraph --useSpinWait --noDataTransfers --exportLayerInfo=yolov7_qat_maskdet_layer.json --profilingVerbosity=detailed --exportProfile=yolov7_qat_maskdet_profile.json +``` + +We can get the fps from the log: +The PTQ performance is : +```bash +[I] Throughput: 206.562 qps +``` +The fp16 performance is : +```bash +[I] Throughput: 139.597 qps +``` +The version 1 QAT performance is: +```bash +[I] Throughput: 180.439 qps +``` +That is not a good performance as we expect, Let's look insight the reason + +### 4) Draw Engine graph + +we use TensorRT opensource tool: [trt-engine-explorer](https://github.com/NVIDIA/TensorRT/tree/main/tools/experimental/trt-engine-explorer) drawing the enqueue graph of TensorRT. This tool take the trtexec exported layer json information as input. +Use the below code to draw the TensorRT-Engine-graph.(edit from `trt-engine-explorer/utils/draw_engine.py`) + +```python +import graphviz +from trex import * +import argparse +import shutil + + +def draw_engine(engine_json_fname: str, engine_profile_fname: str): + graphviz_is_installed = shutil.which("dot") is not None + if not graphviz_is_installed: + print("graphviz is required but it is not installed.\n") + print("To install on Ubuntu:") + print("sudo apt --yes install graphviz") + exit() + + plan = EnginePlan(engine_json_fname, engine_profile_fname) + formatter = layer_type_formatter + display_regions = True + expand_layer_details = False + + graph = to_dot(plan, formatter, + display_regions=display_regions, + expand_layer_details=expand_layer_details) + render_dot(graph, engine_json_fname, 'svg') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--layer', help="name of engine JSON file to draw") + parser.add_argument('--profile', help="name of profile JSON file to draw") + args = parser.parse_args() + draw_engine(engine_json_fname=args.layer,engine_profile_fname=args.profile) +``` +draw the graph: +```bash +$ python draw_engine.py --layer yolov7_qat_layer.json --profile yolov7_qat_profile.json +$ python draw_engine.py --layer yolov7_ptq_layer.json --profile yolov7_ptq_profile.json +``` +we get `yolov7_qat_layer.json.svg` and `yolov7_ptq_layer.json.svg` + +Let's see the difference: + +monkey-patch-qat-conv-fp16-issue_ptqonnxmonkey-patch-qat-conv-fp16-issue_ptqmonkey-patch-qat-conv-fp16-issue_qatonnxmonkey-patch-qat-conv-fp16-issue_qatonnx + +-
pic1: The convolution layers before first concat layer in onnx
+-
pic2: pic1's TensorRT-graph
+-
pic3: the qat-onnx model
+-
pic4: pic3's TensorRt-graph
+-
(click to see full picture)
+ +### 5) Gap analyze and QDQ placement optimization + There are a lot of useless int8->fp16 and fp16->int8 data convert in our QAT model. That is because : TensorRT will enforce the rules of QDQ to ensure consistent accuracy during inference and training(We didn't see any fp32 tensors here becasue TensorRT believes that fp16 will have the same accuracy as fp32) + That is to say: If we want to reduce these useless data format convertion, We must edit our QDQ nodes to suit the fusion rules of TensorRT QAT. + From the PTQ & QAT engine-graph, we can observed that: the concat layer will be reduced in TensorRT and all the input and output of concat will merge to one tensor(marked are red arrows in the below pic). If we do not guarantee the scale of Q&DQ nodes(marked with green circle in the below pic) of these tensors are the same. There will be redundant precision-conversion in our Graph. + + monkey-patch-qat-conv-fp16-issue_qatonnx_edit + +For all the network-struct like this, We need do the same restrict. There is a special scene we need to take care: QDQ can cross some of the layers according to the commute rules from [TensorRT-developer-guide:tensorrt-process-qdq](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#tensorrt-process-qdq). eg. Max-pooling. +the DQ nodes marked with red circle will cross the MaxPool layer and TensorRT will remember the crossed-MaxPooling layer as int8 precision. Now we meet the similar scence as concat: We should restrict the scale of Q&DQ the same as the Q&DQ in the green circle to avoid generate useless data format convertion here. + + monkey-patch-qat-maxpooling-qat.png + +### 6) optimized QAT model's performance +Now we apply all the restriction we have metioned. We can test the performance: + +we still use trtexec to benchmark the onnx model: +```bash +$ /usr/src/tensorrt/bin/trtexec --onnx=yolov7_qat_maskdet.onnx --fp16 --int8 --verbose --saveEngine=yolov7_qat_optimized.engine --workspace=1024000 --warmUp=500 --duration=10 --useCudaGraph --useSpinWait --noDataTransfers --exportLayerInfo=yolov7_qat_optimized_layer.json --profilingVerbosity=detailed --exportProfile=yolov7_qat_optimized_profile.json +[I] Throughput: 207.267 qps +``` +This performance is almost the same as PTQ performance. + +Next we need can finetune training our model to improve the accracy of the model. diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATConv.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATConv.png new file mode 100644 index 00000000..88f2adcd Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATConv.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATFlow.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATFlow.png new file mode 100644 index 00000000..c63778c4 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/QATFlow.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/int8_q_recommended_procedure.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/int8_q_recommended_procedure.png new file mode 100644 index 00000000..e29af4c2 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/int8_q_recommended_procedure.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptq.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptq.png new file mode 100644 index 00000000..670ce611 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptq.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptqonnx.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptqonnx.png new file mode 100644 index 00000000..6a7ad576 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_ptqonnx.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qat.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qat.png new file mode 100644 index 00000000..841ad89f Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qat.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx.png new file mode 100644 index 00000000..e4467a18 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx_edit.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx_edit.png new file mode 100644 index 00000000..6e8c4790 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-conv-fp16-issue_qatonnx_edit.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-maxpooling-qat.png b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-maxpooling-qat.png new file mode 100644 index 00000000..fbaec092 Binary files /dev/null and b/tools/yolo_deepstream/yolov7_qat/doc/imgs/monkey-patch-qat-maxpooling-qat.png differ diff --git a/tools/yolo_deepstream/yolov7_qat/quantization/quantize.py b/tools/yolo_deepstream/yolov7_qat/quantization/quantize.py new file mode 100644 index 00000000..c9ddc591 --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/quantization/quantize.py @@ -0,0 +1,383 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import os +import re +from typing import List, Callable, Union, Dict +from tqdm import tqdm +from copy import deepcopy + +# PyTorch +import torch +import torch.optim as optim +from torch.cuda import amp + +# Pytorch Quantization +from pytorch_quantization import nn as quant_nn +from pytorch_quantization.nn.modules import _utils as quant_nn_utils +from pytorch_quantization import calib +from pytorch_quantization.tensor_quant import QuantDescriptor +from pytorch_quantization import quant_modules +from pytorch_quantization import tensor_quant +from absl import logging as quant_logging + +# Custom Rules +from quantization.rules import find_quantizer_pairs + +class QuantAdd(torch.nn.Module): + def __init__(self, quantization): + super().__init__() + + if quantization: + self._input0_quantizer = quant_nn.TensorQuantizer(QuantDescriptor(num_bits=8, calib_method="histogram")) + self._input1_quantizer = quant_nn.TensorQuantizer(QuantDescriptor(num_bits=8, calib_method="histogram")) + self._input0_quantizer._calibrator._torch_hist = True + self._input1_quantizer._calibrator._torch_hist = True + self._fake_quant = True + self.quantization = quantization + + def forward(self, x, y): + if self.quantization: + # print(f"QAdd {self._input0_quantizer} {self._input1_quantizer}") + return self._input0_quantizer(x) + self._input1_quantizer(y) + return x + y + + +class disable_quantization: + def __init__(self, model): + self.model = model + + def apply(self, disabled=True): + for name, module in self.model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + module._disabled = disabled + + def __enter__(self): + self.apply(True) + + def __exit__(self, *args, **kwargs): + self.apply(False) + + +class enable_quantization: + def __init__(self, model): + self.model = model + + def apply(self, enabled=True): + for name, module in self.model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + module._disabled = not enabled + + def __enter__(self): + self.apply(True) + return self + + def __exit__(self, *args, **kwargs): + self.apply(False) + + +def have_quantizer(module): + for name, module in module.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + return True + + +# Initialize PyTorch Quantization +def initialize(): + quant_desc_input = QuantDescriptor(calib_method="histogram") + quant_nn.QuantConv2d.set_default_quant_desc_input(quant_desc_input) + quant_nn.QuantMaxPool2d.set_default_quant_desc_input(quant_desc_input) + quant_nn.QuantLinear.set_default_quant_desc_input(quant_desc_input) + quant_logging.set_verbosity(quant_logging.ERROR) + + +def transfer_torch_to_quantization(nninstance : torch.nn.Module, quantmodule): + + quant_instance = quantmodule.__new__(quantmodule) + for k, val in vars(nninstance).items(): + setattr(quant_instance, k, val) + + def __init__(self): + quant_desc_input, quant_desc_weight = quant_nn_utils.pop_quant_desc_in_kwargs(self.__class__) + if isinstance(self, quant_nn_utils.QuantInputMixin): + self.init_quantizer(quant_desc_input) + + # Turn on torch_hist to enable higher calibration speeds + if isinstance(self._input_quantizer._calibrator, calib.HistogramCalibrator): + self._input_quantizer._calibrator._torch_hist = True + else: + self.init_quantizer(quant_desc_input, quant_desc_weight) + + # Turn on torch_hist to enable higher calibration speeds + if isinstance(self._input_quantizer._calibrator, calib.HistogramCalibrator): + self._input_quantizer._calibrator._torch_hist = True + self._weight_quantizer._calibrator._torch_hist = True + + __init__(quant_instance) + return quant_instance + + +def quantization_ignore_match(ignore_policy : Union[str, List[str], Callable], path : str) -> bool: + + if ignore_policy is None: return False + if isinstance(ignore_policy, Callable): + return ignore_policy(path) + + if isinstance(ignore_policy, str) or isinstance(ignore_policy, List): + + if isinstance(ignore_policy, str): + ignore_policy = [ignore_policy] + + if path in ignore_policy: return True + for item in ignore_policy: + if re.match(item, path): + return True + return False + + +# For example: YoloV5 Bottleneck +def bottleneck_quant_forward(self, x): + if hasattr(self, "addop"): + return self.addop(x, self.cv2(self.cv1(x))) if self.add else self.cv2(self.cv1(x)) + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +# For example: YoloV5 Bottleneck +def replace_bottleneck_forward(model): + for name, bottleneck in model.named_modules(): + if bottleneck.__class__.__name__ == "Bottleneck": + if bottleneck.add: + if not hasattr(bottleneck, "addop"): + print(f"Add QuantAdd to {name}") + bottleneck.addop = QuantAdd(bottleneck.add) + bottleneck.__class__.forward = bottleneck_quant_forward + + +def replace_to_quantization_module(model : torch.nn.Module, ignore_policy : Union[str, List[str], Callable] = None): + + module_dict = {} + for entry in quant_modules._DEFAULT_QUANT_MAP: + module = getattr(entry.orig_mod, entry.mod_name) + module_dict[id(module)] = entry.replace_mod + + def recursive_and_replace_module(module, prefix=""): + for name in module._modules: + submodule = module._modules[name] + path = name if prefix == "" else prefix + "." + name + recursive_and_replace_module(submodule, path) + + submodule_id = id(type(submodule)) + if submodule_id in module_dict: + ignored = quantization_ignore_match(ignore_policy, path) + if ignored: + print(f"Quantization: {path} has ignored.") + continue + + module._modules[name] = transfer_torch_to_quantization(submodule, module_dict[submodule_id]) + + recursive_and_replace_module(model) + + +def get_attr_with_path(m, path): + def sub_attr(m, names): + name = names[0] + value = getattr(m, name) + + if len(names) == 1: + return value + + return sub_attr(value, names[1:]) + return sub_attr(m, path.split(".")) + + +def apply_custom_rules_to_quantizer(model : torch.nn.Module, export_onnx : Callable): + + # apply rules to graph + export_onnx(model, "quantization-custom-rules-temp.onnx") + pairs = find_quantizer_pairs("quantization-custom-rules-temp.onnx") + for major, sub in pairs: + print(f"Rules: {sub} match to {major}") + get_attr_with_path(model, sub)._input_quantizer = get_attr_with_path(model, major)._input_quantizer + os.remove("quantization-custom-rules-temp.onnx") + + for name, bottleneck in model.named_modules(): + if bottleneck.__class__.__name__ == "Bottleneck": + if bottleneck.add: + print(f"Rules: {name}.add match to {name}.cv1") + major = bottleneck.cv1.conv._input_quantizer + bottleneck.addop._input0_quantizer = major + bottleneck.addop._input1_quantizer = major + + +def calibrate_model(model : torch.nn.Module, dataloader, device, num_batch=25): + + def compute_amax(model, **kwargs): + for name, module in model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if module._calibrator is not None: + if isinstance(module._calibrator, calib.MaxCalibrator): + module.load_calib_amax() + else: + module.load_calib_amax(**kwargs) + + module._amax = module._amax.to(device) + + def collect_stats(model, data_loader, device, num_batch=200): + """Feed data to the network and collect statistics""" + # Enable calibrators + model.eval() + for name, module in model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if module._calibrator is not None: + module.disable_quant() + module.enable_calib() + else: + module.disable() + + # Feed data to the network for collecting stats + with torch.no_grad(): + for i, datas in tqdm(enumerate(data_loader), total=num_batch, desc="Collect stats for calibrating"): + imgs = datas[0].to(device, non_blocking=True).float() / 255.0 + model(imgs) + + if i >= num_batch: + break + + # Disable calibrators + for name, module in model.named_modules(): + if isinstance(module, quant_nn.TensorQuantizer): + if module._calibrator is not None: + module.enable_quant() + module.disable_calib() + else: + module.enable() + + collect_stats(model, dataloader, device, num_batch=num_batch) + compute_amax(model, method="mse") + + +def finetune( + model : torch.nn.Module, train_dataloader, per_epoch_callback : Callable = None, preprocess : Callable = None, + nepochs=10, early_exit_batchs_per_epoch=1000, lrschedule : Dict = None, fp16=True, learningrate=1e-5, + supervision_policy : Callable = None +): + origin_model = deepcopy(model).eval() + disable_quantization(origin_model).apply() + + model.train() + model.requires_grad_(True) + + scaler = amp.GradScaler(enabled=fp16) + optimizer = optim.Adam(model.parameters(), learningrate) + quant_lossfn = torch.nn.MSELoss() + device = next(model.parameters()).device + + if lrschedule is None: + lrschedule = { + 0: 1e-6, + 3: 1e-5, + 8: 1e-6 + } + + def make_layer_forward_hook(l): + def forward_hook(m, input, output): + l.append(output) + return forward_hook + + supervision_module_pairs = [] + for ((mname, ml), (oriname, ori)) in zip(model.named_modules(), origin_model.named_modules()): + if isinstance(ml, quant_nn.TensorQuantizer): continue + + if supervision_policy: + if not supervision_policy(mname, ml): + continue + + supervision_module_pairs.append([ml, ori]) + + + for iepoch in range(nepochs): + + if iepoch in lrschedule: + learningrate = lrschedule[iepoch] + for g in optimizer.param_groups: + g["lr"] = learningrate + + model_outputs = [] + origin_outputs = [] + remove_handle = [] + + for ml, ori in supervision_module_pairs: + remove_handle.append(ml.register_forward_hook(make_layer_forward_hook(model_outputs))) + remove_handle.append(ori.register_forward_hook(make_layer_forward_hook(origin_outputs))) + + model.train() + pbar = tqdm(train_dataloader, desc="QAT", total=early_exit_batchs_per_epoch) + for ibatch, imgs in enumerate(pbar): + + if ibatch >= early_exit_batchs_per_epoch: + break + + if preprocess: + imgs = preprocess(imgs) + + imgs = imgs.to(device) + with amp.autocast(enabled=fp16): + model(imgs) + + with torch.no_grad(): + origin_model(imgs) + + quant_loss = 0 + for index, (mo, fo) in enumerate(zip(model_outputs, origin_outputs)): + quant_loss += quant_lossfn(mo, fo) + + model_outputs.clear() + origin_outputs.clear() + + if fp16: + scaler.scale(quant_loss).backward() + scaler.step(optimizer) + scaler.update() + else: + quant_loss.backward() + optimizer.step() + optimizer.zero_grad() + pbar.set_description(f"QAT Finetuning {iepoch + 1} / {nepochs}, Loss: {quant_loss.detach().item():.5f}, LR: {learningrate:g}") + + # You must remove hooks during onnx export or torch.save + for rm in remove_handle: + rm.remove() + + if per_epoch_callback: + if per_epoch_callback(model, iepoch, learningrate): + break + + +def export_onnx(model, input, file, *args, **kwargs): + + quant_nn.TensorQuantizer.use_fb_fake_quant = True + + model.eval() + with torch.no_grad(): + torch.onnx.export(model, input, file, *args, **kwargs) + + quant_nn.TensorQuantizer.use_fb_fake_quant = False diff --git a/tools/yolo_deepstream/yolov7_qat/quantization/rules.py b/tools/yolo_deepstream/yolov7_qat/quantization/rules.py new file mode 100644 index 00000000..b9730adf --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/quantization/rules.py @@ -0,0 +1,96 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import onnx + +def find_with_input_node(model, name): + for node in model.graph.node: + if len(node.input) > 0 and name in node.input: + return node + +def find_all_with_input_node(model, name): + all = [] + for node in model.graph.node: + if len(node.input) > 0 and name in node.input: + all.append(node) + return all + +def find_with_output_node(model, name): + for node in model.graph.node: + if len(node.output) > 0 and name in node.output: + return node + +def find_with_no_change_parent_node(model, node): + parent = find_with_output_node(model, node.input[0]) + if parent is not None: + if parent.op_type in ["Concat", "MaxPool"]: + return find_with_no_change_parent_node(model, parent) + return parent + +def find_quantizelinear_conv(model, qnode): + dq = find_with_input_node(model, qnode.output[0]) + conv = find_with_input_node(model, dq.output[0]) + return conv + + +def find_quantize_conv_name(model, weight_qname): + dq = find_with_output_node(model, weight_qname) + q = find_with_output_node(model, dq.input[0]) + return ".".join(q.input[0].split(".")[:-1]) + +def find_quantizer_pairs(onnx_file): + + model = onnx.load(onnx_file) + match_pairs = [] + for node in model.graph.node: + if node.op_type == "Concat": + qnodes = find_all_with_input_node(model, node.output[0]) + major = None + for qnode in qnodes: + if qnode.op_type != "QuantizeLinear": + continue + + conv = find_quantizelinear_conv(model, qnode) + if major is None: + major = find_quantize_conv_name(model, conv.input[1]) + else: + match_pairs.append([major, find_quantize_conv_name(model, conv.input[1])]) + + for subnode in model.graph.node: + if len(subnode.input) > 0 and subnode.op_type == "QuantizeLinear" and subnode.input[0] in node.input: + subconv = find_quantizelinear_conv(model, subnode) + match_pairs.append([major, find_quantize_conv_name(model, subconv.input[1])]) + + elif node.op_type == "MaxPool": + qnode = find_with_input_node(model, node.output[0]) + if not (qnode and qnode.op_type == "QuantizeLinear"): + continue + + major = find_quantizelinear_conv(model, qnode) + major = find_quantize_conv_name(model, major.input[1]) + same_input_nodes = find_all_with_input_node(model, node.input[0]) + + for same_input_node in same_input_nodes: + if same_input_node.op_type == "QuantizeLinear": + subconv = find_quantizelinear_conv(model, same_input_node) + match_pairs.append([major, find_quantize_conv_name(model, subconv.input[1])]) + return match_pairs diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/detect-trt.py b/tools/yolo_deepstream/yolov7_qat/scripts/detect-trt.py new file mode 100644 index 00000000..8ae0444e --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/detect-trt.py @@ -0,0 +1,244 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import argparse +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random +import numpy as np +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import check_img_size, check_requirements, check_imshow, non_max_suppression, apply_classifier, \ + scale_coords, xyxy2xywh, strip_optimizer, set_logging, increment_path +from utils.plots import plot_one_box +from utils.torch_utils import select_device, load_classifier, time_synchronized, TracedModel + +import pycuda.autoinit +import pycuda.driver as cuda +import tensorrt as trt +# Allocates all buffers required for an engine, i.e. host/device inputs/outputs. +# Simple helper data class that's a little nicer to use than a 2-tuple. +names = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', \ + 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant',\ + 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', \ + 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',\ + 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', \ + 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite',\ + 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',\ + 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', \ + 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli',\ + 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant',\ + 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',\ + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', \ + 'teddy bear', 'hair drier', 'toothbrush'] + +class HostDeviceMem(object): + def __init__(self, host_mem, device_mem): + self.host = host_mem + self.device = device_mem + + def __str__(self): + return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device) + + def __repr__(self): + return self.__str__() + +def allocate_buffers(engine): + inputs = [] + outputs = [] + bindings = [] + stream = cuda.Stream() + for binding in engine: + size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size + print("binding shape: ", engine.get_binding_shape(binding)) + dtype = trt.nptype(engine.get_binding_dtype(binding)) + # Allocate host and device buffers + host_mem = cuda.pagelocked_empty(size, dtype) + device_mem = cuda.mem_alloc(host_mem.nbytes) + # Append the device buffer to device bindings. + bindings.append(int(device_mem)) + # Append to the appropriate list. + if engine.binding_is_input(binding): + inputs.append(HostDeviceMem(host_mem, device_mem)) + else: + outputs.append(HostDeviceMem(host_mem, device_mem)) + return inputs, outputs, bindings, stream + +def do_inference_v2(context, bindings, inputs, outputs, stream): + # Transfer input data to the GPU. + [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs] + # Run inference. + context.execute_async_v2(bindings=bindings, stream_handle=stream.handle) + # Transfer predictions back from the GPU. + [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs] + # Synchronize the stream + stream.synchronize() + # Return only the host outputs. + return [out.host for out in outputs] + +def detect(save_img=False): + source, view_img, save_txt, imgsz = opt.source, opt.view_img, opt.save_txt, opt.img_size + save_img = not opt.nosave and not source.endswith('.txt') # save inference images + webcam = source.isnumeric() or source.endswith('.txt') or source.lower().startswith( + ('rtsp://', 'rtmp://', 'http://', 'https://')) + + # Directories + save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Initialize + set_logging() + device = select_device(opt.device) # device will be avilable for NMS + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = check_imshow() + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz) + else: + dataset = LoadImages(source, img_size=imgsz, auto=False) + + colors = [[random.randint(0, 255) for _ in range(3)] for _ in names] + + ####### start trt objects + logger = trt.Logger(trt.Logger.INFO) + f = open(opt.engine, 'rb') + runtime = trt.Runtime(logger) + engine = runtime.deserialize_cuda_engine(f.read()) + inputs, outputs, bindings, stream = allocate_buffers(engine) + outputshape = [engine.get_binding_shape(binding) for binding in engine][1] + + t0 = time.time() + for path, img, im0s, vid_cap in dataset: + img = img.astype(np.float32) + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if len(img.shape) == 3: + img = np.expand_dims(img, 0) + + inputs[0].host = img + context = engine.create_execution_context() + trt_outputs = do_inference_v2(context, bindings=bindings, inputs=inputs, outputs=outputs, stream = stream) + trt_outputs = torch.Tensor(trt_outputs[0].reshape(outputshape)) + # Inference + t1 = time_synchronized() + + # pred = trt_outputs + t2 = time_synchronized() + + # Apply NMS + trt_outputs = non_max_suppression(trt_outputs, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t3 = time_synchronized() + + # Process detections + for i, det in enumerate(trt_outputs): # detections per image + if webcam: # batch_size >= 1 + p, s, im0, frame = path[i], '%g: ' % i, im0s[i].copy(), dataset.count + else: + p, s, im0, frame = path, '', im0s, getattr(dataset, 'frame', 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # img.jpg + txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # img.txt + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Write results + for *xyxy, conf, cls in reversed(det): + if save_txt: # Write to file + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if opt.save_conf else (cls, *xywh) # label format + with open(txt_path + '.txt', 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + if save_img or view_img: # Add bbox to image + label = f'{names[int(cls)]} {conf:.2f}' + plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=1) + + # Print time (inference + NMS) + print(f'{s}Done. ({(1E3 * (t2 - t1)):.1f}ms) Inference, ({(1E3 * (t3 - t2)):.1f}ms) NMS') + + # Stream results + if view_img: + cv2.imshow(str(p), im0) + cv2.waitKey(1) # 1 millisecond + + # Save results (image with detections) + if save_img: + if dataset.mode == 'image': + cv2.imwrite(save_path, im0) + print(f" The image with the result is saved in: {save_path}") + else: # 'video' or 'stream' + if vid_path != save_path: # new video + vid_path = save_path + if isinstance(vid_writer, cv2.VideoWriter): + vid_writer.release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path += '.mp4' + vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer.write(im0) + + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + #print(f"Results saved to {save_dir}{s}") + + print(f'Done. ({time.time() - t0:.3f}s)') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--engine', type=str, default='yolov7.engine', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--project', default='runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + opt = parser.parse_args() + print(opt) + + detect() diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/draw-engine.py b/tools/yolo_deepstream/yolov7_qat/scripts/draw-engine.py new file mode 100644 index 00000000..0eb93f7e --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/draw-engine.py @@ -0,0 +1,64 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + + +""" +This script generates an SVG diagram of the input engine graph SVG file. +Note: + THIS SCRIPT DEPENDS ON LIB: https://github.com/NVIDIA/TensorRT/tree/main/tools/experimental/trt-engine-explorer + this script requires graphviz which can be installed manually: + $ sudo apt-get --yes install graphviz + $ python3 -m pip install graphviz networkx +""" + +import graphviz +from trex import * +import argparse +import shutil + + +def draw_engine(engine_json_fname: str, engine_profile_fname: str): + graphviz_is_installed = shutil.which("dot") is not None + if not graphviz_is_installed: + print("graphviz is required but it is not installed.\n") + print("To install on Ubuntu:") + print("sudo apt --yes install graphviz") + exit() + + plan = EnginePlan(engine_json_fname, engine_profile_fname) + formatter = layer_type_formatter + display_regions = True + expand_layer_details = False + + graph = to_dot(plan, formatter, + display_regions=display_regions, + expand_layer_details=expand_layer_details) + render_dot(graph, engine_json_fname, 'svg') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--layer', help="name of engine JSON file to draw") + parser.add_argument('--profile', help="name of profile JSON file to draw") + args = parser.parse_args() + draw_engine(engine_json_fname=args.layer,engine_profile_fname=args.profile) diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.py b/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.py new file mode 100644 index 00000000..b9aa23ba --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.py @@ -0,0 +1,410 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import sys +import os +sys.path.insert(0, os.path.abspath(".")) +pydir = os.path.dirname(__file__) + +import argparse +import json +from pathlib import Path +from threading import Thread + +import numpy as np +import torch +import yaml +from tqdm import tqdm +from models.yolo import Model +from models.experimental import attempt_load +from utils.datasets import create_dataloader +from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, check_requirements, \ + box_iou, non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path, colorstr +from utils.metrics import ap_per_class, ConfusionMatrix +from utils.plots import plot_images, output_to_target, plot_study_txt +from utils.torch_utils import select_device, TracedModel + +import time +import pycuda.autoinit +import pycuda.driver as cuda +import tensorrt as trt +import cv2 + +def time_synchronized(): + return time.time() + +names = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', \ + 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant',\ + 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', \ + 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',\ + 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', \ + 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite',\ + 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',\ + 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', \ + 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli',\ + 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant',\ + 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',\ + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', \ + 'teddy bear', 'hair drier', 'toothbrush'] + +class HostDeviceMem(object): + def __init__(self, host_mem, device_mem): + self.host = host_mem + self.device = device_mem + + def __str__(self): + return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device) + + def __repr__(self): + return self.__str__() + +def allocate_buffers(engine): + inputs = [] + outputs = [] + bindings = [] + stream = cuda.Stream() + for binding in engine: + size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size + print("binding shape: ", engine.get_binding_shape(binding)) + dtype = trt.nptype(engine.get_binding_dtype(binding)) + # Allocate host and device buffers + host_mem = cuda.pagelocked_empty(size, dtype) + device_mem = cuda.mem_alloc(host_mem.nbytes) + # Append the device buffer to device bindings. + bindings.append(int(device_mem)) + # Append to the appropriate list. + if engine.binding_is_input(binding): + inputs.append(HostDeviceMem(host_mem, device_mem)) + else: + outputs.append(HostDeviceMem(host_mem, device_mem)) + return inputs, outputs, bindings, stream + +def do_inference_v2(context, bindings, inputs, outputs, stream): + # Transfer input data to the GPU. + [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs] + # Run inference. + context.execute_async_v2(bindings=bindings, stream_handle=stream.handle) + # Transfer predictions back from the GPU. + [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs] + # Synchronize the stream + stream.synchronize() + # Return only the host outputs. + return [out.host for out in outputs] + +def test(data, + engine_file=None, + batch_size=32, + imgsz=640, + conf_thres=0.001, + iou_thres=0.65, # for NMS + save_json=False, + single_cls=False, + augment=False, + verbose=False, + model=None, + dataloader=None, + save_dir=Path(''), # for saving images + save_txt=False, # for auto-labelling + save_hybrid=False, # for hybrid auto-labelling + save_conf=False, # save auto-label confidences + plots=False, + wandb_logger=None, + compute_loss=None, + half_precision=True, + trace=False, + is_coco=False): + + ####### start trt objects + logger = trt.Logger(trt.Logger.INFO) + runtime = trt.Runtime(logger) + engine = runtime.deserialize_cuda_engine(open(engine_file, 'rb') .read()) + inputs, outputs, bindings, stream = allocate_buffers(engine) + outputshape = [engine.get_binding_shape(binding) for binding in engine][1] + context = engine.create_execution_context() + + # Initialize/load model and set device + training = False + device = torch.device("cpu") + gs = 32 + + # Configure + if isinstance(data, str): + is_coco = data.endswith('coco.yaml') + with open(data) as f: + data = yaml.load(f, Loader=yaml.SafeLoader) + check_dataset(data) # check + nc = 1 if single_cls else int(data['nc']) # number of classes + iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95 + niou = iouv.numel() + + # Logging + log_imgs = 0 + if wandb_logger and wandb_logger.wandb: + log_imgs = min(wandb_logger.log_imgs, 100) + # Dataloader + if not training: + # if device.type != 'cpu': + # model(torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once + task = opt.task if opt.task in ('train', 'val', 'test') else 'val' # path to train/val/test images + dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, pad=0.5, rect=True, workers=0, + prefix=colorstr(f'{task}: '))[1] + seen = 0 + confusion_matrix = ConfusionMatrix(nc=nc) + coco91class = coco80_to_coco91_class() + s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') + p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. + loss = torch.zeros(3, device=device) + jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] + + for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): + + img = (img / 255.0).float() # 0 - 255 to 0.0 - 1.0 + + # dataloader is setup pad=0.5 + input_image = torch.full((3, 672, 672), 114 / 255.0, dtype=torch.float32) + input_image[:, :img.size(1), :img.size(2)] = img + + img = img.unsqueeze(dim=0) + targets = targets.to(device) + nb, _, height, width = img.shape # batch size, channels, height, width + + paths = [paths] + shapes = [shapes] + + # Run model + t = time_synchronized() + #out, train_out = model(img, augment=augment) # inference and training outputs + inputs[0].host = input_image.data.numpy() + trt_outputs = do_inference_v2(context, bindings=bindings, inputs=inputs, outputs=outputs, stream = stream) + # trt_outputs = trt_outputs.reshape((1,25200,85)) + out = torch.Tensor(trt_outputs[0].reshape(outputshape)) + t0 += time_synchronized() - t + + # Run NMS + targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels + lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling + t = time_synchronized() + out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) + t1 += time_synchronized() - t + + # Statistics per image + for si, pred in enumerate(out): + labels = targets[targets[:, 0] == si, 1:] + nl = len(labels) + tcls = labels[:, 0].tolist() if nl else [] # target class + path = Path(paths[si]) + seen += 1 + + if len(pred) == 0: + if nl: + stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) + continue + + # Predictions + predn = pred.clone() + scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred + + # Append to text file + if save_txt: + gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh + for *xyxy, conf, cls in predn.tolist(): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + # W&B logging - Media Panel Plots + if len(wandb_images) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation + if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0: + box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, + "class_id": int(cls), + "box_caption": "%s %.3f" % (names[cls], conf), + "scores": {"class_score": conf}, + "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()] + boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space + wandb_images.append(wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name)) + wandb_logger.log_training_progress(predn, path, names) if wandb_logger and wandb_logger.wandb_run else None + + # Append to pycocotools JSON dictionary + if save_json: + # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... + image_id = int(path.stem) if path.stem.isnumeric() else path.stem + box = xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(pred.tolist(), box.tolist()): + jdict.append({'image_id': image_id, + 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), + 'bbox': [round(x, 3) for x in b], + 'score': round(p[4], 5)}) + + # Assign all predictions as incorrect + correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) + if nl: + detected = [] # target indices + tcls_tensor = labels[:, 0] + + # target boxes + tbox = xywh2xyxy(labels[:, 1:5]) + scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels + if plots: + confusion_matrix.process_batch(predn, torch.cat((labels[:, 0:1], tbox), 1)) + + # Per target class + for cls in torch.unique(tcls_tensor): + ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices + pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices + + # Search for detections + if pi.shape[0]: + # Prediction to target ious + ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices + + # Append detections + detected_set = set() + for j in (ious > iouv[0]).nonzero(as_tuple=False): + d = ti[i[j]] # detected target + if d.item() not in detected_set: + detected_set.add(d.item()) + detected.append(d) + correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn + if len(detected) == nl: # all targets already located in image + break + + # Append statistics (correct, conf, pcls, tcls) + stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) + + # Plot images + if plots and batch_i < 3: + f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels + Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() + f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions + Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() + + # Compute statistics + stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy + if len(stats) and stats[0].any(): + p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) + ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 + mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() + nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class + else: + nt = torch.zeros(1) + + # Print results + pf = '%20s' + '%12i' * 2 + '%12.5g' * 4 # print format + print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) + + # Print results per class + if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): + for i, c in enumerate(ap_class): + print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) + + # Print speeds + t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple + if not training: + print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) + + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir, names=list(names)) + if wandb_logger and wandb_logger.wandb: + val_batches = [wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))] + wandb_logger.log({"Validation": val_batches}) + if wandb_images: + wandb_logger.log({"Bounding Box Debugger/Images": wandb_images}) + + # Save JSON + if save_json and len(jdict): + w = Path(engine_file).stem if engine_file is not None else '' # weights + anno_json = '/datav/dataset/coco/annotations/instances_val2017.json' # annotations json + pred_json = str(save_dir / f"{w}_predictions.json") # predictions json + print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) + with open(pred_json, 'w') as f: + json.dump(jdict, f) + + #try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + + anno = COCO(anno_json) # init annotations api + pred = anno.loadRes(pred_json) # init predictions api + eval = COCOeval(anno, pred, 'bbox') + if is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in dataloader.img_files] # image IDs to evaluate + eval.evaluate() + eval.accumulate() + eval.summarize() + map, map50 = eval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) + # except Exception as e: + # print(f'pycocotools unable to run: {e}, {type(e)}') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog='test.py') + #parser.add_argument('--weights', nargs='+', type=str, default='ptq.pt', help='model.pt path(s)') + # /datav/jingwd/yolov7_qat/runs/train/no123detlre3/weights/qat_014.pt + parser.add_argument('--engine', type=str, default='ptq.engine', help='model.pt path(s)') + parser.add_argument('--data', type=str, default='data/coco.yaml', help='*.data path') + parser.add_argument('--batch-size', type=int, default=1, help='size of each image batch') + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.65, help='IOU threshold for NMS') + parser.add_argument('--task', default='val', help='train, val, test, speed or study') + parser.add_argument('--device', default='0', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--verbose', action='store_true', help='report mAP by class') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file') + parser.add_argument('--project', default='runs/test', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--no-trace', default=True, action='store_true', help='don`t trace model') + parser.add_argument('--cfg', type=str, default='cfg/training/yolov7_qat.yaml', help='model.yaml path') + parser.add_argument('--hyp', type=str, default='data/hyp.scratch.p5-qat.yaml', help='hyperparameters path') + opt = parser.parse_args() + opt.save_json |= opt.data.endswith('coco.yaml') + opt.data = check_file(opt.data) # check file + print(opt) + #check_requirements() + + if opt.task in ('train', 'val', 'test'): # run normally + test(opt.data, + opt.engine, + opt.batch_size, + opt.img_size, + opt.conf_thres, + opt.iou_thres, + opt.save_json, + opt.single_cls, + opt.augment, + opt.verbose, + save_txt=opt.save_txt | opt.save_hybrid, + save_hybrid=opt.save_hybrid, + save_conf=opt.save_conf, + trace=not opt.no_trace, + compute_loss=False + ) + + elif opt.task == 'speed': # speed benchmarks + test(opt.data, opt.engine, opt.batch_size, opt.img_size, 0.25, 0.45, save_json=False, plots=False) diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.sh b/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.sh new file mode 100644 index 00000000..8d5c5267 --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/eval-trt.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script only worked on the Quantization model, otherwise, errors would throw up (Origin model, etc.) +weight=$1 +prefix=${weight%.*} +onnx=${prefix}.onnx +graph=${prefix}.graph +engine=${prefix}.engine + +# onnx must be 672x672 of input +python scripts/qat.py export $weight --dynamic --save=$onnx --size=672 + +# To obtain more QPS can add --fp16 flag for detect layer +trtexec --onnx=$onnx \ + --saveEngine=${engine} --int8 --buildOnly --memPoolSize=workspace:1024MiB \ + --dumpLayerInfo --exportLayerInfo=${graph} --profilingVerbosity=detailed + +python scripts/draw-engine.py ${graph} +python scripts/eval-trt.py --engine=${engine} diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/qat-yolov5.py b/tools/yolo_deepstream/yolov7_qat/scripts/qat-yolov5.py new file mode 100644 index 00000000..1c9699bf --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/qat-yolov5.py @@ -0,0 +1,351 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import sys +import os + +# Add the current directory to PYTHONPATH for yolov5s +sys.path.insert(0, os.path.abspath(".")) +pydir = os.path.dirname(__file__) + +import yaml +import collections +import warnings +import argparse +import json +from pathlib import Path + +# PyTorch +import torch +import torch.nn as nn + +# yolov5s +import val +from models.yolo import Model +from models.common import Conv +from utils.dataloaders import create_dataloader +from utils.downloads import attempt_download +from utils.general import init_seeds, check_dataset + +import quantization.quantize as quantize +from copy import deepcopy + +# Disable all warning +warnings.filterwarnings("ignore") + + +class SummaryTool: + def __init__(self, file): + self.file = file + self.data = [] + + def append(self, item): + self.data.append(item) + json.dump(self.data, open(self.file, "w"), indent=4) + + +# Load yolov5s Model +def load_yolov5s_model(weight, device) -> Model: + + attempt_download(weight) + model = torch.load(weight, map_location=device)["model"] + for m in model.modules(): + if type(m) is nn.Upsample: + m.recompute_scale_factor = None # torch 1.11.0 compatibility + elif type(m) is Conv: + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility + + model.float() + model.eval() + + with torch.no_grad(): + model.fuse() + return model + + +def create_coco_train_dataloader(cocodir, batch_size=10): + + with open("data/hyps/hyp.scratch-low.yaml") as f: + hyp = yaml.load(f, Loader=yaml.SafeLoader) # load hyps + + loader = create_dataloader( + f"{cocodir}/train2017.txt", + imgsz=640, + batch_size=batch_size, + augment=True, hyp=hyp, rect=False, cache=False, stride=32,pad=0, image_weights=False)[0] + return loader + + +def create_coco_val_dataloader(cocodir, batch_size=10, keep_images=None): + + loader = create_dataloader( + f"{cocodir}/val2017.txt", + imgsz=640, + batch_size=batch_size, + augment=False, hyp=None, rect=True, cache=False,stride=32,pad=0.5, image_weights=False)[0] + + def subclass_len(self): + if keep_images is not None: + return keep_images + return len(self.img_files) + + loader.dataset.__len__ = subclass_len + return loader + + +def evaluate_coco(model, dataloader, using_cocotools = False, save_dir=".", conf_thres=0.001, iou_thres=0.65): + + if save_dir and os.path.dirname(save_dir) != "": + os.makedirs(os.path.dirname(save_dir), exist_ok=True) + + model = deepcopy(model) + return val.run( + check_dataset("data/coco.yaml"), + save_dir=Path(save_dir), + dataloader=dataloader, conf_thres=conf_thres,iou_thres=iou_thres,model=model, + plots=False,save_json=using_cocotools)[0][3] + + +def export_onnx(model : Model, file, size=640, dynamic_batch=False, noanchor=False): + + device = next(model.parameters()).device + model.float() + + dummy = torch.zeros(1, 3, size, size, device=device) + model.model[-1].concat = True + grid_old_func = model.model[-1]._make_grid + model.model[-1]._make_grid = lambda *args: [torch.from_numpy(item.cpu().data.numpy()).to(item.device) for item in grid_old_func(*args)] + + if noanchor: + def hook_forward(self, x): + for i in range(self.nl): + x[i] = self.m[i](x[i]) + bs, _, ny, nx = map(int, x[i].shape) + x[i] = x[i].view(bs, self.na, self.no, ny * nx).permute(0, 1, 3, 2).contiguous() + return x + + model.model[-1].__class__.forward = hook_forward + + quantize.export_onnx(model, dummy, file, opset_version=13, + input_names=["images"], output_names=["s8", "s16", "s32"], + dynamic_axes={"images": {0: "batch"}, "s32": {0: "batch"}, "s16": {0: "batch"}, "s8": {0: "batch"}} if dynamic_batch else None + ) + else: + quantize.export_onnx(model, dummy, file, opset_version=13, + input_names=["images"], output_names=["outputs"], + dynamic_axes={"images": {0: "batch"}, "outputs": {0: "batch"}} if dynamic_batch else None + ) + model.model[-1].concat = False + model.model[-1]._make_grid = grid_old_func + + +def cmd_quantize(weight, cocodir, device, ignore_policy, save_ptq, save_qat, supervision_stride, iters, eval_origin, eval_ptq): + quantize.initialize() + + if save_ptq and os.path.dirname(save_ptq) != "": + os.makedirs(os.path.dirname(save_ptq), exist_ok=True) + + if save_qat and os.path.dirname(save_qat) != "": + os.makedirs(os.path.dirname(save_qat), exist_ok=True) + + device = torch.device(device) + model = load_yolov5s_model(weight, device) + train_dataloader = create_coco_train_dataloader(cocodir) + val_dataloader = create_coco_val_dataloader(cocodir) + quantize.replace_bottleneck_forward(model) + quantize.replace_to_quantization_module(model, ignore_policy=ignore_policy) + quantize.apply_custom_rules_to_quantizer(model, export_onnx) + quantize.calibrate_model(model, train_dataloader, device) + + json_save_dir = "." if os.path.dirname(save_ptq) == "" else os.path.dirname(save_ptq) + summary_file = os.path.join(json_save_dir, "summary.json") + summary = SummaryTool(summary_file) + + if eval_origin: + print("Evaluate Origin...") + with quantize.disable_quantization(model): + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append(["Origin", ap]) + + if eval_ptq: + print("Evaluate PTQ...") + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append(["PTQ", ap]) + + if save_ptq: + print(f"Save ptq model to {save_ptq}") + torch.save({"model": model}, save_ptq) + + if save_qat is None: + print("Done as save_qat is None.") + return + + best_ap = 0 + def per_epoch(model, epoch, lr): + + nonlocal best_ap + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append([f"QAT{epoch}", ap]) + + if ap > best_ap: + print(f"Save qat model to {save_qat} @ {ap:.5f}") + best_ap = ap + torch.save({"model": model}, save_qat) + + def preprocess(datas): + return datas[0].to(device).float() / 255.0 + + def supervision_policy(): + supervision_list = [] + for item in model.model: + supervision_list.append(id(item)) + + keep_idx = list(range(0, len(model.model) - 1, supervision_stride)) + keep_idx.append(len(model.model) - 2) + def impl(name, module): + if id(module) not in supervision_list: return False + idx = supervision_list.index(id(module)) + if idx in keep_idx: + print(f"Supervision: {name} will compute loss with origin model during QAT training") + else: + print(f"Supervision: {name} no compute loss during QAT training, that is unsupervised only and doesn't mean don't learn") + return idx in keep_idx + return impl + + quantize.finetune( + model, train_dataloader, per_epoch, early_exit_batchs_per_epoch=iters, + preprocess=preprocess, supervision_policy=supervision_policy()) + + +def cmd_export(weight, save, size, dynamic, noanchor, noqadd): + + quantize.initialize() + if save is None: + name = os.path.basename(weight) + name = name[:name.rfind('.')] + save = os.path.join(os.path.dirname(weight), name + ".onnx") + + model = torch.load(weight, map_location="cpu")["model"] + if not noqadd: + quantize.replace_bottleneck_forward(model) + + export_onnx(model, save, size, dynamic_batch=dynamic, noanchor=noanchor) + print(f"Save onnx to {save}") + + +def cmd_sensitive_analysis(weight, device, cocodir, summary_save, num_image): + + quantize.initialize() + device = torch.device(device) + model = load_yolov5s_model(weight, device) + train_dataloader = create_coco_train_dataloader(cocodir) + val_dataloader = create_coco_val_dataloader(cocodir, keep_images=None if num_image is None or num_image < 1 else num_image) + quantize.replace_to_quantization_module(model) + quantize.calibrate_model(model, train_dataloader) + + summary = SummaryTool(summary_save) + print("Evaluate PTQ...") + ap = evaluate_coco(model, val_dataloader) + summary.append([ap, "PTQ"]) + + print("Sensitive analysis by each layer...") + for i in range(0, len(model.model)): + layer = model.model[i] + if quantize.have_quantizer(layer): + print(f"Quantization disable model.{i}") + quantize.disable_quantization(layer).apply() + ap = evaluate_coco(model, val_dataloader) + summary.append([ap, f"model.{i}"]) + quantize.enable_quantization(layer).apply() + else: + print(f"ignore model.{i} because it is {type(layer)}") + + summary = sorted(summary.data, key=lambda x:x[0], reverse=True) + print("Sensitive summary:") + for n, (ap, name) in enumerate(summary[:10]): + print(f"Top{n}: Using fp16 {name}, ap = {ap:.5f}") + + +def cmd_test(weight, device, cocodir, confidence, nmsthres): + + device = torch.device(device) + model = load_yolov5s_model(weight, device) + val_dataloader = create_coco_val_dataloader(cocodir) + evaluate_coco(model, val_dataloader, True, conf_thres=confidence, iou_thres=nmsthres) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(prog='qat.py') + subps = parser.add_subparsers(dest="cmd") + exp = subps.add_parser("export", help="Export weight to onnx file") + exp.add_argument("weight", type=str, default="yolov5s.pt", help="export pt file") + exp.add_argument("--save", type=str, required=False, help="export onnx file") + exp.add_argument("--size", type=int, default=640, help="export input size") + exp.add_argument("--dynamic", action="store_true", help="export dynamic batch") + exp.add_argument("--noanchor", action="store_true", help="export no anchor nodes") + exp.add_argument("--noqadd", action="store_true", help="export do not add QuantAdd") + + qat = subps.add_parser("quantize", help="PTQ/QAT finetune ...") + qat.add_argument("weight", type=str, nargs="?", default="yolov5s.pt", help="weight file") + qat.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + qat.add_argument("--device", type=str, default="cuda:0", help="device") + qat.add_argument("--ignore-policy", type=str, default="model\.24\.m\.(.*)", help="regx") + qat.add_argument("--ptq", type=str, default="ptq.pt", help="file") + qat.add_argument("--qat", type=str, default=None, help="file") + qat.add_argument("--supervision-stride", type=int, default=1, help="supervision stride") + qat.add_argument("--iters", type=int, default=200, help="iters per epoch") + qat.add_argument("--eval-origin", action="store_true", help="do eval for origin model") + qat.add_argument("--eval-ptq", action="store_true", help="do eval for ptq model") + + sensitive = subps.add_parser("sensitive", help="Sensitive layer analysis") + sensitive.add_argument("weight", type=str, nargs="?", default="yolov5s.pt", help="weight file") + sensitive.add_argument("--device", type=str, default="cuda:0", help="device") + sensitive.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + sensitive.add_argument("--summary", type=str, default="sensitive-summary.json", help="summary save file") + sensitive.add_argument("--num-image", type=int, default=None, help="number of image to evaluate") + + testcmd = subps.add_parser("test", help="Do evaluate") + testcmd.add_argument("weight", type=str, default="yolov5s.pt", help="weight file") + testcmd.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + testcmd.add_argument("--device", type=str, default="cuda:0", help="device") + testcmd.add_argument("--confidence", type=float, default=0.001, help="confidence threshold") + testcmd.add_argument("--nmsthres", type=float, default=0.65, help="nms threshold") + + args = parser.parse_args() + init_seeds(57) + + if args.cmd == "export": + cmd_export(args.weight, args.save, args.size, args.dynamic, args.noanchor, args.noqadd) + elif args.cmd == "quantize": + print(args) + cmd_quantize( + args.weight, args.cocodir, args.device, args.ignore_policy, + args.ptq, args.qat, args.supervision_stride, args.iters, + args.eval_origin, args.eval_ptq + ) + elif args.cmd == "sensitive": + cmd_sensitive_analysis(args.weight, args.device, args.cocodir, args.summary, args.num_image) + elif args.cmd == "test": + cmd_test(args.weight, args.device, args.cocodir, args.confidence, args.nmsthres) + else: + parser.print_help() diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/qat.py b/tools/yolo_deepstream/yolov7_qat/scripts/qat.py new file mode 100644 index 00000000..c7132836 --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/qat.py @@ -0,0 +1,329 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import sys +import os + +# Add the current directory to PYTHONPATH for YoloV7 +sys.path.insert(0, os.path.abspath(".")) +pydir = os.path.dirname(__file__) + +import yaml +import collections +import warnings +import argparse +import json +from pathlib import Path + +# PyTorch +import torch +import torch.nn as nn + +# YoloV7 +import test +from models.yolo import Model +from models.common import Conv +from utils.datasets import create_dataloader +from utils.google_utils import attempt_download +from utils.general import init_seeds + +import quantization.quantize as quantize + +# Disable all warning +warnings.filterwarnings("ignore") + + +class SummaryTool: + def __init__(self, file): + self.file = file + self.data = [] + + def append(self, item): + self.data.append(item) + json.dump(self.data, open(self.file, "w"), indent=4) + + +# Load YoloV7 Model +def load_yolov7_model(weight, device) -> Model: + + attempt_download(weight) + model = torch.load(weight, map_location=device)["model"] + for m in model.modules(): + if type(m) is nn.Upsample: + m.recompute_scale_factor = None # torch 1.11.0 compatibility + elif type(m) is Conv: + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility + + model.float() + model.eval() + + with torch.no_grad(): + model.fuse() + return model + + +def create_coco_train_dataloader(cocodir, batch_size=10): + + with open("data/hyp.scratch.p5.yaml") as f: + hyp = yaml.load(f, Loader=yaml.SafeLoader) # load hyps + + loader = create_dataloader( + f"{cocodir}/train2017.txt", + imgsz=640, + batch_size=batch_size, + opt=collections.namedtuple("Opt", "single_cls")(False), + augment=True, hyp=hyp, rect=False, cache=False, stride=32,pad=0, image_weights=False)[0] + return loader + + +def create_coco_val_dataloader(cocodir, batch_size=10, keep_images=None): + + loader = create_dataloader( + f"{cocodir}/val2017.txt", + imgsz=640, + batch_size=batch_size, + opt=collections.namedtuple("Opt", "single_cls")(False), + augment=False, hyp=None, rect=True, cache=False,stride=32,pad=0.5, image_weights=False)[0] + + def subclass_len(self): + if keep_images is not None: + return keep_images + return len(self.img_files) + + loader.dataset.__len__ = subclass_len + return loader + + +def evaluate_coco(model, dataloader, using_cocotools = False, save_dir=".", conf_thres=0.001, iou_thres=0.65): + + if save_dir and os.path.dirname(save_dir) != "": + os.makedirs(os.path.dirname(save_dir), exist_ok=True) + + return test.test( + "data/coco.yaml", + save_dir=Path(save_dir), + dataloader=dataloader, conf_thres=conf_thres,iou_thres=iou_thres,model=model,is_coco=True, + plots=False,half_precision=True,save_json=using_cocotools)[0][3] + + +def export_onnx(model : Model, file, size=640, dynamic_batch=False): + + device = next(model.parameters()).device + model.float() + + dummy = torch.zeros(1, 3, size, size, device=device) + model.model[-1].concat = True + grid_old_func = model.model[-1]._make_grid + model.model[-1]._make_grid = lambda *args: torch.from_numpy(grid_old_func(*args).data.numpy()) + + quantize.export_onnx(model, dummy, file, opset_version=13, + input_names=["images"], output_names=["outputs"], + dynamic_axes={"images": {0: "batch"}, "outputs": {0: "batch"}} if dynamic_batch else None + ) + model.model[-1].concat = False + model.model[-1]._make_grid = grid_old_func + + +def cmd_quantize(weight, cocodir, device, ignore_policy, save_ptq, save_qat, supervision_stride, iters, eval_origin, eval_ptq): + quantize.initialize() + + if save_ptq and os.path.dirname(save_ptq) != "": + os.makedirs(os.path.dirname(save_ptq), exist_ok=True) + + if save_qat and os.path.dirname(save_qat) != "": + os.makedirs(os.path.dirname(save_qat), exist_ok=True) + + device = torch.device(device) + model = load_yolov7_model(weight, device) + train_dataloader = create_coco_train_dataloader(cocodir) + val_dataloader = create_coco_val_dataloader(cocodir) + quantize.replace_to_quantization_module(model, ignore_policy=ignore_policy) + quantize.apply_custom_rules_to_quantizer(model, export_onnx) + quantize.calibrate_model(model, train_dataloader, device) + + json_save_dir = "." if os.path.dirname(save_ptq) == "" else os.path.dirname(save_ptq) + summary_file = os.path.join(json_save_dir, "summary.json") + summary = SummaryTool(summary_file) + + if eval_origin: + print("Evaluate Origin...") + with quantize.disable_quantization(model): + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append(["Origin", ap]) + + if eval_ptq: + print("Evaluate PTQ...") + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append(["PTQ", ap]) + + if save_ptq: + print(f"Save ptq model to {save_ptq}") + torch.save({"model": model}, save_ptq) + + if save_qat is None: + print("Done as save_qat is None.") + return + + best_ap = 0 + def per_epoch(model, epoch, lr): + + nonlocal best_ap + ap = evaluate_coco(model, val_dataloader, True, json_save_dir) + summary.append([f"QAT{epoch}", ap]) + + if ap > best_ap: + print(f"Save qat model to {save_qat} @ {ap:.5f}") + best_ap = ap + torch.save({"model": model}, save_qat) + + def preprocess(datas): + return datas[0].to(device).float() / 255.0 + + def supervision_policy(): + supervision_list = [] + for item in model.model: + supervision_list.append(id(item)) + + keep_idx = list(range(0, len(model.model) - 1, supervision_stride)) + keep_idx.append(len(model.model) - 2) + def impl(name, module): + if id(module) not in supervision_list: return False + idx = supervision_list.index(id(module)) + if idx in keep_idx: + print(f"Supervision: {name} will compute loss with origin model during QAT training") + else: + print(f"Supervision: {name} no compute loss during QAT training, that is unsupervised only and doesn't mean don't learn") + return idx in keep_idx + return impl + + quantize.finetune( + model, train_dataloader, per_epoch, early_exit_batchs_per_epoch=iters, + preprocess=preprocess, supervision_policy=supervision_policy()) + + +def cmd_export(weight, save, size, dynamic): + + quantize.initialize() + if save is None: + name = os.path.basename(weight) + name = name[:name.rfind('.')] + save = os.path.join(os.path.dirname(weight), name + ".onnx") + + export_onnx(torch.load(weight, map_location="cpu")["model"], save, size, dynamic_batch=dynamic) + print(f"Save onnx to {save}") + + +def cmd_sensitive_analysis(weight, device, cocodir, summary_save, num_image): + + quantize.initialize() + device = torch.device(device) + model = load_yolov7_model(weight, device) + train_dataloader = create_coco_train_dataloader(cocodir) + val_dataloader = create_coco_val_dataloader(cocodir, keep_images=None if num_image is None or num_image < 1 else num_image) + quantize.replace_to_quantization_module(model) + quantize.calibrate_model(model, train_dataloader, device) + + summary = SummaryTool(summary_save) + print("Evaluate PTQ...") + ap = evaluate_coco(model, val_dataloader) + summary.append([ap, "PTQ"]) + + print("Sensitive analysis by each layer...") + for i in range(0, len(model.model)): + layer = model.model[i] + if quantize.have_quantizer(layer): + print(f"Quantization disable model.{i}") + quantize.disable_quantization(layer).apply() + ap = evaluate_coco(model, val_dataloader) + summary.append([ap, f"model.{i}"]) + quantize.enable_quantization(layer).apply() + else: + print(f"ignore model.{i} because it is {type(layer)}") + + summary = sorted(summary.data, key=lambda x:x[0], reverse=True) + print("Sensitive summary:") + for n, (ap, name) in enumerate(summary[:10]): + print(f"Top{n}: Using fp16 {name}, ap = {ap:.5f}") + + +def cmd_test(weight, device, cocodir, confidence, nmsthres): + + device = torch.device(device) + model = load_yolov7_model(weight, device) + val_dataloader = create_coco_val_dataloader(cocodir) + evaluate_coco(model, val_dataloader, True, conf_thres=confidence, iou_thres=nmsthres) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(prog='qat.py') + subps = parser.add_subparsers(dest="cmd") + exp = subps.add_parser("export", help="Export weight to onnx file") + exp.add_argument("weight", type=str, default="yolov7.pt", help="export pt file") + exp.add_argument("--save", type=str, required=False, help="export onnx file") + exp.add_argument("--size", type=int, default=640, help="export input size") + exp.add_argument("--dynamic", action="store_true", help="export dynamic batch") + + qat = subps.add_parser("quantize", help="PTQ/QAT finetune ...") + qat.add_argument("weight", type=str, nargs="?", default="yolov7.pt", help="weight file") + qat.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + qat.add_argument("--device", type=str, default="cuda:0", help="device") + qat.add_argument("--ignore-policy", type=str, default="model\.105\.m\.(.*)", help="regx") + qat.add_argument("--ptq", type=str, default="ptq.pt", help="file") + qat.add_argument("--qat", type=str, default=None, help="file") + qat.add_argument("--supervision-stride", type=int, default=1, help="supervision stride") + qat.add_argument("--iters", type=int, default=200, help="iters per epoch") + qat.add_argument("--eval-origin", action="store_true", help="do eval for origin model") + qat.add_argument("--eval-ptq", action="store_true", help="do eval for ptq model") + + sensitive = subps.add_parser("sensitive", help="Sensitive layer analysis") + sensitive.add_argument("weight", type=str, nargs="?", default="yolov7.pt", help="weight file") + sensitive.add_argument("--device", type=str, default="cuda:0", help="device") + sensitive.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + sensitive.add_argument("--summary", type=str, default="sensitive-summary.json", help="summary save file") + sensitive.add_argument("--num-image", type=int, default=None, help="number of image to evaluate") + + testcmd = subps.add_parser("test", help="Do evaluate") + testcmd.add_argument("weight", type=str, default="yolov7.pt", help="weight file") + testcmd.add_argument("--cocodir", type=str, default="/datav/dataset/coco", help="coco directory") + testcmd.add_argument("--device", type=str, default="cuda:0", help="device") + testcmd.add_argument("--confidence", type=float, default=0.001, help="confidence threshold") + testcmd.add_argument("--nmsthres", type=float, default=0.65, help="nms threshold") + + args = parser.parse_args() + init_seeds(57) + + if args.cmd == "export": + cmd_export(args.weight, args.save, args.size, args.dynamic) + elif args.cmd == "quantize": + print(args) + cmd_quantize( + args.weight, args.cocodir, args.device, args.ignore_policy, + args.ptq, args.qat, args.supervision_stride, args.iters, + args.eval_origin, args.eval_ptq + ) + elif args.cmd == "sensitive": + cmd_sensitive_analysis(args.weight, args.device, args.cocodir, args.summary, args.num_image) + elif args.cmd == "test": + cmd_test(args.weight, args.device, args.cocodir, args.confidence, args.nmsthres) + else: + parser.print_help() diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/quantize_utils.py b/tools/yolo_deepstream/yolov7_qat/scripts/quantize_utils.py new file mode 100644 index 00000000..2d11413a --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/quantize_utils.py @@ -0,0 +1,330 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ + +import onnx_graphsurgeon as gs +from onnx_graphsurgeon.ir.tensor import Variable +import onnx +import numpy as np +import argparse +import logging + +LAYER_ID = 0 +TENSOR_ID = 0 +def get_qparams_constants(node_to_quantize_name, scale_init=0.5, zero_point_init=0): + global LAYER_ID, TENSOR_ID + """ ATTENTION: "node_to_quantize_name" needs to be different every time this function is called. + Otherwise, "scale, zero_point" are overwritten. + TODO: ensure that this happens! The same goes for + "q_out and dq_out = gs.Variable(UNIQUE_NAME)" + + :param node_to_quantize_name: + :param scale_init: + :param zero_point_init: + :return: 2 gs.Constants (scale and zero-point). + """ + scale = gs.Constant( + name=node_to_quantize_name + "_scale" + str(TENSOR_ID), + values=np.array(scale_init, dtype=np.float32)) + TENSOR_ID = TENSOR_ID + 1 + zero_point = gs.Constant( + name=node_to_quantize_name + "_zero_point" + str(TENSOR_ID), + values=np.array(zero_point_init, dtype=np.int8) + ) + TENSOR_ID = TENSOR_ID + 1 + return scale, zero_point + +def quantize_tensor(graph, tensor_to_quantize, scale, name_suffix=""): + global LAYER_ID, TENSOR_ID + output_nodes = tensor_to_quantize['x'].outputs + nodes_and_quantized = [] + nodes_inputidx = [] + + for node in output_nodes: + for idx, inp in enumerate(node.inputs): + if inp.name == tensor_to_quantize['x'].name: + nodes_and_quantized.append(node) + nodes_inputidx.append(idx) + break + + # QuantizeLinear node + q_scale, q_zero_point = get_qparams_constants(tensor_to_quantize['x'].name + "_inp_q" + name_suffix, scale_init=scale) + q_out = gs.Variable(name=tensor_to_quantize['x'].name + "_QuantizeLinear_out" + name_suffix + str(TENSOR_ID)) + TENSOR_ID = TENSOR_ID + 1 + quant_node = gs.Node( + op="QuantizeLinear", + name="QuantI_"+ tensor_to_quantize['x'].name + str(LAYER_ID), + inputs=[tensor_to_quantize["x"], q_scale, q_zero_point], + outputs=[q_out] + ) + LAYER_ID = LAYER_ID + 1 + # DequantizeLinear node + dq_scale, dq_zero_point = get_qparams_constants(tensor_to_quantize['x'].name + "_inp_dq" + name_suffix, scale_init=scale) + dq_out = gs.Variable(name=tensor_to_quantize['x'].name + "_DequantizeLinear_out" + name_suffix + str(TENSOR_ID)) + TENSOR_ID = TENSOR_ID + 1 + dequant_node = gs.Node( + op="DequantizeLinear", + name="DequantI_"+ tensor_to_quantize['x'].name + str(LAYER_ID), + inputs=[q_out, dq_scale, dq_zero_point], + outputs=[dq_out] + ) + LAYER_ID = LAYER_ID + 1 + #shit code + for i, node in enumerate(nodes_and_quantized): + node.inputs[nodes_inputidx[i]] = dq_out + + graph.nodes.extend([quant_node, dequant_node]) + return graph + +def quantize_input(graph, node_to_quantize, node_to_quantize_input, scale, name_suffix=""): + global LAYER_ID, TENSOR_ID + # QuantizeLinear node + q_scale, q_zero_point = get_qparams_constants(node_to_quantize.name + "_inp_q" + name_suffix, scale_init=scale) + q_out = gs.Variable(name=node_to_quantize.name + "_QuantizeLinear_out" + name_suffix + name_suffix + str(TENSOR_ID)) + TENSOR_ID = TENSOR_ID + 1 + quant_node = gs.Node( + op="QuantizeLinear", + name="QuantI_"+ node_to_quantize.name + str(LAYER_ID), + inputs=[node_to_quantize_input["x"], q_scale, q_zero_point], + outputs=[q_out] + ) + LAYER_ID = LAYER_ID + 1 + + # DequantizeLinear node + dq_scale, dq_zero_point = get_qparams_constants(node_to_quantize.name + "_inp_dq" + name_suffix, scale_init=scale) + dq_out = gs.Variable(name=node_to_quantize.name + "_DequantizeLinear_out" + name_suffix + name_suffix + str(TENSOR_ID)) + TENSOR_ID = TENSOR_ID + 1 + dequant_node = gs.Node( + op="DequantizeLinear", + name="DequantI_"+ node_to_quantize.name + str(LAYER_ID), + inputs=[q_out, dq_scale, dq_zero_point], + outputs=[dq_out] + ) + LAYER_ID = LAYER_ID + 1 + + node_to_quantize.inputs[node_to_quantize_input["idx"]] = dq_out + graph.nodes.extend([quant_node, dequant_node]) + + graph.cleanup().toposort() + return graph + + +def quantize_weight(graph, node_to_quantize, node_to_quantize_weight, axis=0, name_suffix=""): + global LAYER_ID, TENSOR_ID + """ + When connected to the weight, the "y_scale" parameter can be recovered directly from the Weight matrix. + See official doc: https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#intro-quantization + + :param graph: + :param node_to_quantize: + :param node_to_quantize_weight: + :param axis: + :param name_suffix: + :return: + """ + shape = node_to_quantize_weight["x"].shape[axis] + # Recover "y_scale" from weight matrix + weight_matrix = node_to_quantize_weight["x"].values + y_scale_arr = [] + # Recover "y_scale" for each batch. If axis != 0, move the desired axis to the be idx=0. + if axis !=0: + weight_matrix = np.moveaxis(weight_matrix, [axis], [0]) + # for bais 1d-weight + if len(weight_matrix.shape) == 1: + weight_matrix = np.expand_dims(weight_matrix, axis=0) + for w in weight_matrix[:]: + dyn_range = max(abs(w.min()), abs(w.max())) + y_scale = dyn_range / 127.0 + y_scale_arr.append(y_scale) + + # QuantizeLinear node + q_scale, q_zero_point = get_qparams_constants( + node_to_quantize.name + "_weight_q" + name_suffix, + scale_init=y_scale_arr, # * np.ones(shape=(shape,)), + zero_point_init=np.zeros(shape=(shape,)), + ) + q_out = gs.Variable(name=node_to_quantize.name + "_QuantizeLinear_weight_out" + name_suffix + str(TENSOR_ID)) + TENSOR_ID = TENSOR_ID + 1 + quant_node = gs.Node( + op="QuantizeLinear", + name="QuantW_"+ node_to_quantize.name + str(LAYER_ID), + inputs=[node_to_quantize_weight["x"], q_scale, q_zero_point], + outputs=[q_out], + attrs={"axis": axis} + ) + LAYER_ID = LAYER_ID + 1 + + + # DequantizeLinear node + dq_scale, dq_zero_point = get_qparams_constants( + node_to_quantize.name + "_weight_dq" + name_suffix, + scale_init=y_scale_arr, # * np.ones(shape=(shape,)), + zero_point_init=np.zeros(shape=(shape,)), + ) + TENSOR_ID = TENSOR_ID + 1 + dq_out = gs.Variable(name=node_to_quantize.name + "_DequantizeLinear_weight_out" + name_suffix + str(TENSOR_ID)) + dequant_node = gs.Node( + op="DequantizeLinear", + name="DequantW_"+ node_to_quantize.name + str(LAYER_ID), + inputs=[q_out, dq_scale, dq_zero_point], + outputs=[dq_out], + attrs={"axis": axis} + ) + LAYER_ID = LAYER_ID + 1 + + node_to_quantize.inputs[node_to_quantize_weight["idx"]] = dq_out + graph.nodes.extend([quant_node, dequant_node]) + + graph.cleanup().toposort() + return graph + +def get_node_to_quantize_infos(node_to_quantize, disableResAdd:bool): + # Separate inputs into activation ('Variable' type) and weight ('Constant' type). + node_to_quantize_input = [] + node_to_quantize_weight = [] + for idx, inp in enumerate(node_to_quantize.inputs): + if isinstance(inp, Variable): + node_to_quantize_input.append({"x": inp, "idx": idx}) + # residual add, will not work with bias add + if node_to_quantize.op == "Add" and (not disableResAdd) and len(node_to_quantize_input) == 2: + node_to_quantize_input = [node_to_quantize_input[0]] + else: # Constant + if ( + len(node_to_quantize_weight) == 0 + and node_to_quantize.op not in ["Add", "BatchNormalization"] + and len(inp.shape) > 1 + ): + # 1) Only quantize the Weight, not Bias + # 2) Do not quantize bias matrix in BiasAdd ops + # 3) Only save weight matrices with shape > 1 (Conv 4D, MatMul 2D) + node_to_quantize_weight.append({"x": inp, "idx": idx}) + + # for bias add after matmul + elif( + len(node_to_quantize_weight) == 0 + and node_to_quantize.op =="Add" + and isinstance(node_to_quantize.inputs[0], gs.Constant)): + node_to_quantize_weight.append({"x": inp, "idx": idx}) + + + return node_to_quantize_input, node_to_quantize_weight + +def quantize_node_automatically(graph, node_to_quantize, scale, disableResAdd:bool): + """ + Quantizes a node according to information in graph.json (generated from the PTQ engine building step. + + :return: + """ + node_to_quantize_input, node_to_quantize_weight = get_node_to_quantize_infos(node_to_quantize, disableResAdd) + + # Quantize inputs + input_was_quantized = False + # Quantizable layer + for i, node_inp in enumerate(node_to_quantize_input): + graph = quantize_input(graph, node_to_quantize, node_inp, scale, name_suffix=str(i)) + input_was_quantized = True + + # Quantize weights + for i, node_weight in enumerate(node_to_quantize_weight): + if input_was_quantized: + graph = quantize_weight( + graph, + node_to_quantize, + node_weight, + axis=1 if node_to_quantize.op in ["MatMul", "ConvTranspose"] else 0, # TODO: Automatize axis detection. Automatize this by checking the expected layer output and extract axis that matches desired dimension. + name_suffix=str(i) + ) + return graph + +def quantize_tensor_automatically(graph, tensor_to_quantize, scale): + """ + Quantizes a tensor + + :return: + """ + tensor_to_quantize = [{'x':tensor_to_quantize},] + # Quantizable tensor + for i, tensor_inp in enumerate(tensor_to_quantize): + graph = quantize_tensor(graph, tensor_inp, scale, name_suffix=str(i)) + return graph + + +def quant_one_node(graph, node_name, scale=0.04370, disableResAdd:bool = False): + nodes = graph.nodes + node_to_quantize = [x for x in nodes if x.name == node_name] + if len(node_to_quantize) == 0: + logging.warning(f'node: ',node_name, "did not found, skip") + if len(node_to_quantize) > 1: + logging.error(f'found multiple node named: ',node_name) + node_to_quantize = node_to_quantize[0] + graph = quantize_node_automatically(graph, node_to_quantize, scale, disableResAdd) + return graph + +def quant_one_tensor(graph, tensor_name, scale=0.04370): + # nodes = graph.nodes + tensors = graph.tensors() + tensor_to_quantize = [tensor for name, tensor in tensors.items() if tensor.name == tensor_name] + if len(tensor_to_quantize) == 0: + logging.warning(f'tensor: ',tensor_name, "did not found, skip") + if len(tensor_to_quantize) > 1: + logging.error(f'found multiple tensor named: ',tensor_name) + + tensor_to_quantize = tensor_to_quantize[0] + graph = quantize_tensor_automatically(graph, tensor_to_quantize, scale) + return graph + +def quant_node_of_list(graph, op_name_list:list, disableResAdd:bool): + for op in op_name_list: + graph = quant_one_node(graph, op, disableResAdd=disableResAdd) + ##TODO: if one element is Conv1:0.03, it should be support + return graph + +def quant_tensor_of_list(graph, tensor_name_list:list): + for tensor in tensor_name_list: + graph = quant_one_tensor(graph, tensor) + return graph + +# def quant_all_nodes_of_type(): + # return None + +def quant_onnx(model_path, output_model_path, nodes_name_to_quant, tensors_name_to_quant, disableResAdd:bool): + model = onnx.load(model_path) + model = onnx.shape_inference.infer_shapes(model) + graph = gs.import_onnx(model) + graph = quant_node_of_list(graph, nodes_name_to_quant, disableResAdd) + graph = quant_tensor_of_list(graph, tensors_name_to_quant) + graph.cleanup() + new_model = gs.export_onnx(graph) + onnx.save(new_model, output_model_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='iso_the onnx model with new input and output') + parser.add_argument('--model', default='model.onnx', type=str, help='the onnx model') + parser.add_argument('--output_model', default='', type=str, help='the output model') + parser.add_argument('--nodes', nargs='+', type=str, help='the input nodes list you want to quant',default=[]) + parser.add_argument('--disableResAdd', action='store_true', help='if enabled this flag, residual add will have two inputs') + + parser.add_argument('--tensors', nargs='+', type=str, help='the tensors list you want to quant',default=[]) + + args = parser.parse_args() + print(args) + quant_onnx(args.model, args.output_model, args.nodes, args.tensors, args.disableResAdd) diff --git a/tools/yolo_deepstream/yolov7_qat/scripts/trt-int8.py b/tools/yolo_deepstream/yolov7_qat/scripts/trt-int8.py new file mode 100644 index 00000000..3271d405 --- /dev/null +++ b/tools/yolo_deepstream/yolov7_qat/scripts/trt-int8.py @@ -0,0 +1,166 @@ +################################################################################ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +################################################################################ +import tensorrt as trt +import pycuda.driver as cuda +import pycuda.autoinit + +import numpy as np +import random +import cv2 + +# For ../common.py +import sys, os +TRT_LOGGER = trt.Logger() + + +def load_yolov7_coco_image(cocodir, topn = None): + + files = os.listdir(cocodir) + files = [file for file in files if file.endswith(".jpg")] + + if topn is not None: + np.random.seed(31) + np.random.shuffle(files) + files = files[:topn] + + datas = [] + + # dataloader is setup pad=0.5 + for i, file in enumerate(files): + if i == 0: continue + if (i + 1) % 200 == 0: + print(f"Load {i + 1} / {len(files)} ...") + + img = cv2.imread(os.path.join(cocodir, file)) + from_ = img.shape[1], img.shape[0] + to_ = 640, 640 + scale = min(to_[0] / from_[0], to_[1] / from_[1]) + + # low accuracy + # M = np.array([ + # [scale, 0, 16], + # [0, scale, 16], # same to pytorch + # ]) + + # more accuracy + M = np.array([ + [scale, 0, -scale * from_[0] * 0.5 + to_[0] * 0.5 + scale * 0.5 - 0.5 + 16], + [0, scale, -scale * from_[1] * 0.5 + to_[1] * 0.5 + scale * 0.5 - 0.5 + 16], # same to pytorch + ]) + input = cv2.warpAffine(img, M, (672, 672), borderValue=(114, 114, 114)) + input = input[..., ::-1].transpose(2, 0, 1)[None] # BGR->RGB, HWC->CHW, CHW->1CHW + input = (input / 255.0).astype(np.float32) + datas.append(input) + + return np.concatenate(datas, axis=0) + + +class MNISTEntropyCalibrator(trt.IInt8EntropyCalibrator2): + def __init__(self, training_data, cache_file, batch_size=64): + # Whenever you specify a custom constructor for a TensorRT class, + # you MUST call the constructor of the parent explicitly. + trt.IInt8EntropyCalibrator2.__init__(self) + + self.cache_file = cache_file + self.batch_size = batch_size + self.current_index = 0 + + # Every time get_batch is called, the next batch of size batch_size will be copied to the device and returned. + if not os.path.exists(cache_file): + + # Allocate enough memory for a whole batch. + self.data = load_yolov7_coco_image(training_data, 1000) + self.device_input = cuda.mem_alloc(self.data[0].nbytes * self.batch_size) + + def get_batch_size(self): + return self.batch_size + + # TensorRT passes along the names of the engine bindings to the get_batch function. + # You don't necessarily have to use them, but they can be useful to understand the order of + # the inputs. The bindings list is expected to have the same ordering as 'names'. + def get_batch(self, names): + if self.current_index + self.batch_size > self.data.shape[0]: + return None + + current_batch = int(self.current_index / self.batch_size) + if current_batch % 10 == 0: + print("Calibrating batch {:}, containing {:} images".format(current_batch, self.batch_size)) + + batch = self.data[self.current_index : self.current_index + self.batch_size].ravel() + cuda.memcpy_htod(self.device_input, batch) + self.current_index += self.batch_size + return [self.device_input] + + def read_calibration_cache(self): + # If there is a cache, use it instead of calibrating again. Otherwise, implicitly return None. + if os.path.exists(self.cache_file): + with open(self.cache_file, "rb") as f: + return f.read() + + def write_calibration_cache(self, cache): + with open(self.cache_file, "wb") as f: + f.write(cache) + + +def build_int8_engine(onnx_file, calib, batch_size=32): + with trt.Builder( + TRT_LOGGER + ) as builder, builder.create_network(1) as network, builder.create_builder_config() as config: + # We set the builder batch size to be the same as the calibrator's, as we use the same batches + # during inference. Note that this is not required in general, and inference batch size is + # independent of calibration batch size. + builder.max_batch_size = batch_size + config.max_workspace_size = 1024 * 1024 * 1024 # 1024 MB + config.set_flag(trt.BuilderFlag.INT8) + config.int8_calibrator = calib + with trt.OnnxParser(network, TRT_LOGGER) as parser: + parser.parse_from_file(onnx_file) + # network.mark_output(model_tensors.find(ModelData.OUTPUT_NAME)) + # Build engine and do int8 calibration. + plan = builder.build_serialized_network(network, config) + return bytes(plan) + + +def replace_suffix(file, new_suffix): + r = file.rfind(".") + return f"{file[:r]}{new_suffix}" + + +def main(): + # Now we create a calibrator and give it the location of our calibration data. + # We also allow it to cache calibration data for faster engine building. + onnxfile = "yolov7.onnx" + calibration_cache = replace_suffix(onnxfile, ".cache") + engine_file = replace_suffix(onnxfile, ".engine") + calib = MNISTEntropyCalibrator("/datav/dataset/coco/images/train2017/", cache_file=calibration_cache) + + # Inference batch size can be different from calibration batch size. + batch_size = 1 + engine_data = build_int8_engine(onnxfile, calib, batch_size) + + with open(engine_file, "wb") as f: + f.write(engine_data) + + +if __name__ == "__main__": + main()