When running a stereo pipeline that uses a YOLO detection node along with depth estimation and feature tracking, the YOLO branch initially processes frames (≈11 FPS) but then stops executing after a few seconds.

At that point, the depth branch reaches 30 FPS and the NOC DDR bandwidth drops (from ~1300 MB/s to ~430 MB/s). This suggests that the YOLO node is no longer processing frames.

I tried setting a different number of inference threads, recompiling with a different number of shaves, etc, but nothing seems to solve it.

Script to reproduce the faulty behavior:

import depthai
import time
import cv2

MODEL_NAME = "utils/dai-yolo/yolov8n_coco_640x352.blob"

nnPath = MODEL_NAME

class DepthAIProcessor:
    def __init__(self):
        pipeline = depthai.Pipeline()

        # YOLO setup
        detectionNetwork = pipeline.create(depthai.node.YoloDetectionNetwork)

        detectionNetwork.setConfidenceThreshold(0.5)
        detectionNetwork.setNumClasses(80)
        detectionNetwork.setCoordinateSize(4)
        detectionNetwork.setIouThreshold(0.5)
        detectionNetwork.setBlobPath(nnPath)
        detectionNetwork.setNumInferenceThreads(2)
        detectionNetwork.input.setQueueSize(1)
        detectionNetwork.input.setBlocking(False)

        nnOut = pipeline.create(depthai.node.XLinkOut)
        nnOut.setStreamName("nn")

        manipLeft = pipeline.create(depthai.node.ImageManip)
        manipLeft.inputImage.setBlocking(False)
        manipLeft.inputImage.setQueueSize(1)
        #manipLeft.initialConfig.setResizeThumbnail(416, 416)
        manipLeft.initialConfig.setResizeThumbnail(640, 352)
        manipLeft.initialConfig.setFrameType(depthai.RawImgFrame.Type.RGB888p)

        manipLeft.out.link(detectionNetwork.input)
        detectionNetwork.out.link(nnOut.input)

        # Depth output
        xoutDepth = pipeline.create(depthai.node.XLinkOut)
        xoutDepth.setStreamName("depth")
        xoutDepth.input.setBlocking(False)
        xoutDepth.input.setQueueSize(3)

        xoutManip = pipeline.create(depthai.node.XLinkOut)
        xoutManip.setStreamName("manip")
        xoutManip.input.setQueueSize(1)
        xoutManip.input.setBlocking(False)
        manipLeft.out.link(xoutManip.input)

        stereo = pipeline.create(depthai.node.StereoDepth)
        stereo.setDepthAlign(depthai.StereoDepthProperties.DepthAlign.RECTIFIED_LEFT)
        stereo.depth.link(xoutDepth.input)   

        FPS = 30
        monoLeft = pipeline.create(depthai.node.MonoCamera)
        monoLeft.setCamera("left")
        monoLeft.setFps(FPS)
        monoLeft.setResolution(depthai.MonoCameraProperties.SensorResolution.THE_400_P)

        monoLeft.out.link(stereo.left)

        monoRight = pipeline.create(depthai.node.MonoCamera)
        monoRight.setCamera("right")
        monoRight.setFps(FPS)
        monoRight.setResolution(depthai.MonoCameraProperties.SensorResolution.THE_400_P)
        monoRight.out.link(stereo.right)


        featureTrackerLeft = pipeline.create(depthai.node.FeatureTracker)
        monoLeft.out.link(featureTrackerLeft.inputImage)
        xoutFeaturesLeft = pipeline.create(depthai.node.XLinkOut)
        featureTrackerLeft.outputFeatures.link(xoutFeaturesLeft.input) # left feature tracker
        xoutFeaturesLeft.setStreamName("trackedFeaturesLeft")


        featureTrackerRight = pipeline.create(depthai.node.FeatureTracker)
        monoRight.out.link(featureTrackerRight.inputImage)
        xoutFeaturesRight = pipeline.create(depthai.node.XLinkOut)
        featureTrackerRight.outputFeatures.link(xoutFeaturesRight.input) # right feature tracker
        xoutFeaturesRight.setStreamName("trackedFeaturesRight")

        stereo.rectifiedLeft.link(manipLeft.inputImage)

        nnOut.input.setBlocking(False)
        nnOut.input.setQueueSize(1)

        self.device = depthai.Device(pipeline)

        self.nn_queue = self.device.getOutputQueue(name="nn", maxSize=1, blocking=False)
        self.depth_queue = self.device.getOutputQueue(name="depth", maxSize=3, blocking=False)
        self.left_features_queue = self.device.getOutputQueue(name="trackedFeaturesLeft", maxSize=1, blocking=False)
        self.right_features_queue = self.device.getOutputQueue(name="trackedFeaturesRight", maxSize=1, blocking=False)
        self.manip_queue = self.device.getOutputQueue(name="manip", maxSize=1, blocking=False)


    def run(self):
        depth_frame_count = 0
        last_fps_time = time.time()
        yolo_frame_count = 0
        yolo_last_fps_time = time.time()
        manip_frame_count = 0
        manip_last_fps_time = time.time()
        fps_interval = 1

        latest_features_left = 0
        latest_features_right = 0

        while True:
            try:                
                in_nn = self.nn_queue.tryGet()
                in_depth = self.depth_queue.tryGet()
                in_manip = self.manip_queue.tryGet()

                in_features_left = self.left_features_queue.tryGet()
                in_features_right = self.right_features_queue.tryGet()

                if in_features_left:
                    latest_features_left = len(in_features_left.trackedFeatures)

                if in_features_right:
                    latest_features_right = len(in_features_right.trackedFeatures)

                if in_depth is not None:
                    current_time = time.time()
                    data_depth = in_depth.getCvFrame()
                    cv2.imshow("depth", data_depth)


                    depth_frame_count += 1
                    if current_time - last_fps_time >= fps_interval:
                        fps = depth_frame_count / (current_time - last_fps_time)
                        print(f"Depth Map FPS: {fps:.2f}")
                        depth_frame_count = 0
                        last_fps_time = current_time

                        print(f"featuresRight {latest_features_right}")
                        print(f"featuresLeft {latest_features_left}")

                if in_nn is not None:
                    yolo_frame_count += 1
                    current_time = time.time()
                    if current_time - yolo_last_fps_time >= fps_interval:
                        yolo_fps = yolo_frame_count / (current_time - yolo_last_fps_time)
                        print(f"YOLO FPS: {yolo_fps:.2f}")
                        yolo_frame_count = 0
                        yolo_last_fps_time = current_time

                if in_manip is not None:
                    cv2.imshow("manip", in_manip.getCvFrame())
                    manip_frame_count += 1
                    current_time = time.time()
                    if current_time - manip_last_fps_time >= fps_interval:
                        manip_fps = manip_frame_count / (current_time - manip_last_fps_time)
                        print(f"Manip FPS: {manip_fps:.2f}")
                        manip_frame_count = 0
                        manip_last_fps_time = current_time

                cv2.waitKey(1)


            except KeyboardInterrupt:
                print("Stopping...")
                break
            except Exception as e:
                print(f"Error: {e}")
                continue

    def close(self):
        self.device.close()

def main():
    processor = DepthAIProcessor()
try:
    processor.run()
except KeyboardInterrupt:
    print("Keyboard Interrupt received")
finally:
    processor.close()

if __name__ == '__main__':
    main()

Example output:

$ DEPTHAI_LEVEL=info python3 script.py
[19443010319B191300] [1.1] [0.898] [system] [info] Memory Usage - DDR: 0.12 / 333.28 MiB, CMX: 2.04 / 2.50 MiB, LeonOS Heap: 7.51 / 81.76 MiB, LeonRT Heap: 2.89 / 39.90 MiB / NOC ddr: 26 MB/s
[19443010319B191300] [1.1] [0.898] [system] [info] Temperatures - Average: 43.73C, CSS: 45.77C, MSS 43.27C, UPA: 43.04C, DSS: 42.81C
[19443010319B191300] [1.1] [0.898] [system] [info] Cpu Usage - LeonOS 70.73%, LeonRT: 1.79%
[2025-02-14 15:17:54.425] [depthai] [info] Logging disabled
[19443010319B191300] [1.1] [1.017] [MonoCamera(6)] [info] Using board socket: -1, id: 1
[19443010319B191300] [1.1] [1.018] [MonoCamera(7)] [info] Using board socket: -1, id: 2
[19443010319B191300] [1.1] [1.018] [FeatureTracker(8)] [info] Init node feature tracker.
[19443010319B191300] [1.1] [1.018] [FeatureTracker(10)] [info] Init node feature tracker.
[19443010319B191300] [1.1] [1.021] [system] [info] SIPP (Signal Image Processing Pipeline) internal buffer size '18432'B, DMA buffer size: '16384'B
[19443010319B191300] [1.1] [1.056] [system] [info] ImageManip internal buffer size '363520'B, shave buffer size '35840'B
[19443010319B191300] [1.1] [1.056] [system] [info] NeuralNetwork allocated resources: shaves: [0-10] cmx slices: [0-10]
MonoCamera allocated resources: no shaves; cmx slices: [13-15]
StereoDepth allocated resources: shaves: [12-12] cmx slices: [11-11]
ImageManip allocated resources: shaves: [15-15] no cmx slices.
FeatureTracker allocated resources: shaves: [13-13] cmx slices: [12-12]

[19443010319B191300] [1.1] [1.069] [DetectionNetwork(0)] [info] Needed resources: shaves: 6, ddr: 4956160
[19443010319B191300] [1.1] [1.069] [DetectionNetwork(0)] [warning] Network compiled for 6 shaves, maximum available 11, compiling for 5 shaves likely will yield in better performance
[19443010319B191300] [1.1] [1.083] [DetectionNetwork(0)] [info] Inference thread count: 2, number of shaves allocated per thread: 6, number of Neural Compute Engines (NCE) allocated per thread: 1
[19443010319B191300] [1.1] [1.246] [StereoDepth(5)] [info] Depth baseline: '7.5000005' cm, focal length: '401.15234'
[19443010319B191300] [1.1] [1.246] [StereoDepth(5)] [info] Depth horizontal FOV: '77.15886', vertical FOV: '52.998356'
[19443010319B191300] [1.1] [1.246] [StereoDepth(5)] [info] Depth intrinsics:
401\.152,    0.000,  318.788,
0\.000,  401.152,  190.277,
0\.000,    0.000,    1.000,
[19443010319B191300] [1.1] [1.899] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 474 MB/s
[19443010319B191300] [1.1] [1.899] [system] [info] Temperatures - Average: 47.07C, CSS: 47.80C, MSS 46.68C, UPA: 48.03C, DSS: 45.77C
[19443010319B191300] [1.1] [1.899] [system] [info] Cpu Usage - LeonOS 46.05%, LeonRT: 44.70%
Depth Map FPS: 25.39
featuresRight 266
featuresLeft 264
Manip FPS: 22.36
YOLO FPS: 8.33
[19443010319B191300] [1.1] [2.900] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1343 MB/s
[19443010319B191300] [1.1] [2.900] [system] [info] Temperatures - Average: 47.97C, CSS: 48.03C, MSS 47.80C, UPA: 49.14C, DSS: 46.90C
[19443010319B191300] [1.1] [2.900] [system] [info] Cpu Usage - LeonOS 40.77%, LeonRT: 34.12%
Depth Map FPS: 21.85
featuresRight 283
featuresLeft 278
Manip FPS: 21.84
YOLO FPS: 12.06
[19443010319B191300] [1.1] [3.901] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1316 MB/s
[19443010319B191300] [1.1] [3.901] [system] [info] Temperatures - Average: 49.14C, CSS: 49.37C, MSS 48.47C, UPA: 50.25C, DSS: 48.47C
[19443010319B191300] [1.1] [3.901] [system] [info] Cpu Usage - LeonOS 41.16%, LeonRT: 34.04%
Depth Map FPS: 21.30
featuresRight 293
featuresLeft 274
Manip FPS: 21.33
YOLO FPS: 9.25
[19443010319B191300] [1.1] [4.902] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1320 MB/s
[19443010319B191300] [1.1] [4.902] [system] [info] Temperatures - Average: 49.53C, CSS: 50.03C, MSS 48.70C, UPA: 50.70C, DSS: 48.70C
[19443010319B191300] [1.1] [4.902] [system] [info] Cpu Usage - LeonOS 42.80%, LeonRT: 34.63%
Depth Map FPS: 22.81
featuresRight 290
featuresLeft 282
Manip FPS: 22.59
YOLO FPS: 10.99
[19443010319B191300] [1.1] [5.903] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1323 MB/s
[19443010319B191300] [1.1] [5.903] [system] [info] Temperatures - Average: 49.75C, CSS: 50.70C, MSS 49.14C, UPA: 50.70C, DSS: 48.47C
[19443010319B191300] [1.1] [5.903] [system] [info] Cpu Usage - LeonOS 41.98%, LeonRT: 34.75%
Depth Map FPS: 21.78
featuresRight 293
featuresLeft 282
Manip FPS: 21.95
YOLO FPS: 11.13

...

[19443010319B191300] [1.1] [43.942] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1327 MB/s
[19443010319B191300] [1.1] [43.942] [system] [info] Temperatures - Average: 53.71C, CSS: 53.99C, MSS 53.77C, UPA: 54.86C, DSS: 52.24C
[19443010319B191300] [1.1] [43.942] [system] [info] Cpu Usage - LeonOS 43.23%, LeonRT: 34.93%
Manip FPS: 22.47
Depth Map FPS: 22.37
featuresRight 295
featuresLeft 284
[19443010319B191300] [1.1] [44.943] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 1140 MB/s
[19443010319B191300] [1.1] [44.943] [system] [info] Temperatures - Average: 52.35C, CSS: 52.68C, MSS 52.46C, UPA: 53.55C, DSS: 50.70C
[19443010319B191300] [1.1] [44.943] [system] [info] Cpu Usage - LeonOS 43.76%, LeonRT: 30.77%
Manip FPS: 28.61
Depth Map FPS: 29.89
featuresRight 296
featuresLeft 284
[19443010319B191300] [1.1] [45.944] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.99 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 432 MB/s
[19443010319B191300] [1.1] [45.944] [system] [info] Temperatures - Average: 52.18C, CSS: 53.12C, MSS 51.80C, UPA: 53.33C, DSS: 50.47C
[19443010319B191300] [1.1] [45.944] [system] [info] Cpu Usage - LeonOS 44.08%, LeonRT: 28.20%
Manip FPS: 30.84
Depth Map FPS: 30.81
featuresRight 294
featuresLeft 281
[19443010319B191300] [1.1] [46.945] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 431 MB/s
[19443010319B191300] [1.1] [46.945] [system] [info] Temperatures - Average: 52.40C, CSS: 53.55C, MSS 52.02C, UPA: 53.33C, DSS: 50.70C
[19443010319B191300] [1.1] [46.945] [system] [info] Cpu Usage - LeonOS 42.78%, LeonRT: 27.95%
Manip FPS: 30.41
Depth Map FPS: 30.14
featuresRight 294
featuresLeft 282
[19443010319B191300] [1.1] [47.946] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 425 MB/s
[19443010319B191300] [1.1] [47.946] [system] [info] Temperatures - Average: 51.91C, CSS: 52.90C, MSS 51.58C, UPA: 52.90C, DSS: 50.25C
[19443010319B191300] [1.1] [47.946] [system] [info] Cpu Usage - LeonOS 41.35%, LeonRT: 27.99%
Manip FPS: 29.97
Depth Map FPS: 29.98
featuresRight 292
featuresLeft 286
[19443010319B191300] [1.1] [48.947] [system] [info] Memory Usage - DDR: 34.82 / 333.28 MiB, CMX: 2.47 / 2.50 MiB, LeonOS Heap: 35.97 / 81.76 MiB, LeonRT Heap: 5.76 / 39.90 MiB / NOC ddr: 425 MB/s
[19443010319B191300] [1.1] [48.947] [system] [info] Temperatures - Average: 51.85C, CSS: 53.12C, MSS 51.14C, UPA: 52.68C, DSS: 50.47C
[19443010319B191300] [1.1] [48.947] [system] [info] Cpu Usage - LeonOS 41.84%, LeonRT: 27.74%

Thank you for your support