• DepthAI-v2
  • No messages from after a short duration with SyncNode

I tried running the following to view RGB, mono, disparity, and IMU outputs. Worked ok at first but I stopped getting updates after about 2 mins. Is this a shave issue?

import depthai as dai
import numpy as np
import cv2
from datetime import timedelta

pipeline = dai.Pipeline()

monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)
color = pipeline.create(dai.node.ColorCamera)
stereo = pipeline.create(dai.node.StereoDepth)
imu = pipeline.create(dai.node.IMU)
sync = pipeline.create(dai.node.Sync)

xoutGrp = pipeline.create(dai.node.XLinkOut)
xoutGrp.setStreamName("xout")

color.setResolution(dai.ColorCameraProperties.SensorResolution.THE_12_MP)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoLeft.setCamera("left")
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoRight.setCamera("right")
imu.enableIMUSensor(dai.IMUSensor.ACCELEROMETER_RAW, 360)
imu.setBatchReportThreshold(10)
imu.setMaxBatchReports(10)

color.setFps(30)
monoLeft.setFps(30)
monoRight.setFps(30)

stereo.setLeftRightCheck(True)
stereo.setExtendedDisparity(True)
stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) # similar to `stereo_depth.initialConfig.setConfidenceThreshold(<threshold, an int>)`, # noqa: E501
stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7) # Options: MEDIAN_OFF, KERNEL_3x3, KERNEL_5x5, KERNEL_7x7 (default) # noqa: E501

stereoConfig = stereo.initialConfig.get()
stereoConfig.postProcessing.speckleFilter.enable = True
stereoConfig.postProcessing.speckleFilter.speckleRange = 50
# stereoConfig.postProcessing.temporalFilter.enable = True
stereoConfig.postProcessing.spatialFilter.enable = True
stereoConfig.postProcessing.spatialFilter.holeFillingRadius = 2
stereoConfig.postProcessing.spatialFilter.numIterations = 1
# stereoConfig.postProcessing.spatialFilter.delta = 20
stereoConfig.postProcessing.thresholdFilter.minRange = 150
stereoConfig.postProcessing.thresholdFilter.maxRange = 1000
stereoConfig.postProcessing.decimationFilter.decimationFactor = 2
stereo.initialConfig.set(stereoConfig)

stereo.setOutputSize(800, 599)
color.setCamera("color")

sync.setSyncThreshold(timedelta(milliseconds=11))

monoLeft.out.link(stereo.left)
monoRight.out.link(stereo.right)

monoLeft.out.link(sync.inputs["left"])
monoRight.out.link(sync.inputs["right"])
stereo.depth.link(sync.inputs["depth"])
stereo.disparity.link(sync.inputs["disparity"])
color.isp.link(sync.inputs["video"])
imu.out.link(sync.inputs["imu"])

sync.out.link(xoutGrp.input)

disparityMultiplier = 255.0 / stereo.initialConfig.getMaxDisparity()
with dai.Device(pipeline, maxUsbSpeed=dai.UsbSpeed.SUPER_PLUS) as device:
    queue = device.getOutputQueue("xout", 1, False)
    while True:
        msgGrp = queue.get()
        for name in ["disparity", "video", "left", "right"]:
            msg = msgGrp[name]
            frame = msg.getCvFrame()
            if name == "disparity":
                frame = (frame * disparityMultiplier).astype(np.uint8)
                frame = cv2.applyColorMap(frame, cv2.COLORMAP_JET)
            cv2.imshow(name, cv2.resize(frame, (800, 599), 
                                        interpolation = cv2.INTER_LINEAR))
            print(f"Device timestamp {name}:" + str(msg.getTimestampDevice()))

        imuMsg = msgGrp["imu"]

        print("Device timestamp imu: " + str(imuMsg.getTimestampDevice()))
        latestRotationVector = imuMsg.packets[-1].rotationVector
        imuF = "{:.4f}"
        print(f"Quaternion: i: {imuF.format(latestRotationVector.i)} j: {imuF.format(latestRotationVector.j)} "
        f"k: {imuF.format(latestRotationVector.k)} real: {imuF.format(latestRotationVector.real)}")

        if cv2.waitKey(1) == ord("q"):
            break

    Hi glitchyordis
    First off, you are sending a lot of data to the host:

    monoLeft.out.link(sync.inputs["left"])
    monoRight.out.link(sync.inputs["right"])
    stereo.depth.link(sync.inputs["depth"])
    stereo.disparity.link(sync.inputs["disparity"])
    color.isp.link(sync.inputs["video"])
    imu.out.link(sync.inputs["imu"])

    There is something blocking the pipeline (either the host is unable to process data fast enough) or the device isn't producing frames fast enough.

    Increasing the threshold: sync.setSyncThreshold(timedelta(milliseconds=80)) seemed to solve the problem. 11ms is way too low for such a complex pipeline.
    You can also try increasing the pool sizes to see if it has an effect. color.setIspNumFramesPool(10)

    Thanks,
    Jaka