Hello!
I am trying to get depth information from the camera by using the code below. it works, but the problem I am facing is that the frames are lagging behind. it works at around 5 FPS, so that would be good enough for me. But the problem is that the Frame recorded now, is lagging behind about 15 frames so it is displayed only 3 seconds later...
Is there a way to solve this problem? What am I doing wrong?
(Please ignore the mess in the code, we are trying multiple different things... 🙂 )
thank you

import math

import cv2
import depthai as dai
import time
import numpy as np

CUSTOM_DEPTH_CALC = False
AVG_QTY = 1
GROUND = 1120
XL = [0, 0]
YL = [0, 0]
XR = [0, 0]
YR = [0, 0]

XL[0] = 0.35
YL[0] = 0.25
XR[0] = 0.55
YR[0] = 0.45

XL[1] = 0.35
YL[1] = 0.55
XR[1] = 0.55
YR[1] = 0.75
MIDDLE = 250

# Create pipeline
pipeline = dai.Pipeline()

# Define sources and outputs
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)
stereo = pipeline.create(dai.node.StereoDepth)
spatialLocationCalculator = pipeline.create(dai.node.SpatialLocationCalculator)
spatialLocationCalculator2 = pipeline.create(dai.node.SpatialLocationCalculator)

xoutDepth = pipeline.create(dai.node.XLinkOut)
xoutSpatialData = pipeline.create(dai.node.XLinkOut)
xinSpatialCalcConfig = pipeline.create(dai.node.XLinkIn)
xinSpatialCalcConfig2 = pipeline.create(dai.node.XLinkIn)

xoutDepth.setStreamName("depth")
xoutSpatialData.setStreamName("spatialData")
xinSpatialCalcConfig.setStreamName("spatialCalcConfig")
xinSpatialCalcConfig2.setStreamName("spatialCalcConfig2")

# Properties
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)

lrcheck = True
subpixel = False

stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
stereo.setLeftRightCheck(lrcheck)
stereo.setSubpixel(subpixel)
stereo.setExtendedDisparity(True)

# Config
topLeft = dai.Point2f(XL[0], YL[0])
bottomRight = dai.Point2f(XR[0], YR[0])

topLeft2 = dai.Point2f(XL[1], YL[1])
bottomRight2 = dai.Point2f(XR[1], YR[1])

config = dai.SpatialLocationCalculatorConfigData()
config.depthThresholds.lowerThreshold = 100
config.depthThresholds.upperThreshold = 1500
config.roi = dai.Rect(topLeft, bottomRight)

config2 = dai.SpatialLocationCalculatorConfigData()
config2.depthThresholds.lowerThreshold = 100
config2.depthThresholds.upperThreshold = 1500
config2.roi = dai.Rect(topLeft2, bottomRight2)

spatialLocationCalculator.inputConfig.setWaitForMessage(False)
spatialLocationCalculator.initialConfig.addROI(config)

spatialLocationCalculator2.inputConfig.setWaitForMessage(False)
spatialLocationCalculator2.initialConfig.addROI(config2)

# Linking
monoLeft.out.link(stereo.left)
monoRight.out.link(stereo.right)

spatialLocationCalculator.passthroughDepth.link(xoutDepth.input)
stereo.depth.link(spatialLocationCalculator.inputDepth)

spatialLocationCalculator.out.link(xoutSpatialData.input)
xinSpatialCalcConfig.out.link(spatialLocationCalculator.inputConfig)

spatialLocationCalculator2.passthroughDepth.link(xoutDepth.input)
stereo.depth.link(spatialLocationCalculator2.inputDepth)

spatialLocationCalculator2.out.link(xoutSpatialData.input)
xinSpatialCalcConfig2.out.link(spatialLocationCalculator2.inputConfig)

# Connect to device and start pipeline
with dai.Device(pipeline) as device:
    depthQueue = device.getOutputQueue(name="depth", maxSize=4, blocking=False)
    spatialCalcQueue = device.getOutputQueue(name="spatialData", maxSize=4, blocking=False)

    device.setIrLaserDotProjectorBrightness(600)  # in mA, 0..1200
    #device.setIrLaserDotProjectorBrightness(100) # in mA, 0..1200

    start_time = time.time()
    counter = 0
    fps = 0

    while True:
        zavg0 = []
        zavg1 = []
        debugPixel = []
        debugPixel0 = []
        debugPixel1 = []

        xmin0 = 1
        xmax0 = 1
        ymin0 = 1
        ymax0 = 1

        xmin1 = 1
        xmax1 = 1
        ymin1 = 1
        ymax1 = 1

        for cnt in range(AVG_QTY):
            depthFrame = depthQueue.get().getFrame()  # depthFrame values are in millimeters

            if CUSTOM_DEPTH_CALC:
                for i in range(2):
                    topLeftX = XL[i] * depthFrame.shape[1]
                    topLeftY = YL[i] * depthFrame.shape[0]
                    bottomRightX = XR[i] * depthFrame.shape[1]
                    bottomRightY = YR[i] * depthFrame.shape[0]
                    n, m = int(abs(topLeftX - bottomRightX)), int(abs(topLeftY - bottomRightY))
                    roi_center = (int((topLeftX + bottomRightX) / 2), int((topLeftY + bottomRightY) / 2))
                    bottom = roi_center[1] + int(np.floor(m / 2))
                    top = roi_center[1] - int(np.ceil(m / 2))
                    left = roi_center[0] - int(np.floor(n / 2))
                    right = roi_center[0] + int(np.ceil(n / 2))
                    depthRoi = depthFrame[top:bottom, left:right]
                    depthRoiFlat = sorted(depthRoi.flatten())
                    while 0 in depthRoiFlat:
                        depthRoiFlat.remove(0)
                    depthRoiSmall = depthRoiFlat[:int(len(depthRoiFlat)/20)]
                    avg = np.average(depthRoiSmall)

                    if i == 0:
                        if len(depthRoiSmall) > 0:
                            zavg0.append(int(avg))
                        else:
                            zavg0.append(0)
                        xmin0 = left
                        ymin0 = top
                        xmax0 = right
                        ymax0 = bottom
                    else:
                        if len(depthRoiSmall) > 0:
                            zavg1.append(int(avg))
                        else:
                            zavg1.append(0)
                        xmin1 = left
                        ymin1 = top
                        xmax1 = right
                        ymax1 = bottom

            if not CUSTOM_DEPTH_CALC:
                for i in range(2):
                    spatialData = spatialCalcQueue.get().getSpatialLocations()
                    for depthData in spatialData:
                        roi = depthData.config.roi
                        roi = roi.denormalize(width=depthFrame.shape[1], height=depthFrame.shape[0])
                        print(roi.topLeft().y, MIDDLE)
                        if roi.topLeft().y < MIDDLE:
                            xmin0 = int(roi.topLeft().x)
                            ymin0 = int(roi.topLeft().y)
                            xmax0 = int(roi.bottomRight().x)
                            ymax0 = int(roi.bottomRight().y)
                            #zavg0.append(int(depthData.spatialCoordinates.z))
                            zavg0.append(int(depthData.depthAverage))
                        else:
                            xmin1 = int(roi.topLeft().x)
                            ymin1 = int(roi.topLeft().y)
                            xmax1 = int(roi.bottomRight().x)
                            ymax1 = int(roi.bottomRight().y)
                            #zavg1.append(int(depthData.spatialCoordinates.z))
                            zavg1.append(int(depthData.depthAverage))

            depthFrameColor = cv2.normalize(depthFrame, None, 255, 0, cv2.NORM_INF, cv2.CV_8UC1)
            depthFrameColor = cv2.equalizeHist(depthFrameColor)
            depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_JET)

        fontType = cv2.FONT_HERSHEY_DUPLEX

        avg0 = 0
        if len(zavg0) > 0:
            avg0 = GROUND - (sum(zavg0) / len(zavg0))
        crates0 = math.ceil((avg0-15) / 30.3)

        avg1 = 0
        if len(zavg1) > 0:
            avg1 = GROUND - (sum(zavg1) / len(zavg1))
        crates1 = math.ceil((avg1-15) / 30.3)

        color = (255, 255, 255)
        textColor = (0, 255, 0)

        cv2.rectangle(depthFrameColor, (xmin0, ymin0), (xmax0, ymax0), color, cv2.FONT_HERSHEY_SCRIPT_SIMPLEX)
        cv2.putText(depthFrameColor, "Crate h: " + "{0:.2f}".format(avg0) + "mm", (xmin0 + 10, ymin0 - 50), fontType, 0.5, textColor)
        cv2.putText(depthFrameColor, "Crates: " + str(crates0), (xmin0 + 10, ymin0 - 100), fontType, 0.5, textColor)

        cv2.rectangle(depthFrameColor, (xmin1, ymin1), (xmax1, ymax1), color, cv2.FONT_HERSHEY_SCRIPT_SIMPLEX)
        cv2.putText(depthFrameColor, "Crate h: " + "{0:.2f}".format(avg1) + "mm", (xmin1 + 10, ymin1 - 50), fontType, 0.5, textColor)
        cv2.putText(depthFrameColor, "Crates: " + str(crates1), (xmin1 + 10, ymin1 - 100), fontType, 0.5, textColor)

        label_fps = "Fps: {:.2f}".format(fps)
        (w1, h1), _ = cv2.getTextSize(label_fps, cv2.FONT_HERSHEY_TRIPLEX, 0.4, 1)
        cv2.putText(depthFrameColor, label_fps, (10, 10), cv2.FONT_HERSHEY_TRIPLEX, 0.4, textColor)

        cv2.imshow("depth", depthFrameColor)

        # Quit
        key = cv2.waitKey(1)
        if key == ord('q'):
            break