I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?

Thanks,

Li

  • hi @erik sorry I didn't notice that.

    
    import cv2
    import depthai as dai
    import numpy as np
    import time
    # Path to the model weights file
    model_weights_path = "yolov8n.blob"
    # Load model weights using dai.OpenVINO.Blob
    custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
    # Create a pipeline
    pipeline = dai.Pipeline()
    res = dai.MonoCameraProperties.SensorResolution.THE_400_P
    # Define IR camera and outputs
    monoL = pipeline.create(dai.node.MonoCamera)
    monoL.setResolution(res)
    monoL.setFps(30)
    manipIR = pipeline.create(dai.node.ImageManip)
    manipIROut = pipeline.create(dai.node.XLinkOut)
    detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
    objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
    trackerIROut = pipeline.create(dai.node.XLinkOut)
    # Define RGB camera and outputs
    rgbCamera = pipeline.create(dai.node.ColorCamera)
    rgbCamera.setPreviewSize(320, 320)
    rgbCamera.setInterleaved(False)
    rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
    rgbCamera.setFps(30)
    manipRGB = pipeline.create(dai.node.ImageManip)
    manipRGBOut = pipeline.create(dai.node.XLinkOut)
    detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
    objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
    trackerRGBOut = pipeline.create(dai.node.XLinkOut)
    # Set stream names for outputs
    trackerIROut.setStreamName("trackletsIR")
    manipIROut.setStreamName('flood_left')
    trackerRGBOut.setStreamName("trackletsRGB")
    manipRGBOut.setStreamName('rgb_out')
    # Script node for frame routing and IR dot/flood alternate
    script = pipeline.create(dai.node.Script)
    script.setProcessor(dai.ProcessorType.LEON_CSS)
    script.setScript("""
    
    floodBright = 0.1
    node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
    while True:
        event = node.io['event'].get()
        Device.setIrFloodLightIntensity(floodBright)
        frameL = node.io['frameL'].get()
        node.io['floodL'].send(frameL)
    
    """)
    # Model-specific settings
    detectionNetworkIR.setBlob(custom_model_blob)
    detectionNetworkIR.setConfidenceThreshold(0.7)
    detectionNetworkRGB.setBlob(custom_model_blob)
    detectionNetworkRGB.setConfidenceThreshold(0.7)
    # YOLO specific parameters for IR
    detectionNetworkIR.setNumClasses(2)
    detectionNetworkIR.setCoordinateSize(4)
    detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
    detectionNetworkIR.setIouThreshold(0.5)
    detectionNetworkIR.input.setBlocking(False)
    # YOLO specific parameters for RGB
    detectionNetworkRGB.setNumClasses(2)
    detectionNetworkRGB.setCoordinateSize(4)
    detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
    detectionNetworkRGB.setIouThreshold(0.5)
    detectionNetworkRGB.input.setBlocking(False)
    # Convert the grayscale frame into a format acceptable by the neural network for IR
    manipIR.initialConfig.setResize(320, 320)
    manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
    # Convert the color frame into a format acceptable by the neural network for RGB
    manipRGB.initialConfig.setResize(320, 320)
    manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
    # Object tracker settings for IR
    objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
    objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
    objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
    objectTrackerIR.setTrackerThreshold(0.5)
    # Object tracker settings for RGB
    objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
    objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
    objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
    objectTrackerRGB.setTrackerThreshold(0.5)
    # Connect nodes in the pipeline for IR
    monoL.out.link(manipIR.inputImage)
    manipIR.out.link(detectionNetworkIR.input)
    detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
    detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
    detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
    objectTrackerIR.out.link(trackerIROut.input)
    monoL.frameEvent.link(script.inputs['event'])
    monoL.out.link(script.inputs['frameL'])
    script.outputs['floodL'].link(manipIROut.input)
    # Connect nodes in the pipeline for RGB
    rgbCamera.preview.link(manipRGB.inputImage)
    manipRGB.out.link(detectionNetworkRGB.input)
    detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
    objectTrackerRGB.out.link(trackerRGBOut.input)
    # Function to normalize frame coordinates
    def frameNorm(frame, bbox):
    
    normVals = np.full(len(bbox), frame.shape[0])
    normVals[::2] = frame.shape[1]
    return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)
    
    # Connect device and start pipeline
    try:
    
    with dai.Device(pipeline) as device:
        startTime = time.monotonic()
        counter = 0
        fps = 0
        frameIR = None
        frameRGB = None
        frame_detectIR = None
        frame_detectRGB = None
        qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
        qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
        trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
        trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
        
    while True: counter += 1 current_time1 = time.monotonic() if (current_time1 - startTime) > 1: fps = counter / (current_time1 - startTime) counter = 0 startTime = current_time1
    # IR camera processing imageQueueDataIR = qIR.tryGet() if imageQueueDataIR is not None: frameIR = imageQueueDataIR.getCvFrame() frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR) frameIR = frameIR[:, 120:520] frame_detectIR = frameIR.copy() cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackIR = trackletsIR.get() trackletsDataIR = trackIR.tracklets for t in trackletsDataIR: roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2) # RGB camera processing imageQueueDataRGB = qRGB.tryGet() if imageQueueDataRGB is not None: frameRGB = imageQueueDataRGB.getCvFrame() frame_detectRGB = frameRGB.copy() cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackRGB = trackletsRGB.get() trackletsDataRGB = trackRGB.tracklets for t in trackletsDataRGB: roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.imshow("RGB detect", frame_detectRGB) cv2.imshow("IR detect", frame_detectIR) if cv2.waitKey(1) == 27: break finally: cv2.destroyAllWindows() ``` [1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)] [2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: Traceback (most recent call last): line 152, in <module> imageQueueDataIR = qIR.tryGet() RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'

hello @erik

I tried to write a program like the following, but it couldn't display frame as expected.

it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.and imageQueueDataIR = qIR.tryGet()

RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'

The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.

This is my code,

import cv2

import depthai as dai

import numpy as np

import time

# Path to the model weights file

model_weights_path = "yolov8n.blob"

# Load model weights using dai.OpenVINO.Blob

custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

# Create a pipeline

pipeline = dai.Pipeline()

res = dai.MonoCameraProperties.SensorResolution.THE_400_P

# Define IR camera and outputs

monoL = pipeline.create(dai.node.MonoCamera)

monoL.setResolution(res)

monoL.setFps(30)

manipIR = pipeline.create(dai.node.ImageManip)

manipIROut = pipeline.create(dai.node.XLinkOut)

detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerIR = pipeline.create(dai.node.ObjectTracker)

trackerIROut = pipeline.create(dai.node.XLinkOut)

# Define RGB camera and outputs

rgbCamera = pipeline.create(dai.node.ColorCamera)

rgbCamera.setPreviewSize(320, 320)

rgbCamera.setInterleaved(False)

rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)

rgbCamera.setFps(30)

manipRGB = pipeline.create(dai.node.ImageManip)

manipRGBOut = pipeline.create(dai.node.XLinkOut)

detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)

trackerRGBOut = pipeline.create(dai.node.XLinkOut)

# Set stream names for outputs

trackerIROut.setStreamName("trackletsIR")

manipIROut.setStreamName('flood-left')

trackerRGBOut.setStreamName("trackletsRGB")

manipRGBOut.setStreamName('rgb-out')

# Script node for frame routing and IR dot/flood alternate

script = pipeline.create(dai.node.Script)

script.setProcessor(dai.ProcessorType.LEON_CSS)

script.setScript("""

floodBright = 0.1

node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

while True:

event = node.io['event'].get()

Device.setIrFloodLightIntensity(floodBright)

frameL = node.io['frameL'].get()

node.io['floodL'].send(frameL)

""")

# Model-specific settings

detectionNetworkIR.setBlob(custom_model_blob)

detectionNetworkIR.setConfidenceThreshold(0.7)

detectionNetworkRGB.setBlob(custom_model_blob)

detectionNetworkRGB.setConfidenceThreshold(0.7)

# YOLO specific parameters for IR

detectionNetworkIR.setNumClasses(2)

detectionNetworkIR.setCoordinateSize(4)

detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkIR.setIouThreshold(0.5)

detectionNetworkIR.input.setBlocking(False)

# YOLO specific parameters for RGB

detectionNetworkRGB.setNumClasses(2)

detectionNetworkRGB.setCoordinateSize(4)

detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkRGB.setIouThreshold(0.5)

detectionNetworkRGB.input.setBlocking(False)

# Convert the grayscale frame into a format acceptable by the neural network for IR

manipIR.initialConfig.setResize(320, 320)

manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Convert the color frame into a format acceptable by the neural network for RGB

manipRGB.initialConfig.setResize(320, 320)

manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Object tracker settings for IR

objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerIR.setTrackerThreshold(0.5)

# Object tracker settings for RGB

objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerRGB.setTrackerThreshold(0.5)

# Connect nodes in the pipeline for IR

monoL.out.link(manipIR.inputImage)

manipIR.out.link(detectionNetworkIR.input)

detectionNetworkIR.out.link(objectTrackerIR.inputDetections)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)

objectTrackerIR.out.link(trackerIROut.input)

monoL.frameEvent.link(script.inputs['event'])

monoL.out.link(script.inputs['frameL'])

script.outputs['floodL'].link(manipIROut.input)

# Connect nodes in the pipeline for RGB

rgbCamera.preview.link(manipRGB.inputImage)

manipRGB.out.link(detectionNetworkRGB.input)

detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)

objectTrackerRGB.out.link(trackerRGBOut.input)

# Function to normalize frame coordinates

def frameNorm(frame, bbox):

normVals = np.full(len(bbox), frame.shape[0])

normVals[::2] = frame.shape[1]

return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

# Connect device and start pipeline

try:

with dai.Device(pipeline) as device:

startTime = time.monotonic()

counter = 0

fps = 0

frameIR = None

frameRGB = None

frame_detectIR = None

frame_detectRGB = None

qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)

qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)

trackletsIR = device.getOutputQueue("trackletsIR", 4, False)

trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)

while True:

counter += 1

current_time1 = time.monotonic()

if (current_time1 - startTime) > 1:

fps = counter / (current_time1 - startTime)

counter = 0

startTime = current_time1

# IR camera processing

imageQueueDataIR = qIR.tryGet()

if imageQueueDataIR is not None:

frameIR = imageQueueDataIR.getCvFrame()

frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)

frameIR = frameIR[:, 120:520]

frame_detectIR = frameIR.copy()

cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackIR = trackletsIR.get()

trackletsDataIR = trackIR.tracklets

for t in trackletsDataIR:

roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)

cv2.imshow("IR detect", frame_detectIR)

# RGB camera processing

imageQueueDataRGB = qRGB.tryGet()

if imageQueueDataRGB is not None:

frameRGB = imageQueueDataRGB.getCvFrame()

frame_detectRGB = frameRGB.copy()

cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackRGB = trackletsRGB.get()

trackletsDataRGB = trackRGB.tracklets

for t in trackletsDataRGB:

roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)

cv2.imshow("RGB detect", frame_detectRGB)

if cv2.waitKey(1) == 27:

break

finally:

cv2.destroyAllWindows()

thanks,

Li

@li_you_chen please provide minimal repro example that's formatted correctly (use ```)

hi @erik sorry I didn't notice that.


import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood_left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb_out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""

floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
    event = node.io['event'].get()
    Device.setIrFloodLightIntensity(floodBright)
    frameL = node.io['frameL'].get()
    node.io['floodL'].send(frameL)

""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):

normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)

# Connect device and start pipeline
try:

with dai.Device(pipeline) as device:
    startTime = time.monotonic()
    counter = 0
    fps = 0
    frameIR = None
    frameRGB = None
    frame_detectIR = None
    frame_detectRGB = None
    qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
    qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
    trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
    trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
    
while True: counter += 1 current_time1 = time.monotonic() if (current_time1 - startTime) > 1: fps = counter / (current_time1 - startTime) counter = 0 startTime = current_time1
# IR camera processing imageQueueDataIR = qIR.tryGet() if imageQueueDataIR is not None: frameIR = imageQueueDataIR.getCvFrame() frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR) frameIR = frameIR[:, 120:520] frame_detectIR = frameIR.copy() cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackIR = trackletsIR.get() trackletsDataIR = trackIR.tracklets for t in trackletsDataIR: roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2) # RGB camera processing imageQueueDataRGB = qRGB.tryGet() if imageQueueDataRGB is not None: frameRGB = imageQueueDataRGB.getCvFrame() frame_detectRGB = frameRGB.copy() cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackRGB = trackletsRGB.get() trackletsDataRGB = trackRGB.tracklets for t in trackletsDataRGB: roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.imshow("RGB detect", frame_detectRGB) cv2.imshow("IR detect", frame_detectIR) if cv2.waitKey(1) == 27: break finally: cv2.destroyAllWindows() ``` [1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)] [2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: Traceback (most recent call last): line 152, in <module> imageQueueDataIR = qIR.tryGet() RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'

@li_you_chen this isn't reproducible, as I do not have your model. Please upload whole zip to drive and share the link.

li_you_chen

  • Up the pool size of mono camera (and color). The latency of the whole pipeline is about 0.5s which means at 30FPS that is 15 frames. You will run out of pool size before the frames are dereferenced. Set it to 15 (or something above 10) and you should have less problems.

  • Script node doesn't have Device.setIrFloodLightIntensity(floodBright) but it does have Device.setIrFloodLightBrightness(floodBright).

Thanks,
Jaka