IR and RGB tracking

Lli_you_chen · Aug 31, 2024

I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?

Thanks,

Li

erik · Aug 31, 2024

Hi @li_you_chen ,
Yep, that's possible. So you can have pipeline like this:

Lli_you_chen · Aug 31, 2024

hello @erik

I tried to write a program like the following, but it couldn't display frame as expected.

it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.and imageQueueDataIR = qIR.tryGet()

RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'

The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.

This is my code,

import cv2

import depthai as dai

import numpy as np

import time

# Path to the model weights file

model_weights_path = "yolov8n.blob"

# Load model weights using dai.OpenVINO.Blob

custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

# Create a pipeline

pipeline = dai.Pipeline()

res = dai.MonoCameraProperties.SensorResolution.THE_400_P

# Define IR camera and outputs

monoL = pipeline.create(dai.node.MonoCamera)

monoL.setResolution(res)

monoL.setFps(30)

manipIR = pipeline.create(dai.node.ImageManip)

manipIROut = pipeline.create(dai.node.XLinkOut)

detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerIR = pipeline.create(dai.node.ObjectTracker)

trackerIROut = pipeline.create(dai.node.XLinkOut)

# Define RGB camera and outputs

rgbCamera = pipeline.create(dai.node.ColorCamera)

rgbCamera.setPreviewSize(320, 320)

rgbCamera.setInterleaved(False)

rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)

rgbCamera.setFps(30)

manipRGB = pipeline.create(dai.node.ImageManip)

manipRGBOut = pipeline.create(dai.node.XLinkOut)

detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)

trackerRGBOut = pipeline.create(dai.node.XLinkOut)

# Set stream names for outputs

trackerIROut.setStreamName("trackletsIR")

manipIROut.setStreamName('flood-left')

trackerRGBOut.setStreamName("trackletsRGB")

manipRGBOut.setStreamName('rgb-out')

# Script node for frame routing and IR dot/flood alternate

script = pipeline.create(dai.node.Script)

script.setProcessor(dai.ProcessorType.LEON_CSS)

script.setScript("""

floodBright = 0.1

node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

while True:

event = node.io['event'].get()

Device.setIrFloodLightIntensity(floodBright)

frameL = node.io['frameL'].get()

node.io['floodL'].send(frameL)

""")

# Model-specific settings

detectionNetworkIR.setBlob(custom_model_blob)

detectionNetworkIR.setConfidenceThreshold(0.7)

detectionNetworkRGB.setBlob(custom_model_blob)

detectionNetworkRGB.setConfidenceThreshold(0.7)

# YOLO specific parameters for IR

detectionNetworkIR.setNumClasses(2)

detectionNetworkIR.setCoordinateSize(4)

detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkIR.setIouThreshold(0.5)

detectionNetworkIR.input.setBlocking(False)

# YOLO specific parameters for RGB

detectionNetworkRGB.setNumClasses(2)

detectionNetworkRGB.setCoordinateSize(4)

detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkRGB.setIouThreshold(0.5)

detectionNetworkRGB.input.setBlocking(False)

# Convert the grayscale frame into a format acceptable by the neural network for IR

manipIR.initialConfig.setResize(320, 320)

manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Convert the color frame into a format acceptable by the neural network for RGB

manipRGB.initialConfig.setResize(320, 320)

manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Object tracker settings for IR

objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerIR.setTrackerThreshold(0.5)

# Object tracker settings for RGB

objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerRGB.setTrackerThreshold(0.5)

# Connect nodes in the pipeline for IR

monoL.out.link(manipIR.inputImage)

manipIR.out.link(detectionNetworkIR.input)

detectionNetworkIR.out.link(objectTrackerIR.inputDetections)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)

objectTrackerIR.out.link(trackerIROut.input)

monoL.frameEvent.link(script.inputs['event'])

monoL.out.link(script.inputs['frameL'])

script.outputs['floodL'].link(manipIROut.input)

# Connect nodes in the pipeline for RGB

rgbCamera.preview.link(manipRGB.inputImage)

manipRGB.out.link(detectionNetworkRGB.input)

detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)

objectTrackerRGB.out.link(trackerRGBOut.input)

# Function to normalize frame coordinates

def frameNorm(frame, bbox):

normVals = np.full(len(bbox), frame.shape[0])

normVals[::2] = frame.shape[1]

return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

# Connect device and start pipeline

try:

with dai.Device(pipeline) as device:

startTime = time.monotonic()

counter = 0

fps = 0

frameIR = None

frameRGB = None

frame_detectIR = None

frame_detectRGB = None

qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)

qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)

trackletsIR = device.getOutputQueue("trackletsIR", 4, False)

trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)

while True:

counter += 1

current_time1 = time.monotonic()

if (current_time1 - startTime) > 1:

fps = counter / (current_time1 - startTime)

counter = 0

startTime = current_time1

# IR camera processing

imageQueueDataIR = qIR.tryGet()

if imageQueueDataIR is not None:

frameIR = imageQueueDataIR.getCvFrame()

frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)

frameIR = frameIR[:, 120:520]

frame_detectIR = frameIR.copy()

cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackIR = trackletsIR.get()

trackletsDataIR = trackIR.tracklets

for t in trackletsDataIR:

roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)

cv2.imshow("IR detect", frame_detectIR)

# RGB camera processing

imageQueueDataRGB = qRGB.tryGet()

if imageQueueDataRGB is not None:

frameRGB = imageQueueDataRGB.getCvFrame()

frame_detectRGB = frameRGB.copy()

cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackRGB = trackletsRGB.get()

trackletsDataRGB = trackRGB.tracklets

for t in trackletsDataRGB:

roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)

cv2.imshow("RGB detect", frame_detectRGB)

if cv2.waitKey(1) == 27:

break

finally:

cv2.destroyAllWindows()

thanks,

Li

erik · Aug 31, 2024

@li_you_chen please provide minimal repro example that's formatted correctly (use ```)

Lli_you_chen · Aug 31, 2024

hi @erik sorry I didn't notice that.


import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood_left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb_out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""

floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
    event = node.io['event'].get()
    Device.setIrFloodLightIntensity(floodBright)
    frameL = node.io['frameL'].get()
    node.io['floodL'].send(frameL)

""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):

normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)

# Connect device and start pipeline
try:

with dai.Device(pipeline) as device:
    startTime = time.monotonic()
    counter = 0
    fps = 0
    frameIR = None
    frameRGB = None
    frame_detectIR = None
    frame_detectRGB = None
    qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
    qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
    trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
    trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
    

    while True:
        counter += 1
        current_time1 = time.monotonic()
        if (current_time1 - startTime) > 1:
            fps = counter / (current_time1 - startTime)
            counter = 0
            startTime = current_time1
        

        # IR camera processing
        imageQueueDataIR = qIR.tryGet()
        if imageQueueDataIR is not None:
            frameIR = imageQueueDataIR.getCvFrame()
            frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)
            frameIR = frameIR[:, 120:520]
            frame_detectIR = frameIR.copy()
            cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
            trackIR = trackletsIR.get()
            trackletsDataIR = trackIR.tracklets
            for t in trackletsDataIR:
                roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])
                x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
                if t.status.name == 'TRACKED':
                    cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)
        # RGB camera processing
        imageQueueDataRGB = qRGB.tryGet()
        if imageQueueDataRGB is not None:
            frameRGB = imageQueueDataRGB.getCvFrame()
            frame_detectRGB = frameRGB.copy()
            cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
            trackRGB = trackletsRGB.get()
            trackletsDataRGB = trackRGB.tracklets
            for t in trackletsDataRGB:
                roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])
                x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
                if t.status.name == 'TRACKED':
                    cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)
            cv2.imshow("RGB detect", frame_detectRGB)
            cv2.imshow("IR detect", frame_detectIR)
        if cv2.waitKey(1) == 27:
            break

finally:

cv2.destroyAllWindows()

```

[1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)]

[2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: 

Traceback (most recent call last): line 152, in <module>

    imageQueueDataIR = qIR.tryGet()

RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'

erik · Aug 31, 2024

@li_you_chen this isn't reproducible, as I do not have your model. Please upload whole zip to drive and share the link.

Lli_you_chen · Aug 31, 2024

@erik

https://drive.google.com/file/d/1xMDcBE--cvusVnXOdSPXPGYgHztUEde-/view?usp=sharing

My model and code are provided here

thank you,

Li

jakaskerl · Sep 3, 2024

li_you_chen

Up the pool size of mono camera (and color). The latency of the whole pipeline is about 0.5s which means at 30FPS that is 15 frames. You will run out of pool size before the frames are dereferenced. Set it to 15 (or something above 10) and you should have less problems.
Script node doesn't have Device.setIrFloodLightIntensity(floodBright) but it does have Device.setIrFloodLightBrightness(floodBright).

Thanks,
Jaka