I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?
Thanks,
Li
I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?
Thanks,
Li
Hi @li_you_chen ,
Yep, that's possible. So you can have pipeline like this:
hello @erik
I tried to write a program like the following, but it couldn't display frame as expected.
it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.
and imageQueueDataIR = qIR.tryGet()
RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'
The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.
This is my code,
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood-left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb-out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
# Connect device and start pipeline
try:
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frameIR = None
frameRGB = None
frame_detectIR = None
frame_detectRGB = None
qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)
trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
while True:
counter += 1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1:
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
# IR camera processing
imageQueueDataIR = qIR.tryGet()
if imageQueueDataIR is not None:
frameIR = imageQueueDataIR.getCvFrame()
frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)
frameIR = frameIR[:, 120:520]
frame_detectIR = frameIR.copy()
cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackIR = trackletsIR.get()
trackletsDataIR = trackIR.tracklets
for t in trackletsDataIR:
roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)
cv2.imshow("IR detect", frame_detectIR)
# RGB camera processing
imageQueueDataRGB = qRGB.tryGet()
if imageQueueDataRGB is not None:
frameRGB = imageQueueDataRGB.getCvFrame()
frame_detectRGB = frameRGB.copy()
cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackRGB = trackletsRGB.get()
trackletsDataRGB = trackRGB.tracklets
for t in trackletsDataRGB:
roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)
cv2.imshow("RGB detect", frame_detectRGB)
if cv2.waitKey(1) == 27:
break
finally:
cv2.destroyAllWindows()
thanks,
Li
@li_you_chen please provide minimal repro example that's formatted correctly (use ```)
hi @erik sorry I didn't notice that.
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood_left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb_out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)
# Connect device and start pipeline
try:
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frameIR = None
frameRGB = None
frame_detectIR = None
frame_detectRGB = None
qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
while True:
counter += 1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1:
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
# IR camera processing
imageQueueDataIR = qIR.tryGet()
if imageQueueDataIR is not None:
frameIR = imageQueueDataIR.getCvFrame()
frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)
frameIR = frameIR[:, 120:520]
frame_detectIR = frameIR.copy()
cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackIR = trackletsIR.get()
trackletsDataIR = trackIR.tracklets
for t in trackletsDataIR:
roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)
# RGB camera processing
imageQueueDataRGB = qRGB.tryGet()
if imageQueueDataRGB is not None:
frameRGB = imageQueueDataRGB.getCvFrame()
frame_detectRGB = frameRGB.copy()
cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackRGB = trackletsRGB.get()
trackletsDataRGB = trackRGB.tracklets
for t in trackletsDataRGB:
roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)
cv2.imshow("RGB detect", frame_detectRGB)
cv2.imshow("IR detect", frame_detectIR)
if cv2.waitKey(1) == 27:
break
finally:
cv2.destroyAllWindows()
```
[1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)]
[2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in:
Traceback (most recent call last): line 152, in <module>
imageQueueDataIR = qIR.tryGet()
RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'
@li_you_chen this isn't reproducible, as I do not have your model. Please upload whole zip to drive and share the link.
https://drive.google.com/file/d/1xMDcBE--cvusVnXOdSPXPGYgHztUEde-/view?usp=sharing
My model and code are provided here
thank you,
Li
Up the pool size of mono camera (and color). The latency of the whole pipeline is about 0.5s which means at 30FPS that is 15 frames. You will run out of pool size before the frames are dereferenced. Set it to 15 (or something above 10) and you should have less problems.
Script node doesn't have Device.setIrFloodLightIntensity(floodBright)
but it does have Device.setIrFloodLightBrightness(floodBright)
.
Thanks,
Jaka