https://drive.google.com/file/d/1xMDcBE--cvusVnXOdSPXPGYgHztUEde-/view?usp=sharing
My model and code are provided here
thank you,
Li
https://drive.google.com/file/d/1xMDcBE--cvusVnXOdSPXPGYgHztUEde-/view?usp=sharing
My model and code are provided here
thank you,
Li
hi @erik sorry I didn't notice that.
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood_left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb_out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)
# Connect device and start pipeline
try:
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frameIR = None
frameRGB = None
frame_detectIR = None
frame_detectRGB = None
qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
while True:
counter += 1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1:
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
# IR camera processing
imageQueueDataIR = qIR.tryGet()
if imageQueueDataIR is not None:
frameIR = imageQueueDataIR.getCvFrame()
frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)
frameIR = frameIR[:, 120:520]
frame_detectIR = frameIR.copy()
cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackIR = trackletsIR.get()
trackletsDataIR = trackIR.tracklets
for t in trackletsDataIR:
roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)
# RGB camera processing
imageQueueDataRGB = qRGB.tryGet()
if imageQueueDataRGB is not None:
frameRGB = imageQueueDataRGB.getCvFrame()
frame_detectRGB = frameRGB.copy()
cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackRGB = trackletsRGB.get()
trackletsDataRGB = trackRGB.tracklets
for t in trackletsDataRGB:
roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)
cv2.imshow("RGB detect", frame_detectRGB)
cv2.imshow("IR detect", frame_detectIR)
if cv2.waitKey(1) == 27:
break
finally:
cv2.destroyAllWindows()
```
[1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)]
[2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in:
Traceback (most recent call last): line 152, in <module>
imageQueueDataIR = qIR.tryGet()
RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'
hello @erik
I tried to write a program like the following, but it couldn't display frame as expected.
it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.
and imageQueueDataIR = qIR.tryGet()
RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'
The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.
This is my code,
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "yolov8n.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define IR camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manipIR = pipeline.create(dai.node.ImageManip)
manipIROut = pipeline.create(dai.node.XLinkOut)
detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
trackerIROut = pipeline.create(dai.node.XLinkOut)
# Define RGB camera and outputs
rgbCamera = pipeline.create(dai.node.ColorCamera)
rgbCamera.setPreviewSize(320, 320)
rgbCamera.setInterleaved(False)
rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
rgbCamera.setFps(30)
manipRGB = pipeline.create(dai.node.ImageManip)
manipRGBOut = pipeline.create(dai.node.XLinkOut)
detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
trackerRGBOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerIROut.setStreamName("trackletsIR")
manipIROut.setStreamName('flood-left')
trackerRGBOut.setStreamName("trackletsRGB")
manipRGBOut.setStreamName('rgb-out')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetworkIR.setBlob(custom_model_blob)
detectionNetworkIR.setConfidenceThreshold(0.7)
detectionNetworkRGB.setBlob(custom_model_blob)
detectionNetworkRGB.setConfidenceThreshold(0.7)
# YOLO specific parameters for IR
detectionNetworkIR.setNumClasses(2)
detectionNetworkIR.setCoordinateSize(4)
detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkIR.setIouThreshold(0.5)
detectionNetworkIR.input.setBlocking(False)
# YOLO specific parameters for RGB
detectionNetworkRGB.setNumClasses(2)
detectionNetworkRGB.setCoordinateSize(4)
detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetworkRGB.setIouThreshold(0.5)
detectionNetworkRGB.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network for IR
manipIR.initialConfig.setResize(320, 320)
manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Convert the color frame into a format acceptable by the neural network for RGB
manipRGB.initialConfig.setResize(320, 320)
manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings for IR
objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerIR.setTrackerThreshold(0.5)
# Object tracker settings for RGB
objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTrackerRGB.setTrackerThreshold(0.5)
# Connect nodes in the pipeline for IR
monoL.out.link(manipIR.inputImage)
manipIR.out.link(detectionNetworkIR.input)
detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
objectTrackerIR.out.link(trackerIROut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipIROut.input)
# Connect nodes in the pipeline for RGB
rgbCamera.preview.link(manipRGB.inputImage)
manipRGB.out.link(detectionNetworkRGB.input)
detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
objectTrackerRGB.out.link(trackerRGBOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
# Connect device and start pipeline
try:
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frameIR = None
frameRGB = None
frame_detectIR = None
frame_detectRGB = None
qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)
trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
while True:
counter += 1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1:
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
# IR camera processing
imageQueueDataIR = qIR.tryGet()
if imageQueueDataIR is not None:
frameIR = imageQueueDataIR.getCvFrame()
frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)
frameIR = frameIR[:, 120:520]
frame_detectIR = frameIR.copy()
cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackIR = trackletsIR.get()
trackletsDataIR = trackIR.tracklets
for t in trackletsDataIR:
roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)
cv2.imshow("IR detect", frame_detectIR)
# RGB camera processing
imageQueueDataRGB = qRGB.tryGet()
if imageQueueDataRGB is not None:
frameRGB = imageQueueDataRGB.getCvFrame()
frame_detectRGB = frameRGB.copy()
cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
trackRGB = trackletsRGB.get()
trackletsDataRGB = trackRGB.tracklets
for t in trackletsDataRGB:
roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])
x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
if t.status.name == 'TRACKED':
cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)
cv2.imshow("RGB detect", frame_detectRGB)
if cv2.waitKey(1) == 27:
break
finally:
cv2.destroyAllWindows()
thanks,
Li
I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?
Thanks,
Li
@erik Here are some pictures for your reference
I am using the code provided above and removing
objectTracker.passthroughTrackerFrame.link(trackerOut.input)
And draw a circle with a fixed radius at the midpoint of roi
Thanks,
Li
Thank you for your reply. You solved my problem. However, when I want to mark a circle on an object, the x-coordinate of the returned object will be offset left and right from the actual position. The middle of the screen will hardly be offset, left and right. The two sides will shift outward more as they get closer to the edge of the screen. I wonder if the image distortion is caused by the wide angle of the camera.
btw , my camera is Oak-d Pro
Thanks,
Li
Hi @jakaskerl
After I modify ,It show
trackletsData = track.tracklets
AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'
trackletsData = track.tracklets
AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'
#!/usr/bin/env python3
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "C:/piegg.blob"
labelMap = ["egg", "stick"]
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manip = pipeline.create(dai.node.ImageManip)
manipOut = pipeline.create(dai.node.XLinkOut)
detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
objectTracker = pipeline.create(dai.node.ObjectTracker)
trackerOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerOut.setStreamName("tracklets")
manipOut.setStreamName('flood-left')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetwork.setBlob(custom_model_blob)
detectionNetwork.setConfidenceThreshold(0.8)
# YOLO specific parameters
detectionNetwork.setNumClasses(2)
detectionNetwork.setCoordinateSize(4)
detectionNetwork.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetwork.setIouThreshold(0.5)
detectionNetwork.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network
manip.initialConfig.setResize(320, 320)
manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings
objectTracker.setDetectionLabelsToTrack([0, 1, 2])
objectTracker.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTracker.setTrackerThreshold(0.85)
# Connect nodes in the pipeline
monoL.out.link(manip.inputImage)
manip.out.link(detectionNetwork.input)
detectionNetwork.out.link(objectTracker.inputDetections)
detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
objectTracker.passthroughTrackerFrame.link(trackerOut.input)
objectTracker.out.link(trackerOut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
# Connect device and start pipeline
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frame = None
qRight = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
tracklets = device.getOutputQueue("tracklets", 4, False)
while True:
counter += 1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1:
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
imageQueueData = qRight.tryGet()
track = tracklets.get()
trackletsData = track.tracklets
if imageQueueData is not None:
frame = imageQueueData.getCvFrame()
if frame is not None:
frame_detect = frame.copy()
cv2.putText(frame_detect, "fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
for t in trackletsData:
# Get region of interest (ROI) and draw circle on detected objects
roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
x1 = int(roi.topLeft().x)
y1 = int(roi.topLeft().y)
x2 = int(roi.bottomRight().x)
y2 = int(roi.bottomRight().y)
cv2.circle(frame_detect, ((x1 + x2) // 2, (y1 + y2) // 2), 15, (0, 0, 255), 2)
# Display the frame with detections
cv2.imshow("ir detect", frame_detect)
# Exit the loop when 'q' is pressed
if cv2.waitKey(1) == ord("q"):
break
cv2.destroyAllWindows()
I want to use Yolo for object tracking , After i refer to example , the trackletsData is empty, else part is run correctly ,please help me figure out where the problem is
import cv2
import depthai as dai
import numpy as np
import time
# Specify the path to your custom model weights
model_weights_path = r"C:\catdog320.blob"
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
numClasses = 80
dim = next(iter(custom_model_blob.networkInputs.values())).dims
output_name, output_tenser = next(iter(custom_model_blob.networkOutputs.items()))
numClasses = output_tenser.dims[2] - 5
# Create pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define camera and output
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manip = pipeline.create(dai.node.ImageManip)
manipOut = pipeline.create(dai.node.XLinkOut)
detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
monoL.setNumFramesPool(24)
xoutNN = pipeline.create(dai.node.XLinkOut)
manipOut.setStreamName('flood-left')
xoutNN.setStreamName("nn")
# Create object tracker
objectTracker = pipeline.create(dai.node.ObjectTracker)
# Create output queue for tracker
xoutTracker = pipeline.create(dai.node.XLinkOut)
xoutTracker.setStreamName("tracklets")
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetwork.setBlob(custom_model_blob)
detectionNetwork.setConfidenceThreshold(0.7)
# YOLO-specific parameters
detectionNetwork.setNumClasses(numClasses)
detectionNetwork.setCoordinateSize(4)
detectionNetwork.setAnchors([])
detectionNetwork.setAnchorMasks({})
detectionNetwork.setIouThreshold(0.5)
detectionNetwork.input.setBlocking(False)
# Convert the grayscale frame into the nn-acceptable form
manip.initialConfig.setResize(320, 320)
# The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case)
manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Link nodes
monoL.out.link(manip.inputImage)
manip.out.link(detectionNetwork.input)
detectionNetwork.out.link(xoutNN.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipOut.input)
# Link tracker
detectionNetwork.out.link(objectTracker.inputDetections)
objectTracker.out.link(xoutTracker.input)
frame_count = 0
midpoints = []
# Connect to device and start pipeline
with dai.Device(pipeline) as device:
startTime = time.monotonic()
counter = 0
fps = 0
frame = None
qRight = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
qDet = device.getOutputQueue("nn", maxSize=4, blocking=False)
qTracker = device.getOutputQueue("tracklets", maxSize=4, blocking=False)
detections = []
def frameNorm(frame, bbox):
normVals = np.full(len(bbox), frame.shape[0])
normVals[::2] = frame.shape[1]
return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
while True:
counter+=1
current_time1 = time.monotonic()
if (current_time1 - startTime) > 1 :
fps = counter / (current_time1 - startTime)
counter = 0
startTime = current_time1
imageQueueData = qRight.tryGet()
detectQueueData = qDet.tryGet()
trackletsQueueData = qTracker.tryGet()
current_time = time.localtime()
time_text = f"{str(current_time.tm_mon).zfill(2)}/{str(current_time.tm_mday).zfill(2)} : {str(current_time.tm_hour).zfill(2)} : {str(current_time.tm_min).zfill(2)} : {str(current_time.tm_sec).zfill(2)}"
if imageQueueData is not None:
frame = imageQueueData.getCvFrame()
frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
cv2.putText(frame, time_text, (420, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 2)
if detectQueueData is not None:
detections = detectQueueData.detections
if frame is not None:
cv2.putText(frame, "fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
for detection in detections:
bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
center = (int((bbox[0] + bbox[2]) / 2), int((bbox[1] + bbox[3]) / 2))
cv2.circle(frame, (center[0], center[1]), 25, (0, 255, 0), 2)
if trackletsQueueData is not None:
tracklets = trackletsQueueData.tracklets
for tracklet in tracklets:
roi = tracklet.roi.denormalize(frame.shape[1], frame.shape[0])
xmin, ymin, xmax, ymax = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)
id = tracklet.id
label = tracklet.label
cv2.putText(frame, f"ID: {id}", (xmin, ymin - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
cv2.putText(frame, f"Detections: {len(detections)}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2)
cv2.imshow("ir detect", frame)
if cv2.waitKey(1) == ord("q"):
break
cv2.destroyAllWindows()
thanks,
Li
Hi @jakaskerl
Yes, I can training yolo model ! It just the object tracking example is using MBNet ,but I want to use yolo
Hi @jakaskerl
I think I will try training MobileMe Ssd model,Do you have any tutorial can training
Hi @jakaskerl
Can i object tracking by yolodetect not mobilenet ssd. Or If I train my own mobilennet ssd model so I can run this
Thanks,
Li
Hi @jakaskerl
it can run ,but can't detect right ,full of wrong boundingbox , even I raise threshold
can i object tracking by yolodetect not mobilenet ssd
Hi @jakaskerl
I can't run the code you send ,but i can run https://docs.luxonis.com/projects/api/en/latest/samples/Yolo/tiny_yolo/
Hi @jakaskerl
After I change model ,it works and print(detections) show [<depthai.ImgDetection object at 0x000002276FD7ECF0>, <depthai.ImgDetection object at 0x000002276FCD8CB0>
it works ,but i have to delete two line(that is not a problem)
#cv2.putText(frame, labelMap[detection.label]….
#cv2.putText(frame, f"{int(detection.confidence * 100)}%…….
I modify this three place
#change my model
nnPathDefault = str((Path(__file__).parent / Path('C:\catdog320.blob')).resolve().absolute())
#MobileNetDetectionNetwork -> YoloDetectionNetwork
detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
detectionNetwork.setBlobPath(nnPathDefault)
it show nothing on the frame
Do you mean this trained model's format is mobilennetssd ?
Hi @jakaskerl
I don’t know how you test my model ,but I’m pretty sure it’s work on this Yolodetectionnetwork,The Only wondering me is why the same model in blob format is not work on the mobilnetdetection ,
I had always use Depthai Tools to convert the model from .pt into .blob
Thanks,
Li
Hi @jakaskerl
Raising the confident threshold doesn't change anything, iou is not allow to setting in this example,
This my model(.pt & .blob) There is no problem with these two models in detection, it just can't do tracking.
Hi @jakaskerl
The link is using the same Depthai Tools to transfer model as I did ,It's just that I'm not using Colab rather and using my laptop,I think there should be no difference,
I refer to the old yoloDetect example and modify it like this
import cv2
import depthai as dai
import argparse
# Specify the path to your own model weights file
model_weights_path = "C:/catdog320.blob"
parser = argparse.ArgumentParser()
parser.add_argument('nnPath', nargs='?', help="Path to mobilenet detection network blob", default=model_weights_path)
parser.add_argument('-ff', '--full_frame', action="store_true", help="Perform tracking on full RGB frame", default=False)
args = parser.parse_args()
fullFrameTracking = args.full_frame
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
numClasses = 80
dim = next(iter(custom_model_blob.networkInputs.values())).dims
output_name, output_tenser = next(iter(custom_model_blob.networkOutputs.items()))
numClasses = output_tenser.dims[2] - 5
labelMap = ["class_%s" % i
for i in range(numClasses)
]
# Create a pipeline
pipeline = dai.Pipeline()
monoL = pipeline.create(dai.node.MonoCamera)
manip = pipeline.create(dai.node.ImageManip)
detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
objectTracker = pipeline.create(dai.node.ObjectTracker)
manipOut = pipeline.create(dai.node.XLinkOut)
trackerOut = pipeline.create(dai.node.XLinkOut)
manipOut.setStreamName('flood-left')
trackerOut.setStreamName("tracklets")
monoL.setNumFramesPool(24)
monoL.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
floodBright = 0.1
node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
while True:
event = node.io['event'].get()
Device.setIrFloodLightIntensity(floodBright)
frameL = node.io['frameL'].get()
node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetwork.setBlob(custom_model_blob)
detectionNetwork.setConfidenceThreshold(0.7)
detectionNetwork.input.setBlocking(False)
objectTracker.setDetectionLabelsToTrack([]) #all
objectTracker.setTrackerType(dai.TrackerType.ZERO_TERM_COLOR_HISTOGRAM)
objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.SMALLEST_ID)
manip.initialConfig.setResize(320, 320)
manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
#link
monoL.out.link(manip.inputImage)
manip.out.link(detectionNetwork.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipOut.input)
if fullFrameTracking:
manip.video.link(objectTracker.inputTrackerFrame)
else:
detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
detectionNetwork.out.link(objectTracker.inputDetections)
objectTracker.out.link(trackerOut.input)
# Connect to the device and start the Pipeline
with dai.Device(pipeline) as device:
preview = device.getOutputQueue("flood-left", 4, False)
tracklets = device.getOutputQueue(name="tracklets", maxSize=4, blocking=False)
while True:
imgFrame = preview.get()
track = tracklets.get()
print('track',track)
color = (255, 0, 0)
frame = imgFrame.getCvFrame()
trackletsData = track.tracklets
print('trackdata',trackletsData)
for t in trackletsData:
print('t',t)
roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
x1 = int(roi.topLeft().x)
y1 = int(roi.topLeft().y)
x2 = int(roi.bottomRight().x)
y2 = int(roi.bottomRight().y)
try:
label = labelMap[t.label]
except:
label = t.label
cv2.putText(frame, str(label), (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
cv2.putText(frame, f"ID: {[t.id]}", (x1 + 10, y1 + 35), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
cv2.putText(frame, t.status.name, (x1 + 10, y1 + 50), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)
cv2.rectangle(frame, (x1, y1), (x2, y2), color, cv2.FONT_HERSHEY_SIMPLEX)
cv2.imshow("tracker", frame)
if cv2.waitKey(1) == ord('q'):
break
The program can run but cannot recognize anything, If adding detectionNetwork.out.link(trackerOut.input),
it will show " trackletsData = track.tracklets AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets' " ,Can you help me debug
Hi @jakaskerl
I had try my own model on Mono & MobilenetSSD ,but It don't detect correct ,The Screen fall of wrong bounding box, I don't know why it doesn't work, But the same model at the old example is working(the old example is not using MobilenetSSD ,is rather like the bottom of https://www.oakchina.cn/2023/02/24/yolov8-blob/ )
I had modify the Mono & MobilenetSSD is
nnPath = str((Path(__file__).parent / Path('C:/egg.blob')).resolve().absolute())
labelMap = ["egg"]
manip.initialConfig.setResize(320, 320)
There’s nothing wrong with using Depthai Tools to transfer yolov8 model, right?
Thanks,
LI