L

li_you_chen

hello @erik

I tried to write a program like the following, but it couldn't display frame as expected.

it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.and imageQueueDataIR = qIR.tryGet()

RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'

The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.

This is my code,

import cv2

import depthai as dai

import numpy as np

import time

# Path to the model weights file

model_weights_path = "yolov8n.blob"

# Load model weights using dai.OpenVINO.Blob

custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

# Create a pipeline

pipeline = dai.Pipeline()

res = dai.MonoCameraProperties.SensorResolution.THE_400_P

# Define IR camera and outputs

monoL = pipeline.create(dai.node.MonoCamera)

monoL.setResolution(res)

monoL.setFps(30)

manipIR = pipeline.create(dai.node.ImageManip)

manipIROut = pipeline.create(dai.node.XLinkOut)

detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerIR = pipeline.create(dai.node.ObjectTracker)

trackerIROut = pipeline.create(dai.node.XLinkOut)

# Define RGB camera and outputs

rgbCamera = pipeline.create(dai.node.ColorCamera)

rgbCamera.setPreviewSize(320, 320)

rgbCamera.setInterleaved(False)

rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)

rgbCamera.setFps(30)

manipRGB = pipeline.create(dai.node.ImageManip)

manipRGBOut = pipeline.create(dai.node.XLinkOut)

detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)

objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)

trackerRGBOut = pipeline.create(dai.node.XLinkOut)

# Set stream names for outputs

trackerIROut.setStreamName("trackletsIR")

manipIROut.setStreamName('flood-left')

trackerRGBOut.setStreamName("trackletsRGB")

manipRGBOut.setStreamName('rgb-out')

# Script node for frame routing and IR dot/flood alternate

script = pipeline.create(dai.node.Script)

script.setProcessor(dai.ProcessorType.LEON_CSS)

script.setScript("""

floodBright = 0.1

node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

while True:

event = node.io['event'].get()

Device.setIrFloodLightIntensity(floodBright)

frameL = node.io['frameL'].get()

node.io['floodL'].send(frameL)

""")

# Model-specific settings

detectionNetworkIR.setBlob(custom_model_blob)

detectionNetworkIR.setConfidenceThreshold(0.7)

detectionNetworkRGB.setBlob(custom_model_blob)

detectionNetworkRGB.setConfidenceThreshold(0.7)

# YOLO specific parameters for IR

detectionNetworkIR.setNumClasses(2)

detectionNetworkIR.setCoordinateSize(4)

detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkIR.setIouThreshold(0.5)

detectionNetworkIR.input.setBlocking(False)

# YOLO specific parameters for RGB

detectionNetworkRGB.setNumClasses(2)

detectionNetworkRGB.setCoordinateSize(4)

detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

detectionNetworkRGB.setIouThreshold(0.5)

detectionNetworkRGB.input.setBlocking(False)

# Convert the grayscale frame into a format acceptable by the neural network for IR

manipIR.initialConfig.setResize(320, 320)

manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Convert the color frame into a format acceptable by the neural network for RGB

manipRGB.initialConfig.setResize(320, 320)

manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

# Object tracker settings for IR

objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerIR.setTrackerThreshold(0.5)

# Object tracker settings for RGB

objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])

objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

objectTrackerRGB.setTrackerThreshold(0.5)

# Connect nodes in the pipeline for IR

monoL.out.link(manipIR.inputImage)

manipIR.out.link(detectionNetworkIR.input)

detectionNetworkIR.out.link(objectTrackerIR.inputDetections)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)

detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)

objectTrackerIR.out.link(trackerIROut.input)

monoL.frameEvent.link(script.inputs['event'])

monoL.out.link(script.inputs['frameL'])

script.outputs['floodL'].link(manipIROut.input)

# Connect nodes in the pipeline for RGB

rgbCamera.preview.link(manipRGB.inputImage)

manipRGB.out.link(detectionNetworkRGB.input)

detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)

detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)

objectTrackerRGB.out.link(trackerRGBOut.input)

# Function to normalize frame coordinates

def frameNorm(frame, bbox):

normVals = np.full(len(bbox), frame.shape[0])

normVals[::2] = frame.shape[1]

return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

# Connect device and start pipeline

try:

with dai.Device(pipeline) as device:

startTime = time.monotonic()

counter = 0

fps = 0

frameIR = None

frameRGB = None

frame_detectIR = None

frame_detectRGB = None

qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)

qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)

trackletsIR = device.getOutputQueue("trackletsIR", 4, False)

trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)

while True:

counter += 1

current_time1 = time.monotonic()

if (current_time1 - startTime) > 1:

fps = counter / (current_time1 - startTime)

counter = 0

startTime = current_time1

# IR camera processing

imageQueueDataIR = qIR.tryGet()

if imageQueueDataIR is not None:

frameIR = imageQueueDataIR.getCvFrame()

frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)

frameIR = frameIR[:, 120:520]

frame_detectIR = frameIR.copy()

cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackIR = trackletsIR.get()

trackletsDataIR = trackIR.tracklets

for t in trackletsDataIR:

roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)

cv2.imshow("IR detect", frame_detectIR)

# RGB camera processing

imageQueueDataRGB = qRGB.tryGet()

if imageQueueDataRGB is not None:

frameRGB = imageQueueDataRGB.getCvFrame()

frame_detectRGB = frameRGB.copy()

cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

trackRGB = trackletsRGB.get()

trackletsDataRGB = trackRGB.tracklets

for t in trackletsDataRGB:

roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])

x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

if t.status.name == 'TRACKED':

cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)

cv2.imshow("RGB detect", frame_detectRGB)

if cv2.waitKey(1) == 27:

break

finally:

cv2.destroyAllWindows()

thanks,

Li

@erik Here are some pictures for your reference

I am using the code provided above and removing

objectTracker.passthroughTrackerFrame.link(trackerOut.input)

And draw a circle with a fixed radius at the midpoint of roi

Thanks,

Li

Hi @jakaskerl

After I modify ,It show

trackletsData = track.tracklets

AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'

trackletsData = track.tracklets
AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'

#!/usr/bin/env python3
import cv2
import depthai as dai
import numpy as np
import time
# Path to the model weights file
model_weights_path = "C:/piegg.blob"
labelMap = ["egg", "stick"]
# Load model weights using dai.OpenVINO.Blob
custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
# Create a pipeline
pipeline = dai.Pipeline()
res = dai.MonoCameraProperties.SensorResolution.THE_400_P
# Define camera and outputs
monoL = pipeline.create(dai.node.MonoCamera)
monoL.setResolution(res)
monoL.setFps(30)
manip = pipeline.create(dai.node.ImageManip)
manipOut = pipeline.create(dai.node.XLinkOut)
detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
objectTracker = pipeline.create(dai.node.ObjectTracker)
trackerOut = pipeline.create(dai.node.XLinkOut)
# Set stream names for outputs
trackerOut.setStreamName("tracklets")
manipOut.setStreamName('flood-left')
# Script node for frame routing and IR dot/flood alternate
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
    floodBright = 0.1
    node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
    while True:
        event = node.io['event'].get()
        Device.setIrFloodLightIntensity(floodBright)
        frameL = node.io['frameL'].get()
        node.io['floodL'].send(frameL)
""")
# Model-specific settings
detectionNetwork.setBlob(custom_model_blob)
detectionNetwork.setConfidenceThreshold(0.8)
# YOLO specific parameters
detectionNetwork.setNumClasses(2)
detectionNetwork.setCoordinateSize(4)
detectionNetwork.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
detectionNetwork.setIouThreshold(0.5)
detectionNetwork.input.setBlocking(False)
# Convert the grayscale frame into a format acceptable by the neural network
manip.initialConfig.setResize(320, 320)
manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
# Object tracker settings
objectTracker.setDetectionLabelsToTrack([0, 1, 2])
objectTracker.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
objectTracker.setTrackerThreshold(0.85)
# Connect nodes in the pipeline
monoL.out.link(manip.inputImage)
manip.out.link(detectionNetwork.input)
detectionNetwork.out.link(objectTracker.inputDetections)
detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
objectTracker.passthroughTrackerFrame.link(trackerOut.input)
objectTracker.out.link(trackerOut.input)
monoL.frameEvent.link(script.inputs['event'])
monoL.out.link(script.inputs['frameL'])
script.outputs['floodL'].link(manipOut.input)
# Function to normalize frame coordinates
def frameNorm(frame, bbox):
    normVals = np.full(len(bbox), frame.shape[0])
    normVals[::2] = frame.shape[1]
    return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
# Connect device and start pipeline
with dai.Device(pipeline) as device:
    startTime = time.monotonic()
    counter = 0
    fps = 0
    frame = None
    qRight = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
    tracklets = device.getOutputQueue("tracklets", 4, False)
    while True:
        counter += 1
        current_time1 = time.monotonic()
        if (current_time1 - startTime) > 1:
            fps = counter / (current_time1 - startTime)
            counter = 0
            startTime = current_time1
        imageQueueData = qRight.tryGet()
        track = tracklets.get()
        trackletsData = track.tracklets
        if imageQueueData is not None:
            frame = imageQueueData.getCvFrame()
        if frame is not None:
            frame_detect = frame.copy()
            cv2.putText(frame_detect, "fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
            
            for t in trackletsData:
                # Get region of interest (ROI) and draw circle on detected objects
                roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
                x1 = int(roi.topLeft().x)
                y1 = int(roi.topLeft().y)
                x2 = int(roi.bottomRight().x)
                y2 = int(roi.bottomRight().y)
                cv2.circle(frame_detect, ((x1 + x2) // 2, (y1 + y2) // 2), 15, (0, 0, 255), 2)
                
            # Display the frame with detections
            cv2.imshow("ir detect", frame_detect)
            # Exit the loop when 'q' is pressed
            if cv2.waitKey(1) == ord("q"):
                break
cv2.destroyAllWindows()

Hi @jakaskerl

it can run ,but can't detect right ,full of wrong boundingbox , even I raise threshold

can i object tracking by yolodetect not mobilenet ssd

Hi @jakaskerl

I had try my own model on Mono & MobilenetSSD ,but It don't detect correct ,The Screen fall of wrong bounding box, I don't know why it doesn't work, But the same model at the old example is working(the old example is not using MobilenetSSD ,is rather like the bottom of https://www.oakchina.cn/2023/02/24/yolov8-blob/ )

I had modify the Mono & MobilenetSSD is

nnPath = str((Path(__file__).parent / Path('C:/egg.blob')).resolve().absolute())

labelMap = ["egg"]

manip.initialConfig.setResize(320, 320)

There’s nothing wrong with using Depthai Tools to transfer yolov8 model, right?

Thanks,

LI