L
li_you_chen

  • 5 days ago
  • Joined Mar 31, 2024
  • 1 best answer
  • hi @erik sorry I didn't notice that.

    
    import cv2
    import depthai as dai
    import numpy as np
    import time
    # Path to the model weights file
    model_weights_path = "yolov8n.blob"
    # Load model weights using dai.OpenVINO.Blob
    custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
    # Create a pipeline
    pipeline = dai.Pipeline()
    res = dai.MonoCameraProperties.SensorResolution.THE_400_P
    # Define IR camera and outputs
    monoL = pipeline.create(dai.node.MonoCamera)
    monoL.setResolution(res)
    monoL.setFps(30)
    manipIR = pipeline.create(dai.node.ImageManip)
    manipIROut = pipeline.create(dai.node.XLinkOut)
    detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
    objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
    trackerIROut = pipeline.create(dai.node.XLinkOut)
    # Define RGB camera and outputs
    rgbCamera = pipeline.create(dai.node.ColorCamera)
    rgbCamera.setPreviewSize(320, 320)
    rgbCamera.setInterleaved(False)
    rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
    rgbCamera.setFps(30)
    manipRGB = pipeline.create(dai.node.ImageManip)
    manipRGBOut = pipeline.create(dai.node.XLinkOut)
    detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
    objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
    trackerRGBOut = pipeline.create(dai.node.XLinkOut)
    # Set stream names for outputs
    trackerIROut.setStreamName("trackletsIR")
    manipIROut.setStreamName('flood_left')
    trackerRGBOut.setStreamName("trackletsRGB")
    manipRGBOut.setStreamName('rgb_out')
    # Script node for frame routing and IR dot/flood alternate
    script = pipeline.create(dai.node.Script)
    script.setProcessor(dai.ProcessorType.LEON_CSS)
    script.setScript("""
    
    floodBright = 0.1
    node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
    while True:
        event = node.io['event'].get()
        Device.setIrFloodLightIntensity(floodBright)
        frameL = node.io['frameL'].get()
        node.io['floodL'].send(frameL)
    
    """)
    # Model-specific settings
    detectionNetworkIR.setBlob(custom_model_blob)
    detectionNetworkIR.setConfidenceThreshold(0.7)
    detectionNetworkRGB.setBlob(custom_model_blob)
    detectionNetworkRGB.setConfidenceThreshold(0.7)
    # YOLO specific parameters for IR
    detectionNetworkIR.setNumClasses(2)
    detectionNetworkIR.setCoordinateSize(4)
    detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
    detectionNetworkIR.setIouThreshold(0.5)
    detectionNetworkIR.input.setBlocking(False)
    # YOLO specific parameters for RGB
    detectionNetworkRGB.setNumClasses(2)
    detectionNetworkRGB.setCoordinateSize(4)
    detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
    detectionNetworkRGB.setIouThreshold(0.5)
    detectionNetworkRGB.input.setBlocking(False)
    # Convert the grayscale frame into a format acceptable by the neural network for IR
    manipIR.initialConfig.setResize(320, 320)
    manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
    # Convert the color frame into a format acceptable by the neural network for RGB
    manipRGB.initialConfig.setResize(320, 320)
    manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
    # Object tracker settings for IR
    objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
    objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
    objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
    objectTrackerIR.setTrackerThreshold(0.5)
    # Object tracker settings for RGB
    objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
    objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
    objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
    objectTrackerRGB.setTrackerThreshold(0.5)
    # Connect nodes in the pipeline for IR
    monoL.out.link(manipIR.inputImage)
    manipIR.out.link(detectionNetworkIR.input)
    detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
    detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
    detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
    objectTrackerIR.out.link(trackerIROut.input)
    monoL.frameEvent.link(script.inputs['event'])
    monoL.out.link(script.inputs['frameL'])
    script.outputs['floodL'].link(manipIROut.input)
    # Connect nodes in the pipeline for RGB
    rgbCamera.preview.link(manipRGB.inputImage)
    manipRGB.out.link(detectionNetworkRGB.input)
    detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
    objectTrackerRGB.out.link(trackerRGBOut.input)
    # Function to normalize frame coordinates
    def frameNorm(frame, bbox):
    
    normVals = np.full(len(bbox), frame.shape[0])
    normVals[::2] = frame.shape[1]
    return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)
    
    # Connect device and start pipeline
    try:
    
    with dai.Device(pipeline) as device:
        startTime = time.monotonic()
        counter = 0
        fps = 0
        frameIR = None
        frameRGB = None
        frame_detectIR = None
        frame_detectRGB = None
        qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
        qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
        trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
        trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
        
    while True: counter += 1 current_time1 = time.monotonic() if (current_time1 - startTime) > 1: fps = counter / (current_time1 - startTime) counter = 0 startTime = current_time1
    # IR camera processing imageQueueDataIR = qIR.tryGet() if imageQueueDataIR is not None: frameIR = imageQueueDataIR.getCvFrame() frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR) frameIR = frameIR[:, 120:520] frame_detectIR = frameIR.copy() cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackIR = trackletsIR.get() trackletsDataIR = trackIR.tracklets for t in trackletsDataIR: roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2) # RGB camera processing imageQueueDataRGB = qRGB.tryGet() if imageQueueDataRGB is not None: frameRGB = imageQueueDataRGB.getCvFrame() frame_detectRGB = frameRGB.copy() cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackRGB = trackletsRGB.get() trackletsDataRGB = trackRGB.tracklets for t in trackletsDataRGB: roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.imshow("RGB detect", frame_detectRGB) cv2.imshow("IR detect", frame_detectIR) if cv2.waitKey(1) == 27: break finally: cv2.destroyAllWindows() ``` [1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)] [2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: Traceback (most recent call last): line 152, in <module> imageQueueDataIR = qIR.tryGet() RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'
  • hello @erik

    I tried to write a program like the following, but it couldn't display frame as expected.

    it show Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: C:\Users\USER\YOLO\.cache\depthai\crashdumps\c02add63364f33fafa5876087ab5a5831b5d7c42\crash_dump.json - please report to developers.and imageQueueDataIR = qIR.tryGet()

    RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood-left' (X_LINK_ERROR)'

    The IR screen will be displayed first, then it will crash. It will try to open the lens again after listening to the computer prompts, and then close the program completely , please help me solve it.

    This is my code,

    import cv2

    import depthai as dai

    import numpy as np

    import time

    # Path to the model weights file

    model_weights_path = "yolov8n.blob"

    # Load model weights using dai.OpenVINO.Blob

    custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

    # Create a pipeline

    pipeline = dai.Pipeline()

    res = dai.MonoCameraProperties.SensorResolution.THE_400_P

    # Define IR camera and outputs

    monoL = pipeline.create(dai.node.MonoCamera)

    monoL.setResolution(res)

    monoL.setFps(30)

    manipIR = pipeline.create(dai.node.ImageManip)

    manipIROut = pipeline.create(dai.node.XLinkOut)

    detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)

    objectTrackerIR = pipeline.create(dai.node.ObjectTracker)

    trackerIROut = pipeline.create(dai.node.XLinkOut)

    # Define RGB camera and outputs

    rgbCamera = pipeline.create(dai.node.ColorCamera)

    rgbCamera.setPreviewSize(320, 320)

    rgbCamera.setInterleaved(False)

    rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)

    rgbCamera.setFps(30)

    manipRGB = pipeline.create(dai.node.ImageManip)

    manipRGBOut = pipeline.create(dai.node.XLinkOut)

    detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)

    objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)

    trackerRGBOut = pipeline.create(dai.node.XLinkOut)

    # Set stream names for outputs

    trackerIROut.setStreamName("trackletsIR")

    manipIROut.setStreamName('flood-left')

    trackerRGBOut.setStreamName("trackletsRGB")

    manipRGBOut.setStreamName('rgb-out')

    # Script node for frame routing and IR dot/flood alternate

    script = pipeline.create(dai.node.Script)

    script.setProcessor(dai.ProcessorType.LEON_CSS)

    script.setScript("""

    floodBright = 0.1

    node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

    while True:

    event = node.io['event'].get()

    Device.setIrFloodLightIntensity(floodBright)

    frameL = node.io['frameL'].get()

    node.io['floodL'].send(frameL)

    """)

    # Model-specific settings

    detectionNetworkIR.setBlob(custom_model_blob)

    detectionNetworkIR.setConfidenceThreshold(0.7)

    detectionNetworkRGB.setBlob(custom_model_blob)

    detectionNetworkRGB.setConfidenceThreshold(0.7)

    # YOLO specific parameters for IR

    detectionNetworkIR.setNumClasses(2)

    detectionNetworkIR.setCoordinateSize(4)

    detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

    detectionNetworkIR.setIouThreshold(0.5)

    detectionNetworkIR.input.setBlocking(False)

    # YOLO specific parameters for RGB

    detectionNetworkRGB.setNumClasses(2)

    detectionNetworkRGB.setCoordinateSize(4)

    detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})

    detectionNetworkRGB.setIouThreshold(0.5)

    detectionNetworkRGB.input.setBlocking(False)

    # Convert the grayscale frame into a format acceptable by the neural network for IR

    manipIR.initialConfig.setResize(320, 320)

    manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

    # Convert the color frame into a format acceptable by the neural network for RGB

    manipRGB.initialConfig.setResize(320, 320)

    manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

    # Object tracker settings for IR

    objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])

    objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

    objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

    objectTrackerIR.setTrackerThreshold(0.5)

    # Object tracker settings for RGB

    objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])

    objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)

    objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)

    objectTrackerRGB.setTrackerThreshold(0.5)

    # Connect nodes in the pipeline for IR

    monoL.out.link(manipIR.inputImage)

    manipIR.out.link(detectionNetworkIR.input)

    detectionNetworkIR.out.link(objectTrackerIR.inputDetections)

    detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)

    detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)

    objectTrackerIR.out.link(trackerIROut.input)

    monoL.frameEvent.link(script.inputs['event'])

    monoL.out.link(script.inputs['frameL'])

    script.outputs['floodL'].link(manipIROut.input)

    # Connect nodes in the pipeline for RGB

    rgbCamera.preview.link(manipRGB.inputImage)

    manipRGB.out.link(detectionNetworkRGB.input)

    detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)

    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)

    detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)

    objectTrackerRGB.out.link(trackerRGBOut.input)

    # Function to normalize frame coordinates

    def frameNorm(frame, bbox):

    normVals = np.full(len(bbox), frame.shape[0])

    normVals[::2] = frame.shape[1]

    return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

    # Connect device and start pipeline

    try:

    with dai.Device(pipeline) as device:

    startTime = time.monotonic()

    counter = 0

    fps = 0

    frameIR = None

    frameRGB = None

    frame_detectIR = None

    frame_detectRGB = None

    qIR = device.getOutputQueue("flood-left", maxSize=4, blocking=False)

    qRGB = device.getOutputQueue("rgb-out", maxSize=4, blocking=False)

    trackletsIR = device.getOutputQueue("trackletsIR", 4, False)

    trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)

    while True:

    counter += 1

    current_time1 = time.monotonic()

    if (current_time1 - startTime) > 1:

    fps = counter / (current_time1 - startTime)

    counter = 0

    startTime = current_time1

    # IR camera processing

    imageQueueDataIR = qIR.tryGet()

    if imageQueueDataIR is not None:

    frameIR = imageQueueDataIR.getCvFrame()

    frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR)

    frameIR = frameIR[:, 120:520]

    frame_detectIR = frameIR.copy()

    cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

    trackIR = trackletsIR.get()

    trackletsDataIR = trackIR.tracklets

    for t in trackletsDataIR:

    roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0])

    x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

    if t.status.name == 'TRACKED':

    cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2)

    cv2.imshow("IR detect", frame_detectIR)

    # RGB camera processing

    imageQueueDataRGB = qRGB.tryGet()

    if imageQueueDataRGB is not None:

    frameRGB = imageQueueDataRGB.getCvFrame()

    frame_detectRGB = frameRGB.copy()

    cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

    trackRGB = trackletsRGB.get()

    trackletsDataRGB = trackRGB.tracklets

    for t in trackletsDataRGB:

    roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0])

    x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

    if t.status.name == 'TRACKED':

    cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2)

    cv2.imshow("RGB detect", frame_detectRGB)

    if cv2.waitKey(1) == 27:

    break

    finally:

    cv2.destroyAllWindows()

    thanks,

    Li

  • I want to use IR and RGB for object tracking at the same time in Oak-d Pro. Both images have their own tracking results. Is this possible?

    Thanks,

    Li

    • hi @erik sorry I didn't notice that.

      
      import cv2
      import depthai as dai
      import numpy as np
      import time
      # Path to the model weights file
      model_weights_path = "yolov8n.blob"
      # Load model weights using dai.OpenVINO.Blob
      custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
      # Create a pipeline
      pipeline = dai.Pipeline()
      res = dai.MonoCameraProperties.SensorResolution.THE_400_P
      # Define IR camera and outputs
      monoL = pipeline.create(dai.node.MonoCamera)
      monoL.setResolution(res)
      monoL.setFps(30)
      manipIR = pipeline.create(dai.node.ImageManip)
      manipIROut = pipeline.create(dai.node.XLinkOut)
      detectionNetworkIR = pipeline.create(dai.node.YoloDetectionNetwork)
      objectTrackerIR = pipeline.create(dai.node.ObjectTracker)
      trackerIROut = pipeline.create(dai.node.XLinkOut)
      # Define RGB camera and outputs
      rgbCamera = pipeline.create(dai.node.ColorCamera)
      rgbCamera.setPreviewSize(320, 320)
      rgbCamera.setInterleaved(False)
      rgbCamera.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
      rgbCamera.setFps(30)
      manipRGB = pipeline.create(dai.node.ImageManip)
      manipRGBOut = pipeline.create(dai.node.XLinkOut)
      detectionNetworkRGB = pipeline.create(dai.node.YoloDetectionNetwork)
      objectTrackerRGB = pipeline.create(dai.node.ObjectTracker)
      trackerRGBOut = pipeline.create(dai.node.XLinkOut)
      # Set stream names for outputs
      trackerIROut.setStreamName("trackletsIR")
      manipIROut.setStreamName('flood_left')
      trackerRGBOut.setStreamName("trackletsRGB")
      manipRGBOut.setStreamName('rgb_out')
      # Script node for frame routing and IR dot/flood alternate
      script = pipeline.create(dai.node.Script)
      script.setProcessor(dai.ProcessorType.LEON_CSS)
      script.setScript("""
      
      floodBright = 0.1
      node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
      while True:
          event = node.io['event'].get()
          Device.setIrFloodLightIntensity(floodBright)
          frameL = node.io['frameL'].get()
          node.io['floodL'].send(frameL)
      
      """)
      # Model-specific settings
      detectionNetworkIR.setBlob(custom_model_blob)
      detectionNetworkIR.setConfidenceThreshold(0.7)
      detectionNetworkRGB.setBlob(custom_model_blob)
      detectionNetworkRGB.setConfidenceThreshold(0.7)
      # YOLO specific parameters for IR
      detectionNetworkIR.setNumClasses(2)
      detectionNetworkIR.setCoordinateSize(4)
      detectionNetworkIR.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
      detectionNetworkIR.setIouThreshold(0.5)
      detectionNetworkIR.input.setBlocking(False)
      # YOLO specific parameters for RGB
      detectionNetworkRGB.setNumClasses(2)
      detectionNetworkRGB.setCoordinateSize(4)
      detectionNetworkRGB.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
      detectionNetworkRGB.setIouThreshold(0.5)
      detectionNetworkRGB.input.setBlocking(False)
      # Convert the grayscale frame into a format acceptable by the neural network for IR
      manipIR.initialConfig.setResize(320, 320)
      manipIR.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
      # Convert the color frame into a format acceptable by the neural network for RGB
      manipRGB.initialConfig.setResize(320, 320)
      manipRGB.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
      # Object tracker settings for IR
      objectTrackerIR.setDetectionLabelsToTrack([0, 1, 2])
      objectTrackerIR.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
      objectTrackerIR.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
      objectTrackerIR.setTrackerThreshold(0.5)
      # Object tracker settings for RGB
      objectTrackerRGB.setDetectionLabelsToTrack([0, 1, 2])
      objectTrackerRGB.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
      objectTrackerRGB.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
      objectTrackerRGB.setTrackerThreshold(0.5)
      # Connect nodes in the pipeline for IR
      monoL.out.link(manipIR.inputImage)
      manipIR.out.link(detectionNetworkIR.input)
      detectionNetworkIR.out.link(objectTrackerIR.inputDetections)
      detectionNetworkIR.passthrough.link(objectTrackerIR.inputTrackerFrame)
      detectionNetworkIR.passthrough.link(objectTrackerIR.inputDetectionFrame)
      objectTrackerIR.out.link(trackerIROut.input)
      monoL.frameEvent.link(script.inputs['event'])
      monoL.out.link(script.inputs['frameL'])
      script.outputs['floodL'].link(manipIROut.input)
      # Connect nodes in the pipeline for RGB
      rgbCamera.preview.link(manipRGB.inputImage)
      manipRGB.out.link(detectionNetworkRGB.input)
      detectionNetworkRGB.out.link(objectTrackerRGB.inputDetections)
      detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputTrackerFrame)
      detectionNetworkRGB.passthrough.link(objectTrackerRGB.inputDetectionFrame)
      objectTrackerRGB.out.link(trackerRGBOut.input)
      # Function to normalize frame coordinates
      def frameNorm(frame, bbox):
      
      normVals = np.full(len(bbox), frame.shape[0])
      normVals[::2] = frame.shape[1]
      return (np.clip(np.array(bbox), 0, 1) \* normVals).astype(int)
      
      # Connect device and start pipeline
      try:
      
      with dai.Device(pipeline) as device:
          startTime = time.monotonic()
          counter = 0
          fps = 0
          frameIR = None
          frameRGB = None
          frame_detectIR = None
          frame_detectRGB = None
          qIR = device.getOutputQueue("flood_left", maxSize=4, blocking=False)
          qRGB = device.getOutputQueue("rgb_out", maxSize=4, blocking=False)
          trackletsIR = device.getOutputQueue("trackletsIR", 4, False)
          trackletsRGB = device.getOutputQueue("trackletsRGB", 4, False)
          
      while True: counter += 1 current_time1 = time.monotonic() if (current_time1 - startTime) > 1: fps = counter / (current_time1 - startTime) counter = 0 startTime = current_time1
      # IR camera processing imageQueueDataIR = qIR.tryGet() if imageQueueDataIR is not None: frameIR = imageQueueDataIR.getCvFrame() frameIR = cv2.cvtColor(frameIR, cv2.COLOR_GRAY2BGR) frameIR = frameIR[:, 120:520] frame_detectIR = frameIR.copy() cv2.putText(frame_detectIR, "IR fps: {:.2f}".format(fps), (2, frameIR.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackIR = trackletsIR.get() trackletsDataIR = trackIR.tracklets for t in trackletsDataIR: roi = t.roi.denormalize(frameIR.shape[1], frameIR.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectIR, (x1, y1), (x2, y2), (25, 25, 255), 2) # RGB camera processing imageQueueDataRGB = qRGB.tryGet() if imageQueueDataRGB is not None: frameRGB = imageQueueDataRGB.getCvFrame() frame_detectRGB = frameRGB.copy() cv2.putText(frame_detectRGB, "RGB fps: {:.2f}".format(fps), (2, frameRGB.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1) trackRGB = trackletsRGB.get() trackletsDataRGB = trackRGB.tracklets for t in trackletsDataRGB: roi = t.roi.denormalize(frameRGB.shape[1], frameRGB.shape[0]) x1, y1, x2, y2 = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y) if t.status.name == 'TRACKED': cv2.rectangle(frame_detectRGB, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.imshow("RGB detect", frame_detectRGB) cv2.imshow("IR detect", frame_detectIR) if cv2.waitKey(1) == 27: break finally: cv2.destroyAllWindows() ``` [1844301001CC3AF500] [1.1] [1.716] [Script(12)] [warning] IR drivers detected: [('LM3644', 2, 99)] [2024-08-31 22:46:58.045] [depthai] [error] Device with id 1844301001CC3AF500 has crashed. Crash dump logs are stored in: Traceback (most recent call last): line 152, in <module> imageQueueDataIR = qIR.tryGet() RuntimeError: Communication exception - possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'flood_left' (X_LINK_ERROR)'
  • @erik

    Thank you so much, You solved a big problem for me, have a nice day

    Thanks,

    Li

  • @erik Here are some pictures for your reference

    I am using the code provided above and removing

    objectTracker.passthroughTrackerFrame.link(trackerOut.input)

    And draw a circle with a fixed radius at the midpoint of roi

    Thanks,

    Li

  • @eric-soderberg ,

    Thank you for your reply. You solved my problem. However, when I want to mark a circle on an object, the x-coordinate of the returned object will be offset left and right from the actual position. The middle of the screen will hardly be offset, left and right. The two sides will shift outward more as they get closer to the edge of the screen. I wonder if the image distortion is caused by the wide angle of the camera.

    btw , my camera is Oak-d Pro

    Thanks,

    Li

    • erik replied to this.
    • Hi @jakaskerl

      After I modify ,It show

      trackletsData = track.tracklets

      AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'

      
      trackletsData = track.tracklets
      AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets'
      
      #!/usr/bin/env python3
      import cv2
      import depthai as dai
      import numpy as np
      import time
      # Path to the model weights file
      model_weights_path = "C:/piegg.blob"
      labelMap = ["egg", "stick"]
      # Load model weights using dai.OpenVINO.Blob
      custom_model_blob = dai.OpenVINO.Blob(model_weights_path)
      # Create a pipeline
      pipeline = dai.Pipeline()
      res = dai.MonoCameraProperties.SensorResolution.THE_400_P
      # Define camera and outputs
      monoL = pipeline.create(dai.node.MonoCamera)
      monoL.setResolution(res)
      monoL.setFps(30)
      manip = pipeline.create(dai.node.ImageManip)
      manipOut = pipeline.create(dai.node.XLinkOut)
      detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)
      objectTracker = pipeline.create(dai.node.ObjectTracker)
      trackerOut = pipeline.create(dai.node.XLinkOut)
      # Set stream names for outputs
      trackerOut.setStreamName("tracklets")
      manipOut.setStreamName('flood-left')
      # Script node for frame routing and IR dot/flood alternate
      script = pipeline.create(dai.node.Script)
      script.setProcessor(dai.ProcessorType.LEON_CSS)
      script.setScript("""
          floodBright = 0.1
          node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')
          while True:
              event = node.io['event'].get()
              Device.setIrFloodLightIntensity(floodBright)
              frameL = node.io['frameL'].get()
              node.io['floodL'].send(frameL)
      """)
      # Model-specific settings
      detectionNetwork.setBlob(custom_model_blob)
      detectionNetwork.setConfidenceThreshold(0.8)
      # YOLO specific parameters
      detectionNetwork.setNumClasses(2)
      detectionNetwork.setCoordinateSize(4)
      detectionNetwork.setAnchorMasks({"side26": [1, 2, 3], "side13": [3, 4, 5]})
      detectionNetwork.setIouThreshold(0.5)
      detectionNetwork.input.setBlocking(False)
      # Convert the grayscale frame into a format acceptable by the neural network
      manip.initialConfig.setResize(320, 320)
      manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)
      # Object tracker settings
      objectTracker.setDetectionLabelsToTrack([0, 1, 2])
      objectTracker.setTrackerType(dai.TrackerType.SHORT_TERM_IMAGELESS)
      objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.UNIQUE_ID)
      objectTracker.setTrackerThreshold(0.85)
      # Connect nodes in the pipeline
      monoL.out.link(manip.inputImage)
      manip.out.link(detectionNetwork.input)
      detectionNetwork.out.link(objectTracker.inputDetections)
      detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
      detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
      objectTracker.passthroughTrackerFrame.link(trackerOut.input)
      objectTracker.out.link(trackerOut.input)
      monoL.frameEvent.link(script.inputs['event'])
      monoL.out.link(script.inputs['frameL'])
      script.outputs['floodL'].link(manipOut.input)
      # Function to normalize frame coordinates
      def frameNorm(frame, bbox):
          normVals = np.full(len(bbox), frame.shape[0])
          normVals[::2] = frame.shape[1]
          return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
      # Connect device and start pipeline
      with dai.Device(pipeline) as device:
          startTime = time.monotonic()
          counter = 0
          fps = 0
          frame = None
          qRight = device.getOutputQueue("flood-left", maxSize=4, blocking=False)
          tracklets = device.getOutputQueue("tracklets", 4, False)
          while True:
              counter += 1
              current_time1 = time.monotonic()
              if (current_time1 - startTime) > 1:
                  fps = counter / (current_time1 - startTime)
                  counter = 0
                  startTime = current_time1
              imageQueueData = qRight.tryGet()
              track = tracklets.get()
              trackletsData = track.tracklets
              if imageQueueData is not None:
                  frame = imageQueueData.getCvFrame()
              if frame is not None:
                  frame_detect = frame.copy()
                  cv2.putText(frame_detect, "fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)
                  
                  for t in trackletsData:
                      # Get region of interest (ROI) and draw circle on detected objects
                      roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
                      x1 = int(roi.topLeft().x)
                      y1 = int(roi.topLeft().y)
                      x2 = int(roi.bottomRight().x)
                      y2 = int(roi.bottomRight().y)
                      cv2.circle(frame_detect, ((x1 + x2) // 2, (y1 + y2) // 2), 15, (0, 0, 255), 2)
                      
                  # Display the frame with detections
                  cv2.imshow("ir detect", frame_detect)
                  # Exit the loop when 'q' is pressed
                  if cv2.waitKey(1) == ord("q"):
                      break
      cv2.destroyAllWindows()
    • I want to use Yolo for object tracking , After i refer to example , the trackletsData is empty, else part is run correctly ,please help me figure out where the problem is

      import cv2

      import depthai as dai

      import numpy as np

      import time

      # Specify the path to your custom model weights

      model_weights_path = r"C:\catdog320.blob"

      # Load model weights using dai.OpenVINO.Blob

      custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

      numClasses = 80

      dim = next(iter(custom_model_blob.networkInputs.values())).dims

      output_name, output_tenser = next(iter(custom_model_blob.networkOutputs.items()))

      numClasses = output_tenser.dims[2] - 5

      # Create pipeline

      pipeline = dai.Pipeline()

      res = dai.MonoCameraProperties.SensorResolution.THE_400_P

      # Define camera and output

      monoL = pipeline.create(dai.node.MonoCamera)

      monoL.setResolution(res)

      monoL.setFps(30)

      manip = pipeline.create(dai.node.ImageManip)

      manipOut = pipeline.create(dai.node.XLinkOut)

      detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)

      monoL.setNumFramesPool(24)

      xoutNN = pipeline.create(dai.node.XLinkOut)

      manipOut.setStreamName('flood-left')

      xoutNN.setStreamName("nn")

      # Create object tracker

      objectTracker = pipeline.create(dai.node.ObjectTracker)

      # Create output queue for tracker

      xoutTracker = pipeline.create(dai.node.XLinkOut)

      xoutTracker.setStreamName("tracklets")

      # Script node for frame routing and IR dot/flood alternate

      script = pipeline.create(dai.node.Script)

      script.setProcessor(dai.ProcessorType.LEON_CSS)

      script.setScript("""

      floodBright = 0.1

      node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

      while True:

      event = node.io['event'].get()

      Device.setIrFloodLightIntensity(floodBright)

      frameL = node.io['frameL'].get()

      node.io['floodL'].send(frameL)

      """)

      # Model-specific settings

      detectionNetwork.setBlob(custom_model_blob)

      detectionNetwork.setConfidenceThreshold(0.7)

      # YOLO-specific parameters

      detectionNetwork.setNumClasses(numClasses)

      detectionNetwork.setCoordinateSize(4)

      detectionNetwork.setAnchors([])

      detectionNetwork.setAnchorMasks({})

      detectionNetwork.setIouThreshold(0.5)

      detectionNetwork.input.setBlocking(False)

      # Convert the grayscale frame into the nn-acceptable form

      manip.initialConfig.setResize(320, 320)

      # The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case)

      manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

      # Link nodes

      monoL.out.link(manip.inputImage)

      manip.out.link(detectionNetwork.input)

      detectionNetwork.out.link(xoutNN.input)

      monoL.frameEvent.link(script.inputs['event'])

      monoL.out.link(script.inputs['frameL'])

      script.outputs['floodL'].link(manipOut.input)

      # Link tracker

      detectionNetwork.out.link(objectTracker.inputDetections)

      objectTracker.out.link(xoutTracker.input)

      frame_count = 0

      midpoints = []

      # Connect to device and start pipeline

      with dai.Device(pipeline) as device:

      startTime = time.monotonic()

      counter = 0

      fps = 0

      frame = None

      qRight = device.getOutputQueue("flood-left", maxSize=4, blocking=False)

      qDet = device.getOutputQueue("nn", maxSize=4, blocking=False)

      qTracker = device.getOutputQueue("tracklets", maxSize=4, blocking=False)

      detections = []

      def frameNorm(frame, bbox):

      normVals = np.full(len(bbox), frame.shape[0])

      normVals[::2] = frame.shape[1]

      return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)

      while True:

      counter+=1

      current_time1 = time.monotonic()

      if (current_time1 - startTime) > 1 :

      fps = counter / (current_time1 - startTime)

      counter = 0

      startTime = current_time1

      imageQueueData = qRight.tryGet()

      detectQueueData = qDet.tryGet()

      trackletsQueueData = qTracker.tryGet()

      current_time = time.localtime()

      time_text = f"{str(current_time.tm_mon).zfill(2)}/{str(current_time.tm_mday).zfill(2)} : {str(current_time.tm_hour).zfill(2)} : {str(current_time.tm_min).zfill(2)} : {str(current_time.tm_sec).zfill(2)}"

      if imageQueueData is not None:

      frame = imageQueueData.getCvFrame()

      frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)

      cv2.putText(frame, time_text, (420, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 2)

      if detectQueueData is not None:

      detections = detectQueueData.detections

      if frame is not None:

      cv2.putText(frame, "fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (255, 255, 0), 1)

      for detection in detections:

      bbox = frameNorm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))

      center = (int((bbox[0] + bbox[2]) / 2), int((bbox[1] + bbox[3]) / 2))

      cv2.circle(frame, (center[0], center[1]), 25, (0, 255, 0), 2)

      if trackletsQueueData is not None:

      tracklets = trackletsQueueData.tracklets

      for tracklet in tracklets:

      roi = tracklet.roi.denormalize(frame.shape[1], frame.shape[0])

      xmin, ymin, xmax, ymax = int(roi.topLeft().x), int(roi.topLeft().y), int(roi.bottomRight().x), int(roi.bottomRight().y)

      id = tracklet.id

      label = tracklet.label

      cv2.putText(frame, f"ID: {id}", (xmin, ymin - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)

      cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)

      cv2.putText(frame, f"Detections: {len(detections)}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2)

      cv2.imshow("ir detect", frame)

      if cv2.waitKey(1) == ord("q"):

      break

      cv2.destroyAllWindows()

      thanks,

      Li

    • Hi @jakaskerl

      Yes, I can training yolo model ! It just the object tracking example is using MBNet ,but I want to use yolo

    • Hi @jakaskerl

      Can i object tracking by yolodetect not mobilenet ssd. Or If I train my own mobilennet ssd model so I can run this

      Thanks,

      Li

    • Hi @jakaskerl

      it can run ,but can't detect right ,full of wrong boundingbox , even I raise threshold

      can i object tracking by yolodetect not mobilenet ssd

      • Hi @jakaskerl

        After I change model ,it works and print(detections) show [<depthai.ImgDetection object at 0x000002276FD7ECF0>, <depthai.ImgDetection object at 0x000002276FCD8CB0>  

        it works ,but i have to delete two line(that is not a problem)

        #cv2.putText(frame, labelMap[detection.label]….

        #cv2.putText(frame, f"{int(detection.confidence * 100)}%…….

        I modify this three place

        #change my model

        nnPathDefault = str((Path(__file__).parent / Path('C:\catdog320.blob')).resolve().absolute())

        #MobileNetDetectionNetwork -> YoloDetectionNetwork

        detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)

        detectionNetwork.setBlobPath(nnPathDefault)

        it show nothing on the frame

        Do you mean this trained model's format is mobilennetssd ?

      • Hi @jakaskerl

        I don’t know how you test my model ,but I’m pretty sure it’s work on this Yolodetectionnetwork,The Only wondering me is why the same model in blob format is not work on the mobilnetdetection ,

        I had always use Depthai Tools to convert the model from .pt into .blob

        Thanks,

        Li

      • Hi @jakaskerl

        The link is using the same Depthai Tools to transfer model as I did ,It's just that I'm not using Colab rather and using my laptop,I think there should be no difference,

        I refer to the old yoloDetect example and modify it like this

        import cv2

        import depthai as dai

        import argparse

        # Specify the path to your own model weights file

        model_weights_path = "C:/catdog320.blob"

        parser = argparse.ArgumentParser()

        parser.add_argument('nnPath', nargs='?', help="Path to mobilenet detection network blob", default=model_weights_path)

        parser.add_argument('-ff', '--full_frame', action="store_true", help="Perform tracking on full RGB frame", default=False)

        args = parser.parse_args()

        fullFrameTracking = args.full_frame

        # Load model weights using dai.OpenVINO.Blob

        custom_model_blob = dai.OpenVINO.Blob(model_weights_path)

        numClasses = 80

        dim = next(iter(custom_model_blob.networkInputs.values())).dims

        output_name, output_tenser = next(iter(custom_model_blob.networkOutputs.items()))

        numClasses = output_tenser.dims[2] - 5

        labelMap = ["class_%s" % i

        for i in range(numClasses)

        ]

        # Create a pipeline

        pipeline = dai.Pipeline()

        monoL = pipeline.create(dai.node.MonoCamera)

        manip = pipeline.create(dai.node.ImageManip)

        detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)

        objectTracker = pipeline.create(dai.node.ObjectTracker)

        manipOut = pipeline.create(dai.node.XLinkOut)

        trackerOut = pipeline.create(dai.node.XLinkOut)

        manipOut.setStreamName('flood-left')

        trackerOut.setStreamName("tracklets")

        monoL.setNumFramesPool(24)

        monoL.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

        script = pipeline.create(dai.node.Script)

        script.setProcessor(dai.ProcessorType.LEON_CSS)

        script.setScript("""

        floodBright = 0.1

        node.warn(f'IR drivers detected: {str(Device.getIrDrivers())}')

        while True:

        event = node.io['event'].get()

        Device.setIrFloodLightIntensity(floodBright)

        frameL = node.io['frameL'].get()

        node.io['floodL'].send(frameL)

        """)

        # Model-specific settings

        detectionNetwork.setBlob(custom_model_blob)

        detectionNetwork.setConfidenceThreshold(0.7)

        detectionNetwork.input.setBlocking(False)

        objectTracker.setDetectionLabelsToTrack([]) #all

        objectTracker.setTrackerType(dai.TrackerType.ZERO_TERM_COLOR_HISTOGRAM)

        objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.SMALLEST_ID)

        manip.initialConfig.setResize(320, 320)

        manip.initialConfig.setFrameType(dai.ImgFrame.Type.BGR888p)

        #link

        monoL.out.link(manip.inputImage)

        manip.out.link(detectionNetwork.input)

        monoL.frameEvent.link(script.inputs['event'])

        monoL.out.link(script.inputs['frameL'])

        script.outputs['floodL'].link(manipOut.input)

        if fullFrameTracking:

        manip.video.link(objectTracker.inputTrackerFrame)

        else:

        detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)

        detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)

        detectionNetwork.out.link(objectTracker.inputDetections)

        objectTracker.out.link(trackerOut.input)

        # Connect to the device and start the Pipeline

        with dai.Device(pipeline) as device:

        preview = device.getOutputQueue("flood-left", 4, False)

        tracklets = device.getOutputQueue(name="tracklets", maxSize=4, blocking=False)

        while True:

        imgFrame = preview.get()

        track = tracklets.get()

        print('track',track)

        color = (255, 0, 0)

        frame = imgFrame.getCvFrame()

        trackletsData = track.tracklets

        print('trackdata',trackletsData)

        for t in trackletsData:

        print('t',t)

        roi = t.roi.denormalize(frame.shape[1], frame.shape[0])

        x1 = int(roi.topLeft().x)

        y1 = int(roi.topLeft().y)

        x2 = int(roi.bottomRight().x)

        y2 = int(roi.bottomRight().y)

        try:

        label = labelMap[t.label]

        except:

        label = t.label

        cv2.putText(frame, str(label), (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)

        cv2.putText(frame, f"ID: {[t.id]}", (x1 + 10, y1 + 35), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)

        cv2.putText(frame, t.status.name, (x1 + 10, y1 + 50), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255)

        cv2.rectangle(frame, (x1, y1), (x2, y2), color, cv2.FONT_HERSHEY_SIMPLEX)

        cv2.imshow("tracker", frame)

        if cv2.waitKey(1) == ord('q'):

        break

        The program can run but cannot recognize anything, If adding detectionNetwork.out.link(trackerOut.input),

        it will show " trackletsData = track.tracklets AttributeError: 'depthai.ImgDetections' object has no attribute 'tracklets' " ,Can you help me debug

      • Hi @jakaskerl

        I had try my own model on Mono & MobilenetSSD ,but It don't detect correct ,The Screen fall of wrong bounding box, I don't know why it doesn't work, But the same model at the old example is working(the old example is not using MobilenetSSD ,is rather like the bottom of https://www.oakchina.cn/2023/02/24/yolov8-blob/ )

        I had modify the Mono & MobilenetSSD is

        nnPath = str((Path(__file__).parent / Path('C:/egg.blob')).resolve().absolute())

        labelMap = ["egg"]

        manip.initialConfig.setResize(320, 320)

        There’s nothing wrong with using Depthai Tools to transfer yolov8 model, right?

        Thanks,

        LI