A

awetzel

I’ve been attempting to get this two-stage neural network model working and I have been simplifying but nothing works. It appears to get 4 frames in and then hangs up. I have attempted using the loopback approach and other sync approaches as found in different online examples and experiments. Would you mind looking at the simplified pipeline I am running and let me know if anything is broken, or if it is simply hanging up because the second network is too slow?

from os.path import dirname
import depthai as dai
import cv2


def frame_norm(img, bounding_box):
    norm_vals = np.full(len(bounding_box), img.shape[0])
    norm_vals[::2] = img.shape[1]
    return (np.clip(np.array(bounding_box), 0, 1) * norm_vals).astype(int)


PATH = os.path.abspath(os.getcwd())
anchors = [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0, 198.0, 373.0, 326.0]
masks = {"side40": [0, 1, 2], "side20": [3, 4, 5], "side10": [6, 7, 8]}

# Create Pipeline
pipeline = dai.Pipeline()

# Create Cam
cam = pipeline.create(dai.node.ColorCamera)
cam.setPreviewSize(int(1920), int(1080))
cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)
cam.setInterleaved(False)
cam.setFps(10)

# Image Manip
manip = pipeline.createImageManip()
manip.initialConfig.setResize(320, 320)
manip.setMaxOutputFrameSize(320*320*3)
manip.initialConfig.setKeepAspectRatio(False)

# Create mouse Neural Network
mouse_nn = pipeline.create(dai.node.YoloDetectionNetwork)
mouse_nn.setBlob(os.path.join(PATH, 'mouse.blob'))
mouse_nn.setConfidenceThreshold(.7)
mouse_nn.setNumClasses(1)
mouse_nn.setCoordinateSize(4)
mouse_nn.setAnchors(anchors)
mouse_nn.setAnchorMasks(masks)

# Create scroll_wheel Neural Network
scroll_wheel_nn = pipeline.create(dai.node.YoloDetectionNetwork)
scroll_wheel_nn.setBlob(os.path.join(PATH, 'scroll.blob'))
scroll_wheel_nn.setConfidenceThreshold(.7)
scroll_wheel_nn.setNumClasses(1)
scroll_wheel_nn.setCoordinateSize(4)
scroll_wheel_nn.setAnchors(anchors)
scroll_wheel_nn.setAnchorMasks(masks)

# Create Script
script = pipeline.create(dai.node.Script)
script.setScript("""
def limit_roi(det):
    if det.xmin <= 0: det.xmin = 0.001
    if det.ymin <= 0: det.ymin = 0.001
    if det.xmax >= 1: det.xmax = 0.999
    if det.ymax >= 1: det.ymax = 0.999
    

while True:
    mouse_det = node.io['mouse'].get().detections
    for det in mouse_det:
        limit_roi(det)
        cfg = ImageManipConfig()
        cfg.setCropRect(det.xmin, det.ymin, det.xmax, det.ymax)
        cfg.setResize(320, 320)
        cfg.setKeepAspectRatio(False)
        node.io['manip_cfg'].send(cfg)
        

""")


# Create scroll_wheel Manip
scroll_wheel_manip = pipeline.create(dai.node.ImageManip)
scroll_wheel_manip.initialConfig.setResize(320, 320)
scroll_wheel_manip.inputConfig.setWaitForMessage(True)

# Label
cam_x = pipeline.create(dai.node.XLinkOut)
cam_x.setStreamName('cam')
scroll_wheel_x = pipeline.create(dai.node.XLinkOut)
scroll_wheel_x.setStreamName('scroll_wheel')

# Link
cam.preview.link(cam_x.input)
scroll_wheel_nn.out.link(scroll_wheel_x.input)
cam.preview.link(manip.inputImage)
cam.preview.link(scroll_wheel_manip.inputImage)
manip.out.link(mouse_nn.input)  # manip to mouse_nn input
scroll_wheel_manip.out.link(scroll_wheel_nn.input)
mouse_nn.out.link(script.inputs['mouse'])
script.outputs['manip_cfg'].link(scroll_wheel_manip.inputConfig)  # sends manip_cfg to scroll_wheel input

with dai.Device(pipeline) as device:
    cam_q = device.getOutputQueue('cam', maxSize=4, blocking=False)
    scroll_wheel_q = device.getOutputQueue('scroll_wheel', maxSize=4, blocking=False)

    mouse_color = (0, 255, 0)
    scroll_wheel_color = (255, 0, 255)

    while True:
        frame = cam_q.get().getCvFrame()
        scroll_wheel_det = scroll_wheel_q.get().detections

        for detection in scroll_wheel_det:
            bbox = frame_norm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
            print(f'BBOX: {bbox}')

`