I’ve been attempting to get this two-stage neural network model working and I have been simplifying but nothing works. It appears to get 4 frames in and then hangs up. I have attempted using the loopback approach and other sync approaches as found in different online examples and experiments. Would you mind looking at the simplified pipeline I am running and let me know if anything is broken, or if it is simply hanging up because the second network is too slow?
from os.path import dirname
import depthai as dai
import cv2
def frame_norm(img, bounding_box):
norm_vals = np.full(len(bounding_box), img.shape[0])
norm_vals[::2] = img.shape[1]
return (np.clip(np.array(bounding_box), 0, 1) * norm_vals).astype(int)
PATH = os.path.abspath(os.getcwd())
anchors = [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0, 198.0, 373.0, 326.0]
masks = {"side40": [0, 1, 2], "side20": [3, 4, 5], "side10": [6, 7, 8]}
# Create Pipeline
pipeline = dai.Pipeline()
# Create Cam
cam = pipeline.create(dai.node.ColorCamera)
cam.setPreviewSize(int(1920), int(1080))
cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)
cam.setInterleaved(False)
cam.setFps(10)
# Image Manip
manip = pipeline.createImageManip()
manip.initialConfig.setResize(320, 320)
manip.setMaxOutputFrameSize(320*320*3)
manip.initialConfig.setKeepAspectRatio(False)
# Create mouse Neural Network
mouse_nn = pipeline.create(dai.node.YoloDetectionNetwork)
mouse_nn.setBlob(os.path.join(PATH, 'mouse.blob'))
mouse_nn.setConfidenceThreshold(.7)
mouse_nn.setNumClasses(1)
mouse_nn.setCoordinateSize(4)
mouse_nn.setAnchors(anchors)
mouse_nn.setAnchorMasks(masks)
# Create scroll_wheel Neural Network
scroll_wheel_nn = pipeline.create(dai.node.YoloDetectionNetwork)
scroll_wheel_nn.setBlob(os.path.join(PATH, 'scroll.blob'))
scroll_wheel_nn.setConfidenceThreshold(.7)
scroll_wheel_nn.setNumClasses(1)
scroll_wheel_nn.setCoordinateSize(4)
scroll_wheel_nn.setAnchors(anchors)
scroll_wheel_nn.setAnchorMasks(masks)
# Create Script
script = pipeline.create(dai.node.Script)
script.setScript("""
def limit_roi(det):
if det.xmin <= 0: det.xmin = 0.001
if det.ymin <= 0: det.ymin = 0.001
if det.xmax >= 1: det.xmax = 0.999
if det.ymax >= 1: det.ymax = 0.999
while True:
mouse_det = node.io['mouse'].get().detections
for det in mouse_det:
limit_roi(det)
cfg = ImageManipConfig()
cfg.setCropRect(det.xmin, det.ymin, det.xmax, det.ymax)
cfg.setResize(320, 320)
cfg.setKeepAspectRatio(False)
node.io['manip_cfg'].send(cfg)
""")
# Create scroll_wheel Manip
scroll_wheel_manip = pipeline.create(dai.node.ImageManip)
scroll_wheel_manip.initialConfig.setResize(320, 320)
scroll_wheel_manip.inputConfig.setWaitForMessage(True)
# Label
cam_x = pipeline.create(dai.node.XLinkOut)
cam_x.setStreamName('cam')
scroll_wheel_x = pipeline.create(dai.node.XLinkOut)
scroll_wheel_x.setStreamName('scroll_wheel')
# Link
cam.preview.link(cam_x.input)
scroll_wheel_nn.out.link(scroll_wheel_x.input)
cam.preview.link(manip.inputImage)
cam.preview.link(scroll_wheel_manip.inputImage)
manip.out.link(mouse_nn.input) # manip to mouse_nn input
scroll_wheel_manip.out.link(scroll_wheel_nn.input)
mouse_nn.out.link(script.inputs['mouse'])
script.outputs['manip_cfg'].link(scroll_wheel_manip.inputConfig) # sends manip_cfg to scroll_wheel input
with dai.Device(pipeline) as device:
cam_q = device.getOutputQueue('cam', maxSize=4, blocking=False)
scroll_wheel_q = device.getOutputQueue('scroll_wheel', maxSize=4, blocking=False)
mouse_color = (0, 255, 0)
scroll_wheel_color = (255, 0, 255)
while True:
frame = cam_q.get().getCvFrame()
scroll_wheel_det = scroll_wheel_q.get().detections
for detection in scroll_wheel_det:
bbox = frame_norm(frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax))
print(f'BBOX: {bbox}')
`
