Try running 2 models altogether on a OAK D Pro PoE

Hi,

I tried to run 2 of my trained YOLO models concurrently on an OAK D Pro PoE so that I could combine results retrieved from both models and then do some calculations as needed. The problem I am meeting is that it can only output results from one model when running the script, but both models can run well individually. Can you please take a look at my simplified script to see what is causing such a problem?

This is my full script:

import depthai as dai
import numpy as np

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define a source - color camera
camRgb = pipeline.createColorCamera()
camRgb.setPreviewSize(416, 416)  # Set to match NN input size
camRgb.setInterleaved(False)
camRgb.setFps(5)

# Main neural network (nn)
nn = pipeline.create(dai.node.YoloDetectionNetwork)
nn.setConfidenceThreshold(0.3)
nn.setIouThreshold(0.5)
nn.setBlobPath('/Documents/yolo10_motion_detection_416_6shaves/best_openvino_2022.1_6shave.blob')
nn.input.setBlocking(False)
nn.setNumClasses(4)
nn.setCoordinateSize(4)

# Link camera to the main neural network
manip_nn = pipeline.create(dai.node.ImageManip)
manip_nn.initialConfig.setResize(416, 416)
camRgb.preview.link(manip_nn.inputImage)

manip_bkt = pipeline.create(dai.node.ImageManip)
manip_bkt.initialConfig.setResize(224, 224)  # Resize to match `nn_bkt` input
camRgb.preview.link(manip_bkt.inputImage)

nn_bkt = pipeline.create(dai.node.YoloDetectionNetwork)
nn_bkt.setConfidenceThreshold(0.05)
nn_bkt.setIouThreshold(0.5)
nn_bkt.setBlobPath('/Documents/yolo11_seg_cap2_224_6shaves/best_openvino_2022.1_6shave.blob')
nn_bkt.input.setBlocking(False)
nn_bkt.setNumClasses(1)
nn_bkt.setCoordinateSize(4)

# Link resized frames to the additional neural network
manip_nn.out.link(nn.input)
manip_bkt.out.link(nn_bkt.input)

nn.setNumInferenceThreads(1)  # Allow parallel execution
nn_bkt.setNumInferenceThreads(1)

# Script node to process detections and conditionally trigger nn_bkt
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)

# Link outputs from both neural networks to the script node
nn_bkt.out.link(script.inputs['bkt_detections'])  # Additional neural network
nn.out.link(script.inputs['nn_detections'])

script.setScript("""
import time
import http.client
import json
import urllib.parse
from datetime import datetime, timedelta

labelMap = ['bucket loading', 'emptying', 'swing empty', 'swing full']
node.warn(f"Started")
  
while True:                 
    try:
        bkt_detections = node.io['bkt_detections'].tryGet()
        nn_detections = node.io['nn_detections'].tryGet()
                 
        if nn_detections is None: 
            continue
            
        if len(nn_detections.detections) > 0:
            node.warn(f'{labelMap[nn_detections.detections[0].label]}') # it sometimes outputs multiple results so just et the first one as a trial
                 
        if bkt_detections is None: 
            node.warn("No bucket detection available")
            continue
        
        if len(bkt_detections.detections) > 0:
            node.warn(f'Bucket detection')

    except Exception as e:
        node.warn(f"Error in main loop: {str(e)}")
                 
# Signal end of script execution
node.warn('Out of loop, finishing')
node.io['end'].send(Buffer(32))
""")

# XLinkOut node to signal the end of the script
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName('end')
script.outputs['end'].link(xout.input)

# Connect to the device with pipeline
with dai.Device(pipeline) as device:
    device.getOutputQueue("end").get()  # Wait for the "end" msg

I would appreciate any kind thoughts from you

Cheer,

Austin