• DepthAI
  • Is it possible to use a Neural Network on stills (and not a RGB stream)?

I'm currently capturing stills from the OAK-1-PoE on trigger. I want to use my neural net on these stills and not a full rgb stream, as there is too much delay and lag in the rgb stream.

Why does my code (below) not print a prediction? I want q_nn to be filled with predictions every time the same trigger that takes the capture goes off, without using the rgb stream.

from pathlib import Path
import depthai
import cv2
import numpy as np
import sys

CLASSES = ['SAMPLE1', 'SAMPLE2', 'SAMPLE3', 'SAMPLE4', 'SAMPLE5', 'SAMPLE6']

nnPath = str((Path(__file__).parent / Path('./best_captured_openvino_2022.1_5shave.blob')).resolve().absolute())
if len(sys.argv) > 1:
    nnPath = sys.argv[1]

if not Path(nnPath).exists():
    raise FileNotFoundError(f'Required file/s not found')

pipeline = depthai.Pipeline()
cam_rgb = pipeline.createColorCamera()

still_out = pipeline.createXLinkOut()
still_out.setStreamName("still")
cam_rgb.still.link(still_out.input)

control_in = pipeline.createXLinkIn()
control_in.setStreamName('control')
control_in.out.link(cam_rgb.inputControl)

detection_nn = pipeline.createNeuralNetwork()
detection_nn.setBlobPath(nnPath)

xout_nn = pipeline.createXLinkOut()
xout_nn.setStreamName("nn")
detection_nn.out.link(xout_nn.input)

device = depthai.Device(pipeline)
q_still = device.getOutputQueue(name="still", maxSize=1, blocking=False)
q_control = device.getInputQueue("control")
q_nn = device.getOutputQueue("nn")
ctrl = depthai.CameraControl()
ctrl.setCaptureStill(True)
q_control.send(ctrl)

while True:
    if cv2.waitKey(1) == ord('c'): 
        q_control.send(ctrl)
    in_still = q_still.tryGet()
    in_nn = q_nn.tryGet() 
    frame = None

    if in_still is not None:
        frame = in_still.getCvFrame()
    if in_nn is not None:
            data = in_nn.getFirstLayerFp16()
            result_pos = np.argmax(data)
            result = CLASSES[result_pos]
            print(result)

            frame = cv2.putText(frame, result, (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
    if frame is not None:       
        cv2.imshow("preview", frame)

    if cv2.waitKey(1) == ord('q'):
            break