Hello, I am trying to run inference with OAK 4 S using a custom YOLO model but feeding images that are loaded locally. in the code that I am providing I am just alternating randomly between two frames but ideally this should be a stream of images. I could run the pipeline using the benchmark node but it seems to be picking up only the first image and not using next images despite feeding them to the input queue. I also tried using xlinkin and xlinkout but it seems to be not working for me and I am lacking documentation for these two nodes on the new version of the API. Here is the code that I created.
import depthai as dai
import numpy as np
import time
import cv2
import random
device = dai.Device()
modelPath = "/home/asker/oak4/depthai-core/examples/python/.depthai_cached_models/bk_gh_od_02_12_24_640_best_objects.rvc4.tar.xz"modelArchive = dai.NNArchive(modelPath)
inputSize = modelArchive.getInputSize()type = modelArchive.getConfig().model.inputs[0].preprocessing.daiTypeif type: try: frameType = dai.ImgFrame.Type.BGR888i except AttributeError: type = None
testImg = cv2.imread("/home/asker/oak4/depthai-core/test.jpg")print(inputSize)testImg = cv2.resize(testImg, (inputSize[0], inputSize[1]))# Construct the input (white) image for benchmarkingimg = np.ones((inputSize[1], inputSize[0], 3), np.uint8) * 255inputFrame = dai.ImgFrame()inputFrame.setCvFrame(testImg, frameType)
with dai.Pipeline(device) as p: benchmarkOut = p.create(dai.node.BenchmarkOut) benchmarkOut.setRunOnHost(False) # The node can run on host or on device benchmarkOut.setFps(-1) # As fast as possible
neuralNetwork = p.create(dai.node.DetectionNetwork).build( benchmarkOut.out, modelArchive) # labelMap = neuralNetwork.getClasses()
benchmarkIn = p.create(dai.node.BenchmarkIn) benchmarkIn.setRunOnHost(False) # The node can run on host or on device benchmarkIn.sendReportEveryNMessages(100) benchmarkIn.logReportsAsWarnings(False) neuralNetwork.out.link(benchmarkIn.input)
outputQueue = benchmarkIn.report.createOutputQueue() inputQueue = benchmarkOut.input.createInputQueue()
qRgb = neuralNetwork.passthrough.createOutputQueue() qDet = neuralNetwork.out.createOutputQueue()
p.start()
frame = None detections = [] startTime = time.monotonic() counter = 0 color2 = (255, 255, 255)
# nn data, being the bounding box locations, are in <0..1> range - they need to be normalized with frame width/height def frameNorm(frame, bbox): normVals = np.full(len(bbox), frame.shape[0]) normVals[::2] = frame.shape[1] return (np.clip(np.array(bbox), 0, 1) * normVals).astype(int)
def displayFrame(name, frame): color = (255, 0, 0) for detection in detections: bbox = frameNorm( frame, (detection.xmin, detection.ymin, detection.xmax, detection.ymax), ) print(f"{bbox=}") cv2.putText( frame, "class", (bbox[0] + 10, bbox[1] + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255, ) cv2.putText( frame, f"{int(detection.confidence * 100)}%", (bbox[0] + 10, bbox[1] + 40), cv2.FONT_HERSHEY_TRIPLEX, 0.5, 255, ) cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2) # Show the frame cv2.imshow(name, frame)
while p.isRunning(): if random.choice([True, False]): inputFrame.setCvFrame(testImg, frameType) else: inputFrame.setCvFrame(img, frameType) inputQueue.send(inputFrame) # Send the input image only once # benchmarkReport = outputQueue.get() # time.sleep(0.01) inRgb: dai.ImgFrame = qRgb.get() inDet: dai.ImgDetections = qDet.get() if inRgb is not None: frame = inRgb.getCvFrame() cv2.putText( frame, "NN fps: {:.2f}".format( counter / (time.monotonic() - startTime)), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.4, color2, )
if inDet is not None: detections = inDet.detections print(inDet.detections) counter += 1
if frame is not None: displayFrame("rgb", frame) print("FPS: {:.2f}".format( counter / (time.monotonic() - startTime))) if cv2.waitKey(1) == ord("q"): p.stop() break # assert isinstance(benchmarkReport, dai.BenchmarkReport) # print(f"FPS is {benchmarkReport.fps}")
The question is what is the best way to feed images to a neural network using the new pipeline? should I use Xlink or is there a new node that can handle this? if it is Xlink, can you provide some guidance on how to use it and how to define the pipeline (queues, links, etc)?
Thanks in advance.
