I'm currently trying to find the optimal configuration for high framerate, low latency 720p stereo using an Oak D pro. First of all, latency generally increases with FPS. This is probably known. I adapted this script from the docs to stereo:
import time
import depthai as dai
import numpy as np
RES = dai.MonoCameraProperties.SensorResolution.THE_720_P
FPS = 75
# Create pipeline
pipeline = dai.Pipeline()
# This might improve reducing the latency on some systems
pipeline.setXLinkChunkSize(0)
def get_link_out(socket: dai.CameraBoardSocket, stream_name: str) -> dai.node.XLinkOut:
cam_mono = pipeline.create(dai.node.MonoCamera)
cam_mono.setBoardSocket(socket)
cam_mono.setFps(FPS)
cam_mono.setResolution(RES)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName(stream_name)
xout.input.setBlocking(False)
xout.input.setQueueSize(1)
cam_mono.out.link(xout.input)
return xout
xout_l = get_link_out(dai.CameraBoardSocket.CAM_B, "out")
xout_r = get_link_out(dai.CameraBoardSocket.CAM_C, "out2")
# Connect to device and start pipeline
with dai.Device(pipeline) as device:
q = device.getOutputQueue(name="out", maxSize=1, blocking=False)
q2 = device.getOutputQueue(name="out2", maxSize=1, blocking=False)
print("Waiting for 100 frames...")
for _ in range(100):
_ = q.get()
_ = q2.get()
latencies = np.array([])
start = time.time()
frames = 0
while True:
imgFrame = q.get()
imgFrame2 = q2.get()
assert isinstance(imgFrame, dai.ImgFrame)
frames += 1
elapsed = time.time() - start
fps = frames / elapsed
latencyMs = (dai.Clock.now() - imgFrame.getTimestamp()).total_seconds() * 1000
latencies = np.append(latencies, latencyMs)
print('Latency: {:.2f} ms, Average latency: {:.2f} ms, Std: {:.2f}, FPS: {:.2f}'.format(latencyMs, np.average(latencies),
np.std(latencies), fps))
At 75 fps, i got: Average latency: 19.58 ms, Std: 0.28, FPS: 74.99
I found that pushing the framerate beyond 75 increases the latency. To push it a bit further, I used the image manip node to crop the frame before sending it to the host. I am working on a tracking application where I can adjust the cropping window dynamically around the tracked object.
import time
import depthai as dai
import numpy as np
RES = dai.MonoCameraProperties.SensorResolution.THE_720_P
FPS = 100
CROP = .25
# Create pipeline
pipeline = dai.Pipeline()
# This might improve reducing the latency on some systems
pipeline.setXLinkChunkSize(0)
def get_link_out(socket: dai.CameraBoardSocket, stream_name: str) -> dai.node.XLinkOut:
cam_mono = pipeline.create(dai.node.MonoCamera)
cam_mono.setBoardSocket(socket)
cam_mono.setFps(FPS)
cam_mono.setResolution(RES)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName(stream_name)
xout.input.setBlocking(False)
xout.input.setQueueSize(1)
if (CROP < 1):
manip = pipeline.create(dai.node.ImageManip)
manip.initialConfig.setCropRect(.5 - CROP * .5, .5 - CROP * .5, .5 + CROP * .5, .5 + CROP * .5)
manip.inputImage.setBlocking(False)
manip.inputImage.setQueueSize(1)
cam_mono.out.link(manip.inputImage)
manip.out.link(xout.input)
else:
cam_mono.out.link(xout.input)
return xout
xout_l = get_link_out(dai.CameraBoardSocket.CAM_B, "out_l")
xout_r = get_link_out(dai.CameraBoardSocket.CAM_C, "out_r")
# Connect to device and start pipeline
with dai.Device(pipeline) as device:
print(device.getUsbSpeed())
q_l = device.getOutputQueue(name="out_l", maxSize=1, blocking=False)
q_r = device.getOutputQueue(name="out_r", maxSize=1, blocking=False)
print("Waiting for 100 frames...")
for _ in range(100):
_ = q_l.get()
_ = q_r.get()
diffs = np.array([])
frames = 0
start = time.time()
while True:
imgFrame = q_l.get()
imgFrame2 = q_r.get()
assert isinstance(imgFrame, dai.ImgFrame)
frames += 1
elapsed = time.time() - start
fps = frames / elapsed
latencyMs = (dai.Clock.now() - imgFrame.getTimestamp()).total_seconds() * 1000
diffs = np.append(diffs, latencyMs)
print('Latency: {:.2f} ms, Average latency: {:.2f} ms, Std: {:.2f}, FPS: {:.2f}'.format(latencyMs, np.average(diffs),
np.std(diffs), fps))
With this, I was able to push the framerate to 100 and lower latency as well: Average latency: 14.02 ms, Std: 0.11, FPS: 100.00 Then I adapted the script to DepthAI v3 (btw the example in the docs says it's v3, but is v2):
import time
import depthai as dai
import numpy as np
RES = (1280, 720)
FPS = 100
CROP = .25
def create_cam(socket: dai.CameraBoardSocket) -> dai.MessageQueue:
cam = pipeline.create(dai.node.Camera).build(socket)
if CROP < 1:
manip = pipeline.create(dai.node.ImageManip)
manip.initialConfig.addCrop(int(RES[0] * (1 - CROP) * .5), int(RES[1] * (1 - CROP) * .5), int(RES[0] * CROP), int(RES[1] * CROP))
manip.inputImage.setBlocking(False)
manip.inputImage.setMaxSize(1)
cam.requestOutput(RES, fps=FPS).link(manip.inputImage)
return manip.out.createOutputQueue(1, False)
else:
return cam.requestOutput(RES, fps=FPS).createOutputQueue(1, False)
with dai.Pipeline() as pipeline:
# This might improve reducing the latency on some systems
pipeline.setXLinkChunkSize(0)
q_l = create_cam(dai.CameraBoardSocket.CAM_B)
q_r = create_cam(dai.CameraBoardSocket.CAM_C)
pipeline.start()
print("Waiting for 100 frames...")
for _ in range(100):
_ = q_l.get()
_ = q_r.get()
latencies = np.array([])
start = time.time()
frames = 0
while True:
left = q_l.get()
right = q_r.get()
arrival_time = dai.Clock.now().total_seconds()
if isinstance(left, dai.ImgFrame) and isinstance(right, dai.ImgFrame):
frames += 1
elapsed = time.time() - start
fps = frames / elapsed
latencyMs = (dai.Clock.now() - left.getTimestamp()).total_seconds() * 1000
latencies = np.append(latencies, latencyMs)
print('Latency: {:.2f} ms, Average latency: {:.2f} ms, Std: {:.2f}, FPS: {:.2f}'.format(latencyMs,
np.average(
latencies),
np.std(
latencies),
fps))
But now, the latency is much higher and I am not even getting the full 100 fps: Average latency: 33.06 ms, Std: 6.77, FPS: 93.82 With the sync node it's quite a bit better, but still not close to depthai v2:
import time
import depthai as dai
import numpy as np
RES = (1280, 720)
FPS = 100
CROP = .25
def create_cam(socket: dai.CameraBoardSocket, name: str):
cam = pipeline.create(dai.node.Camera).build(socket)
if CROP < 1:
manip = pipeline.create(dai.node.ImageManip)
manip.initialConfig.addCrop(int(RES[0] * (1 - CROP) * .5), int(RES[1] * (1 - CROP) * .5), int(RES[0] * CROP), int(RES[1] * CROP))
manip.inputImage.setBlocking(False)
manip.inputImage.setMaxSize(1)
cam.requestOutput(RES, fps=FPS).link(manip.inputImage)
manip.out.link(sync.inputs[name])
else:
cam.requestOutput(RES, fps=FPS).link(sync.inputs[name])
sync.inputs[name].setBlocking(False)
sync.inputs[name].setMaxSize(1)
with dai.Pipeline() as pipeline:
# This might improve reducing the latency on some systems
pipeline.setXLinkChunkSize(0)
sync = pipeline.create(dai.node.Sync)
sync.setRunOnHost(True)
create_cam(dai.CameraBoardSocket.CAM_B, "left")
create_cam(dai.CameraBoardSocket.CAM_C, "right")
synced_q = sync.out.createOutputQueue(maxSize=1, blocking=False)
pipeline.start()
print("Waiting for 100 frames...")
for _ in range(100):
_ = synced_q.get()
latencies = np.array([])
frames = 0
start = time.time()
while True:
message_group = synced_q.get()
arrival_time = dai.Clock.now().total_seconds()
if isinstance(message_group, dai.MessageGroup):
left = message_group["left"]
right = message_group["right"]
if isinstance(left, dai.ImgFrame) and isinstance(right, dai.ImgFrame):
frames += 1
elapsed = time.time() - start
fps = frames / elapsed
latencyMs = (dai.Clock.now() - left.getTimestamp()).total_seconds() * 1000
latencies = np.append(latencies, latencyMs)
print('Latency: {:.2f} ms, Average latency: {:.2f} ms, Std: {:.2f}, FPS: {:.2f}'.format(latencyMs,
np.average(
latencies),
np.std(
latencies),
fps))
Average latency: 26.25 ms, Std: 5.29, FPS: 95.38
So what's up with that? Did ImageManip get slower in v3?
Some info about my setup: CachyOS Linux, Ryzen 7800X3D, depthai 2.32.0.0 and 3.8.0, Python 3.12.13, Oak D Pro connected using the USB cable that it came with to a 10Gbit USB port.
