- Edited
Hello,
I am using an external IMU to use its pitch angle and display it in the same frame along with the NN detection result by using OAK-D camera. I am trying to understand the latency values between camera-NN-host-IMU. I would like to make sure that the latency calculations I've done here are accurate. Is there more accurate way to implement these? Here is the main() functionality I am using:
`
def main():
global imu_thread_running, imu_timestamp
pitch_angle=0.0 # just to initialize
pipeline, labelMap, SHAPE = cameraSetupModel(MODEL, SYNCNN)
outVideo = videoWrite(shape=SHAPE)
imu_thread = threading.Thread(target=read_pitch_angle_thread, daemon=True)
imu_thread.start()
with depthai.Device(pipeline) as device:
q_rgb = device.getOutputQueue("rgb", maxSize=6, blocking=False) # Also default values (maxSize=30, blocking=True), better for H265 for no artifacts
q_nn = device.getOutputQueue("nn", maxSize=6, blocking=False)
fps = FPSHandler()
frame = None
detections = []
shape = (3, SHAPE, SHAPE)
while True:
firstTime = time.time()
if SYNCNN:
inRgb = q_rgb.get()
inDet = q_nn.get()
else:
inRgb = q_rgb.tryGet()
inDet = q_nn.tryGet()
#since image was captured until the frame was received on the host computer.
latencyMs = (depthai.Clock.now() - inRgb.getTimestamp()).total_seconds() * 1000
print(f"[LATENCY] IMAGE CAPTURING: {latencyMs:.4f} ms")
camera_timestamp = time.time() # Timestamp for camera frame capture
latencyMs2 = (depthai.Clock.now() - inDet.getTimestamp()).total_seconds() * 1000
print(f"[LATENCY] DETECTION PROCESS: {latencyMs2:.4f} ms")
nn_timestamp = time.time() # Timestamp for NN processing results
if inRgb is not None:
frame = inRgb.getCvFrame()
latencyGetFrame = (depthai.Clock.now()-inRgb.getTimestamp()).total_seconds() * 1000
frameTime = time.time() # Timestamp for frame retrieval
print(f"[LATENCY] GET CV FRAME LATENCY: {latencyGetFrame:.4f} ms")
if inDet is not None:
detections = inDet.detections
fps.next_iter()
cv2.putText(frame, "Fps: {:.2f}".format(fps.fps()), (2, SHAPE - 4), cv2.FONT_HERSHEY_TRIPLEX, 1, color=(0, 255, 0))
if frame is not None:
# Read pitch angle from serial port
with pitch_lock:
pitch_angle = latest_pitch_angle
imu_time = imu_timestamp # Get the timestamp for the latest IMU data
latencyIMUFrame = (depthai.Clock.now()-inRgb.getTimestamp()).total_seconds() * 1000
print(f"[LATENCY] IMU-FRAME: {latencyIMUFrame:.4f} ms")
print(f"[LATENCY CLASSIC] IMU-FRAME: {(time.time()-imu_time)*1000:.4f} ms")
host_timestamp = inRgb.getTimestamp() # Timestamp for host processing
displayFrame("Frame", frame, detections=detections,pitch_angle=pitch_angle,labelMap=labelMap)
# Write frame to the output video
outVideo.write(frame)
# breakpoint()
latencyHost = (depthai.Clock.now()-host_timestamp).total_seconds() * 1000
print(f"[LATENCY] HOST DELAY: {latencyHost:.4f} ms")
print(f"[LATENCY CLASSIC] ALL LOOP DELAY: {(time.time()-firstTime)*1000:.4f} ms")
print("=====================================")
if cv2.waitKey(1) == ord('q'):
break
imu_thread_running = False
ser.close()
# Release the VideoWriter object and close all OpenCV windows
outVideo.release()
cv2.destroyAllWindows()
if name == "main":
main()
`
Thank you in advance for your help!
Bests,
Cem