Multi synchronized recording with OAK-1 w POE

Jean-Camille

I'd like to record video from 2 OAK-1 w POE camera synchronized. I need to synchronized them because I need to use them on real time later on, but need to record for having sample to evaluate performance. I modify a code from the depthai-experiement github gen2-syncing/host-multiple-OAK-sync.py.

The problem I encounter is that the frame-rate is very low and unstable, like I have an average of 15 fps for a (640,360) frame. I'd like to have a stable 30 fps at this resolution of 1080p if possible.

My host systems are jetson AGX orin and Orin Nano ( I've tried on both)

Here is my code if someone can help:

edit: I need to use openCV since the frame format is used later for processing

#!/usr/bin/env python3

import cv2
import math
import depthai as dai
import contextlib
import argparse
from datetime import timedelta
from datetime import datetime
from time import time_ns
import os
import csv

parser = argparse.ArgumentParser(epilog='Press C to capture a set of frames.')
parser.add_argument('-f', '--fps', type=float, default=30,
                    help='Camera sensor FPS, applied to all cams')

args = parser.parse_args()

cam_socket_opts = {
    'rgb'  : dai.CameraBoardSocket.CAM_A
}
cam_instance = {
    'rgb'  : 0
}

def create_pipeline(cam_list):
    # Start defining a pipeline
    pipeline = dai.Pipeline()
    cam = {}
    xout = {}
    for c in cam_list:
        print(f'value of c is : {c}')
        xout[c] = pipeline.create(dai.node.XLinkOut)
        xout[c].setStreamName(c)
        if c == 'rgb':
            cam[c] = pipeline.create(dai.node.ColorCamera)
            cam[c].setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
            cam[c].setIspScale(1, 3)  # 1920x1080 -> 1280x720
            cam[c].isp.link(xout[c].input)
        cam[c].setBoardSocket(cam_socket_opts[c])
        cam[c].setFps(args.fps)
    return pipeline


# https://docs.python.org/3/library/contextlib.html#contextlib.ExitStack
with contextlib.ExitStack() as stack:
    device_infos = dai.Device.getAllAvailableDevices()

    if len(device_infos) == 0: raise RuntimeError("No devices found!")
    else: print("Found", len(device_infos), "devices")
    queues = []
    path = f'videos/{datetime.now().strftime("%Y-%m-%d_%H-%M")}'
    os.makedirs(path)
    files = []
    csv_file = f'{path}/fps.csv'
    video_writers = {}
    ts_p = {}
    fps_data = {}

    for index, device_info in enumerate(device_infos):
        # Note: the pipeline isn't set here, as we don't know yet what device it is.
        # The extra arguments passed are required by the existing overload variants
        openvino_version = dai.OpenVINO.Version.VERSION_2021_4
        usb2_mode = False
        # device = stack.enter_context(dai.Device(openvino_version, device_info, usb2_mode))
        device = stack.enter_context(dai.Device(device_info, dai.UsbSpeed.HIGH))

        stereo = 1 < len(device.getConnectedCameras())
        cam_list = {'rgb', 'left', 'right'} if stereo else {'rgb'}

        # Get a customized pipeline based on identified device type
        device.startPipeline(create_pipeline(cam_list))

        # Output queue will be used to get the rgb frames from the output defined above
        for cam in cam_list:
            queues.append({
                'queue': device.getOutputQueue(name=cam, maxSize=4, blocking=False),
                'msgs': [], # Frame msgs
                'mx': device.getMxId(),
                'cam': cam
            })
        files.append(f'{path}/{index}.mp4')
        ts_p[device.getMxId()] = time_ns()
        fps_data[device.getMxId()] = []
        # video_writers[device.getMxId()] = cv2.VideoWriter(files[index], cv2.VideoWriter_fourcc(*'mp4v'), 30, (640*3,360*3))
        video_writers[device.getMxId()] = cv2.VideoWriter(files[index], cv2.VideoWriter_fourcc(*'mp4v'), 30, (640,360))
        # video_writers[device.getMxId()] = cv2.VideoWriter(files[index], cv2.VideoWriter_fourcc(*'mp4v'), 30, (640//2,360//2))
        print(device.getMxId())


    def check_sync(queues, timestamp):
        matching_frames = []
        for q in queues:
            for i, msg in enumerate(q['msgs']):
                time_diff = abs(msg.getTimestamp() - timestamp)
                # So below 17ms @ 30 FPS => frames are in sync
                if time_diff <= timedelta(milliseconds=math.ceil(500 / args.fps)):
                    matching_frames.append(i)
                    break

        if len(matching_frames) == len(queues):
            # We have all frames synced. Remove the excess ones
            for i, q in enumerate(queues):
                q['msgs'] = q['msgs'][matching_frames[i]:]
            return True
        else:
            return False

    while True:
        for q in queues:
            new_msg = q['queue'].tryGet()
            if new_msg is not None:
                q['msgs'].append(new_msg)
                if check_sync(queues, new_msg.getTimestamp()):
                    for q in queues:
                        frame = q['msgs'].pop(0).getCvFrame()
                        ts = time_ns()
                        # ADD TIME MEASUREMENT FOR FPS SHOWN IN THE CODE
                        fps = 1000000000/(ts - ts_p[q['mx']])
                        cv2.putText(frame, f"fps: {round(fps,1)}", (10,40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 1)
                        fps_data[q['mx']].append(fps)
                        ts_p[q['mx']] = ts
                        cv2.imshow(f"{q['cam']} - {q['mx']}", frame)
                        # print(frame.shape)
                        video_writers[q['mx']].write(frame)
        if cv2.waitKey(1) == ord('q'):
            l = []
            for key, value in video_writers.items():
                value.release()
                print(f'{key} && {value}')
                l.append(fps_data[key])
            rows = zip(*l)
            # for i in range(len(l[0]):
                # rows.append
            with open(csv_file, 'w', newline='') as file:
                writer = csv.writer(file)
                writer.writerow(video_writers.keys())
                writer.writerows(rows)
            print(f'The data has been written to {csv_file}')
            break

jakaskerl

Hi Jean-Camille
Since you are not using videoEncoder nodes, i'd say it's a bandwidth issue you are facing. Does lowering the resolution improve FPS?

Could you try adding encoding to lower the bandwidth requirements?

Thanks,
Jaka

Jean-Camille

Thank you for your reply, the VideoEncoder was a good idea and I have no trouble with the video anymore.
However, I am trying to get the timestamp of every frame from the multiple camera in the host timestamp system. I have found that there is a sync node for syncing the timestamp with the host but I cannot access to it.

I have found that we can use the clock.nowHost() in the script node to get the timestamp in host time system, but I am not sure how to transfer it in the pipeline.

My code is as following, and I am trying to generate a csv file with the timestamp of each frame to synchronize the POE camera with an eyetracker in post-processing.

#!/usr/bin/env python3

import cv2
import math
import depthai as dai
import contextlib
import argparse
from datetime import timedelta
from datetime import datetime
from time import time_ns
import time
import os
import csv
import av
from fractions import Fraction

parser = argparse.ArgumentParser(epilog='Press C to capture a set of frames.')
parser.add_argument('-f', '--fps', type=float, default=30,
                    help='Camera sensor FPS, applied to all cams')

args = parser.parse_args()

cam_socket_opts = {
    'rgb'  : dai.CameraBoardSocket.CAM_A
}
cam_instance = {
    'rgb'  : 0
}

def create_pipeline(cam_list):
    # Start defining a pipeline
    pipeline = dai.Pipeline()
    cam = {}
    xout = {}
    enc = {}
    for c in cam_list:
        xout[c] = pipeline.create(dai.node.XLinkOut)
        xout[c].setStreamName(c)
        if c == 'rgb':
            cam[c] = pipeline.create(dai.node.ColorCamera)
            cam[c].setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

            cam[c].setBoardSocket(cam_socket_opts[c])
            cam[c].setFps(args.fps)

            # cam[c].setIspScale(1, 3)  # 1920x1080 -> 1280x720
            enc[c] = pipeline.create(dai.node.VideoEncoder)
            enc[c].setDefaultProfilePreset(cam[c].getFps(), dai.VideoEncoderProperties.Profile.H265_MAIN)
            cam[c].video.link(enc[c].input)
            # cam[c].isp.link(xout[c].input)
            enc[c].bitstream.link(xout[c].input)
    return pipeline


# https://docs.python.org/3/library/contextlib.html#contextlib.ExitStack
with contextlib.ExitStack() as stack:
    device_infos = dai.Device.getAllAvailableDevices()

    if len(device_infos) == 0: raise RuntimeError("No devices found!")
    else: print("Found", len(device_infos), "devices")
    queues = []
    path = f'videos/{datetime.now().strftime("%Y-%m-%d_%H-%M")}'
    os.makedirs(path)
    files = {}
    csv_file = f'{path}/fps.csv'
    video_writers = {}
    ts_p = {}
    fps_data = {}
    timestamps = {}

    output_containers = {}
    streams = {}

    for index, device_info in enumerate(device_infos):
        # Note: the pipeline isn't set here, as we don't know yet what device it is.
        # The extra arguments passed are required by the existing overload variants
        openvino_version = dai.OpenVINO.Version.VERSION_2021_4
        usb2_mode = False
        # device = stack.enter_context(dai.Device(openvino_version, device_info, usb2_mode))
        device = stack.enter_context(dai.Device(device_info, dai.UsbSpeed.HIGH))

        device.setTimesync(timedelta(seconds=5), 10, False)

        stereo = 1 < len(device.getConnectedCameras())
        cam_list = {'rgb', 'left', 'right'} if stereo else {'rgb'}

        # Get a customized pipeline based on identified device type
        device.startPipeline(create_pipeline(cam_list))

        # Output queue will be used to get the rgb frames from the output defined above
        for cam in cam_list:
            queues.append({
                'queue': device.getOutputQueue(name=cam, maxSize=30, blocking=False),
                'msgs': [], # Frame msgs
                'mx': device.getMxId(),
                'cam': cam
            })
            timestamps[device.getMxId()] = []
            timestamps[device.getMxId()+'_host'] = []

        files[device.getMxId()] = f'{path}/{index}.mp4'
        print(f'Generate files : {files[device.getMxId()]}')
        output_containers[device.getMxId()] = av.open(files[device.getMxId()], 'w')
        streams[device.getMxId()] = output_containers[device.getMxId()].add_stream('hevc', rate=30)
        streams[device.getMxId()].time_base = Fraction(1, 1000 * 1000) # Microseconds    
        print(device.getMxId())



    def check_sync(queues, timestamp):
        matching_frames = []
        for q in queues:
            for i, msg in enumerate(q['msgs']):
                time_diff = abs(msg.getTimestamp() - timestamp)
                # So below 17ms @ 30 FPS => frames are in sync
                # if time_diff <= timedelta(milliseconds=math.ceil(500 / args.fps)):
                if time_diff <= timedelta(milliseconds=math.ceil(500 / 1)):
                    matching_frames.append(i)
                    break

        if len(matching_frames) == len(queues):
            # We have all frames synced. Remove the excess ones
            for i, q in enumerate(queues):
                q['msgs'] = q['msgs'][matching_frames[i]:]
                timestamps[q['mx']].append(q['msgs'][0].getTimestamp().total_seconds())
                timestamps[q['mx']+'_host'].append(time.time_ns()/1000000000)
            return True
        else:
            return False

    start = time.time()
    try:
        while True:

            for q in queues:
                new_msg = q['queue'].tryGet()
                if new_msg is not None:
                    q['msgs'].append(new_msg)
                    if check_sync(queues, new_msg.getTimestamp()):
                    # if True:
                        for q in queues:
                            data = q['msgs'].pop(0).getData()
                            packet = av.Packet(data)
                            packet.pts = int ( (time.time() - start) * 1000 * 1000)
                            output_containers[q['mx']].mux_one(packet)

    except KeyboardInterrupt:
        # Keyboard interrupt (Ctrl + C) detected
        with open(csv_file, 'w') as csvfile:
            print(f'writing fps data in {csv_file} ...')
            # Créer un objet writer CSV
            csv_writer = csv.writer(csvfile)

            header = list(timestamps)
            csv_writer.writerow(header)

            for i in range(len(timestamps[header[0]])):
                row = [timestamps[header[0]][i], timestamps[header[1]][i], timestamps[header[2]][i],timestamps[header[3]][i]]
                csv_writer.writerow(row)
            print(type(row[0]))


        print('data has been written, end of program')

How would you get the timestamp in the host time system for each frame ?
Thanks,
J-C

jakaskerl

Hi @Jean-Camille
message.getTimestamp() will give you the host side timestamp (the time a which the message was received by the host. This will give you the timedelta from the moment the host (PC) was turned on.
messgage.getTimestampDevice() will give you the time at which the image was taken relative to device's monotonic clock.

This is a quick script I got from GPT:

import psutil
import datetime

def convert_delta_to_actual_time(time_delta):
    # Get the PC's boot time
    boot_time_timestamp = psutil.boot_time()
    boot_time = datetime.datetime.fromtimestamp(boot_time_timestamp)

    # Convert the delta since boot to a datetime.timedelta object if it's not already
    if not isinstance(time_delta, datetime.timedelta):
        # Assuming time_delta is in seconds, modify accordingly if it's in a different unit
        time_delta = datetime.timedelta(seconds=time_delta)

    # Calculate the actual time
    actual_time = boot_time + time_delta

    return actual_time

# Example usage
time_delta = datetime.timedelta(seconds=12345)  # Replace with your actual time delta
actual_time = convert_delta_to_actual_time(time_delta)
print("Actual Time:", actual_time)

Let me know if it works. Might need some parsing first though.

Thanks,
Jaka

Jean-Camille

Hi,
I've tried the code and different iteration with GPT, but there is always a difference between the boot_time + time_delta compare to the system time with time.time() or time.time_ns().
Here is an example of output:

Here the first column in the boot_time + time_delta and the _host is the time.time_ns()/1000000000
Any idea of what it could be ?
Thanks,

jakaskerl

Hi Jean-Camille
That's because the dai.Clock.now() is tied to time.monotonic(), not time.time().

# Function to convert DepthAI timestamp to actual wall clock time
def convert_to_wall_clock_time(depthai_ts, ref_wall_clock, ref_depthai_ts):
    # Calculate the time difference between the DepthAI timestamp and the reference DepthAI timestamp
    delta_ts = depthai_ts - ref_depthai_ts

    # Convert the DepthAI time difference to a datetime.timedelta object
    delta_td = datetime.timedelta(seconds=delta_ts.total_seconds())

    # Add the timedelta to the wall clock reference time
    actual_wall_clock_time = ref_wall_clock + delta_td

    return actual_wall_clock_time

where

ref_wall_clock = datetime.datetime.now()
ref_depthai_ts = dai.Clock.now()

Thanks,
Jaka

Jean-Camille

Hi,
I just understood the difference between the time system and the time use by the depthai,
Many thanks !