• DepthAI-v2
  • problem in syncing rgb frame and depth frame in script node

Hi, i am trying to sync RGB frame with depth frame in script node. I tried several ways, but i cannot make it work. can you please help me on how to make it work? below is my whole script

import depthai as dai

import os

import struct

import datetime

pipeline = dai.Pipeline()
fps = 20  

board = dai.BoardConfig()board.emmc = Truepipeline.setBoardConfig(board)

cam_rgb = pipeline.create(dai.node.ColorCamera)cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)cam_rgb.setIspScale(1, 3, 10, 27)cam_rgb.setFps(fps)
cam_rgb.initialControl.setManualFocus(150)
monoLeft = pipeline.create(dai.node.MonoCamera)monoRight = pipeline.create(dai.node.MonoCamera)stereo = pipeline.create(dai.node.StereoDepth)
monoLeft.setFps(fps)monoRight.setFps(fps)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
config = stereo.initialConfig.get()config.postProcessing.spatialFilter.holeFillingRadius = 5config.postProcessing.spatialFilter.numIterations = 1config.postProcessing.decimationFilter.decimationFactor = 1
stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A) stereo.initialConfig.set(config)stereo.initialConfig.setConfidenceThreshold(250)stereo.setLeftRightCheck(True)stereo.setSubpixel(False)

monoLeft.out.link(stereo.left)monoRight.out.link(stereo.right)

script_save = pipeline.create(dai.node.Script)script_save.setProcessor(dai.ProcessorType.LEON_CSS)script_save.setScript("""import osimport structimport time
index = 0
ROOT_MEDIA_DIR = '/media/mmcsd-0-0/'os.chdir(ROOT_MEDIA_DIR)                                            frame_buffer = {"jpeg": [], "depth": []}TIMESTAMP_THRESHOLD = 0.05
def create_sess_directory():    session_exists = False    sess_idx = []
    for file in os.listdir():        if 'session' in file:            session_exists = True            file_idx = file.split('_')[1]            sess_idx.append(file_idx)
    if session_exists:        last_sess_idx = max(sess_idx)        next_sess_idx = f'{int(last_sess_idx) + 1:03d}'        node.warn(f'Last session idx: {last_sess_idx}. Next session idx: {next_sess_idx}')    else:        node.warn(f'No session directories found. Creating a new one...')        next_sess_idx = '001'                          next_sess_dir = f'session_{next_sess_idx}'    os.mkdir(next_sess_dir)    session_exists = True                          node.warn(f"Session dir created: session_{next_sess_idx}")        return next_sess_dir
def save_jpeg(jpeg_frame, path):    capture_time = jpeg_frame.getTimestampDevice().total_seconds()      start_time = time.time()    with open(path, 'wb') as f:        f.write(jpeg_frame.getData())    end_time = time.time()    save_time = end_time - start_time    return capture_time, save_time
def save_depth(depth_frame, path):    capture_time = depth_frame.getTimestampDevice().total_seconds()      start_time = time.time()    depth_data = depth_frame.getData()    with open(path, 'wb') as f:        for i in range(0, len(depth_data), 2):  # each depth value is 2 bytes            depth_value = struct.unpack_from('H', depth_data, i)[0]            f.write(struct.pack('H', depth_value))    end_time = time.time()    save_time = end_time - start_time    return capture_time, save_time
def log_to_file(log_path, message):    with open(log_path, 'a') as log_file:          log_file.write(message + '\\n')
next_sess_dir = create_sess_directory()
log_file_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, 'log.txt')with open(log_file_path, 'w') as log_file:    log_file.write("Session Log\\n")    log_file.write("====================\\n")
while True:     jpeg_frame = node.io['jpeg'].get()    depth_frame = node.io['depth'].get()
    if jpeg_frame:        rgb_timestamp = jpeg_frame.getTimestampDevice().total_seconds()        frame_buffer["jpeg"].append((rgb_timestamp, jpeg_frame))        node.warn(f"Received RGB Frame at {rgb_timestamp:.6f}")
    if depth_frame:        depth_timestamp = depth_frame.getTimestampDevice().total_seconds()        frame_buffer["depth"].append((depth_timestamp, depth_frame))        node.warn(f"Received Depth Frame at {depth_timestamp:.6f}")
    while frame_buffer["jpeg"] and frame_buffer["depth"]:        rgb_timestamp, rgb_frame = frame_buffer["jpeg"][0]        closest_depth_match = min(frame_buffer["depth"], key=lambda x: abs(x[0] - rgb_timestamp))        depth_timestamp, depth_frame = closest_depth_match
        if abs(rgb_timestamp - depth_timestamp) <= TIMESTAMP_THRESHOLD:            frame_buffer["jpeg"].pop(0)            frame_buffer["depth"].remove(closest_depth_match)
            while True:                jpg_fname = f'{str(index)}_rgb.jpg'                depthmap_fname = f'{str(index)}_depth.npy'
                jpeg_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, jpg_fname)                depth_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, depthmap_fname)
                if not os.path.exists(jpeg_path) and not os.path.exists(depth_path):                    break                index += 1
            rgb_capture_time, rgb_save_time = save_jpeg(rgb_frame, jpeg_path)            depth_capture_time, depth_save_time = save_depth(depth_frame, depth_path)
            log_to_file(log_file_path, f'RGB saved: {jpeg_path}')            log_to_file(log_file_path, f'  Capture time: {rgb_capture_time:.2f} seconds')            log_to_file(log_file_path, f'  Saving time: {rgb_save_time:.4f} seconds')
            log_to_file(log_file_path, f'Depth saved: {depth_path}')            log_to_file(log_file_path, f'  Capture time: {depth_capture_time:.2f} seconds')            log_to_file(log_file_path, f'  Saving time: {depth_save_time:.4f} seconds')
            index += 1  """)

jpegEncoder = pipeline.create(dai.node.VideoEncoder)jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)
cam_rgb.video.link(jpegEncoder.input)jpegEncoder.bitstream.link(script_save.inputs['jpeg'])stereo.depth.link(script_save.inputs['depth'])
(f, bl) = dai.DeviceBootloader.getFirstAvailableDevice()bootloader = dai.DeviceBootloader(bl)
progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')bootloader.flash(progress, pipeline)
print("Pipeline flashed successfully! Now the camera will run automatically when powered on.")

    KyriakiKylili
    Why not just use a sync node? This is difficult to debug.

    Thanks,
    Jaka

    Hi Jaka, thank you for your reply. This is the code with the sync.node. The problem i have is that nothing is saved on the memory of the camera probably because it never gets synchronous frames, or something is wrong with linking in the code below or the various streams. Additonally the camera keeps restarting all the time.

    import depthai as dai

    import os

    import struct

    from datetime import timedelta

    pipeline = dai.Pipeline()
    fps = 15

    board = dai.BoardConfig()

    board.emmc = True

    pipeline.setBoardConfig(board)

    cam_rgb = pipeline.create(dai.node.ColorCamera)

    cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)

    cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

    cam_rgb.setIspScale(1, 3, 10, 27)

    cam_rgb.setFps(fps)

    cam_rgb.initialControl.setManualFocus(150)

    monoLeft = pipeline.create(dai.node.MonoCamera)

    monoRight = pipeline.create(dai.node.MonoCamera)

    monoLeft.setFps(fps)

    monoRight.setFps(fps)

    monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

    monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)

    monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

    monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)

    stereo = pipeline.create(dai.node.StereoDepth)

    stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
    config = stereo.initialConfig.get()

    config.postProcessing.spatialFilter.holeFillingRadius = 5

    config.postProcessing.spatialFilter.numIterations = 1

    config.postProcessing.decimationFilter.decimationFactor = 1
    stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)

    stereo.initialConfig.set(config)

    stereo.setLeftRightCheck(True)

    stereo.setSubpixel(False)

    stereo.initialConfig.setConfidenceThreshold(220)

    monoLeft.out.link(stereo.left)

    monoRight.out.link(stereo.right)

    sync = pipeline.create(dai.node.Sync)

    sync.setSyncThreshold(timedelta(milliseconds=500))

    sync.inputs["jpeg"].setBlocking(False)

    sync.inputs["jpeg"].setQueueSize(10)

    sync.inputs["depth"].setBlocking(False)

    sync.inputs["depth"].setQueueSize(10)

    jpegEncoder = pipeline.create(dai.node.VideoEncoder)

    jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)

    cam_rgb.video.link(jpegEncoder.input)

    jpegEncoder.bitstream.link(sync.inputs['jpeg'])

    stereo.depth.link(sync.inputs['depth'])

    script_save = pipeline.create(dai.node.Script)

    sync.out.link(script_save.inputs['sync'])

    script_save.setProcessor(dai.ProcessorType.LEON_CSS)

    script_save.setScript("""

    import os

    import struct

    import time

    index = 0
    …..
    while True:    

    synced_frames = node.io['sync'].get()       

    jpeg_frame = synced_frames.getMessage('jpeg') 

    rgb_capture_time, rgb_save_time = save_jpeg(jpeg_frame, jpeg_path)       

    depth_frame = synced_frames.getMessage('depth') 

    depth_capture_time, depth_save_time = save_depth(depth_frame, depth_path)

    ………
    """)

    (f, bl) = dai.DeviceBootloader.getFirstAvailableDevice()

    bootloader = dai.DeviceBootloader(bl)
    progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')

    bootloader.flash(progress, pipeline)

      KyriakiKylili

      • Does it work in normal mode?
      • Likely EMMC can not be accessed which causes a fatal error crashing the device (constant restarts)

      Does your device have EMMC storage?

      Thanks,
      Jaka

      No is not working with normal mode. The synchronised frames never get into the script node. Yes my camera has emmc storage. It is the OAK-D POE model. Please help me structure the code properly for getting the synchroised frames into the script node.

      Below is my updated script:

      import depthai as dai

      import os

      import struct

      from datetime import timedelta

      # Create the pipeline

      pipeline = dai.Pipeline()
      # Board configuration to enable EMMC

      board = dai.BoardConfig()board.emmc = True

      pipeline.setBoardConfig(board)
      fps = 20
      # I/O configuration

      cam_rgb = pipeline.create(dai.node.ColorCamera)

      cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)

      cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

      cam_rgb.setIspScale(1, 3, 10, 27)

      cam_rgb.setFps(fps)cam_rgb.initialControl.setSharpness(1)

      cam_rgb.initialControl.setAutoExposureLimit(6000)

      cam_rgb.initialControl.setAutoExposureCompensation(2)

      cam_rgb.initialControl.setAutoFocusMode(dai.RawCameraControl.AutoFocusMode.AUTO)

      monoLeft = pipeline.create(dai.node.MonoCamera)

      monoRight = pipeline.create(dai.node.MonoCamera)

      stereo = pipeline.create(dai.node.StereoDepth)
      # Properties

      monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

      monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)

      monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

      monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)

      stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
      monoLeft.setFps(fps)monoRight.setFps(fps)
      # Stereo Depth Configuration

      stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)

       # Align depth to RGB

      stereo.setOutputSize(640, 400)

      stereo.setLeftRightCheck(True)

      stereo.setSubpixel(False)
      # Configure post-processing

      config = stereo.initialConfig.get()

      config.postProcessing.spatialFilter.holeFillingRadius = 5

      config.postProcessing.spatialFilter.numIterations = 1

      config.postProcessing.decimationFilter.decimationFactor = 1

      stereo.initialConfig.set(config)

      stereo.initialConfig.setConfidenceThreshold(250)

      sync = pipeline.create(dai.node.Sync)

      sync.setSyncThreshold(timedelta(milliseconds=30))  
      # Linking

      monoLeft.out.link(stereo.left)

      monoRight.out.link(stereo.right)

      sync.inputs["jpeg"].setBlocking(False)

      sync.inputs["jpeg"].setQueueSize(4)

      sync.inputs["depth"].setBlocking(False)

      sync.inputs["depth"].setQueueSize(4)
      jpegEncoder = pipeline.create(dai.node.VideoEncoder)

      jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)

      cam_rgb.video.link(jpegEncoder.input)

      jpegEncoder.bitstream.link(sync.inputs['jpeg'])

      # Script to save RGB and Depth data on EMMC

      script_save = pipeline.create(dai.node.Script)

      script_save.setProcessor(dai.ProcessorType.LEON_CSS)

      sync.out.link(script_save.inputs["jpeg"])

      sync.out.link(script_save.inputs["depth"])

      stereo.depth.link(script_save.inputs['depth'])

      script_save.setScript("""import os

      import struct

      import time
      index = 0
      ….

      while True:   

      jpg_fname = f'{index}_rgb.jpg'   

      depthmap_fname = f'{index}_depth.npy'   

      jpeg_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, jpg_fname)   

      depth_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, depthmap_fname)       

      if not os.path.exists(jpeg_path) and not os.path.exists(depth_path):       

      try:           

      jpeg_frame = node.io['jpeg'].get()   

      ….     

      depth_frame = node.io['depth'].get()           
      …..

      """)
      with dai.Device(pipeline) as device:   

      print("Pipeline running...")   

      while True:     

       pass