problem in syncing rgb frame and depth frame in script node

KyriakiKylili

Hi, i am trying to sync RGB frame with depth frame in script node. I tried several ways, but i cannot make it work. can you please help me on how to make it work? below is my whole script

import depthai as dai

import os

import struct

import datetime

pipeline = dai.Pipeline()
fps = 20

board = dai.BoardConfig()board.emmc = Truepipeline.setBoardConfig(board)

cam_rgb = pipeline.create(dai.node.ColorCamera)cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)cam_rgb.setIspScale(1, 3, 10, 27)cam_rgb.setFps(fps)
cam_rgb.initialControl.setManualFocus(150)
monoLeft = pipeline.create(dai.node.MonoCamera)monoRight = pipeline.create(dai.node.MonoCamera)stereo = pipeline.create(dai.node.StereoDepth)
monoLeft.setFps(fps)monoRight.setFps(fps)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
config = stereo.initialConfig.get()config.postProcessing.spatialFilter.holeFillingRadius = 5config.postProcessing.spatialFilter.numIterations = 1config.postProcessing.decimationFilter.decimationFactor = 1
stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A) stereo.initialConfig.set(config)stereo.initialConfig.setConfidenceThreshold(250)stereo.setLeftRightCheck(True)stereo.setSubpixel(False)

monoLeft.out.link(stereo.left)monoRight.out.link(stereo.right)

script_save = pipeline.create(dai.node.Script)script_save.setProcessor(dai.ProcessorType.LEON_CSS)script_save.setScript("""import osimport structimport time
index = 0
ROOT_MEDIA_DIR = '/media/mmcsd-0-0/'os.chdir(ROOT_MEDIA_DIR) frame_buffer = {"jpeg": [], "depth": []}TIMESTAMP_THRESHOLD = 0.05
def create_sess_directory(): session_exists = False sess_idx = []
for file in os.listdir(): if 'session' in file: session_exists = True file_idx = file.split('_')[1] sess_idx.append(file_idx)
if session_exists: last_sess_idx = max(sess_idx) next_sess_idx = f'{int(last_sess_idx) + 1:03d}' node.warn(f'Last session idx: {last_sess_idx}. Next session idx: {next_sess_idx}') else: node.warn(f'No session directories found. Creating a new one...') next_sess_idx = '001' next_sess_dir = f'session_{next_sess_idx}' os.mkdir(next_sess_dir) session_exists = True node.warn(f"Session dir created: session_{next_sess_idx}") return next_sess_dir
def save_jpeg(jpeg_frame, path): capture_time = jpeg_frame.getTimestampDevice().total_seconds()    start_time = time.time() with open(path, 'wb') as f: f.write(jpeg_frame.getData()) end_time = time.time() save_time = end_time - start_time return capture_time, save_time
def save_depth(depth_frame, path): capture_time = depth_frame.getTimestampDevice().total_seconds()    start_time = time.time() depth_data = depth_frame.getData() with open(path, 'wb') as f: for i in range(0, len(depth_data), 2): # each depth value is 2 bytes depth_value = struct.unpack_from('H', depth_data, i)[0] f.write(struct.pack('H', depth_value)) end_time = time.time() save_time = end_time - start_time return capture_time, save_time
def log_to_file(log_path, message): with open(log_path, 'a') as log_file:    log_file.write(message + '\\n')
next_sess_dir = create_sess_directory()
log_file_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, 'log.txt')with open(log_file_path, 'w') as log_file: log_file.write("Session Log\\n") log_file.write("====================\\n")
while True:   jpeg_frame = node.io['jpeg'].get() depth_frame = node.io['depth'].get()
if jpeg_frame: rgb_timestamp = jpeg_frame.getTimestampDevice().total_seconds() frame_buffer["jpeg"].append((rgb_timestamp, jpeg_frame)) node.warn(f"Received RGB Frame at {rgb_timestamp:.6f}")
if depth_frame: depth_timestamp = depth_frame.getTimestampDevice().total_seconds() frame_buffer["depth"].append((depth_timestamp, depth_frame)) node.warn(f"Received Depth Frame at {depth_timestamp:.6f}")
while frame_buffer["jpeg"] and frame_buffer["depth"]: rgb_timestamp, rgb_frame = frame_buffer["jpeg"][0] closest_depth_match = min(frame_buffer["depth"], key=lambda x: abs(x[0] - rgb_timestamp)) depth_timestamp, depth_frame = closest_depth_match
if abs(rgb_timestamp - depth_timestamp) <= TIMESTAMP_THRESHOLD: frame_buffer["jpeg"].pop(0) frame_buffer["depth"].remove(closest_depth_match)
while True: jpg_fname = f'{str(index)}_rgb.jpg' depthmap_fname = f'{str(index)}_depth.npy'
jpeg_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, jpg_fname) depth_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, depthmap_fname)
if not os.path.exists(jpeg_path) and not os.path.exists(depth_path): break index += 1
rgb_capture_time, rgb_save_time = save_jpeg(rgb_frame, jpeg_path) depth_capture_time, depth_save_time = save_depth(depth_frame, depth_path)
log_to_file(log_file_path, f'RGB saved: {jpeg_path}') log_to_file(log_file_path, f' Capture time: {rgb_capture_time:.2f} seconds') log_to_file(log_file_path, f' Saving time: {rgb_save_time:.4f} seconds')
log_to_file(log_file_path, f'Depth saved: {depth_path}') log_to_file(log_file_path, f' Capture time: {depth_capture_time:.2f} seconds') log_to_file(log_file_path, f' Saving time: {depth_save_time:.4f} seconds')
index += 1 """)

jpegEncoder = pipeline.create(dai.node.VideoEncoder)jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)
cam_rgb.video.link(jpegEncoder.input)jpegEncoder.bitstream.link(script_save.inputs['jpeg'])stereo.depth.link(script_save.inputs['depth'])
(f, bl) = dai.DeviceBootloader.getFirstAvailableDevice()bootloader = dai.DeviceBootloader(bl)
progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')bootloader.flash(progress, pipeline)
print("Pipeline flashed successfully! Now the camera will run automatically when powered on.")

jakaskerl

KyriakiKylili
Why not just use a sync node? This is difficult to debug.

Thanks,
Jaka

KyriakiKylili

Hi Jaka, thank you for your reply. This is the code with the sync.node. The problem i have is that nothing is saved on the memory of the camera probably because it never gets synchronous frames, or something is wrong with linking in the code below or the various streams. Additonally the camera keeps restarting all the time.

import depthai as dai

import os

import struct

from datetime import timedelta

pipeline = dai.Pipeline()
fps = 15

board = dai.BoardConfig()

board.emmc = True

pipeline.setBoardConfig(board)

cam_rgb = pipeline.create(dai.node.ColorCamera)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)

cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setIspScale(1, 3, 10, 27)

cam_rgb.setFps(fps)

cam_rgb.initialControl.setManualFocus(150)

monoLeft = pipeline.create(dai.node.MonoCamera)

monoRight = pipeline.create(dai.node.MonoCamera)

monoLeft.setFps(fps)

monoRight.setFps(fps)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)

monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)

stereo = pipeline.create(dai.node.StereoDepth)

stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
config = stereo.initialConfig.get()

config.postProcessing.spatialFilter.holeFillingRadius = 5

config.postProcessing.spatialFilter.numIterations = 1

config.postProcessing.decimationFilter.decimationFactor = 1
stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)

stereo.initialConfig.set(config)

stereo.setLeftRightCheck(True)

stereo.setSubpixel(False)

stereo.initialConfig.setConfidenceThreshold(220)

monoLeft.out.link(stereo.left)

monoRight.out.link(stereo.right)

sync = pipeline.create(dai.node.Sync)

sync.setSyncThreshold(timedelta(milliseconds=500))

sync.inputs["jpeg"].setBlocking(False)

sync.inputs["jpeg"].setQueueSize(10)

sync.inputs["depth"].setBlocking(False)

sync.inputs["depth"].setQueueSize(10)

jpegEncoder = pipeline.create(dai.node.VideoEncoder)

jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)

cam_rgb.video.link(jpegEncoder.input)

jpegEncoder.bitstream.link(sync.inputs['jpeg'])

stereo.depth.link(sync.inputs['depth'])

script_save = pipeline.create(dai.node.Script)

sync.out.link(script_save.inputs['sync'])

script_save.setProcessor(dai.ProcessorType.LEON_CSS)

script_save.setScript("""

import os

import struct

import time

index = 0
…..
while True:

synced_frames = node.io['sync'].get()

jpeg_frame = synced_frames.getMessage('jpeg')

rgb_capture_time, rgb_save_time = save_jpeg(jpeg_frame, jpeg_path)

depth_frame = synced_frames.getMessage('depth')

depth_capture_time, depth_save_time = save_depth(depth_frame, depth_path)

………
""")

(f, bl) = dai.DeviceBootloader.getFirstAvailableDevice()

bootloader = dai.DeviceBootloader(bl)
progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')

bootloader.flash(progress, pipeline)

jakaskerl

KyriakiKylili

Does it work in normal mode?
Likely EMMC can not be accessed which causes a fatal error crashing the device (constant restarts)

Does your device have EMMC storage?

Thanks,
Jaka

KyriakiKylili

No is not working with normal mode. The synchronised frames never get into the script node. Yes my camera has emmc storage. It is the OAK-D POE model. Please help me structure the code properly for getting the synchroised frames into the script node.

Below is my updated script:

import depthai as dai

import os

import struct

from datetime import timedelta

# Create the pipeline

pipeline = dai.Pipeline()
# Board configuration to enable EMMC

board = dai.BoardConfig()board.emmc = True

pipeline.setBoardConfig(board)
fps = 20
# I/O configuration

cam_rgb = pipeline.create(dai.node.ColorCamera)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)

cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setIspScale(1, 3, 10, 27)

cam_rgb.setFps(fps)cam_rgb.initialControl.setSharpness(1)

cam_rgb.initialControl.setAutoExposureLimit(6000)

cam_rgb.initialControl.setAutoExposureCompensation(2)

cam_rgb.initialControl.setAutoFocusMode(dai.RawCameraControl.AutoFocusMode.AUTO)

monoLeft = pipeline.create(dai.node.MonoCamera)

monoRight = pipeline.create(dai.node.MonoCamera)

stereo = pipeline.create(dai.node.StereoDepth)
# Properties

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)

monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)

stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
monoLeft.setFps(fps)monoRight.setFps(fps)
# Stereo Depth Configuration

stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)

# Align depth to RGB

stereo.setOutputSize(640, 400)

stereo.setLeftRightCheck(True)

stereo.setSubpixel(False)
# Configure post-processing

config = stereo.initialConfig.get()

config.postProcessing.spatialFilter.holeFillingRadius = 5

config.postProcessing.spatialFilter.numIterations = 1

config.postProcessing.decimationFilter.decimationFactor = 1

stereo.initialConfig.set(config)

stereo.initialConfig.setConfidenceThreshold(250)

sync = pipeline.create(dai.node.Sync)

sync.setSyncThreshold(timedelta(milliseconds=30))
# Linking

monoLeft.out.link(stereo.left)

monoRight.out.link(stereo.right)

sync.inputs["jpeg"].setBlocking(False)

sync.inputs["jpeg"].setQueueSize(4)

sync.inputs["depth"].setBlocking(False)

sync.inputs["depth"].setQueueSize(4)
jpegEncoder = pipeline.create(dai.node.VideoEncoder)

jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)

cam_rgb.video.link(jpegEncoder.input)

jpegEncoder.bitstream.link(sync.inputs['jpeg'])

# Script to save RGB and Depth data on EMMC

script_save = pipeline.create(dai.node.Script)

script_save.setProcessor(dai.ProcessorType.LEON_CSS)

sync.out.link(script_save.inputs["jpeg"])

sync.out.link(script_save.inputs["depth"])

stereo.depth.link(script_save.inputs['depth'])

script_save.setScript("""import os

import struct

import time
index = 0
….

while True:

jpg_fname = f'{index}_rgb.jpg'

depthmap_fname = f'{index}_depth.npy'

jpeg_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, jpg_fname)

depth_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, depthmap_fname)

if not os.path.exists(jpeg_path) and not os.path.exists(depth_path):

try:

jpeg_frame = node.io['jpeg'].get()

….

depth_frame = node.io['depth'].get()
…..

""")
with dai.Device(pipeline) as device:

print("Pipeline running...")

while True:

pass

jakaskerl

KyriakiKylili
Because you are not linking depth to the sync node 🙂. I suggest you use luxonis/depthai_pipeline_graph to visualize the pipeline so this doesn't happen.

Thanks,
Jaka