- Edited
Hi, i am trying to sync RGB frame with depth frame in script node. I tried several ways, but i cannot make it work. can you please help me on how to make it work? below is my whole script
import depthai as dai
import os
import struct
import datetime
pipeline = dai.Pipeline()
fps = 20
board = dai.BoardConfig()board.emmc = Truepipeline.setBoardConfig(board)
cam_rgb = pipeline.create(dai.node.ColorCamera)cam_rgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)cam_rgb.setIspScale(1, 3, 10, 27)cam_rgb.setFps(fps)
cam_rgb.initialControl.setManualFocus(150)
monoLeft = pipeline.create(dai.node.MonoCamera)monoRight = pipeline.create(dai.node.MonoCamera)stereo = pipeline.create(dai.node.StereoDepth)
monoLeft.setFps(fps)monoRight.setFps(fps)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_5x5)
config = stereo.initialConfig.get()config.postProcessing.spatialFilter.holeFillingRadius = 5config.postProcessing.spatialFilter.numIterations = 1config.postProcessing.decimationFilter.decimationFactor = 1
stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A) stereo.initialConfig.set(config)stereo.initialConfig.setConfidenceThreshold(250)stereo.setLeftRightCheck(True)stereo.setSubpixel(False)
monoLeft.out.link(stereo.left)monoRight.out.link(stereo.right)
script_save = pipeline.create(dai.node.Script)script_save.setProcessor(dai.ProcessorType.LEON_CSS)script_save.setScript("""import osimport structimport time
index = 0
ROOT_MEDIA_DIR = '/media/mmcsd-0-0/'os.chdir(ROOT_MEDIA_DIR) frame_buffer = {"jpeg": [], "depth": []}TIMESTAMP_THRESHOLD = 0.05
def create_sess_directory(): session_exists = False sess_idx = []
for file in os.listdir(): if 'session' in file: session_exists = True file_idx = file.split('_')[1] sess_idx.append(file_idx)
if session_exists: last_sess_idx = max(sess_idx) next_sess_idx = f'{int(last_sess_idx) + 1:03d}' node.warn(f'Last session idx: {last_sess_idx}. Next session idx: {next_sess_idx}') else: node.warn(f'No session directories found. Creating a new one...') next_sess_idx = '001' next_sess_dir = f'session_{next_sess_idx}' os.mkdir(next_sess_dir) session_exists = True node.warn(f"Session dir created: session_{next_sess_idx}") return next_sess_dir
def save_jpeg(jpeg_frame, path): capture_time = jpeg_frame.getTimestampDevice().total_seconds() start_time = time.time() with open(path, 'wb') as f: f.write(jpeg_frame.getData()) end_time = time.time() save_time = end_time - start_time return capture_time, save_time
def save_depth(depth_frame, path): capture_time = depth_frame.getTimestampDevice().total_seconds() start_time = time.time() depth_data = depth_frame.getData() with open(path, 'wb') as f: for i in range(0, len(depth_data), 2): # each depth value is 2 bytes depth_value = struct.unpack_from('H', depth_data, i)[0] f.write(struct.pack('H', depth_value)) end_time = time.time() save_time = end_time - start_time return capture_time, save_time
def log_to_file(log_path, message): with open(log_path, 'a') as log_file: log_file.write(message + '\\n')
next_sess_dir = create_sess_directory()
log_file_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, 'log.txt')with open(log_file_path, 'w') as log_file: log_file.write("Session Log\\n") log_file.write("====================\\n")
while True: jpeg_frame = node.io['jpeg'].get() depth_frame = node.io['depth'].get()
if jpeg_frame: rgb_timestamp = jpeg_frame.getTimestampDevice().total_seconds() frame_buffer["jpeg"].append((rgb_timestamp, jpeg_frame)) node.warn(f"Received RGB Frame at {rgb_timestamp:.6f}")
if depth_frame: depth_timestamp = depth_frame.getTimestampDevice().total_seconds() frame_buffer["depth"].append((depth_timestamp, depth_frame)) node.warn(f"Received Depth Frame at {depth_timestamp:.6f}")
while frame_buffer["jpeg"] and frame_buffer["depth"]: rgb_timestamp, rgb_frame = frame_buffer["jpeg"][0] closest_depth_match = min(frame_buffer["depth"], key=lambda x: abs(x[0] - rgb_timestamp)) depth_timestamp, depth_frame = closest_depth_match
if abs(rgb_timestamp - depth_timestamp) <= TIMESTAMP_THRESHOLD: frame_buffer["jpeg"].pop(0) frame_buffer["depth"].remove(closest_depth_match)
while True: jpg_fname = f'{str(index)}_rgb.jpg' depthmap_fname = f'{str(index)}_depth.npy'
jpeg_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, jpg_fname) depth_path = os.path.join(ROOT_MEDIA_DIR, next_sess_dir, depthmap_fname)
if not os.path.exists(jpeg_path) and not os.path.exists(depth_path): break index += 1
rgb_capture_time, rgb_save_time = save_jpeg(rgb_frame, jpeg_path) depth_capture_time, depth_save_time = save_depth(depth_frame, depth_path)
log_to_file(log_file_path, f'RGB saved: {jpeg_path}') log_to_file(log_file_path, f' Capture time: {rgb_capture_time:.2f} seconds') log_to_file(log_file_path, f' Saving time: {rgb_save_time:.4f} seconds')
log_to_file(log_file_path, f'Depth saved: {depth_path}') log_to_file(log_file_path, f' Capture time: {depth_capture_time:.2f} seconds') log_to_file(log_file_path, f' Saving time: {depth_save_time:.4f} seconds')
index += 1 """)
jpegEncoder = pipeline.create(dai.node.VideoEncoder)jpegEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG)
cam_rgb.video.link(jpegEncoder.input)jpegEncoder.bitstream.link(script_save.inputs['jpeg'])stereo.depth.link(script_save.inputs['depth'])
(f, bl) = dai.DeviceBootloader.getFirstAvailableDevice()bootloader = dai.DeviceBootloader(bl)
progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')bootloader.flash(progress, pipeline)
print("Pipeline flashed successfully! Now the camera will run automatically when powered on.")