Delay with images

dvaro

ChatGPT Plus

Good morning,
I am configuring an Oak-D Pro PoE, and what I want it to do is, when the program receives a message via MQTT, to capture two images: one RGB and one depth.
However, I see that there is a significant delay between the image I save and when I send the signal, as I notice that the object is far behind the detection area in the received image.

This is my code:

import paho.mqtt.client as mqtt

import json

import time

import os

import shutil

import fcntl

import socket

import cv2

import depthai as dai

import numpy as np

from datetime import timedelta

# TCP Variables

socket_connection = None

socket_connected = False

# Per no permetre mes d'una instancia del programa

lock_file = open('/tmp/capture_image.lock', 'w')

try:

fcntl.lockf(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)

except IOError:

sys.exit(0)

# Funcio per crear la imatge de profunditat

def colorizeDepth(frameDepth):

frameDepth_p = np.clip(frameDepth, 0, 2\*\*24 - 1).astype(np.uint32)

R = (frameDepth_p >> 16) & 0xFF

G = (frameDepth_p >> 8) & 0xFF

B = frameDepth_p & 0xFF

encoded_frameDepth = np.stack([B, G, R], axis=-1).astype(np.uint8)

return encoded_frameDepth

# Funcio per enviar un missatge per socket TCP

def send_message_tcp(filename, server_ip, server_port):

global socket_connection, socket_connected

if not connect_to_tcp(server_ip, server_port):

    write_to_log(logfiles_path_json, logfile_name_error_json, "Error connectant amb el servidor TCP")

    return

try:

    socket_connection.sendall(filename.encode('utf-8'))

except Exception as e:

    write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error enviant missatge per TCP: {e}")

    write_to_log(logfiles_path_json, logfile_name_error_json, f"Error enviant missatge per TCP: {e}")

    try:

        socket_connection.close()

    except:

        pass

    socket_connection = None

    socket_connected = False

# Funcio per connectar al socket TCP per enviar el path de la fotografia

def connect_to_tcp(server_ip, server_port):

global socket_connection, socket_connected

if socket_connected:

    return True

try:

    socket_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    socket_connection.connect((server_ip, server_port))

    write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Connexio realitzada al socket correctament")

    socket_connected = True

    return True

except Exception as e:

    socket_connected = False

    socket_connection = None

    write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error al connectar al servidor TCP: {e}")

    write_to_log(logfiles_path_json, logfile_name_error_json, f"Error al connectar al servidor TCP: {e}")

    return False

# Funcio per escriure en el fitxer de logs

def write_to_log(pathfile, filename, log_message):

try:

    new_filename = time.strftime("%Y-%m-%d", time.localtime()) + "_" + filename

    if not os.path.exists(pathfile + new_filename):

        with open(pathfile + new_filename, 'w'):

            pass 

    with open(pathfile + new_filename, 'a') as log:

        current_time = time.strftime("%H%M%S", time.localtime())

        log_entry = f"{current_time}: {log_message}"

        log.write(log_entry + "\\n")

except Exception as e:

    pass

# Funcio per realitzar la fotografia

def take_picture():

try:

    timestamp = time.strftime("%Y%m%d_%H%M%S")

    messageGroup = queue.get()

    frameRgb = messageGroup["rgb"]

    frameDepth = messageGroup["depth_aligned"]

    

    if frameDepth is not None:

        cvFrame = frameRgb.getCvFrame()

        alignedDepthColorized = colorizeDepth(frameDepth.getFrame())

        

        nameColor = f"{save_path_json}{timestamp}{name_image_rgb_json}"

        nameDepth = f"{save_path_json}{timestamp}{name_image_depth_json}"

        

        cv2.imwrite(nameColor, cvFrame)

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Finalitzat la captura de la imatge {nameColor}")

        cv2.imwrite(nameDepth, alignedDepthColorized)

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Finalitzat la captura de la imatge {nameDepth}")

        

        #Enviem les imatges per TCP

        tcp_filename = f"{path_windows_json}{timestamp}{name_image_rgb_json}{character_to_split_images_json}{path_windows_json}{timestamp}{name_image_depth_json}"

        send_message_tcp(tcp_filename, server_tcp_ip_json, server_tcp_port_json)

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Enviat per TCP les imatges")    

    

except Exception as e:

    write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error fent la foto: {e}")

    write_to_log(logfiles_path_json, logfile_name_error_json, f"Error fent la foto: {e}")

# Funcio per connectar amb el mosquitto, broker del MQTT

def on_connect(client, userdata, flags, rc):

if rc == 0:

    write_to_log(logfiles_path_json, logfile_name_traceability_json, "Connectat al broker MQTT")

    client.subscribe(mqtt_take_picture_json)

else:

    write_to_log(logfiles_path_json, logfile_name_traceability_json, "Error: No s'ha connectat al broker MQTT")

    write_to_log(logfiles_path_json, logfile_name_error_json, "No s'ha connectat al broker MQTT")

# Funcio per gestionar els missatges que es reben

def on_message(client, userdata, msg):

take_picture()

# Funcio per gestionar si es desconecta la cua MQTT, que es torni a reconectar automaticament

def on_disconnect(client, userdata, rc):

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Desconnectat del servidor MQTT, missatge {rc}")

while True:

    try:

        client.reconnect()

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Reconnexio al MQTT realitzada correctament")

        break

    except Exception as e:

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error: Al intentar reconnectar al MQTT s'ha obtingut aquest error: {e}")

        write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Al intentar reconnectar al MQTT s'ha obtingut aquest error: {e}")

        time.sleep(5)

# Variables globals

save_path_json = ""

width_resolution_rgb_json = 0

height_resolution_rgb_json = 0

width_resolution_depth_json = 0

height_resolution_depth_json = 0

FPS_json = 0

name_image_rgb_json = ""

name_image_depth_json = ""

logfiles_path_json = ""

logfile_name_traceability_json = ""

logfile_name_error_json = ""

path_windows_json = ""

jsonData = ""

mqtt_take_picture_json = ""

mqtt_ip_json = ""

mqtt_port_json = 0

mqtt_keepalive_json = 60

try:

with open("appsettings.json", "r") as file:

    jsonData = json.load(file)

except Exception as e:

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error obrint appsettings: {e}")

write_to_log(logfiles_path_json, logfile_name_error_json, f"Error obrint appsettings: {e}")

# Llegim els valors del json de configuracio

save_path_json = jsonData["basePath"]

width_resolution_rgb_json = jsonData["widthResolutionRGB"]

height_resolution_rgb_json = jsonData["heightResolutionRGB"]

enable_undistortion_json = jsonData["enableUndistortion"]

width_resolution_left_right_json = jsonData["widthResolutionLeftRight"]

height_resolution_left_right_json = jsonData["heightResolutionLeftRight"]

width_resolution_stereo_json = jsonData["widthResolutionStereo"]

height_resolution_stereo_json = jsonData["heightResolutionStereo"]

FPS_json = jsonData["FPS"]

name_image_rgb_json = jsonData["nameImageRGB"]

name_image_depth_json = jsonData["nameImageDepth"]

logfiles_path_json = jsonData["logfilesPath"]

logfile_name_traceability_json = jsonData["logFileNameTraceability"]

logfile_name_error_json = jsonData["logFileNameError"]

server_tcp_ip_json = jsonData["serverTcpIp"]

server_tcp_port_json = jsonData["serverTcpPort"]

path_windows_json = jsonData["pathWindows"]

character_to_split_images_json = jsonData["characterToSplitImages"]

dot_projector_intensity_json = jsonData["dotProjectorIntensity"]

flood_light_intensity_json = jsonData["floodLightIntensity"]

set_sharpness_json = jsonData["setSharpness"]

set_luma_denoise_json = jsonData["setLumaDenoise"]

set_chroma_denoise_json = jsonData["setChromaDenoise"]

set_auto_exposure_limit_json = jsonData["setAutoExposureLimit"]

set_rectify_edge_fill_color_json = jsonData["setRectifyEdgeFillColor"]

set_left_right_check_json = jsonData["setLeftRightCheck"]

set_extended_disparity_json = jsonData["setExtendedDisparity"]

set_subpixel_json = jsonData["setSubpixel"]

num_iterations_json = jsonData["numIterations"]

min_range_json = jsonData["minRange"]

max_range_json = jsonData["maxRange"]

mqtt_take_picture_json = jsonData["MQTT"]["takePicture"]

mqtt_ip_json = jsonData["MQTT"]["ip"]

mqtt_port_json = jsonData["MQTT"]["port"]

mqtt_keepalive_json = jsonData["MQTT"]["keepAlive"]

# Guardem els valors que hem llegit al log

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Inicialitzant aplicacio")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Base Path: {save_path_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Width Resolution RGB: {width_resolution_rgb_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Height Resolution RGB: {height_resolution_rgb_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Undistortion Enabled: {enable_undistortion_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Width Resolution Left-Right: {width_resolution_left_right_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Height Resolution Left-Right: {height_resolution_left_right_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Width Resolution Stereo: {width_resolution_stereo_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Height Resolution Stereo: {height_resolution_stereo_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"FPS: {FPS_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Name Image RGB: {name_image_rgb_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Name Image Depth: {name_image_depth_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Logfiles Path: {logfiles_path_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Logfile Name Traceability: {logfile_name_traceability_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Logfile Name Error: {logfile_name_error_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Server IP: {server_tcp_ip_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Server Port: {server_tcp_port_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Path Windows: {path_windows_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Character To Split Images: {character_to_split_images_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Dot Projector Intensity (IR): {dot_projector_intensity_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Flood Light Intensity: {flood_light_intensity_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Sharpness: {set_sharpness_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Luma Denoise: {set_luma_denoise_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Chroma Denoise: {set_chroma_denoise_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Auto-exposure Limit (ms): {set_auto_exposure_limit_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Rectify Edge Fill Color: {set_rectify_edge_fill_color_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Left Right Check: {set_left_right_check_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Extended Disparity: {set_extended_disparity_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Set Subpixel: {set_subpixel_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Number Of Iterations: {num_iterations_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Distance Min Range: {min_range_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Distance Max Range: {max_range_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"MQTT take picture: {mqtt_take_picture_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"MQTT IP: {mqtt_ip_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"MQTT port: {mqtt_port_json}")

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"MQTT keep alive: {mqtt_keepalive_json}")

# Iniciem la camara

try:

device_info = dai.DeviceInfo("192.168.18.78")

device = dai.Device(device_info)

pipeline = dai.Pipeline(device)   

platform = pipeline.getDefaultDevice().getPlatform()



device.setIrLaserDotProjectorIntensity(dot_projector_intensity_json)

device.setIrFloodLightIntensity(flood_light_intensity_json)

# Definim les fonts i les sortides

camRgb = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)

camRgb.initialControl.setSharpness(set_sharpness_json)

camRgb.initialControl.setLumaDenoise(set_luma_denoise_json)

camRgb.initialControl.setChromaDenoise(set_chroma_denoise_json)

camRgb.initialControl.setAutoExposureLimit(set_auto_exposure_limit_json)

left = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)

right = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C)

stereo = pipeline.create(dai.node.StereoDepth)

stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DETAIL)

stereo.setRectifyEdgeFillColor(set_rectify_edge_fill_color_json)

stereo.setLeftRightCheck(set_left_right_check_json)

stereo.setExtendedDisparity(set_extended_disparity_json)

stereo.setSubpixel(set_subpixel_json) 

stereo.initialConfig.postProcessing.spatialFilter.numIterations = num_iterations_json

stereo.initialConfig.postProcessing.thresholdFilter.minRange = min_range_json

stereo.initialConfig.postProcessing.thresholdFilter.maxRange = max_range_json





sync = pipeline.create(dai.node.Sync)

if platform == dai.Platform.RVC4:

    align = pipeline.create(dai.node.ImageAlign)

sync.setSyncThreshold(timedelta(seconds=1/(2\*FPS_json)))



rgbOut = camRgb.requestOutput(size = (width_resolution_rgb_json, height_resolution_rgb_json), fps = FPS_json, enableUndistortion=enable_undistortion_json)

leftOut = left.requestOutput(size = (width_resolution_left_right_json, height_resolution_left_right_json), fps = FPS_json)

rightOut = right.requestOutput(size = (width_resolution_left_right_json, height_resolution_left_right_json), fps = FPS_json)

stereo.setOutputSize(width_resolution_stereo_json, height_resolution_stereo_json)





# Linquem totes les cameres perque tinguin la mateixa cua per gestionarles

rgbOut.link(sync.inputs["rgb"])

leftOut.link(stereo.left)

rightOut.link(stereo.right)

if platform == dai.Platform.RVC4:

    stereo.depth.link(align.input)

    rgbOut.link(align.inputAlignTo)

    align.outputAligned.link(sync.inputs["depth_aligned"])

else:

    stereo.depth.link(sync.inputs["depth_aligned"])

    rgbOut.link(stereo.inputAlignTo)

queue = sync.out.createOutputQueue(maxSize=1)

   

pipeline.start()

print("Preparat")



write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Camara iniciada correctament")

except Exception as e:

write_to_log(logfiles_path_json, logfile_name_traceability_json, f"Error al inicialitzar la camara: {e}")

write_to_log(logfiles_path_json, logfile_name_error_json, f"Error al inicialitzar la camara: {e}")

# Creem el client MQTT

client = mqtt.Client(protocol=mqtt.MQTTv311)

client.on_connect = on_connect

client.on_message = on_message

client.on_disconnect = on_disconnect

# Connnectem al mosquitto

client.connect(mqtt_ip_json, mqtt_port_json, mqtt_keepalive_json)

# Bucle infinit per poder gestionar els missatges

client.loop_forever()

Do you know how I could eliminate this delay? I need the depth image at 1248x936 and the RGB image at 3744x2808.

I don’t need to view the stream; I just need the images to be captured when I receive the signal.

jakaskerl

dvaro
Let's try decreasing the queue sizes as mentioned in the email, then we can see if something else is also causing a problem.

Thanks,
Jaka