Multiple cameras freeze when encoding

Lucas

Hi everyone.
I have the following code:

import sys
import depthai as dai
import time
import ffmpeg
import threading
import logging
from datetime import datetime
import paho.mqtt.client as mqtt
import json

logging.basicConfig(format="[tn=%(threadName)s] level=%(levelname)s %(message)s")
L = logging.getLogger(__name__)
L.setLevel(logging.DEBUG)


class Oak(threading.Thread):
    def __init__(self, settings, device_info):
        super().__init__(name=device_info.mxid)
        self.settings = settings
        self.device_info = device_info
        self.id = self.device_info.mxid
        L.info(f"init Oak, mxid: {self.id}")

        # Initialize the camera
        self.pipeline = dai.Pipeline()
        self.cam_rgb = self.pipeline.createColorCamera()
        self.cam_rgb.setVideoSize(settings.get("resolution"))
        self.cam_rgb.setFps(settings.get("framerate"))

        self.ready = threading.Event()
        self.record = threading.Event()
        self.record_msg = {"start": False, "duration": 0}
        self.shutdown_event = threading.Event()
        self.now = datetime.now().strftime("%Y%m%d-%H%M%S")

        # Create a VideoEncoder node
        video_encoder = self.pipeline.createVideoEncoder()
        video_encoder.setDefaultProfilePreset(
            self.cam_rgb.getFps(), dai.VideoEncoderProperties.Profile.H264_MAIN
        )

        # Link the color camera's output to the encoder input
        self.cam_rgb.video.link(video_encoder.input)

        # Create an XLinkOut node for the encoded video stream
        xout_video = self.pipeline.createXLinkOut()
        xout_video.setStreamName("video")
        video_encoder.bitstream.link(xout_video.input)

        self.last_msg = None
        self.mqtt_client = mqtt.Client(self.id)
        self.mqtt_client.on_connect = self.on_connect
        self.mqtt_client.on_message = self.msg_handler
        self.mqtt_client.connect_async("localhost")
        self.mqtt_client.loop_start()

    def is_key_frame(self, sequence_number):
        return False if sequence_number % self.settings["framerate"] else True

    def ffmpeg_pipe_init(self):
        filename = f"video_{self.id}_{self.now}.mp4"
        L.info(f"start ffmpeg pipe, filename: {filename}")
        self.ffmpeg_pipe = (
            ffmpeg.input("pipe:", framerate=30)
            .output(filename, vcodec="copy", pix_fmt="yuv420")
            .global_args("-hide_banner")
            .overwrite_output()
            .run_async(pipe_stdin=True)
        )

    def msg_handler(self, mosq, obj, msg):
        msg = json.loads(msg.payload.decode("utf-8"))
        if msg.get("msgId") != self.last_msg:
            print(f"mqtt message: {msg}")
            self.record_msg = msg["payload"]
            if self.record_msg["start"]:
                self.record.set()
            self.last_msg = msg.get("msgId")

    def on_connect(self, mosq, obj, rc, properties):
        print(f"connected to localhost mqtt broker")
        self.mqtt_client.subscribe("testy", 2)

    def run(self):
        while True:
            L.info(f"about to load pipeline")
            with dai.Device(self.pipeline, self.device_info) as self.device:
                L.info(f"loaded pipeline to {self.id} {time.time()}")
                self.queue = self.device.getOutputQueue(
                    name="video", maxSize=30, blocking=False
                )
                L.info(f"set ready flag to {self.id} {time.time()}")
                first_frame = (
                    None  # first frame of the recording clip, which must be a key frame
                )
                self.ffmpeg_pipe = None
                frame = None
                self.ready.set()
                while True:
                    if self.queue.has():
                        frame = self.queue.get()
                        if frame is not None:
                            sequence_number = frame.getSequenceNum()
                            if not self.record_msg["start"]:
                                L.info(f"sequence_number: {sequence_number}")
                            if self.record.is_set():
                                if first_frame == None and self.is_key_frame(
                                    sequence_number
                                ):
                                    L.info(f"first_frame: {first_frame}")
                                    first_frame = sequence_number
                                    self.ffmpeg_pipe_init()

                                if first_frame is not None:
                                    current_frame = sequence_number - first_frame
                                    L.info(f"recording current_frame {current_frame}")
                                    L.debug(f"frame: {frame}, type: {type(frame)}")
                                    f = frame.getData()
                                    L.debug(f"getting here? frame: {f}")
                                    self.ffmpeg_pipe.stdin.write(f)

                                    cut_frame = (
                                        self.settings["framerate"]
                                        * self.record_msg["duration"]
                                        - 1
                                    )  # end the recording clip 1 frame before the next key frame
                                    if current_frame >= cut_frame:
                                        L.debug(
                                            f"getting here current frame>=cut_frame?"
                                        )
                                        self.ffmpeg_pipe.stdin.close()
                                        self.ffmpeg_pipe.wait()
                                        self.record.clear()
                                        L.info(
                                            f"finish recording, current_frame: {current_frame}, cut_frame: {cut_frame}"
                                        )
                                        self.ffmpeg_pipe = None
                                        first_frame = None
                                        self.record_msg = {
                                            "start": False,
                                            "duration": 0,
                                        }

                                L.debug(
                                    f"first_frame: {first_frame}, sequence_number: {sequence_number}"
                                )

                        time.sleep(0.01)
                        if self.shutdown_event.is_set():
                            self.device.close()
                            L.info(
                                f"shutdown event is set, closed device {self.device.isClosed()}"
                            )
                            break
            if self.shutdown_event.is_set():
                L.info(f"shutdown_event is set")
                break


if __name__ == "__main__":
    available_devices = dai.Device.getAllAvailableDevices()
    L.info(f"available devices: {len(available_devices)} - {available_devices}")
    cameras = []
    for device_info in available_devices:
        time.sleep(1)  # currently required due to XLink race issues
        camera = Oak({"framerate": 30, "resolution": (1920, 1080)}, device_info)
        camera.start()
        cameras.append(camera)
        L.info(f"bootloader: {dai.DeviceBootloader(device_info).getVersion()}")
        L.info(f"device state: {device_info.state}")

    L.info(f"cameras {cameras}")

    if len(cameras) == 0:
        L.error(f"no cameras found, exiting")
        sys.exit()

    L.info(f"waiting for cams")
    while not all(cam.ready.is_set() for cam in cameras):
        time.sleep(1)

    L.info(f"all cams ready")
    try:
        while True:
            time.sleep(5)
            # this could possibly monitor cameras and restart them if crashed
    except KeyboardInterrupt:
        L.info(f"keyboardinterrupt exiting")
    finally:
        for cam in cameras:
            L.info(f"device closed?: {cam.device.isClosed()}")
            cam.shutdown_event.set()

I'm trying to get multiple cameras to save the videos directly to an mp4 container.
It seems to work fine with one camera, but when there is more than 1 camera, it'll freeze on the line:

f = frame.getData()
as if it can't get the frame data to pass it to ffmpeg. I've tried following the example using av instead, from luxonis/depthai-experimentsblob/master/gen2-container-encoding/main.py , but the same thing happens.
Any idea what I might be doing wrong?

Thanks!

jakaskerl

Hi @Lucas
Please write a minimal reproducible example. This current code has a lot of functionality that could cause the block to the pipeline.

Thanks,
Jaka

Lucas

Hi @jakaskerl, thank you for your reply.
Here's an MRE I managed to write:

import sys
import depthai as dai
import time
import threading
import logging
from datetime import datetime
import json

logging.basicConfig(format='[tn=%(threadName)s] level=%(levelname)s %(message)s')
L = logging.getLogger(__name__)
L.setLevel(logging.DEBUG)

class Oak(threading.Thread):
    def __init__(self,settings, device_info):
        super().__init__(name=device_info.mxid)
        self.settings = settings
        self.device_info = device_info
        self.id = self.device_info.mxid
        L.info(f"init Oak, mxid: {self.id}")
    
        # Initialize the camera
        self.pipeline = dai.Pipeline()
        self.cam_rgb = self.pipeline.createColorCamera()
        self.cam_rgb.setVideoSize(settings.get('resolution'))
        self.cam_rgb.setFps(settings.get('framerate'))
   
        self.record = False
        self.shutdown_event = False
        self.queue = False

        # Create a VideoEncoder node
        video_encoder = self.pipeline.createVideoEncoder()
        video_encoder.setDefaultProfilePreset(self.cam_rgb.getFps(),dai.VideoEncoderProperties.Profile.H264_MAIN)

        # Link the color camera's output to the encoder input
        self.cam_rgb.video.link(video_encoder.input)

        # Create an XLinkOut node for the encoded video stream
        xout_video = self.pipeline.createXLinkOut()
        xout_video.setStreamName("video")
        video_encoder.bitstream.link(xout_video.input)


    def is_key_frame(self, sequence_number):
        return False if sequence_number % self.settings['framerate'] else True
   
    def run(self):
        while True:
            L.info(f"about to load pipeline")
            with dai.Device(self.pipeline, self.device_info) as self.device:
                L.info(f"loaded pipeline to {self.id} {time.time()}")
                self.queue = self.device.getOutputQueue(name='video', maxSize=240, blocking=False)
                first_frame = None # first frame of the recording clip, which must be a key frame
                frame = None
                while True:
                    if self.queue.has():
                        frame = self.queue.get()
                        if frame is not None:
                            sequence_number = frame.getSequenceNum()

                            if self.record:

                                if first_frame == None and self.is_key_frame(sequence_number):
                                    first_frame = sequence_number
                                    L.info(f"set {self.id} to record for 5 seconds")
                                    L.info(f"first_frame: {first_frame}")
                                    self.now = datetime.now().strftime("%Y%m%d-%H%M%S")
                                    filename = f"video_{self.id}_{self.now}.h264"
                                    file = open(filename, 'wb')


                                if first_frame is not None:
                                    current_frame = sequence_number - first_frame
                                    try:
                                        frame.getData().tofile(file)
                                    except Exception as e:
                                        L.error(f"Error occurred: {e}")


                                    cut_frame = self.settings['framerate'] * 5 - 1 # end the recording clip 1 frame before the next key frame
                                    if current_frame >= cut_frame:
                                        file.close()
                                        L.info(f"finish recording, current_frame: {current_frame}, cut_frame: {cut_frame}")
                                        first_frame = None
                                        self.shutdown_event = True
                                        self.record = False
                                
                        if self.shutdown_event:
                            self.device.close()
                            L.info(f"shutdown event is set, closed device {self.device.isClosed()}")
                            break
                    time.sleep(0.01)
            if self.shutdown_event:
                L.info(f"shutdown_event is set")
                break



if __name__ == "__main__":

    available_devices = dai.Device.getAllAvailableDevices()
    L.info(f"available devices: {len(available_devices)} - {available_devices}")
    cameras = []
    for device_info in available_devices:
        time.sleep(1) # currently required due to XLink race issues
        camera = Oak({'framerate':30, 'resolution':(1920,1080)}, device_info)
        camera.start()
        cameras.append(camera)
        L.info(f"bootloader: {dai.DeviceBootloader(device_info).getVersion()}")
        L.info(f"device state: {device_info.state}")
    

    while not all(cam.queue for cam in cameras):
        L.info(f"waiting for camera queues to have frames")
        time.sleep(1)

    L.info("all cameras ready")

    for cam in cameras:
        if not cam.record:
            L.info(f"set {cam.id} to record")
            cam.record = True

    try:
        while True:
            time.sleep(5)
            # this could possibly monitor cameras and restart them if crashed
    except KeyboardInterrupt:
        L.info(f"keyboardinterrupt exiting")
    finally:
        for cam in cameras: 
            L.info(f"device closed?: {cam.device.isClosed()}")
            cam.shutdown_event = True

I removed the ffmpeg pipe and the logic dealing with mqtt messages to record.
Now the streams are going directly to h264 files.

The problem seems to be when I wait for the cameras to have the queues set, and then set the cam.record = Truefor each camera.
If I remove those checks, because one camera is set up right after the other, it seems to record fine.
However, if I wait, the script will freeze, the files created will be 0 bytes, and nothing will happen until the script is forcefully terminated.

Any ideas?

Thanks again

jakaskerl

Hi @Lucas
The code you have sent works for me. Only thing I did was ensure both cameras are getting enough power.

Thanks,
Jaka

Lucas

Hi @jakaskerl

Thanks for testing.
I think it has run a couple of times for me as well, but most of the times it freezes.

The cameras are powered properly with POE injectors, powered by power supplies connected to ac.
I'll see if I can try different injectors.
I tested the script on a mac and on a raspberry pi with the same results.
On the raspberry pi I tested it inside a docker container.
If the script is run with a single camera, it works fine.

Is there any reason why the pipeline would block? What else can I add to it to help with debugging?

Thank you again.

jakaskerl

Hi @Lucas
Do you have a POE switch nearby, which you could use? I found that the injectors can sometimes be a bit unstable.

Thanks,
Jaka

Lucas

Hi @jakaskerl

Yes, I tested it with different injectors and a Poe switch, to the same result.

It does work for me sometimes, but other times it'll freeze. It's not reliable, and I can't pinpoint why it's happening.

I'll leave the script running on a loop from the shell with a delay between the runs, to wait for the cameras, and it'll work a few times and then randomly will freeze.

It freezes when it wants to get the data from the frame, with getData(), and it'll only happen when I wait for both cameras to be ready. If I record straight away there's no problem?

It is really strange, and it doesn't seem to be power related.

jakaskerl

Lucas Will check on monday, if I can trace the issue.

Thanks,
Jaka

jakaskerl

Hi @Lucas
I could not reproduce your issue, running the script 20 times.
This leads me to believe there is a hardware problem most likely. There looks to be a bottleneck somewhere in your system.

Low PoE speed
Host speed for writing frames

Could you check that please?

My setup:
MacOS sonoma, depthai==2.24.0.0. Netgear unmanaged poe switch. OAK-D poe, OAK-D-pro POE. 1000Mbps bandwidth.

Thanks,
Jaka

Lucas

Thanks @jakaskerl

That doesn't sound too good for me.

I'll keep testing with different equipment to see what I can find out, and I'll report back if I find the cause.

Cheers

Lucas

Hi @jakaskerl

Still no good news.
I tried with 4 new different oak-1-poe and oak-1-poe W, and a ubiquiti USW-Lite-8-POE switch.
I tried on a raspberry pi and on MacOS running ventura 13.6.2.
The issue persists. Is it possible that is related to the oak-1-poe and not the oak-d? I have no other cameras to test, but this happened in 3 oak-1-poe and 3 oak-1-poe W.

The only new thing I discovered is that the issue seems to be related to waiting at the very beginning.
If I set record=True straight away, it works.
If I wait for the cameras to have a queue, it freezes.
If I wait a random amount of time, it freezes.
If I wait 3 minutes, it seems to work?

Here's an output with debug enabled on the cameras, when the script froze (the cameras seem to keep going, because the info output keeps printing):

[2024-01-11 15:50:58.251] [depthai] [debug] Python bindings - version: 2.24.0.0 from 2023-12-13 15:38:19 +0100 build: 2023-12-13 23:34:03 +0000                                                                                             
[2024-01-11 15:50:58.251] [depthai] [debug] Library information - version: 2.24.0, commit: 6628488ef8956f73f1c7bf4c8f1da218ad327a6f from 2023-12-13 14:45:09 +0100, build: 2023-12-13 23:34:02 +0000, libusb enabled: true                  
[2024-01-11 15:50:58.263] [depthai] [debug] Initialize - finished
[2024-01-11 15:50:58.312] [depthai] [debug] Resources - Archive 'depthai-bootloader-fwp-0.0.26.tar.xz' open: 2ms, archive read: 58ms                                                                                                        
[2024-01-11 15:50:58.578] [depthai] [debug] Resources - Archive 'depthai-device-fwp-a95f582a61ec9bdbd0f72dec84822455872ffaf7.tar.xz' open: 2ms, archive read: 324ms                                                                         
[tn=MainThread] level=INFO available devices: 2 - [DeviceInfo(name=192.168.3.227, mxid=14442C105184EBCF00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), DeviceInfo(name=192.168.3.226, mxid=14442C104130EBCF00, X_LIN
K_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS)]   
[tn=MainThread] level=INFO init Oak, mxid: 14442C105184EBCF00    
[2024-01-11 15:50:59.783] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.227, mxid=14442C105184EBCF00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only                 
[2024-01-11 15:50:59.797] [depthai] [debug] Connected bootloader version 0.0.26
[2024-01-11 15:50:59.797] [depthai] [debug] DeviceBootloader about to be closed...
[2024-01-11 15:50:59.797] [depthai] [debug] XLinkResetRemote of linkId: (0)    
[2024-01-11 15:51:01.289] [depthai] [debug] DeviceBootloader closed, 1492      
[tn=MainThread] level=INFO bootloader: 0.0.26                    
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER    
[tn=14442C105184EBCF00] level=INFO about to load pipeline        
mre.py:51: DeprecationWarning: Use constructor taking 'UsbSpeed' instead                                                                                                                                     with dai.Device(self.pipeline, self.device_info) as self.device:             
[2024-01-11 15:51:01.297] [host] [debug] Device - OpenVINO version: universal  
[14442C105184EBCF00] [192.168.3.227] [1704941461.297] [host] [debug] Device - BoardConfig: {"camera":[],"emmc":null,"gpio":[],"imu":null,"logDevicePrints":null,"logPath":null,"logSizeMax":null,"logVerbosity":null,"network":{"mtu":0,"xli
nkTcpNoDelay":true},"nonExclusiveMode":false,"pcieInternalClock":null,"sysctl":[],"uart":[],"usb":{"flashBootedPid":63037,"flashBootedVid":999,"manufacturer":"","maxSpeed":4,"pid":63035,"productName":"","vid":999},"usb3PhyInternalClock"
:null,"uvc":null,"watchdogInitialDelayMs":null,"watchdogTimeoutMs":null}       
libnop:                                                          
0000: b9 12 b9 07 81 e7 03 81 3b f6 81 e7 03 81 3d f6 04 bd 00 bd 00 b9 02 00 01 ba 00 be be bb 00 bb                                                                                                                                       
0020: 00 be be be be be be be 00 bb 00 be be                     
[2024-01-11 15:51:01.301] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.227, mxid=14442C105184EBCF00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only                 
[tn=MainThread] level=INFO init Oak, mxid: 14442C104130EBCF00    
[2024-01-11 15:51:02.293] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.226, mxid=14442C104130EBCF00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only                 
[2024-01-11 15:51:02.308] [depthai] [debug] Connected bootloader version 0.0.26
[2024-01-11 15:51:02.308] [depthai] [debug] DeviceBootloader about to be closed...
[2024-01-11 15:51:02.309] [depthai] [debug] XLinkResetRemote of linkId: (1)    
[2024-01-11 15:51:03.801] [depthai] [debug] DeviceBootloader closed, 1493      
[tn=MainThread] level=INFO bootloader: 0.0.26                    
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER    
[tn=14442C104130EBCF00] level=INFO about to load pipeline        
[2024-01-11 15:51:03.802] [host] [debug] Device - OpenVINO version: universal  
[tn=MainThread] level=INFO waiting for camera queues to have frames            
[14442C104130EBCF00] [192.168.3.226] [1704941463.802] [host] [debug] Device - BoardConfig: {"camera":[],"emmc":null,"gpio":[],"imu":null,"logDevicePrints":null,"logPath":null,"logSizeMax":null,"logVerbosity":null,"network":{"mtu":0,"xli
nkTcpNoDelay":true},"nonExclusiveMode":false,"pcieInternalClock":null,"sysctl":[],"uart":[],"usb":{"flashBootedPid":63037,"flashBootedVid":999,"manufacturer":"","maxSpeed":4,"pid":63035,"productName":"","vid":999},"usb3PhyInternalClock"
:null,"uvc":null,"watchdogInitialDelayMs":null,"watchdogTimeoutMs":null}       
libnop:                                                          
0000: b9 12 b9 07 81 e7 03 81 3b f6 81 e7 03 81 3d f6 04 bd 00 bd 00 b9 02 00 01 ba 00 be be bb 00 bb                                                                                                                                       
0020: 00 be be be be be be be 00 bb 00 be be                                                                                 
[2024-01-11 15:51:03.809] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.226, mxid=14442C104130EBCF00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only                 
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[2024-01-11 15:51:08.046] [depthai] [debug] Connected bootloader version 0.0.26
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C105184EBCF00] [192.168.3.227] [1704941468.944] [host] [debug] Booting FW with Bootloader. Version 0.0.26, Time taken: 897ms
[2024-01-11 15:51:08.944] [depthai] [debug] DeviceBootloader about to be closed...
[2024-01-11 15:51:08.944] [depthai] [debug] XLinkResetRemote of linkId: (2)
[tn=MainThread] level=INFO waiting for camera queues to have frames
[2024-01-11 15:51:10.167] [depthai] [debug] DeviceBootloader closed, 1222
[2024-01-11 15:51:10.169] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.227, mxid=14442C105184EBCF00, X_LINK_BOOTED, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only
[tn=MainThread] level=INFO waiting for camera queues to have frames
[2024-01-11 15:51:11.070] [depthai] [debug] Connected bootloader version 0.0.26
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C104130EBCF00] [192.168.3.226] [1704941471.972] [host] [debug] Booting FW with Bootloader. Version 0.0.26, Time taken: 901ms
[2024-01-11 15:51:11.972] [depthai] [debug] DeviceBootloader about to be closed...
[2024-01-11 15:51:11.972] [depthai] [debug] XLinkResetRemote of linkId: (3)
[tn=MainThread] level=INFO waiting for camera queues to have frames
[2024-01-11 15:51:13.198] [depthai] [debug] DeviceBootloader closed, 1226
[2024-01-11 15:51:13.204] [depthai] [debug] Searching for booted device: DeviceInfo(name=192.168.3.226, mxid=14442C104130EBCF00, X_LINK_BOOTED, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), name used as hint only
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C105184EBCF00] [192.168.3.227] [6.852] [system] [info] Memory Usage - DDR: 0.12 / 333.46 MiB, CMX: 2.04 / 2.50 MiB, LeonOS Heap: 25.28 / 82.56 MiB, LeonRT Heap: 2.89 / 40.68 MiB / NOC ddr: 0 MB/s
[14442C105184EBCF00] [192.168.3.227] [6.852] [system] [info] Temperatures - Average: 34.40C, CSS: 36.06C, MSS 33.45C, UPA: 33.69C, DSS: 34.40C
[14442C105184EBCF00] [192.168.3.227] [6.852] [system] [info] Cpu Usage - LeonOS 32.38%, LeonRT: 0.39%
[14442C105184EBCF00] [192.168.3.227] [1704941476.365] [host] [debug] Schema dump: {"connections":[{"node1Id":1,"node1Output":"bitstream","node1OutputGroup":"","node2Id":2,"node2Input":"in","node2InputGroup":""},{"node1Id":0,"node1Output
":"video","node1OutputGroup":"","node2Id":1,"node2Input":"in","node2InputGroup":""}],"globalProperties":{"calibData":null,"cameraTuningBlobSize":null,"cameraTuningBlobUri":"","leonCssFrequencyHz":700000000.0,"leonMssFrequencyHz":7000000
00.0,"pipelineName":null,"pipelineVersion":null,"sippBufferSize":18432,"sippDmaBufferSize":16384,"xlinkChunkSize":-1},"nodes":[[0,{"id":0,"ioInfo":[[["","inputConfig"],{"blocking":false,"group":"","id":6,"name":"inputConfig","queueSize"
:8,"type":3,"waitForMessage":false}],[["","frameEvent"],{"blocking":false,"group":"","id":7,"name":"frameEvent","queueSize":8,"type":0,"waitForMessage":false}],[["","video"],{"blocking":false,"group":"","id":12,"name":"video","queueSize
":8,"type":0,"waitForMessage":false}],[["","raw"],{"blocking":false,"group":"","id":8,"name":"raw","queueSize":8,"type":0,"waitForMessage":false}],[["","inputControl"],{"blocking":true,"group":"","id":5,"name":"inputControl","queueSize"
:8,"type":3,"waitForMessage":false}],[["","isp"],{"blocking":false,"group":"","id":9,"name":"isp","queueSize":8,"type":0,"waitForMessage":false}],[["","still"],{"blocking":false,"group":"","id":10,"name":"still","queueSize":8,"type":0,"
waitForMessage":false}],[["","preview"],{"blocking":false,"group":"","id":11,"name":"preview","queueSize":8,"type":0,"waitForMessage":false}]],"name":"ColorCamera","properties":[185,26,185,30,0,3,0,0,0,185,3,0,0,0,185,5,0,0,0,0,0,185,5,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,3,0,0,0,185,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,189,0,255,0,1,0,129,44,1,129,44,1,133,128,7,133,56,4,255,255,0,136,0,0,240,65,0,136,0,0,128,191,136,0,0,128,191,1,185,4,0,0,0,0,3,3,4,4,4,190]}],[1,{"id":1,"i
oInfo":[[["","bitstream"],{"blocking":false,"group":"","id":4,"name":"bitstream","queueSize":8,"type":0,"waitForMessage":false}],[["","in"],{"blocking":true,"group":"","id":2,"name":"in","queueSize":4,"type":3,"waitForMessage":true}],[[
"","out"],{"blocking":false,"group":"","id":3,"name":"out","queueSize":8,"type":0,"waitForMessage":false}]],"name":"VideoEncoder","properties":[185,11,0,30,0,0,0,0,2,80,0,0,136,0,0,240,65]}],[2,{"id":2,"ioInfo":[[["","in"],{"blocking":t
rue,"group":"","id":1,"name":"in","queueSize":8,"type":3,"waitForMessage":true}]],"name":"XLinkOut","properties":[185,3,136,0,0,128,191,189,5,118,105,100,101,111,0]}]]}
[14442C105184EBCF00] [192.168.3.227] [1704941476.365] [host] [debug] Asset map dump: {"map":{}}
[14442C105184EBCF00] [192.168.3.227] [6.886] [system] [info] SIPP (Signal Image Processing Pipeline) internal buffer size '18432'B, DMA buffer size: '16384'B
[14442C105184EBCF00] [192.168.3.227] [6.921] [system] [info] ImageManip internal buffer size '285440'B, shave buffer size '34816'B
[tn=14442C105184EBCF00] level=INFO loaded pipeline to 14442C105184EBCF00 1704941476.465984
[14442C105184EBCF00] [192.168.3.227] [6.921] [system] [info] ColorCamera allocated resources: no shaves; cmx slices: [13-15] 
ImageManip allocated resources: shaves: [15-15] no cmx slices. 
[14442C105184EBCF00] [192.168.3.227] [7.052] [VideoEncoder(1)] [info] Bitrate: auto-set to 8500000 bit/s
[14442C105184EBCF00] [192.168.3.227] [7.055] [VideoEncoder(1)] [debug] Bitstream frame size: 3170304, num frames in pool: 4
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C105184EBCF00] [192.168.3.227] [7.853] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.21 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 524 MB/s
[14442C105184EBCF00] [192.168.3.227] [7.853] [system] [info] Temperatures - Average: 35.88C, CSS: 37.94C, MSS 35.83C, UPA: 34.16C, DSS: 35.59C
[14442C105184EBCF00] [192.168.3.227] [7.853] [system] [info] Cpu Usage - LeonOS 29.66%, LeonRT: 8.62%
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C105184EBCF00] [192.168.3.227] [8.855] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 1045 MB/s
[14442C105184EBCF00] [192.168.3.227] [8.855] [system] [info] Temperatures - Average: 36.18C, CSS: 38.41C, MSS 35.35C, UPA: 34.64C, DSS: 36.30C
[14442C105184EBCF00] [192.168.3.227] [8.855] [system] [info] Cpu Usage - LeonOS 25.32%, LeonRT: 4.26%
[tn=MainThread] level=INFO waiting for camera queues to have frames
[14442C105184EBCF00] [192.168.3.227] [9.856] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 972 MB/s
[14442C105184EBCF00] [192.168.3.227] [9.856] [system] [info] Temperatures - Average: 36.35C, CSS: 38.88C, MSS 35.35C, UPA: 34.88C, DSS: 36.30C
[14442C105184EBCF00] [192.168.3.227] [9.856] [system] [info] Cpu Usage - LeonOS 23.39%, LeonRT: 4.25%
[14442C104130EBCF00] [192.168.3.226] [6.878] [system] [info] Memory Usage - DDR: 0.12 / 333.46 MiB, CMX: 2.04 / 2.50 MiB, LeonOS Heap: 25.28 / 82.56 MiB, LeonRT Heap: 2.89 / 40.68 MiB / NOC ddr: 0 MB/s
[14442C104130EBCF00] [192.168.3.226] [6.878] [system] [info] Temperatures - Average: 34.28C, CSS: 36.06C, MSS 33.93C, UPA: 33.69C, DSS: 33.45C
[14442C104130EBCF00] [192.168.3.226] [6.878] [system] [info] Cpu Usage - LeonOS 29.01%, LeonRT: 0.39%
[14442C104130EBCF00] [192.168.3.226] [1704941479.414] [host] [debug] Schema dump: {"connections":[{"node1Id":1,"node1Output":"bitstream","node1OutputGroup":"","node2Id":2,"node2Input":"in","node2InputGroup":""},{"node1Id":0,"node1Output
":"video","node1OutputGroup":"","node2Id":1,"node2Input":"in","node2InputGroup":""}],"globalProperties":{"calibData":null,"cameraTuningBlobSize":null,"cameraTuningBlobUri":"","leonCssFrequencyHz":700000000.0,"leonMssFrequencyHz":7000000
00.0,"pipelineName":null,"pipelineVersion":null,"sippBufferSize":18432,"sippDmaBufferSize":16384,"xlinkChunkSize":-1},"nodes":[[0,{"id":0,"ioInfo":[[["","inputConfig"],{"blocking":false,"group":"","id":6,"name":"inputConfig","queueSize"
:8,"type":3,"waitForMessage":false}],[["","frameEvent"],{"blocking":false,"group":"","id":7,"name":"frameEvent","queueSize":8,"type":0,"waitForMessage":false}],[["","video"],{"blocking":false,"group":"","id":12,"name":"video","queueSize
":8,"type":0,"waitForMessage":false}],[["","raw"],{"blocking":false,"group":"","id":8,"name":"raw","queueSize":8,"type":0,"waitForMessage":false}],[["","inputControl"],{"blocking":true,"group":"","id":5,"name":"inputControl","queueSize"
:8,"type":3,"waitForMessage":false}],[["","isp"],{"blocking":false,"group":"","id":9,"name":"isp","queueSize":8,"type":0,"waitForMessage":false}],[["","still"],{"blocking":false,"group":"","id":10,"name":"still","queueSize":8,"type":0,"
waitForMessage":false}],[["","preview"],{"blocking":false,"group":"","id":11,"name":"preview","queueSize":8,"type":0,"waitForMessage":false}]],"name":"ColorCamera","properties":[185,26,185,30,0,3,0,0,0,185,3,0,0,0,185,5,0,0,0,0,0,185,5,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,185,3,0,0,0,185,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,189,0,255,0,1,0,129,44,1,129,44,1,133,128,7,133,56,4,255,255,0,136,0,0,240,65,0,136,0,0,128,191,136,0,0,128,191,1,185,4,0,0,0,0,3,3,4,4,4,190]}],[1,{"id":1,"i
oInfo":[[["","bitstream"],{"blocking":false,"group":"","id":4,"name":"bitstream","queueSize":8,"type":0,"waitForMessage":false}],[["","in"],{"blocking":true,"group":"","id":2,"name":"in","queueSize":4,"type":3,"waitForMessage":true}],[[
"","out"],{"blocking":false,"group":"","id":3,"name":"out","queueSize":8,"type":0,"waitForMessage":false}]],"name":"VideoEncoder","properties":[185,11,0,30,0,0,0,0,2,80,0,0,136,0,0,240,65]}],[2,{"id":2,"ioInfo":[[["","in"],{"blocking":t
rue,"group":"","id":1,"name":"in","queueSize":8,"type":3,"waitForMessage":true}]],"name":"XLinkOut","properties":[185,3,136,0,0,128,191,189,5,118,105,100,101,111,0]}]]}
[14442C104130EBCF00] [192.168.3.226] [1704941479.414] [host] [debug] Asset map dump: {"map":{}}
[14442C104130EBCF00] [192.168.3.226] [6.910] [system] [info] SIPP (Signal Image Processing Pipeline) internal buffer size '18432'B, DMA buffer size: '16384'B
[14442C104130EBCF00] [192.168.3.226] [6.945] [system] [info] ImageManip internal buffer size '285440'B, shave buffer size '34816'B
[14442C104130EBCF00] [192.168.3.226] [6.945] [system] [info] ColorCamera allocated resources: no shaves; cmx slices: [13-15] 
ImageManip allocated resources: shaves: [15-15] no cmx slices. 
[tn=14442C104130EBCF00] level=INFO loaded pipeline to 14442C104130EBCF00 1704941479.513939
[14442C104130EBCF00] [192.168.3.226] [7.076] [VideoEncoder(1)] [info] Bitrate: auto-set to 8500000 bit/s
[14442C104130EBCF00] [192.168.3.226] [7.079] [VideoEncoder(1)] [debug] Bitstream frame size: 3170304, num frames in pool: 4
[tn=MainThread] level=INFO all cameras ready
[tn=MainThread] level=INFO set 14442C105184EBCF00 to record 
[tn=MainThread] level=INFO set 14442C104130EBCF00 to record 
[14442C105184EBCF00] [192.168.3.227] [10.857] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 974 MB/s
[14442C105184EBCF00] [192.168.3.227] [10.857] [system] [info] Temperatures - Average: 36.71C, CSS: 38.88C, MSS 36.30C, UPA: 35.11C, DSS: 36.53C
[14442C105184EBCF00] [192.168.3.227] [10.857] [system] [info] Cpu Usage - LeonOS 26.59%, LeonRT: 4.28%
[14442C104130EBCF00] [192.168.3.226] [7.879] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.21 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 493 MB/s
[14442C104130EBCF00] [192.168.3.226] [7.879] [system] [info] Temperatures - Average: 35.64C, CSS: 38.18C, MSS 34.40C, UPA: 34.40C, DSS: 35.59C
[14442C104130EBCF00] [192.168.3.226] [7.879] [system] [info] Cpu Usage - LeonOS 29.95%, LeonRT: 8.65%
[tn=14442C105184EBCF00] level=INFO recording 5 seconds from first_frame: 120
[tn=14442C104130EBCF00] level=INFO recording 5 seconds from first_frame: 30
[14442C105184EBCF00] [192.168.3.227] [11.858] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 979 MB/s
[14442C105184EBCF00] [192.168.3.227] [11.858] [system] [info] Temperatures - Average: 36.12C, CSS: 37.48C, MSS 35.59C, UPA: 35.59C, DSS: 35.83C
[14442C105184EBCF00] [192.168.3.227] [11.858] [system] [info] Cpu Usage - LeonOS 26.54%, LeonRT: 4.32%
[14442C104130EBCF00] [192.168.3.226] [8.880] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.21 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 1037 MB/s
[14442C104130EBCF00] [192.168.3.226] [8.880] [system] [info] Temperatures - Average: 36.00C, CSS: 38.18C, MSS 35.83C, UPA: 34.64C, DSS: 35.35C
[14442C104130EBCF00] [192.168.3.226] [8.880] [system] [info] Cpu Usage - LeonOS 25.50%, LeonRT: 4.28%
[14442C105184EBCF00] [192.168.3.227] [12.859] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 976 MB/s
[14442C105184EBCF00] [192.168.3.227] [12.859] [system] [info] Temperatures - Average: 36.71C, CSS: 38.65C, MSS 36.06C, UPA: 35.35C, DSS: 36.77C
[14442C105184EBCF00] [192.168.3.227] [12.859] [system] [info] Cpu Usage - LeonOS 26.41%, LeonRT: 4.27%
[14442C104130EBCF00] [192.168.3.226] [9.881] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 973 MB/s
[14442C104130EBCF00] [192.168.3.226] [9.881] [system] [info] Temperatures - Average: 36.00C, CSS: 38.18C, MSS 35.11C, UPA: 34.64C, DSS: 36.06C
[14442C104130EBCF00] [192.168.3.226] [9.881] [system] [info] Cpu Usage - LeonOS 26.66%, LeonRT: 4.33%
[14442C105184EBCF00] [192.168.3.227] [13.860] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 975 MB/s
[14442C105184EBCF00] [192.168.3.227] [13.860] [system] [info] Temperatures - Average: 36.77C, CSS: 38.65C, MSS 36.06C, UPA: 35.83C, DSS: 36.53C
[14442C105184EBCF00] [192.168.3.227] [13.860] [system] [info] Cpu Usage - LeonOS 26.27%, LeonRT: 4.28%
[14442C104130EBCF00] [192.168.3.226] [10.882] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 974 MB/s
[14442C104130EBCF00] [192.168.3.226] [10.882] [system] [info] Temperatures - Average: 36.29C, CSS: 38.41C, MSS 34.88C, UPA: 35.11C, DSS: 36.77C
[14442C104130EBCF00] [192.168.3.226] [10.882] [system] [info] Cpu Usage - LeonOS 26.51%, LeonRT: 4.31%
[14442C105184EBCF00] [192.168.3.227] [14.861] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 975 MB/s
[14442C105184EBCF00] [192.168.3.227] [14.861] [system] [info] Temperatures - Average: 36.41C, CSS: 38.88C, MSS 35.83C, UPA: 35.35C, DSS: 35.59C
[14442C105184EBCF00] [192.168.3.227] [14.861] [system] [info] Cpu Usage - LeonOS 26.35%, LeonRT: 4.26%
[14442C104130EBCF00] [192.168.3.226] [11.883] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 970 MB/s
[14442C104130EBCF00] [192.168.3.226] [11.883] [system] [info] Temperatures - Average: 36.23C, CSS: 38.41C, MSS 35.35C, UPA: 34.88C, DSS: 36.30C
[14442C104130EBCF00] [192.168.3.226] [11.883] [system] [info] Cpu Usage - LeonOS 26.47%, LeonRT: 4.31%
[14442C105184EBCF00] [192.168.3.227] [15.862] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 975 MB/s
[14442C105184EBCF00] [192.168.3.227] [15.862] [system] [info] Temperatures - Average: 36.53C, CSS: 38.41C, MSS 35.59C, UPA: 35.35C, DSS: 36.77C
[14442C105184EBCF00] [192.168.3.227] [15.862] [system] [info] Cpu Usage - LeonOS 26.90%, LeonRT: 4.28%
[14442C104130EBCF00] [192.168.3.226] [12.884] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 974 MB/s
[14442C104130EBCF00] [192.168.3.226] [12.884] [system] [info] Temperatures - Average: 36.35C, CSS: 38.88C, MSS 36.06C, UPA: 34.16C, DSS: 36.30C
[14442C104130EBCF00] [192.168.3.226] [12.884] [system] [info] Cpu Usage - LeonOS 26.22%, LeonRT: 4.27%
[14442C105184EBCF00] [192.168.3.227] [16.863] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 976 MB/s
[14442C105184EBCF00] [192.168.3.227] [16.863] [system] [info] Temperatures - Average: 37.06C, CSS: 38.65C, MSS 36.30C, UPA: 35.83C, DSS: 37.48C
[14442C105184EBCF00] [192.168.3.227] [16.863] [system] [info] Cpu Usage - LeonOS 26.92%, LeonRT: 4.28%
[14442C104130EBCF00] [192.168.3.226] [13.885] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 969 MB/s
[14442C104130EBCF00] [192.168.3.226] [13.885] [system] [info] Temperatures - Average: 36.47C, CSS: 38.88C, MSS 35.59C, UPA: 34.88C, DSS: 36.53C
[14442C104130EBCF00] [192.168.3.226] [13.885] [system] [info] Cpu Usage - LeonOS 26.42%, LeonRT: 4.30%
[14442C105184EBCF00] [192.168.3.227] [17.864] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 976 MB/s
[14442C105184EBCF00] [192.168.3.227] [17.864] [system] [info] Temperatures - Average: 36.71C, CSS: 38.41C, MSS 35.83C, UPA: 36.06C, DSS: 36.53C
[14442C105184EBCF00] [192.168.3.227] [17.864] [system] [info] Cpu Usage - LeonOS 26.61%, LeonRT: 4.30%
[14442C104130EBCF00] [192.168.3.226] [14.886] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 975 MB/s
[14442C104130EBCF00] [192.168.3.226] [14.886] [system] [info] Temperatures - Average: 36.53C, CSS: 38.88C, MSS 35.59C, UPA: 35.35C, DSS: 36.30C
[14442C104130EBCF00] [192.168.3.226] [14.886] [system] [info] Cpu Usage - LeonOS 26.79%, LeonRT: 4.28%
[14442C105184EBCF00] [192.168.3.227] [18.865] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 974 MB/s
[14442C105184EBCF00] [192.168.3.227] [18.865] [system] [info] Temperatures - Average: 36.71C, CSS: 38.65C, MSS 35.59C, UPA: 36.30C, DSS: 36.30C
[14442C105184EBCF00] [192.168.3.227] [18.865] [system] [info] Cpu Usage - LeonOS 26.21%, LeonRT: 4.26%
[14442C104130EBCF00] [192.168.3.226] [15.887] [system] [info] Memory Usage - DDR: 54.50 / 333.46 MiB, CMX: 2.39 / 2.50 MiB, LeonOS Heap: 39.46 / 82.56 MiB, LeonRT Heap: 3.76 / 40.68 MiB / NOC ddr: 970 MB/s
[14442C104130EBCF00] [192.168.3.226] [15.887] [system] [info] Temperatures - Average: 36.47C, CSS: 38.18C, MSS 36.06C, UPA: 34.64C, DSS: 37.01C
[14442C104130EBCF00] [192.168.3.226] [15.887] [system] [info] Cpu Usage - LeonOS 26.67%, LeonRT: 4.31%

jakaskerl

Hi @Lucas
Tried again, this time with 4 cameras.

[tn=MainThread] level=INFO available devices: 4 - [DeviceInfo(name=192.168.118.108, mxid=1844301011ACFB0F00, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), DeviceInfo(name=192.168.118.113, mxid=1944301061057F1300, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), DeviceInfo(name=192.168.118.110, mxid=184430107151C31200, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS), DeviceInfo(name=192.168.118.111, mxid=184430107141C31200, X_LINK_BOOTLOADER, X_LINK_TCP_IP, X_LINK_MYRIAD_X, X_LINK_SUCCESS)]
[tn=MainThread] level=INFO init Oak, mxid: 1844301011ACFB0F00
[tn=1844301011ACFB0F00] level=INFO about to load pipeline
[tn=MainThread] level=INFO bootloader: 0.0.26
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER
/Users/jaka/Desktop/tests/camera_encoding_freeze.py:50: DeprecationWarning: Use constructor taking 'UsbSpeed' instead
  with dai.Device(self.pipeline, self.device_info) as self.device:
[tn=MainThread] level=INFO init Oak, mxid: 1944301061057F1300
[tn=1944301061057F1300] level=INFO about to load pipeline
[tn=MainThread] level=INFO bootloader: 0.0.26
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER
[tn=MainThread] level=INFO init Oak, mxid: 184430107151C31200
[tn=184430107151C31200] level=INFO about to load pipeline
[tn=MainThread] level=INFO bootloader: 0.0.26
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER
[tn=MainThread] level=INFO init Oak, mxid: 184430107141C31200
[tn=184430107141C31200] level=INFO about to load pipeline
[tn=MainThread] level=INFO bootloader: 0.0.26
[tn=MainThread] level=INFO device state: XLinkDeviceState.X_LINK_BOOTLOADER
[tn=MainThread] level=INFO cam queues ready: False
[tn=MainThread] level=INFO cam queues ready: False
[tn=MainThread] level=INFO cam queues ready: False
[tn=MainThread] level=INFO cam queues ready: False
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=1844301011ACFB0F00] level=INFO loaded pipeline to 1844301011ACFB0F00 1704983388.812039
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=1944301061057F1300] level=INFO loaded pipeline to 1944301061057F1300 1704983398.7158751
[tn=184430107151C31200] level=INFO loaded pipeline to 184430107151C31200 1704983399.153722
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=MainThread] level=INFO waiting for camera queues to have frames
[tn=184430107141C31200] level=INFO loaded pipeline to 184430107141C31200 1704983403.762336
[tn=MainThread] level=INFO all cameras ready
[tn=MainThread] level=INFO set 1844301011ACFB0F00 to record
[tn=MainThread] level=INFO set 1944301061057F1300 to record
[tn=MainThread] level=INFO set 184430107151C31200 to record
[tn=MainThread] level=INFO set 184430107141C31200 to record
[tn=184430107151C31200] level=INFO set 184430107151C31200 to record for 5 seconds
[tn=184430107151C31200] level=INFO first_frame: 150
[tn=1944301061057F1300] level=INFO set 1944301061057F1300 to record for 5 seconds
[tn=1944301061057F1300] level=INFO first_frame: 180
[tn=184430107141C31200] level=INFO set 184430107141C31200 to record for 5 seconds
[tn=184430107141C31200] level=INFO first_frame: 30
[tn=1844301011ACFB0F00] level=INFO set 1844301011ACFB0F00 to record for 5 seconds
[tn=1844301011ACFB0F00] level=INFO first_frame: 480
[tn=184430107151C31200] level=INFO finish recording, current_frame: 149, cut_frame: 149
[tn=1944301061057F1300] level=INFO finish recording, current_frame: 149, cut_frame: 149
[tn=184430107141C31200] level=INFO finish recording, current_frame: 149, cut_frame: 149
[tn=1844301011ACFB0F00] level=INFO finish recording, current_frame: 149, cut_frame: 149
[tn=1944301061057F1300] level=INFO shutdown event is set, closed device True
[tn=1944301061057F1300] level=INFO shutdown_event is set
[tn=184430107151C31200] level=INFO shutdown event is set, closed device True
[tn=184430107151C31200] level=INFO shutdown_event is set
[tn=184430107141C31200] level=INFO shutdown event is set, closed device True
[tn=184430107141C31200] level=INFO shutdown_event is set
[tn=1844301011ACFB0F00] level=INFO shutdown event is set, closed device True
[tn=1844301011ACFB0F00] level=INFO shutdown_event is set
^C[tn=MainThread] level=INFO keyboardinterrupt exiting
[tn=MainThread] level=INFO device closed?: True
[tn=MainThread] level=INFO device closed?: True
[tn=MainThread] level=INFO device closed?: True
[tn=MainThread] level=INFO device closed?: True

Worked flawlessly 5 times I ran this. I did have problems in the beginning, since two of the devices were running bootloader 0.0.18. I have updated both to 0.0.26, I the problems disappeared.

Any chance the booloader version could be causing the issues you are experiencing?

Thanks,
Jaka

Lucas

Hi @jakaskerl

All of the cameras I tested are on firmware 0.0.26, and the depthai version I'm running is 2.24.0.0

Was any of the 4 cameras you tested an OAK-1-POE or W by any chance?

If I add a 1 second delay between setting self.record = True to each camera, it seems to work.
Any less than that and it doesn't.

If I don't wait for the camera queues, and just set a random delay before triggering, it seems that if it's 3 minutes, it works as well, but lower than that it freezes.

The problem seems to be when the cameras are both triggered simultaneously, but only early on when starting the script.

I tried using threading.Event() for self.record and a self.queue_ready event, just to make sure that the issue wasn't communication between threads, but the result is the same.

This really does not make any sense.

Can you think of something else I can look into?

Thank you again for all your help.

Lucas

jakaskerl

Hi @Lucas
You opened a can of worms here. Turns out when I tried running 1-POEs together on 0.0.22, only one was recognized. But then updated the bootloader for all devices to 0.0.26 (!ALL DEVICES!) it seemed to work. But you need a time.sleep(.5) or similar when connecting to the device, otherwise you get __bootloader error.

Here is your modified code. This one worked for me.

import sys
import depthai as dai
import time
import threading
import logging
from datetime import datetime
import json

logging.basicConfig(format='[tn=%(threadName)s] level=%(levelname)s %(message)s')
L = logging.getLogger(__name__)
L.setLevel(logging.DEBUG)

class Oak(threading.Thread):
    def __init__(self,settings, device_info):
        super().__init__(name=device_info.mxid)
        self.settings = settings
        self.device_info = device_info
        self.id = self.device_info.mxid
        L.info(f"init Oak, mxid: {self.id}")
    
        # Initialize the camera
        self.pipeline = dai.Pipeline()
        self.cam_rgb = self.pipeline.createColorCamera()
        self.cam_rgb.setVideoSize(settings.get('resolution'))
        self.cam_rgb.setFps(settings.get('framerate'))
   
        self.record = False
        self.shutdown_event = False
        self.queue = False

        # Create a VideoEncoder node
        video_encoder = self.pipeline.createVideoEncoder()
        video_encoder.setDefaultProfilePreset(self.cam_rgb.getFps(),dai.VideoEncoderProperties.Profile.H264_MAIN)

        # Link the color camera's output to the encoder input
        self.cam_rgb.video.link(video_encoder.input)

        # Create an XLinkOut node for the encoded video stream
        xout_video = self.pipeline.createXLinkOut()
        xout_video.setStreamName("video")
        video_encoder.bitstream.link(xout_video.input)


    def is_key_frame(self, sequence_number):
        return False if sequence_number % self.settings['framerate'] else True
   
    def run(self):
        while True:

            L.info(f"about to load pipeline")
            # with dai.Device(self.pipeline, self.device_info) as self.device:
            self.device = dai.Device(self.pipeline, self.device_info)
            L.info(f"loaded pipeline to {self.id} {time.time()}")
            while not self.device.isPipelineRunning():
                time.sleep(0.01)
                if time.monotonic() > 100:
                    L.error(f"timeout waiting for pipeline to start")
                    sys.exit(1)
            L.info(f"loaded pipeline to {self.id} {time.time()}")
            self.queue = self.device.getOutputQueue(name='video', maxSize=30, blocking=False)
            first_frame = None # first frame of the recording clip, which must be a key frame
            frame = None
            while True:
                if self.queue.has():
                    frame = self.queue.get()
                    if frame is not None:
                        sequence_number = frame.getSequenceNum()

                        if self.record:

                            if first_frame == None and self.is_key_frame(sequence_number):
                                first_frame = sequence_number
                                L.info(f"set {self.id} to record for 5 seconds")
                                L.info(f"first_frame: {first_frame}")
                                self.now = datetime.now().strftime("%Y%m%d-%H%M%S")
                                filename = f"video_{self.id}_{self.now}.h264"
                                file = open(filename, 'wb')


                            if first_frame is not None:
                                current_frame = sequence_number - first_frame
                                try:
                                    frame.getData().tofile(file)
                                except Exception as e:
                                    L.error(f"Error occurred: {e}")


                                cut_frame = self.settings['framerate'] * 5 - 1 # end the recording clip 1 frame before the next key frame
                                if current_frame >= cut_frame:
                                    file.close()
                                    L.info(f"finish recording, current_frame: {current_frame}, cut_frame: {cut_frame}")
                                    first_frame = None
                                    self.shutdown_event = True
                                    self.record = False
                            
                    if self.shutdown_event:
                        self.device.close()
                        L.info(f"shutdown event is set, closed device {self.device.isClosed()}")
                        break
                time.sleep(0.01)
            if self.shutdown_event:
                L.info(f"shutdown_event is set")
                break



if __name__ == "__main__":

    available_devices = dai.Device.getAllAvailableDevices()
    L.info(f"available devices: {len(available_devices)} - {available_devices}")
    cameras = []
    for device_info in available_devices:
        time.sleep(.5) # currently required due to XLink race issues
        camera = Oak({'framerate':5, 'resolution':(1920,1080)}, device_info)
        camera.start()
        cameras.append(camera)
        L.info(f"bootloader: {dai.DeviceBootloader(device_info).getVersion()}")
        L.info(f"device state: {device_info.state}")
    
    for cam in cameras:
        L.info(f"cam queues ready: {cam.queue}")
    
    while not all(cam.queue for cam in cameras):
        L.info(f"waiting for camera queues to have frames")
        time.sleep(1)

    L.info("all cameras ready")

    for cam in cameras:
        if not cam.record:
            L.info(f"set {cam.id} to record")
            cam.record = True

    try:
        while True:
            time.sleep(1)
            # this could possibly monitor cameras and restart them if crashed
    except KeyboardInterrupt:
        L.info(f"keyboardinterrupt exiting")
    finally:
        for cam in cameras: 
            L.info(f"device closed?: {cam.device.isClosed()}")
            cam.shutdown_event = True

Thanks,
Jaka

Lucas

Hi @jakaskerl

I ran your script on a loop and it didn't freeze a single time.
It seems to be working just fine.

What do you reckon the issue is?
I see that you've added device.isPipelineRunning() that I wasn't using.
And that you don't use the context manager to load the device anymore.

Thank you very much.

Lucas

Hi again Jaka, after more testing, I found that the script didn't freeze on the mac, but it kept freezing on the raspberry pi.
I also tried using the context manager again on the mac, and it froze, one time.

I'm sorry for the can of worms, and appreciate your help.

Thank you

jakaskerl

Hi @Lucas
I think it might work if you change getData to getFrame.

Thanks,
Jaka

Lucas

Hi @jakaskerl

That seems to have done, thank you very much for all your help!

Cheers