Firmware Crash with dump, HELP PLEASE

Jjkchen46033 · 2025-03-04T21:19:53+00:00

I routinely get crashes while running a pipeline of 3 camera nodes, 3 jpeg encoders, one for color, two for mono, encoder's output to scripts. fps is 10 for all 3. Able to stream mjpeg to 3 different ruls, but would crash if running it for about 25 to 30 minutes. Dump report json says "frame pointer", "thread", "waiting for semaphore".

The host message said "ping was missed"

Here is crash dump:

jakaskerl · 2025-03-05T13:28:41+00:00

jkchen46033
Can you show the pipeline you are using?

Thanks,
Jaka

Jjkchen46033 · 2025-03-05T15:48:47+00:00

Here you go. purpose flash 3 cams into mjpeg script node. three urls.

#!/usr/bin/env python3

import depthai as dai
import time

# Start defining a pipeline
pipeline = dai.Pipeline()

cam = pipeline.create(dai.node.ColorCamera)
cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_720_P)
jpeg = pipeline.create(dai.node.VideoEncoder)
cam.setFps(10)
jpeg.setDefaultProfilePreset(10, dai.VideoEncoderProperties.Profile.MJPEG)
cam.video.link(jpeg.input)

leftCam = pipeline.create(dai.node.MonoCamera)
leftCam.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P)
leftCam.setCamera("left")
leftCam.setFps(10)
leftEncoder = pipeline.create(dai.node.VideoEncoder)
leftEncoder.setDefaultProfilePreset(10, dai.VideoEncoderProperties.Profile.MJPEG)
leftCam.out.link(leftEncoder.input)

rightCam = pipeline.create(dai.node.MonoCamera)
rightCam.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P)
rightCam.setCamera("right")
rightCam.setFps(10)
rightEncoder = pipeline.create(dai.node.VideoEncoder)
rightEncoder.setDefaultProfilePreset(10, dai.VideoEncoderProperties.Profile.MJPEG)
rightCam.out.link(rightEncoder.input)

# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
    import time
    import socket
    import fcntl
    import struct
    from socketserver import ThreadingMixIn
    from http.server import BaseHTTPRequestHandler, HTTPServer

    PORT = 80

    def get_ip_address(ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(
            s.fileno(),
            -1071617759,  # SIOCGIFADDR
            struct.pack('256s', ifname[:15].encode())
        )[20:24])

    class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
        pass

    class HTTPHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            if self.path == '/':
                self.send_response(200)
                self.end_headers()
                self.wfile.write(b'<h1>[DepthAI] Hello, world!</h1><p>Click <a href="img">here</a> for an image</p>')
            elif self.path == '/mjpeg':
                try:
                    self.send_response(200)
                    self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                    self.end_headers()
                    fpsCounter = 0
                    timeCounter = time.time()
                    while True:
                        jpegImage = node.io['jpeg'].get()
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()

                        fpsCounter = fpsCounter + 1
                        if time.time() - timeCounter > 1:
                            node.warn(f'FPS: {fpsCounter}')
                            fpsCounter = 0
                            timeCounter = time.time()
                except Exception as ex:
                    node.warn(str(ex))
            elif self.path == '/left':
                try:
                    self.send_response(200)
                    self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                    self.end_headers()
                    fpsCounter = 0
                    timeCounter = time.time()
                    while True:
                        jpegImage = node.io['leftStream'].get()
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()
                except Exception as ex:
                    node.warn(str(ex))
            elif self.path == '/right':
                try:
                    self.send_response(200)
                    self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                    self.end_headers()
                    fpsCounter = 0
                    timeCounter = time.time()
                    while True:
                        jpegImage = node.io['rightStream'].get()
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()
                except Exception as ex:
                    node.warn(str(ex))

    with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
        node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
        httpd.serve_forever()
""")

# encoding output stream linked to script input
jpeg.bitstream.link(script.inputs['jpeg'])
leftEncoder.bitstream.link(script.inputs['leftStream'])
rightEncoder.bitstream.link(script.inputs['rightStream'])

# Connect to device with pipeline
with dai.Device(pipeline) as device:
    while not device.isClosed():
        time.sleep(1)

jakaskerl · 2025-03-06T11:01:02+00:00

jkchen46033 Does it work longer if you lower image resolution? Also try increasing the setNumFramesPool() for each camera node.

Thanks,
Jaka

Jjkchen46033 · 2025-03-07T22:15:08+00:00

Here a new crash of similar program, ie 3 mjpeg streams, 10fps, 720p for each, into script node. Crashed after minutes. Probably 10 minutes. PLEASE HELP.

[18443010C1CBE2F400] [192.170.10.222] [1741381820.067] [host] [warning] Monitor thread (device: 18443010C1CBE2F400 [192.170.10.222]) - ping was missed, closing the device connection
[18443010C1CBE2F400] [192.170.10.222] [1206.384] [Script(6)] [warning] left: 567994812.4833648
[18443010C1CBE2F400] [192.170.10.222] [1741381830.861] [host] [warning] Device crashed, but no crash dump could be extracted.
terminate called without an active exception
Stack trace (most recent call last):
#28 Object "[0xffffffffffffffff]", at 0xffffffffffffffff, in
#27 Object "python3", at 0x60d9dd0e9d64, in _start
#26 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7073f4e29e3f, in __libc_start_main
#25 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7073f4e29d8f, in
#24 Object "python3", at 0x60d9dd0e9e6c, in Py_BytesMain
#23 Object "python3", at 0x60d9dd10fe5d, in Py_RunMain
#22 Object "python3", at 0x60d9dd11b866, in _PyRun_AnyFileObject
#21 Object "python3", at 0x60d9dd11bb87, in _PyRun_SimpleFileObject
#20 Object "python3", at 0x60d9dd11c5a4, in
#19 Object "python3", at 0x60d9dd11700e, in
#18 Object "python3", at 0x60d9dd11c807, in
#17 Object "python3", at 0x60d9dd0f5e25, in PyEval_EvalCode
#16 Object "python3", at 0x60d9dd0f5f55, in
#15 Object "python3", at 0x60d9dd01192e, in _PyEval_EvalFrameDefault
#14 Object "python3", at 0x60d9dd034a6a, in
#13 Object "python3", at 0x60d9dd01d75a, in _PyObject_MakeTpCall
#12 Object "python3", at 0x60d9dd026e11, in
#11 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f2f5763f, in
#10 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f2f90532, in
#9 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f31bcd14, in dai:eviceBase::close()
#8 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f31b6214, in dai:evice::closeImpl()
#7 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f31d09be, in dai:eviceBase::closeImpl()
#6 Object "/home/jk.chen/.local/lib/python3.10/site-packages/depthai.cpython-310-x86_64-linux-gnu.so", at 0x7073f31bd0b3, in dai:eviceBase::_DeviceBase()
#5 Object "/lib/x86_64-linux-gnu/libstdc++.so.6", at 0x7073f2aae276, in std::terminate()
#4 Object "/lib/x86_64-linux-gnu/libstdc++.so.6", at 0x7073f2aae20b, in
#3 Object "/lib/x86_64-linux-gnu/libstdc++.so.6", at 0x7073f2aa2b9d, in
#2 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7073f4e287f2, in abort
#1 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7073f4e42475, in raise
#0 Object "/lib/x86_64-linux-gnu/libc.so.6", at 0x7073f4e969fc, in pthread_kill
Aborted (Signal sent by tkill() 45604 1676005160)
Aborted (core dumped)
jk.chen@corp.sarcos.com@SLC5CD4493XDV:~/dev/new_camera/script_node$

jakaskerl · 2025-03-09T22:16:27+00:00

jkchen46033
MRE please.. Seems like something is awfully wrong with the code to get a core dump..

Thanks,
Jaka