OAK-D PoE mjpeg server edge detector feed freezing after few seconds

Gareth

Hello,

I have been attempting to get an http server running on the OAK-D PoE to display both RGB video and edge detection. I have followed the script_mjpeg_server and edge_detector examples provided in the depthai-python repository, making minor modifications to combine them. While the RGB mjpeg feed appears to function just fine, an issue arises when I try to view the edge detector feed. It will display fine for the first second or so before freezing on a frame. After that point, I can no longer even load the page that should be displaying the edge detector feed. The RGB feed still works as normal while this is happening. When I modify the code to only process and display the edge detection it still freezes. If anyone could shed some light on what I may be doing wrong I would greatly appreciate it.

My current code is as follows:

#!/usr/bin/env python3

import depthai as dai
import time

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define a source - color camera
camRgb = pipeline.create(dai.node.ColorCamera)
# VideoEncoder
rgb_jpeg = pipeline.create(dai.node.VideoEncoder)
rgb_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

########## EDGE DETECT ##########
edge_jpeg = pipeline.create(dai.node.VideoEncoder)
edge_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
# Define sources and outputs
#camRgb = pipeline.create(dai.node.ColorCamera)
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)

edgeDetectorLeft = pipeline.create(dai.node.EdgeDetector)
edgeDetectorRight = pipeline.create(dai.node.EdgeDetector)
edgeDetectorRgb = pipeline.create(dai.node.EdgeDetector)

xoutEdgeLeft = pipeline.create(dai.node.XLinkOut)
xoutEdgeRight = pipeline.create(dai.node.XLinkOut)
xoutEdgeRgb = pipeline.create(dai.node.XLinkOut)
xinEdgeCfg = pipeline.create(dai.node.XLinkIn)

edgeLeftStr = "edge left"
edgeRightStr = "edge right"
edgeRgbStr = "edge rgb"
edgeCfgStr = "edge cfg"

xoutEdgeLeft.setStreamName(edgeLeftStr)
xoutEdgeRight.setStreamName(edgeRightStr)
xoutEdgeRgb.setStreamName(edgeRgbStr)
xinEdgeCfg.setStreamName(edgeCfgStr)

# Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.RGB)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)

edgeDetectorRgb.setMaxOutputFrameSize(camRgb.getVideoWidth() * camRgb.getVideoHeight())

# Linking
monoLeft.out.link(edgeDetectorLeft.inputImage)
monoRight.out.link(edgeDetectorRight.inputImage)
#camRgb.video.link(edgeDetectorRgb.inputImage)

edgeDetectorLeft.outputImage.link(xoutEdgeLeft.input)
edgeDetectorRight.outputImage.link(xoutEdgeRight.input)
edgeDetectorRgb.outputImage.link(xoutEdgeRgb.input)

xinEdgeCfg.out.link(edgeDetectorLeft.inputConfig)
xinEdgeCfg.out.link(edgeDetectorRight.inputConfig)
xinEdgeCfg.out.link(edgeDetectorRgb.inputConfig)
########## EDGE DETECT ##########

# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
import time
import socket
import fcntl
import struct
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer

PORT = 8080

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        -1071617759,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15].encode())
    )[20:24])

class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
    pass

class HTTPHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/':
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b'<h1>Matin Detector</h1><p>Click <a href="img1">here</a> to check for him</p><p>Click <a href="img2">here</a> to see edges</p>')
        elif self.path == '/img1':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['rgb_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))
        elif self.path == '/img2':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['edge_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))        

with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
    node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
    httpd.serve_forever()
""")

# Connections
camRgb.video.link(rgb_jpeg.input)
rgb_jpeg.bitstream.link(script.inputs['rgb_jpeg'])

edgeDetectorLeft.outputImage.link(edge_jpeg.input)
edge_jpeg.bitstream.link(script.inputs['edge_jpeg'])

dev_info = dai.DeviceInfo("10.10.80.25")

# Connect to device with pipeline
with dai.Device(pipeline, dev_info) as device:
    while not device.isClosed():

        time.sleep(1)

jakaskerl

Hi Gareth
This is your pipeline graph:

You have a bunch of reduntant xLinkOuts as well as Edge detector nodes active, which slows down the device. Remove them and you should get a smooth feed.

Thanks,
Jaka

Gareth

jakaskerl

Even after removing all references to other pipelines, the stream still freezes moments after opening it in a browser. If it were too many active nodes slowing down the device I would expect the RGB feed to freeze as well but it still keeps roughly 30 FPS even after the edge detection feed freezes. I modified the server script slightly to denote which FPS is being displayed in the console and as the screenshot below shows, the edge detection appears to stop after a single report.

I assumed that maybe this was an issue of the edgeDetectorLeft output image not being renewed after the initial load so I added

edgeLeftQueue = device.getOutputQueue(edgeLeftStr, 1, False)

to the internal while loop just above the sleep and that seems to have solved my initial issue while presenting another. The FPS of both cameras seems to rapidly deteriorate until I get a missed ping error and the connection closes

Could this be an issue with the sleep duration? I have posted my modified code below for reference.

#!/usr/bin/env python3

import depthai as dai
import time

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define a source - color camera
camRgb = pipeline.create(dai.node.ColorCamera)
# VideoEncoder
rgb_jpeg = pipeline.create(dai.node.VideoEncoder)
rgb_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
#camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

########## EDGE DETECT ##########
edge_jpeg = pipeline.create(dai.node.VideoEncoder)
edge_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
# Define sources and outputs
monoLeft = pipeline.create(dai.node.MonoCamera)

edgeDetectorLeft = pipeline.create(dai.node.EdgeDetector)

xoutEdgeLeft = pipeline.create(dai.node.XLinkOut)

edgeLeftStr = "edge left"

xoutEdgeLeft.setStreamName(edgeLeftStr)

# Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.RGB)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)

# Linking
monoLeft.out.link(edgeDetectorLeft.inputImage)

edgeDetectorLeft.outputImage.link(xoutEdgeLeft.input)
########## EDGE DETECT ##########

# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
import time
import socket
import fcntl
import struct
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer

PORT = 8080

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        -1071617759,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15].encode())
    )[20:24])

class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
    pass

class HTTPHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/':
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b'<h1>Matin Detector</h1><p>Click <a href="img1">here</a> to check for him</p><p>Click <a href="img2">here</a> to see edges</p>')
        elif self.path == '/img1':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['rgb_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'RGB FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))
        elif self.path == '/img2':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['edge_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'Edge FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))        

with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
    node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
    httpd.serve_forever()
""")

# Connections
camRgb.video.link(rgb_jpeg.input)
rgb_jpeg.bitstream.link(script.inputs['rgb_jpeg'])

edgeDetectorLeft.outputImage.link(edge_jpeg.input)
edge_jpeg.bitstream.link(script.inputs['edge_jpeg'])

dev_info = dai.DeviceInfo("10.10.80.25")

# Connect to device with pipeline
with dai.Device(pipeline, dev_info) as device:
    while not device.isClosed():
        edgeLeftQueue = device.getOutputQueue(edgeLeftStr, 1, False)
        time.sleep(1)

jakaskerl

Hi Gareth
This code runs fine on my side:

#!/usr/bin/env python3

import depthai as dai
import time

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define a source - color camera
camRgb = pipeline.create(dai.node.ColorCamera)
# VideoEncoder
rgb_jpeg = pipeline.create(dai.node.VideoEncoder)
rgb_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

########## EDGE DETECT ##########
edge_jpeg = pipeline.create(dai.node.VideoEncoder)
edge_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
# Define sources and outputs
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)

edgeDetectorLeft = pipeline.create(dai.node.EdgeDetector)




# Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.RGB)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)

########## EDGE DETECT ##########

# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
import time
import socket
import fcntl
import struct
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer

PORT = 8080

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        -1071617759,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15].encode())
    )[20:24])

class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
    pass

class HTTPHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/':
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b'<h1>Matin Detector</h1><p>Click <a href="img1">here</a> to check for him</p><p>Click <a href="img2">here</a> to see edges</p>')
        elif self.path == '/img1':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['rgb_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'RGB FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))
            
        elif self.path == '/img2':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['edge_jpeg'].tryGet()
                    #node.warn('edge_jpeg')
                    if jpegImage is not None:
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()

                        fpsCounter = fpsCounter + 1
                        if time.time() - timeCounter > 1:
                            node.warn(f'FPS: {fpsCounter}')
                            fpsCounter = 0
                            timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))        

with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
    node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
    httpd.serve_forever()
""")

# Connections
monoLeft.out.link(edgeDetectorLeft.inputImage)
edgeDetectorLeft.outputImage.link(edge_jpeg.input)
edge_jpeg.bitstream.link(script.inputs['edge_jpeg'])
script.inputs['edge_jpeg'].setBlocking(False)

camRgb.video.link(rgb_jpeg.input)
rgb_jpeg.bitstream.link(script.inputs['rgb_jpeg'])
script.inputs['rgb_jpeg'].setBlocking(False)


#set all links to non-blocking

#dev_info = dai.DeviceInfo("10.10.80.25")

# Connect to device with pipeline
with dai.Device(pipeline) as device:
    while True:
        time.sleep(1)

Thanks,
Jaka

Gareth

jakaskerl

So blocking was the issue! Thank you so much!

jakaskerl

Hi Gareth
Not just blocking apparently. It was the first thing I tried but was still getting the freezing. Only when I combined both was I able to get it to run normally.

Thanks,
Jaka