Linking depth colormap to http server on OAK-D PoE camera via video encoder

GGareth · Jun 29, 2023

Hello,

I am trying to display a depth color map via http server similar to the script_mjpeg_server example but am struggling to find a way to link the processed image back to the server on the camera. Is this possible to do in a manner like the example? I've read through the python documentation wrt depthai.node.StereoDepth and depthai.node.VideoEncoder but I can't find a way to link stereo depth outputs to the video encoder in a way that actually displays what I want. I have posted my code below which currently displays both RGB and edge detection via http but only displays depth through cv2.imshow(). If anyone could offer suggestions as to how I might accomplish my goal, or let me know it can't be done, I'd greatly appreciate it.

#!/usr/bin/env python3

import depthai as dai
import time
import cv2
import numpy as np


# Start defining a pipeline
pipeline = dai.Pipeline()

# Define sources
camRgb = pipeline.create(dai.node.ColorCamera)
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)

# Camera Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

# VideoEncoders
rgb_jpeg = pipeline.create(dai.node.VideoEncoder)
rgb_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
edge_jpeg = pipeline.create(dai.node.VideoEncoder)
edge_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
depth_jpeg = pipeline.create(dai.node.VideoEncoder)
depth_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)

########## EDGE DETECT ##########
# Define sources and outputs
# edgeDetectorRgb = pipeline.create(dai.node.EdgeDetector)
# edgeDetectorRgb.setMaxOutputFrameSize(camRgb.getVideoWidth() * camRgb.getVideoHeight())

edgeDetectorLeft = pipeline.create(dai.node.EdgeDetector)


########## DEPTH CALC ##########
# # Define sources and outputs
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)

xout.setStreamName("disparity")

# Create a node that will produce the depth map (using disparity output as it's easier to visualize depth this way)
depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
# Options: MEDIAN_OFF, KERNEL_3x3, KERNEL_5x5, KERNEL_7x7 (default)
depth.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)
depth.setLeftRightCheck(True)
depth.setExtendedDisparity(False)
depth.setSubpixel(False)

# Linking
monoLeft.out.link(depth.left)
monoRight.out.link(depth.right)
depth.disparity.link(xout.input)

# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
import time
import socket
import fcntl
import struct
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer

PORT = 8080

def get_ip_address(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        -1071617759,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15].encode())
    )[20:24])

class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
    pass

class HTTPHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/':
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b'<h1>Matin Detector</h1><p>Click <a href="img1">here</a> to check for him</p><p>Click <a href="img2">here</a> to see edges</p><p>Click <a href="img3">here</a> to see DEPTH</p>')
        elif self.path == '/img1':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['rgb_jpeg'].get()
                    self.wfile.write("--jpgboundary".encode())
                    self.wfile.write(bytes([13, 10]))
                    self.send_header('Content-type', 'image/jpeg')
                    self.send_header('Content-length', str(len(jpegImage.getData())))
                    self.end_headers()
                    self.wfile.write(jpegImage.getData())
                    self.end_headers()

                    fpsCounter = fpsCounter + 1
                    if time.time() - timeCounter > 1:
                        node.warn(f'RGB FPS: {fpsCounter}')
                        fpsCounter = 0
                        timeCounter = time.time()
                if jpegImage is None:
                        jpegImage = node.io['rgb_jpeg'].tryGet()
            except Exception as ex:
                node.warn(str(ex))
            
        elif self.path == '/img2':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['edge_jpeg'].tryGet()
                    #node.warn('edge_jpeg')
                    if jpegImage is not None:
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()

                        fpsCounter = fpsCounter + 1
                        if time.time() - timeCounter > 1:
                            node.warn(f'Edge FPS: {fpsCounter}')
                            fpsCounter = 0
                            timeCounter = time.time()
                    if jpegImage is None:
                        jpegImage = node.io['edge_jpeg'].tryGet()
            except Exception as ex:
                node.warn(str(ex))        

        elif self.path == '/img3':
            try:
                self.send_response(200)
                self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                self.end_headers()
                fpsCounter = 0
                timeCounter = time.time()
                while True:
                    jpegImage = node.io['depth_jpeg'].tryGet()
                    #node.warn('depth_jpeg')
                    if jpegImage is not None:
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()

                        fpsCounter = fpsCounter + 1
                        if time.time() - timeCounter > 1:
                            node.warn(f'Edge FPS: {fpsCounter}')
                            fpsCounter = 0
                            timeCounter = time.time()
            except Exception as ex:
                node.warn(str(ex))         

with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
    node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
    httpd.serve_forever()
""")

# Connections
# RGB
camRgb.video.link(rgb_jpeg.input)
rgb_jpeg.bitstream.link(script.inputs['rgb_jpeg'])
script.inputs['rgb_jpeg'].setBlocking(False)

# Edge
# camRgb.video.link(edgeDetectorRgb.inputImage)
# edgeDetectorRgb.outputImage.link(edge_jpeg.input)

monoLeft.out.link(edgeDetectorLeft.inputImage)
edgeDetectorLeft.outputImage.link(edge_jpeg.input)

edge_jpeg.bitstream.link(script.inputs['edge_jpeg'])
script.inputs['edge_jpeg'].setBlocking(False)

# Depth
# How do we get the depth map frames to the encoder?
depth_jpeg.bitstream.link(script.inputs['depth_jpeg'])
script.inputs['depth_jpeg'].setBlocking(False)


dev_info = dai.DeviceInfo("10.10.80.25")

retries = 0
max_retries = 15
while retries < max_retries:
    try:        
        # Connect to device with pipeline
        with dai.Device(pipeline, dev_info) as device:
            
            q = device.getOutputQueue(name="disparity", maxSize=4, blocking=False)
            

            retries = 0
            print("Yay, the camera's on!")
            while True:

                inDisparity = q.get()  # blocking call, will wait until a new data has arrived
                frame = inDisparity.getFrame()
                # Normalization for better visualization
                frame = (frame * (255 / depth.initialConfig.getMaxDisparity())).astype(np.uint8)

                cv2.imshow("disparity", frame)

                # Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
                frame = cv2.applyColorMap(frame, cv2.COLORMAP_JET)
                cv2.imshow("disparity_color", frame)

                if cv2.waitKey(1) == ord('q'):
                    retries = max_retries
                    break
                #time.sleep(1)

    except:
        retries += 1
        print("Oopsie, no camera for you. (", retries, ")")
        time.sleep(1000)
if retries == max_retries:
    print("No dice on the connection, chief. Go check the wires or something.")

jakaskerl · Jun 29, 2023

Hi Gareth
You can pass it the disparity: depth.disparity.link(depth_jpeg.input). Make sure you don't turn on subpixel, because the encoder can only support GRAY8 or NV12.

Thanks,
Jaka

GGareth · Jun 29, 2023

jakaskerl

Thank you. Is there a way to adjust the color grading on the camera side or is it locked in at the grayscale? Additionally, is there a way to pass back images that have been processed via opencv? There are a lot of other things in the examples and experiments repositories that I'd like to try to implement in this manner as well, but so far I've not seen a way to.

jakaskerl · Jun 30, 2023

Hi Gareth
I think the stereo node is locked in grayscale, that's why we do the colormap on host side.
But, I think it might be possible to apply a color map to the disparity frame by passing it through a manip node. [Setting colormap]. You would have to also convert the frame to nv12 so the encoder can then process it correctly.

You can also backfeed the frames from the host. Most nodes expect an ImgFrame input, so if you convert the CV frame to dai.ImgFrame, you will be able to pass it back to the device.

Thanks,
Jaka