Hello,
I am trying to display a depth color map via http server similar to the script_mjpeg_server example but am struggling to find a way to link the processed image back to the server on the camera. Is this possible to do in a manner like the example? I've read through the python documentation wrt depthai.node.StereoDepth and depthai.node.VideoEncoder but I can't find a way to link stereo depth outputs to the video encoder in a way that actually displays what I want. I have posted my code below which currently displays both RGB and edge detection via http but only displays depth through cv2.imshow(). If anyone could offer suggestions as to how I might accomplish my goal, or let me know it can't be done, I'd greatly appreciate it.
#!/usr/bin/env python3
import depthai as dai
import time
import cv2
import numpy as np
# Start defining a pipeline
pipeline = dai.Pipeline()
# Define sources
camRgb = pipeline.create(dai.node.ColorCamera)
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)
# Camera Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.CAM_A)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.CAM_B)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.CAM_C)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
# VideoEncoders
rgb_jpeg = pipeline.create(dai.node.VideoEncoder)
rgb_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
edge_jpeg = pipeline.create(dai.node.VideoEncoder)
edge_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
depth_jpeg = pipeline.create(dai.node.VideoEncoder)
depth_jpeg.setDefaultProfilePreset(camRgb.getFps(), dai.VideoEncoderProperties.Profile.MJPEG)
########## EDGE DETECT ##########
# Define sources and outputs
# edgeDetectorRgb = pipeline.create(dai.node.EdgeDetector)
# edgeDetectorRgb.setMaxOutputFrameSize(camRgb.getVideoWidth() * camRgb.getVideoHeight())
edgeDetectorLeft = pipeline.create(dai.node.EdgeDetector)
########## DEPTH CALC ##########
# # Define sources and outputs
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName("disparity")
# Create a node that will produce the depth map (using disparity output as it's easier to visualize depth this way)
depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
# Options: MEDIAN_OFF, KERNEL_3x3, KERNEL_5x5, KERNEL_7x7 (default)
depth.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)
depth.setLeftRightCheck(True)
depth.setExtendedDisparity(False)
depth.setSubpixel(False)
# Linking
monoLeft.out.link(depth.left)
monoRight.out.link(depth.right)
depth.disparity.link(xout.input)
# Script node
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
script.setScript("""
import time
import socket
import fcntl
import struct
from socketserver import ThreadingMixIn
from http.server import BaseHTTPRequestHandler, HTTPServer
PORT = 8080
def get_ip_address(ifname):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return socket.inet_ntoa(fcntl.ioctl(
s.fileno(),
-1071617759, # SIOCGIFADDR
struct.pack('256s', ifname[:15].encode())
)[20:24])
class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
pass
class HTTPHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/':
self.send_response(200)
self.end_headers()
self.wfile.write(b'<h1>Matin Detector</h1><p>Click <a href="img1">here</a> to check for him</p><p>Click <a href="img2">here</a> to see edges</p><p>Click <a href="img3">here</a> to see DEPTH</p>')
elif self.path == '/img1':
try:
self.send_response(200)
self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
self.end_headers()
fpsCounter = 0
timeCounter = time.time()
while True:
jpegImage = node.io['rgb_jpeg'].get()
self.wfile.write("--jpgboundary".encode())
self.wfile.write(bytes([13, 10]))
self.send_header('Content-type', 'image/jpeg')
self.send_header('Content-length', str(len(jpegImage.getData())))
self.end_headers()
self.wfile.write(jpegImage.getData())
self.end_headers()
fpsCounter = fpsCounter + 1
if time.time() - timeCounter > 1:
node.warn(f'RGB FPS: {fpsCounter}')
fpsCounter = 0
timeCounter = time.time()
if jpegImage is None:
jpegImage = node.io['rgb_jpeg'].tryGet()
except Exception as ex:
node.warn(str(ex))
elif self.path == '/img2':
try:
self.send_response(200)
self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
self.end_headers()
fpsCounter = 0
timeCounter = time.time()
while True:
jpegImage = node.io['edge_jpeg'].tryGet()
#node.warn('edge_jpeg')
if jpegImage is not None:
self.wfile.write("--jpgboundary".encode())
self.wfile.write(bytes([13, 10]))
self.send_header('Content-type', 'image/jpeg')
self.send_header('Content-length', str(len(jpegImage.getData())))
self.end_headers()
self.wfile.write(jpegImage.getData())
self.end_headers()
fpsCounter = fpsCounter + 1
if time.time() - timeCounter > 1:
node.warn(f'Edge FPS: {fpsCounter}')
fpsCounter = 0
timeCounter = time.time()
if jpegImage is None:
jpegImage = node.io['edge_jpeg'].tryGet()
except Exception as ex:
node.warn(str(ex))
elif self.path == '/img3':
try:
self.send_response(200)
self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
self.end_headers()
fpsCounter = 0
timeCounter = time.time()
while True:
jpegImage = node.io['depth_jpeg'].tryGet()
#node.warn('depth_jpeg')
if jpegImage is not None:
self.wfile.write("--jpgboundary".encode())
self.wfile.write(bytes([13, 10]))
self.send_header('Content-type', 'image/jpeg')
self.send_header('Content-length', str(len(jpegImage.getData())))
self.end_headers()
self.wfile.write(jpegImage.getData())
self.end_headers()
fpsCounter = fpsCounter + 1
if time.time() - timeCounter > 1:
node.warn(f'Edge FPS: {fpsCounter}')
fpsCounter = 0
timeCounter = time.time()
except Exception as ex:
node.warn(str(ex))
with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
httpd.serve_forever()
""")
# Connections
# RGB
camRgb.video.link(rgb_jpeg.input)
rgb_jpeg.bitstream.link(script.inputs['rgb_jpeg'])
script.inputs['rgb_jpeg'].setBlocking(False)
# Edge
# camRgb.video.link(edgeDetectorRgb.inputImage)
# edgeDetectorRgb.outputImage.link(edge_jpeg.input)
monoLeft.out.link(edgeDetectorLeft.inputImage)
edgeDetectorLeft.outputImage.link(edge_jpeg.input)
edge_jpeg.bitstream.link(script.inputs['edge_jpeg'])
script.inputs['edge_jpeg'].setBlocking(False)
# Depth
# How do we get the depth map frames to the encoder?
depth_jpeg.bitstream.link(script.inputs['depth_jpeg'])
script.inputs['depth_jpeg'].setBlocking(False)
dev_info = dai.DeviceInfo("10.10.80.25")
retries = 0
max_retries = 15
while retries < max_retries:
try:
# Connect to device with pipeline
with dai.Device(pipeline, dev_info) as device:
q = device.getOutputQueue(name="disparity", maxSize=4, blocking=False)
retries = 0
print("Yay, the camera's on!")
while True:
inDisparity = q.get() # blocking call, will wait until a new data has arrived
frame = inDisparity.getFrame()
# Normalization for better visualization
frame = (frame * (255 / depth.initialConfig.getMaxDisparity())).astype(np.uint8)
cv2.imshow("disparity", frame)
# Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
frame = cv2.applyColorMap(frame, cv2.COLORMAP_JET)
cv2.imshow("disparity_color", frame)
if cv2.waitKey(1) == ord('q'):
retries = max_retries
break
#time.sleep(1)
except:
retries += 1
print("Oopsie, no camera for you. (", retries, ")")
time.sleep(1000)
if retries == max_retries:
print("No dice on the connection, chief. Go check the wires or something.")