Hi,
I am attempting to create a pipeline that outputs a pointcloud and an image when a HTTP request is sent to a server hosted in the Script node of the camera pipeline. Due to the camera operating on a somewhat unstable network, I am trying to run the camera in 'Standalone' mode.
I get the pipeline up and running properly, but I face the following issues:
1) When a request is sent, the retrieved image is no real-time and has ~5sec latency. How can this be mitigated and images retrieved closer to real-time?
2) The image and pipeline seem to not be properly synced, I had to remove the sync node in order to use the Script node. Is there any known way to link them up to receive synced pointclouds and images in the Script node?
The code I used is as follows:
Setup camera and pipeline in standalone mode
import depthai as dai
import time
from datetime import timedelta
def create_pipeline(camera_ip, camera_port, script_path):
pipeline = dai.Pipeline()
RGB_SOCKET = dai.CameraBoardSocket.CAM_C
TOF_SOCKET = dai.CameraBoardSocket.CAM_A
# Create nodes
camRgb = pipeline.create(dai.node.ColorCamera)
tof = pipeline.create(dai.node.ToF)
camTof = pipeline.create(dai.node.Camera)
align = pipeline.create(dai.node.ImageAlign)
pointcloud = pipeline.create(dai.node.PointCloud)
script = pipeline.create(dai.node.Script)
# Configure nodes
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) camRgb.setBoardSocket(RGB_SOCKET)
camRgb.setIspScale(1,2)
FPS = 1
camRgb.setFps(FPS)
# ToF settings
camTof.setFps(FPS)
camTof.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG) camTof.setBoardSocket(TOF_SOCKET)
# Higher number => faster processing. 1 shave core can do 30FPS.
tof.setNumShaves(1)
# Median filter, kernel size 3x3
tof.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_3x3)
tofConfig = tof.initialConfig.get() # Temporal filter averages shuffle/non-shuffle frequencies tofConfig.enablePhaseShuffleTemporalFilter = True # Phase unwrapping, for longer range. tofConfig.phaseUnwrappingLevel = 2
tofConfig.phaseUnwrapErrorThreshold = 500
tof.initialConfig.set(tofConfig)
pointcloud.initialConfig.setSparse(False)
script.setProcessor(dai.ProcessorType.LEON_CSS)
with open(script_path, "r") as f:
script_content = f.read()
script_content = script_content.replace("IP_PLACEHOLDER",camera_ip).replace("PORT_PLACEHOLDER",str(camera_port))
print(f"Script Content: {script_content}")
script.setScript(script_content)
# Linking
print("Linking nodes...")
camRgb.isp.link(script.inputs["rgb"])
camRgb.isp.link(align.inputAlignTo)
camTof.raw.link(tof.input)
tof.depth.link(align.input)
align.outputAligned.link(pointcloud.inputDepth) pointcloud.outputPointCloud.link(script.inputs["pcl"])
script.outputs['out'].link(camRgb.inputControl)
print("Nodes linked") return pipeline
if __name__ == "__main__":
camera_ip = '192.168.100.100'
camera_port = '8080'
script_path = "./http_server_script.py"
pipeline = create_pipeline(camera_ip, camera_port, script_path)
devices = dai.DeviceBootloader.getAllAvailableDevices()
print(devices)
for device in devices:
if device.name == camera_ip:
bootloader = dai.DeviceBootloader(device)
progress = lambda p : print(f'Flashing progress: {p*100:.1f}%')
bootloader.flash(progress, pipeline)
HTTP script for Script node:
from http.server import BaseHTTPRequestHandler
import socketserver
import socket
import fcntl
import struct
ctrl = CameraControl() # Assuming CameraControl can handle point cloudsctrl.setCaptureStill(True)
def get_ip_address(ifname):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return socket.inet_ntoa(fcntl.ioctl( s.fileno(), -1071617759, # SIOCGIFADDR struct.pack('256s', ifname[:15].encode()) )[20:24])
class HTTPHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/':
self.send_response(200)
self.end_headers()
self.wfile.write(b'<h1>[DepthAI] Hello, world!</h1><p>Click <a href="img">here</a> for an image</p><p>Click <a href="pointcloud">here</a> for a pointcloud</p>')
elif self.path == '/img':
# Capture the image from the camera
node.io['out'].send(ctrl)
image = node.io['rgb'].get()
self.send_response(200)
self.send_header('Content-Type', 'image/jpeg')
self.send_header('Content-Length', str(len(image.getData())))
self.end_headers()
print(f"Image size: {len(image.getData())}")
self.wfile.write(image.getData())
elif self.path == '/pointcloud':
# Capture the point cloud data
node.io['out'].send(ctrl)
pointcloud = node.io['pcl'].get()
self.send_response(200)
self.send_header('Content-Type', 'application/octet-stream') # Binary file self.send_header('Content-Length', str(len(pointcloud.getData()))) self.end_headers()
self.wfile.write(pointcloud.getData())
else:
self.send_response(404)
self.end_headers()
self.wfile.write(b'Url not found...')
with socketserver.TCPServer(("IP_PLACEHOLDER", PORT_PLACEHOLDER), HTTPHandler) as httpd:
node.warn(f"Serving at {get_ip_address('re0')}:{PORT_PLACEHOLDER}")
httpd.serve_forever()