Hi!

I have the below script which creates a pipeline for an HTTP server to serve an MJPEG stream on a 4P-PoE camera board. I have three cameras attached on ports A,C, and D. Because of memory and CPU limitations on the board, I can only run two cams at a time though: a wide angle and a fisheye cam.

I'd like to zoom lossless into the wide angle view by taking the full ISP view and crop into it and also have switch between that cam and the 360 degree. The script only ever sends down one of the streams, also due to CPU limitations.

So far, so good, the script works.

However, I'd also like to rotate the frames by 180 degrees. Hence, I was including the lines

cam1.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG)
and
cam2.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG)

With these, the image does rotate, but I get artifacts on one side of the frames on both cameras:

Hence, I was wondering what would be the best way to achieve the rotation? The script already feels overengineered a bit, I'd be happy to get feedback on what I could improve as well.

Secondly, I still perceive the issue that the camera view is too blue. We installed some units on customer's boats and they complain that everything always looks like a rainy day. I made a post (https://discuss.luxonis.com/d/3915-4p-poe-cant-ping-no-available-devices/18) about it some time ago, but white balance modes and temperatures did not help. Is there anything I can do?

Thanks!

(you can toogle to the different modes 'wide', 'zoom', 'three-sixty' via:

curl -X POST http://169.254.1.222:8080 -d "wide"

)

import depthai as dai
import cv2

# Constants
SCENE_SIZE = (1920, 1080)

# Start defining a pipeline
pipeline = dai.Pipeline()

# Define the first source - color camera
cam1 = pipeline.create(dai.node.ColorCamera)
cam1.setBoardSocket(dai.CameraBoardSocket.CAM_C)
cam1.setResolution(dai.ColorCameraProperties.SensorResolution.THE_12_MP)
cam1.setInterleaved(False)
#cam1.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
cam1.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG)

#cam1.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)

# Define the second source - color camera
cam2 = pipeline.create(dai.node.ColorCamera)
cam2.setBoardSocket(dai.CameraBoardSocket.CAM_A)
cam2.setResolution(dai.ColorCameraProperties.SensorResolution.THE_12_MP)
cam2.setIspScale(3, 4)
cam2.setInterleaved(False)
#cam1.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
cam2.setImageOrientation(dai.CameraImageOrientation.ROTATE_180_DEG)

# Create ImageManip node for the first camera
manip1 = pipeline.create(dai.node.ImageManip)
manip1.setMaxOutputFrameSize(SCENE_SIZE[0] * SCENE_SIZE[1] * 3)
manip1.initialConfig.setFrameType(dai.RawImgFrame.Type.NV12)
cam1.isp.link(manip1.inputImage)

# Create ImageManip node for the second camera
manip2 = pipeline.create(dai.node.ImageManip)
manip2.initialConfig.setFrameType(dai.RawImgFrame.Type.NV12)
manip2.initialConfig.setCenterCrop(1.0, 1)
manip2.initialConfig.setResize(1600, 1600)
manip2.setMaxOutputFrameSize(1600 * 1600 * 3)
#manip2.initialConfig.setResize(1920, 1080)
#manip2.setMaxOutputFrameSize(1920 * 1080 * 3)
cam2.isp.link(manip2.inputImage)

# Combined Script node for handling toggles and frame routing
script = pipeline.create(dai.node.Script)
script.setScript(f"""
ORIGINAL_SIZE = (4056, 3040) 
SCENE_SIZE = (1920, 1080) # 1080P

cfg = ImageManipConfig()
size = Size2f(SCENE_SIZE[0], SCENE_SIZE[1])
toggle_crop = False
toggle_camera = False

def send_config():
    rect = RotatedRect()
    rect.size = size

    if toggle_crop:
        # Crop configuration
        rect.center = Point2f(ORIGINAL_SIZE[0] * 0.5, ORIGINAL_SIZE[1] * 0.5)
        rect.size = Size2f(ORIGINAL_SIZE[0] * 0.5, ORIGINAL_SIZE[1] * 0.5)
        node.warn("Toggled to crop mode")
    else:
        # Downscale configuration
        rect.center = Point2f(ORIGINAL_SIZE[0] * 0.5, ORIGINAL_SIZE[1] * 0.5)
        rect.size = Size2f(ORIGINAL_SIZE[0], ORIGINAL_SIZE[1])
        node.warn("Toggled to downscale mode")

    cfg.setCropRotatedRect(rect, False)
    cfg.setResize(SCENE_SIZE[0], SCENE_SIZE[1])
    cfg.setFrameType(ImgFrame.Type.NV12)
    node.io['cfg_out'].send(cfg)

send_config()

while True:
    toggle_crop_msg = node.io['toggle_crop'].tryGet()
    if toggle_crop_msg is not None:
        toggle_crop = not toggle_crop
        send_config()
    
    toggle_camera_msg = node.io['toggle_camera'].tryGet()
    if toggle_camera_msg is not None:
        toggle_camera = not toggle_camera
        node.warn(f"Toggled to {{'camera 1' if toggle_camera else 'camera 2'}}")
    
    if toggle_camera:
        frame = node.io['modified'].tryGet()
        if frame is not None:
            node.io['encoder1'].send(frame)
    else:
        frame = node.io['unmodified'].tryGet()
        if frame is not None:
            node.io['encoder2'].send(frame)
""")

# Link manip nodes to the script node
manip1.out.link(script.inputs['modified'])
manip2.out.link(script.inputs['unmodified'])

# Create two VideoEncoder nodes
videoEnc1 = pipeline.create(dai.node.VideoEncoder)
videoEnc1.setDefaultProfilePreset(30, dai.VideoEncoderProperties.Profile.MJPEG)

videoEnc2 = pipeline.create(dai.node.VideoEncoder)
videoEnc2.setDefaultProfilePreset(30, dai.VideoEncoderProperties.Profile.MJPEG)

# Link script outputs to respective encoders
script.outputs['encoder1'].link(videoEnc1.input)
script.outputs['encoder2'].link(videoEnc2.input)

socket_script = pipeline.create(dai.node.Script)
socket_script.setProcessor(dai.ProcessorType.LEON_CSS)

socket_script.setScript("""
    import time
    import socket
    import fcntl
    import struct
    from socketserver import ThreadingMixIn
    from http.server import BaseHTTPRequestHandler, HTTPServer
    PORT = 8080

    def get_ip_address(ifname):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(
            s.fileno(),
            -1071617759,  # SIOCGIFADDR
            struct.pack('256s', ifname[:15].encode())
        )[20:24])

    which_cam = {"now" : "frame2"}
    which_zoom = {"now" : "wide"}
    name_to_number = {"wide":"1","zoom":"3","infrared":"1","three-sixty":"2","docking":"2"}
    class ThreadingSimpleServer(ThreadingMixIn, HTTPServer):
        pass

    class HTTPHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            if self.path == '/':
                self.send_response(200)
                self.end_headers()
                self.wfile.write(b'<h1>[DepthAI] Hello, world!</h1><p>Click <a href="img">here</a> for an image</p>')
            elif self.path == '/img':
                try:
                    self.send_response(200)
                    self.send_header('Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary')
                    self.end_headers()
                    fpsCounter = 0
                    timeCounter = time.time()
                    while True:
                        jpegImage = node.io[which_cam["now"]].get()
                        self.wfile.write("--jpgboundary".encode())
                        self.wfile.write(bytes([13, 10]))
                        self.send_header('Content-type', 'image/jpeg')
                        self.send_header('Content-length', str(len(jpegImage.getData())))
                        self.end_headers()
                        self.wfile.write(jpegImage.getData())
                        self.end_headers()

                        fpsCounter = fpsCounter + 1
                        if time.time() - timeCounter > 1:
                            node.warn(f'FPS: {fpsCounter}')
                            fpsCounter = 0
                            timeCounter = time.time()
                except Exception as ex:
                    node.warn(str(ex))
                
        def do_POST(self):
            # Handle POST request
            content_length = int(self.headers['Content-Length'])
            post_data = self.rfile.read(content_length)
            sent_number = post_data.decode('utf-8')
            if sent_number in list(name_to_number.keys()):
                 sent_number = name_to_number[sent_number]
            else:
                return
            node.warn(sent_number)
            node.warn(str(int(sent_number)))
            
            if int(sent_number) in [1,2]:
                node.warn('jo0')
                if which_zoom["now"] == "zoom":
                    node.warn('jo1')
                    which_zoom["now"] = "wide"
                    node.warn("switched to wide")
                    toggle_crop = Buffer(1)
                    node.io['toggle_crop'].send(toggle_crop)
                if which_cam["now"] != "frame" + str(int(sent_number)):
                    node.warn('jo2')
                    which_cam["now"] = "frame" + str(int(sent_number))
                    node.warn("switching cam to frame ?")
                    toggle_camera = Buffer(1)
                    node.io['toggle_camera'].send(toggle_camera)
                    #time.sleep(2)
            elif int(sent_number) == 3:
                if which_zoom["now"] == "wide":
                    node.warn('jo3')
                    which_zoom["now"] = "zoom"
                    node.warn("switched to zoom")
                    toggle_crop = Buffer(1)
                    node.io['toggle_crop'].send(toggle_crop)
                if which_cam["now"] == "frame2":
                    node.warn('jo4')
                    which_cam["now"] = "frame1"
                    node.warn("switching cam to frame 1")
                    toggle_camera = Buffer(1)
                    node.io['toggle_camera'].send(toggle_camera)
                    #time.sleep(2)            
            
            
            
            # Respond to the POST request
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            self.wfile.write(b'<h1>POST request received</h1>')                
        

    with ThreadingSimpleServer(("", PORT), HTTPHandler) as httpd:
        node.warn(f"Serving at {get_ip_address('re0')}:{PORT}")
        httpd.serve_forever()
""")

# Link toggle commands to the main script node
socket_script.outputs['toggle_crop'].link(script.inputs['toggle_crop'])
socket_script.outputs['toggle_camera'].link(script.inputs['toggle_camera'])

script.outputs['cfg_out'].link(manip1.inputConfig)

# Link each VideoEncoder to the Script node
videoEnc1.bitstream.link(socket_script.inputs['frame1'])
videoEnc2.bitstream.link(socket_script.inputs['frame2'])


# Configure the Script node to handle multiple streams
socket_script.inputs['frame1'].setBlocking(False)
socket_script.inputs['frame1'].setQueueSize(1)
socket_script.inputs['frame2'].setBlocking(False)
socket_script.inputs['frame2'].setQueueSize(1)

with dai.Device(pipeline) as device:
    print("Connected")
    import time
    while True:
        time.sleep(1)