Long Range Crop instead of downscale

oscarthorn

Hi!

I'm using the LR sensor and want to crop to get it down the needed width of 1280 for stereo-matching thus reducing fov instead of downscaling. Just wanted to make sure I'm doing it in the correct way since there are no examples. I mostly use c++ but I did this test in python.

#!/usr/bin/env python3

import cv2
import numpy as np
import depthai as dai

crop = True
fps = 30

# Create pipeline
pipeline = dai.Pipeline()
device = dai.Device()
queueNames = []

# Define sources and outputs
camRgb = pipeline.create(dai.node.Camera)
left = pipeline.create(dai.node.ColorCamera)
right = pipeline.create(dai.node.ColorCamera)
stereo = pipeline.create(dai.node.StereoDepth)
cropRgb = pipeline.create(dai.node.ImageManip)
cropLeft = pipeline.create(dai.node.ImageManip)
cropRight = pipeline.create(dai.node.ImageManip)

# Crop range
topLeft = dai.Point2f((1/6), (1/6))
bottomRight = dai.Point2f((5/6), (5/6))

cropRgb.initialConfig.setCropRect(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y)
cropRgb.setMaxOutputFrameSize(1280*800*3)
cropLeft.initialConfig.setCropRect(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y)
cropLeft.setMaxOutputFrameSize(1280*800*3)
cropRight.initialConfig.setCropRect(topLeft.x, topLeft.y, bottomRight.x, bottomRight.y)
cropRight.setMaxOutputFrameSize(1280*800*3)

rgbOut = pipeline.create(dai.node.XLinkOut)
disparityOut = pipeline.create(dai.node.XLinkOut)

rgbOut.setStreamName("rgb")
queueNames.append("rgb")
disparityOut.setStreamName("depth")
queueNames.append("depth")

#Properties
rgbCamSocket = dai.CameraBoardSocket.CAM_A
camRgb.setBoardSocket(rgbCamSocket)
camRgb.setSize(1280, 800)
camRgb.setFps(fps)

# For now, RGB needs fixed focus to properly align with depth.
# This value was used during calibration
try:
    calibData = device.readCalibration2()
    lensPosition = calibData.getLensPosition(rgbCamSocket)
    if lensPosition:
        camRgb.initialControl.setManualFocus(lensPosition)
except:
    raise
left.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)
left.setCamera("left")
left.setFps(fps)

right.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)
right.setCamera("right")
right.setFps(fps)

stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
stereo.setOutputSize(1280, 800)
stereo.setLeftRightCheck(True)
stereo.setSubpixel(True)
stereo.setSubpixelFractionalBits(5)

stereo.setDepthAlign(rgbCamSocket)

if crop:
    camRgb.video.link(cropRgb.inputImage)
    left.isp.link(cropLeft.inputImage)
    right.isp.link(cropRight.inputImage)

    cropLeft.out.link(stereo.left)
    cropRight.out.link(stereo.right)

    cropRgb.out.link(rgbOut.input)

    stereo.disparity.link(disparityOut.input)
else:
    left.setIspScale(2, 3)
    right.setIspScale(2, 3)

    camRgb.video.link(rgbOut.input)
    left.isp.link(stereo.left)
    right.isp.link(stereo.right)
    stereo.disparity.link(disparityOut.input)

maxDisp = stereo.initialConfig.getMaxDisparity()

with device:
    device.startPipeline(pipeline)

    while not device.isClosed():
        queueNames = device.getQueueEvents()
        for q in queueNames:
            message = device.getOutputQueue(q).get()
            # Display arrived frames
            if type(message) == dai.ImgFrame:
                frame = message.getCvFrame()
                if 'depth' in q:
                    disp = (frame * (255.0 / maxDisp)).astype(np.uint8)
                    disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET)
                    cv2.imshow(q, disp)
                else:
                    cv2.imshow(q, frame)
        if cv2.waitKey(1) == ord('q'):
            break

jakaskerl

oscarthorn
You can directly use the ColorCamera's setPreview size to crop to 800p. It is faster as well. For Camera (I assume you want this because it has undistort functionality built-in) you should be able to set it via setVideoSize().

So you probably don't need image manips.

Thanks,
Jaka

oscarthorn

jakaskerl

Thanks for the help, sorry for the slow response 🙂

I did what you suggested but now I'm getting the following error:
Left input image stride ('3840') should be equal to its width ('1280'). Skipping frame!

Code:

#!/usr/bin/env python3
from collections import deque
import cv2
import numpy as np
import depthai as dai
import time

fps = 30

# Create pipeline
pipeline = dai.Pipeline()
device = dai.Device()
queueNames = []

# Define sources and outputs
camRgb = pipeline.create(dai.node.Camera)
left = pipeline.create(dai.node.ColorCamera)
right = pipeline.create(dai.node.ColorCamera)
stereo = pipeline.create(dai.node.StereoDepth)

rgbOut = pipeline.create(dai.node.XLinkOut)
disparityOut = pipeline.create(dai.node.XLinkOut)

rgbOut.setStreamName("rgb")
queueNames.append("rgb")
disparityOut.setStreamName("depth")
queueNames.append("depth")

#Properties
rgbCamSocket = dai.CameraBoardSocket.CAM_A
camRgb.setBoardSocket(rgbCamSocket)
camRgb.setSize(1280, 800)
camRgb.setFps(fps)
camRgb.setPreviewSize(1280, 800)

# For now, RGB needs fixed focus to properly align with depth.
# This value was used during calibration
try:
    calibData = device.readCalibration2()
    lensPosition = calibData.getLensPosition(rgbCamSocket)
    if lensPosition:
        camRgb.initialControl.setManualFocus(lensPosition)
except:
    raise
left.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)
left.setCamera("left")
left.setFps(fps)
left.setPreviewSize(1280, 800)

right.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)
right.setCamera("right")
right.setFps(fps)
right.setPreviewSize(1280, 800)

stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
stereo.setOutputSize(1280, 800)
stereo.setLeftRightCheck(True)
stereo.setSubpixel(True)
stereo.setSubpixelFractionalBits(5)

stereo.setDepthAlign(rgbCamSocket)

camRgb.preview.link(rgbOut.input)
left.preview.link(stereo.left)
right.preview.link(stereo.right)
stereo.disparity.link(disparityOut.input)

maxDisp = stereo.initialConfig.getMaxDisparity()

prev_frame_time = 0
new_frame_time = 0
average_fps = deque(maxlen=30)
with device:
    device.startPipeline(pipeline)

    while not device.isClosed():
        queueNames = device.getQueueEvents()
        for q in queueNames:
            message = device.getOutputQueue(q).get()
            # Display arrived frames
            if type(message) == dai.ImgFrame:
                frame = message.getCvFrame()
                if 'depth' in q:
                    disp = (frame * (255.0 / maxDisp)).astype(np.uint8)
                    disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET)
                    cv2.imshow(q, disp)
                else:
                    new_frame_time = time.time()
                    average_fps.append(1/(new_frame_time-prev_frame_time))
                    prev_frame_time = new_frame_time
                    cv2.putText(frame, str(np.mean(average_fps)), (7, 70), cv2.FONT_HERSHEY_SIMPLEX , 3, (100, 255, 0), 3, cv2.LINE_AA)
                    cv2.imshow(q, frame)
        if cv2.waitKey(1) == ord('q'):
            break

jakaskerl

oscarthorn
Hey, seems like a depthai issue. I will check it tomorrow since I don't have LR nearby right now.

Thanks,
Jaka

jakaskerl

oscarthorn
Link isp output, not preview. Then it should work.
Also consider using left.setIspScale(2, 3) instead of setPreviewSize.

Thanks,
Jaka

oscarthorn

jakaskerl Thanks, but that does not crop the image. If I just link to isp instead of preview then that is not cropped so the stereo module gives: Maximum supported input image width for stereo is 1280. Skipping frame! and if I use setIspScale then that downscales to the correct resolution but I want a cropped image.

To reiterate I want to crop the stereo pair images to 1280x800 so that it works with the stereo node and I want to crop instead of downscale because I want to reduce the fov to get better long range accuracy.

jakaskerl

Hi @oscarthorn
Then use video output along with setVideoSize(1280, 800).

Thanks,
Jaka

oscarthorn

jakaskerl Thanks, that solves the problem!