setDisparityShift destroys alignment

Robengaard

Hi guys,

I'm using a OAK-D Pro POE AF, I'm trying to play around with the depth.initialConfig.setDisparityShift since I'm almost below 50 centimeters away from my object. However it seems like it shifts the depth image when using it at 30.

Without setDisparityShift.

With setDisparityShift set to 15.

With setDisparityShift set to 30.

setDisparityShift set to 45.

Can anyone explain this behavior?

Br Martin

jakaskerl

Hi Robengaard ,
Hm, very strange. Could you post a MRE code of this issue so we can debug/test?

Thanks,
Jaka

erik

Robengaard I believe this is a known limitation. Could you also post a feature request on depthai-core about this?
Thanks, Erik

Robengaard

Hi jakaskerl

import time

import depthai as dai

import numpy as np

from matplotlib import pyplot as plt

import cv2

dry_run=None,

input_path=None,

init_size=2,

pause=1,

use_syslog=False,

calibration=False,

device = None

use_syslog = use_syslog

calibration = calibration

__last_frame_num = -1

rgb_queue = None

depth_queue = None

syslog_queue = None

focal_length = None

mono_left_queue = None

mono_right_queue = None

pipeline = dai.Pipeline()

cam_rgb = pipeline.create(dai.node.ColorCamera)

rgb_out = pipeline.create(dai.node.XLinkOut)

rgb_out.setStreamName("rgb")

rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)

rgb_ctrl_in.setStreamName("rgb-ctrl")

# Settings

cam_rgb.setIspScale(2, 5)

cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)

cam_rgb.setFps(15)

# Linking

cam_rgb.isp.link(rgb_out.input)

rgb_ctrl_in.out.link(cam_rgb.inputControl)

# Setup nodes

mono_left = pipeline.create(dai.node.MonoCamera)

mono_right = pipeline.create(dai.node.MonoCamera)

depth = pipeline.create(dai.node.StereoDepth)

xout = pipeline.create(dai.node.XLinkOut)

xout.setStreamName("depth")

left_out = pipeline.create(dai.node.XLinkOut)

left_out.setStreamName("left")

right_out = pipeline.create(dai.node.XLinkOut)

right_out.setStreamName("right")

left_ctrl_in = pipeline.create(dai.node.XLinkIn)

left_ctrl_in.setStreamName("left-ctrl")

right_ctrl_in = pipeline.create(dai.node.XLinkIn)

right_ctrl_in.setStreamName("right-ctrl")

# Settings

mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)

mono_left.setFps(15)

mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)

mono_right.setFps(15)

depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)

depth.setLeftRightCheck(True)

depth.setDepthAlign(dai.CameraBoardSocket.RGB)

depth.initialConfig.setConfidenceThreshold(

180

)

depth.setExtendedDisparity(True)

depth.setSubpixel(True)

depth.initialConfig.setDisparityShift(

45

)

# Linking

mono_left.out.link(depth.left)

mono_right.out.link(depth.right)

mono_left.out.link(left_out.input)

mono_right.out.link(right_out.input)

depth.depth.link(xout.input)

left_ctrl_in.out.link(mono_left.inputControl)

right_ctrl_in.out.link(mono_right.inputControl)

print("Searching local network for camera")

# info = dai.DeviceInfo(config.camera["ip"])

#else:

# while not found:

(found, info) = dai.DeviceBootloader.getFirstAvailableDevice()

print("Connecting to %s (ip: %s)" % (info.mxid, info.name))

success = False

wait_count = 2

while not success:

try:

device = dai.Device(pipeline, info)

success = True

except dai.XLinkWriteError as e:

print("Failed " + str(e))

except RuntimeError as e:

print("Failed " + str(e))

if wait_count <= 0:

print("No more retries")

raise

print("Retrying after wait")

wait_count -= 1

time.sleep(10)

print("Connected")

calib_data = device.readCalibration()

intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

focal_length = intrinsics[0][0]

depth_queue = device.getOutputQueue(

name="depth", maxSize=1, blocking=False

)

rgb_queue = device.getOutputQueue(

name="rgb", maxSize=1, blocking=False

)

mono_left_queue = device.getOutputQueue(

name="left", maxSize=1, blocking=False

)

mono_right_queue = device.getOutputQueue(

name="right", maxSize=1, blocking=False

)

for i in range(0,2):

frame = None

while frame is None:

frame = rgb_queue.tryGet()

rgb = frame.getCvFrame()

frame = None

while frame is None:

frame = depth_queue.tryGet()

depth_img = frame.getFrame()

fig, ax = plt.subplots(1, 2, figsize=(14, 6))

ax[0].imshow(rgb)

ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")

plt.show()

backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)

backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)

dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)

plt.imshow(dst)

time.sleep(10)

I have tried to remove as much code as possible.

Br Martin

Robengaard

Hi jakaskerl
Here is it in a better format. Sorry…

import time
import depthai as dai
import numpy as np
from matplotlib import pyplot as plt
import cv2



dry_run=None,
input_path=None,
init_size=2,
pause=1,
use_syslog=False,
calibration=False,
device = None
use_syslog = use_syslog
calibration = calibration
__last_frame_num = -1
rgb_queue = None
depth_queue = None
syslog_queue = None
focal_length = None
mono_left_queue = None
mono_right_queue = None


pipeline = dai.Pipeline()


cam_rgb = pipeline.create(dai.node.ColorCamera)
rgb_out = pipeline.create(dai.node.XLinkOut)
rgb_out.setStreamName("rgb")

rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)
rgb_ctrl_in.setStreamName("rgb-ctrl")

# Settings
cam_rgb.setIspScale(2, 5)
cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)
cam_rgb.setFps(15)

# Linking
cam_rgb.isp.link(rgb_out.input)
rgb_ctrl_in.out.link(cam_rgb.inputControl)

# Setup nodes
mono_left = pipeline.create(dai.node.MonoCamera)
mono_right = pipeline.create(dai.node.MonoCamera)
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName("depth")

left_out = pipeline.create(dai.node.XLinkOut)
left_out.setStreamName("left")

right_out = pipeline.create(dai.node.XLinkOut)
right_out.setStreamName("right")

left_ctrl_in = pipeline.create(dai.node.XLinkIn)
left_ctrl_in.setStreamName("left-ctrl")
right_ctrl_in = pipeline.create(dai.node.XLinkIn)
right_ctrl_in.setStreamName("right-ctrl")

# Settings
mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)
mono_left.setFps(15)
mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)
mono_right.setFps(15)

depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)
depth.setLeftRightCheck(True)
depth.setDepthAlign(dai.CameraBoardSocket.RGB)

depth.initialConfig.setConfidenceThreshold(
    180
)
depth.setExtendedDisparity(True)
depth.setSubpixel(True)
depth.initialConfig.setDisparityShift(
    45
)


# Linking
mono_left.out.link(depth.left)
mono_right.out.link(depth.right)

mono_left.out.link(left_out.input)
mono_right.out.link(right_out.input)

depth.depth.link(xout.input)

left_ctrl_in.out.link(mono_left.inputControl)
right_ctrl_in.out.link(mono_right.inputControl)


print("Searching local network for camera")

#    info = dai.DeviceInfo(config.camera["ip"])
#else:
#    while not found:
(found, info) = dai.DeviceBootloader.getFirstAvailableDevice()
print("Connecting to %s (ip: %s)" % (info.mxid, info.name))
success = False
wait_count = 2
while not success:
    try:
        device = dai.Device(pipeline, info)
        success = True
    except dai.XLinkWriteError as e:
        print("Failed " + str(e))
    except RuntimeError as e:
        print("Failed " + str(e))
        if wait_count <= 0:
            print("No more retries")
            raise
        print("Retrying after wait")
        wait_count -= 1
        time.sleep(10)

print("Connected")
calib_data = device.readCalibration()
intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

focal_length = intrinsics[0][0]
depth_queue = device.getOutputQueue(
    name="depth", maxSize=1, blocking=False
)
rgb_queue = device.getOutputQueue(
    name="rgb", maxSize=1, blocking=False
)

mono_left_queue = device.getOutputQueue(
    name="left", maxSize=1, blocking=False
)

mono_right_queue = device.getOutputQueue(
    name="right", maxSize=1, blocking=False
)

for i in range(0,2):
    
    frame = None
    while frame is None:
        
        frame = rgb_queue.tryGet()
    
    
    
    rgb = frame.getCvFrame()
    
    frame = None
    while frame is None:
        frame = depth_queue.tryGet()
    depth_img = frame.getFrame()
    
    
    fig, ax = plt.subplots(1, 2, figsize=(14, 6))
    ax[0].imshow(rgb)
    ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")
    plt.show()
    
    backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)
    backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)
    dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)
    plt.imshow(dst)
    time.sleep(10)

Robengaard

Hi erik

How do I find that this is a known limitation?

It is not mentioned in https://docs.luxonis.com/projects/api/en/latest/tutorials/configuring-stereo-depth/#disparity-shift

or here: https://docs.luxonis.com/projects/api/en/latest/references/python/?highlight=setDisparityShift#depthai.StereoDepthConfig.setDisparityShift

Br Martin

erik

Robengaard Could you make MRE (minimal repro example)? This is far from minimal..

Robengaard

erik

Maybe this is better.

# -*- coding: utf-8 -*-
"""
Created on Sat May 13 10:23:57 2023

@author: Martin
"""

import time
import depthai as dai
import numpy as np
from matplotlib import pyplot as plt
import cv2


pipeline = dai.Pipeline()


cam_rgb = pipeline.create(dai.node.ColorCamera)
rgb_out = pipeline.create(dai.node.XLinkOut)
rgb_out.setStreamName("rgb")

rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)
rgb_ctrl_in.setStreamName("rgb-ctrl")

# Settings
cam_rgb.setIspScale(2, 5)
cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)
cam_rgb.setFps(15)

# Linking
cam_rgb.isp.link(rgb_out.input)
rgb_ctrl_in.out.link(cam_rgb.inputControl)

# Setup nodes
mono_left = pipeline.create(dai.node.MonoCamera)
mono_right = pipeline.create(dai.node.MonoCamera)
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName("depth")

left_out = pipeline.create(dai.node.XLinkOut)
left_out.setStreamName("left")

right_out = pipeline.create(dai.node.XLinkOut)
right_out.setStreamName("right")

left_ctrl_in = pipeline.create(dai.node.XLinkIn)
left_ctrl_in.setStreamName("left-ctrl")
right_ctrl_in = pipeline.create(dai.node.XLinkIn)
right_ctrl_in.setStreamName("right-ctrl")

# Settings
mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)
mono_left.setFps(15)
mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)
mono_right.setFps(15)

depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)
depth.setLeftRightCheck(True)
depth.setDepthAlign(dai.CameraBoardSocket.RGB)

depth.initialConfig.setConfidenceThreshold(
    180
)
depth.setExtendedDisparity(True)
depth.setSubpixel(True)
depth.initialConfig.setDisparityShift(
    45
)


# Linking
mono_left.out.link(depth.left)
mono_right.out.link(depth.right)

mono_left.out.link(left_out.input)
mono_right.out.link(right_out.input)

depth.depth.link(xout.input)

left_ctrl_in.out.link(mono_left.inputControl)
right_ctrl_in.out.link(mono_right.inputControl)

print("Searching local network for camera")

(found, info) = dai.DeviceBootloader.getFirstAvailableDevice()
print("Connecting to %s (ip: %s)" % (info.mxid, info.name))
device = dai.Device(pipeline, info)
success = False


calib_data = device.readCalibration()
intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

focal_length = intrinsics[0][0]
depth_queue = device.getOutputQueue(
    name="depth", maxSize=1, blocking=False
)
rgb_queue = device.getOutputQueue(
    name="rgb", maxSize=1, blocking=False
)

mono_left_queue = device.getOutputQueue(
    name="left", maxSize=1, blocking=False
)

mono_right_queue = device.getOutputQueue(
    name="right", maxSize=1, blocking=False
)

for i in range(0,2):
    
    frame = None
    while frame is None:
        
        frame = rgb_queue.tryGet()
    
    
    
    rgb = frame.getCvFrame()
    
    frame = None
    while frame is None:
        frame = depth_queue.tryGet()
    depth_img = frame.getFrame()
    
    
    fig, ax = plt.subplots(1, 2, figsize=(14, 6))
    ax[0].imshow(rgb)
    ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")
    plt.show()
    
    backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)
    backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)
    dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)
    plt.imshow(dst)
    time.sleep(10)

erik

Hi Robengaard ,
Unfortunately, this is a known limitation, as per docs here: https://docs.luxonis.com/projects/api/en/latest/components/nodes/stereo_depth/#limitations
If you'd like to get updated when this gets fixed, I'd suggest following the feature request on depthai-core: luxonis/depthai-core831
Thanks, Erik