• setDisparityShift destroys alignment

Hi guys,

I'm using a OAK-D Pro POE AF, I'm trying to play around with the depth.initialConfig.setDisparityShift since I'm almost below 50 centimeters away from my object. However it seems like it shifts the depth image when using it at 30.

Without setDisparityShift.

With setDisparityShift set to 15.

With setDisparityShift set to 30.

setDisparityShift set to 45.

Can anyone explain this behavior?

Br Martin

      Robengaard I believe this is a known limitation. Could you also post a feature request on depthai-core about this?
      Thanks, Erik

          Hi jakaskerl

          import time

          import depthai as dai

          import numpy as np

          from matplotlib import pyplot as plt

          import cv2

          dry_run=None,

          input_path=None,

          init_size=2,

          pause=1,

          use_syslog=False,

          calibration=False,

          device = None

          use_syslog = use_syslog

          calibration = calibration

          __last_frame_num = -1

          rgb_queue = None

          depth_queue = None

          syslog_queue = None

          focal_length = None

          mono_left_queue = None

          mono_right_queue = None

          pipeline = dai.Pipeline()

          cam_rgb = pipeline.create(dai.node.ColorCamera)

          rgb_out = pipeline.create(dai.node.XLinkOut)

          rgb_out.setStreamName("rgb")

          rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)

          rgb_ctrl_in.setStreamName("rgb-ctrl")

          # Settings

          cam_rgb.setIspScale(2, 5)

          cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

          cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)

          cam_rgb.setFps(15)

          # Linking

          cam_rgb.isp.link(rgb_out.input)

          rgb_ctrl_in.out.link(cam_rgb.inputControl)

          # Setup nodes

          mono_left = pipeline.create(dai.node.MonoCamera)

          mono_right = pipeline.create(dai.node.MonoCamera)

          depth = pipeline.create(dai.node.StereoDepth)

          xout = pipeline.create(dai.node.XLinkOut)

          xout.setStreamName("depth")

          left_out = pipeline.create(dai.node.XLinkOut)

          left_out.setStreamName("left")

          right_out = pipeline.create(dai.node.XLinkOut)

          right_out.setStreamName("right")

          left_ctrl_in = pipeline.create(dai.node.XLinkIn)

          left_ctrl_in.setStreamName("left-ctrl")

          right_ctrl_in = pipeline.create(dai.node.XLinkIn)

          right_ctrl_in.setStreamName("right-ctrl")

          # Settings

          mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

          mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)

          mono_left.setFps(15)

          mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)

          mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)

          mono_right.setFps(15)

          depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)

          depth.setLeftRightCheck(True)

          depth.setDepthAlign(dai.CameraBoardSocket.RGB)

          depth.initialConfig.setConfidenceThreshold(

          180

          )

          depth.setExtendedDisparity(True)

          depth.setSubpixel(True)

          depth.initialConfig.setDisparityShift(

          45

          )

          # Linking

          mono_left.out.link(depth.left)

          mono_right.out.link(depth.right)

          mono_left.out.link(left_out.input)

          mono_right.out.link(right_out.input)

          depth.depth.link(xout.input)

          left_ctrl_in.out.link(mono_left.inputControl)

          right_ctrl_in.out.link(mono_right.inputControl)

          print("Searching local network for camera")

          # info = dai.DeviceInfo(config.camera["ip"])

          #else:

          # while not found:

          (found, info) = dai.DeviceBootloader.getFirstAvailableDevice()

          print("Connecting to %s (ip: %s)" % (info.mxid, info.name))

          success = False

          wait_count = 2

          while not success:

          try:

          device = dai.Device(pipeline, info)

          success = True

          except dai.XLinkWriteError as e:

          print("Failed " + str(e))

          except RuntimeError as e:

          print("Failed " + str(e))

          if wait_count <= 0:

          print("No more retries")

          raise

          print("Retrying after wait")

          wait_count -= 1

          time.sleep(10)

          print("Connected")

          calib_data = device.readCalibration()

          intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

          focal_length = intrinsics[0][0]

          depth_queue = device.getOutputQueue(

          name="depth", maxSize=1, blocking=False

          )

          rgb_queue = device.getOutputQueue(

          name="rgb", maxSize=1, blocking=False

          )

          mono_left_queue = device.getOutputQueue(

          name="left", maxSize=1, blocking=False

          )

          mono_right_queue = device.getOutputQueue(

          name="right", maxSize=1, blocking=False

          )

          for i in range(0,2):

          frame = None

          while frame is None:

          frame = rgb_queue.tryGet()

          rgb = frame.getCvFrame()

          frame = None

          while frame is None:

          frame = depth_queue.tryGet()

          depth_img = frame.getFrame()

          fig, ax = plt.subplots(1, 2, figsize=(14, 6))

          ax[0].imshow(rgb)

          ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")

          plt.show()

          backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)

          backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)

          dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)

          plt.imshow(dst)

          time.sleep(10)

          I have tried to remove as much code as possible.

          Br Martin

            Hi jakaskerl
            Here is it in a better format. Sorry…

            import time
import depthai as dai
import numpy as np
from matplotlib import pyplot as plt
import cv2



dry_run=None,
input_path=None,
init_size=2,
pause=1,
use_syslog=False,
calibration=False,
device = None
use_syslog = use_syslog
calibration = calibration
__last_frame_num = -1
rgb_queue = None
depth_queue = None
syslog_queue = None
focal_length = None
mono_left_queue = None
mono_right_queue = None


pipeline = dai.Pipeline()


cam_rgb = pipeline.create(dai.node.ColorCamera)
rgb_out = pipeline.create(dai.node.XLinkOut)
rgb_out.setStreamName("rgb")

rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)
rgb_ctrl_in.setStreamName("rgb-ctrl")

# Settings
cam_rgb.setIspScale(2, 5)
cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)
cam_rgb.setFps(15)

# Linking
cam_rgb.isp.link(rgb_out.input)
rgb_ctrl_in.out.link(cam_rgb.inputControl)

# Setup nodes
mono_left = pipeline.create(dai.node.MonoCamera)
mono_right = pipeline.create(dai.node.MonoCamera)
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName("depth")

left_out = pipeline.create(dai.node.XLinkOut)
left_out.setStreamName("left")

right_out = pipeline.create(dai.node.XLinkOut)
right_out.setStreamName("right")

left_ctrl_in = pipeline.create(dai.node.XLinkIn)
left_ctrl_in.setStreamName("left-ctrl")
right_ctrl_in = pipeline.create(dai.node.XLinkIn)
right_ctrl_in.setStreamName("right-ctrl")

# Settings
mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)
mono_left.setFps(15)
mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)
mono_right.setFps(15)

depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)
depth.setLeftRightCheck(True)
depth.setDepthAlign(dai.CameraBoardSocket.RGB)

depth.initialConfig.setConfidenceThreshold(
    180
)
depth.setExtendedDisparity(True)
depth.setSubpixel(True)
depth.initialConfig.setDisparityShift(
    45
)


# Linking
mono_left.out.link(depth.left)
mono_right.out.link(depth.right)

mono_left.out.link(left_out.input)
mono_right.out.link(right_out.input)

depth.depth.link(xout.input)

left_ctrl_in.out.link(mono_left.inputControl)
right_ctrl_in.out.link(mono_right.inputControl)


print("Searching local network for camera")

#    info = dai.DeviceInfo(config.camera["ip"])
#else:
#    while not found:
(found, info) = dai.DeviceBootloader.getFirstAvailableDevice()
print("Connecting to %s (ip: %s)" % (info.mxid, info.name))
success = False
wait_count = 2
while not success:
    try:
        device = dai.Device(pipeline, info)
        success = True
    except dai.XLinkWriteError as e:
        print("Failed " + str(e))
    except RuntimeError as e:
        print("Failed " + str(e))
        if wait_count <= 0:
            print("No more retries")
            raise
        print("Retrying after wait")
        wait_count -= 1
        time.sleep(10)

print("Connected")
calib_data = device.readCalibration()
intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

focal_length = intrinsics[0][0]
depth_queue = device.getOutputQueue(
    name="depth", maxSize=1, blocking=False
)
rgb_queue = device.getOutputQueue(
    name="rgb", maxSize=1, blocking=False
)

mono_left_queue = device.getOutputQueue(
    name="left", maxSize=1, blocking=False
)

mono_right_queue = device.getOutputQueue(
    name="right", maxSize=1, blocking=False
)

for i in range(0,2):
    
    frame = None
    while frame is None:
        
        frame = rgb_queue.tryGet()
    
    
    
    rgb = frame.getCvFrame()
    
    frame = None
    while frame is None:
        frame = depth_queue.tryGet()
    depth_img = frame.getFrame()
    
    
    fig, ax = plt.subplots(1, 2, figsize=(14, 6))
    ax[0].imshow(rgb)
    ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")
    plt.show()
    
    backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)
    backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)
    dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)
    plt.imshow(dst)
    time.sleep(10)
            • erik replied to this.

                erik

                Maybe this is better.

                # -*- coding: utf-8 -*-
"""
Created on Sat May 13 10:23:57 2023

@author: Martin
"""

import time
import depthai as dai
import numpy as np
from matplotlib import pyplot as plt
import cv2


pipeline = dai.Pipeline()


cam_rgb = pipeline.create(dai.node.ColorCamera)
rgb_out = pipeline.create(dai.node.XLinkOut)
rgb_out.setStreamName("rgb")

rgb_ctrl_in = pipeline.create(dai.node.XLinkIn)
rgb_ctrl_in.setStreamName("rgb-ctrl")

# Settings
cam_rgb.setIspScale(2, 5)
cam_rgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)

cam_rgb.setBoardSocket(dai.CameraBoardSocket.RGB)
cam_rgb.setFps(15)

# Linking
cam_rgb.isp.link(rgb_out.input)
rgb_ctrl_in.out.link(cam_rgb.inputControl)

# Setup nodes
mono_left = pipeline.create(dai.node.MonoCamera)
mono_right = pipeline.create(dai.node.MonoCamera)
depth = pipeline.create(dai.node.StereoDepth)
xout = pipeline.create(dai.node.XLinkOut)
xout.setStreamName("depth")

left_out = pipeline.create(dai.node.XLinkOut)
left_out.setStreamName("left")

right_out = pipeline.create(dai.node.XLinkOut)
right_out.setStreamName("right")

left_ctrl_in = pipeline.create(dai.node.XLinkIn)
left_ctrl_in.setStreamName("left-ctrl")
right_ctrl_in = pipeline.create(dai.node.XLinkIn)
right_ctrl_in.setStreamName("right-ctrl")

# Settings
mono_left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_left.setBoardSocket(dai.CameraBoardSocket.LEFT)
mono_left.setFps(15)
mono_right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
mono_right.setBoardSocket(dai.CameraBoardSocket.RIGHT)
mono_right.setFps(15)

depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)
depth.setLeftRightCheck(True)
depth.setDepthAlign(dai.CameraBoardSocket.RGB)

depth.initialConfig.setConfidenceThreshold(
    180
)
depth.setExtendedDisparity(True)
depth.setSubpixel(True)
depth.initialConfig.setDisparityShift(
    45
)


# Linking
mono_left.out.link(depth.left)
mono_right.out.link(depth.right)

mono_left.out.link(left_out.input)
mono_right.out.link(right_out.input)

depth.depth.link(xout.input)

left_ctrl_in.out.link(mono_left.inputControl)
right_ctrl_in.out.link(mono_right.inputControl)

print("Searching local network for camera")

(found, info) = dai.DeviceBootloader.getFirstAvailableDevice()
print("Connecting to %s (ip: %s)" % (info.mxid, info.name))
device = dai.Device(pipeline, info)
success = False


calib_data = device.readCalibration()
intrinsics = calib_data.getCameraIntrinsics(dai.CameraBoardSocket.RIGHT)

focal_length = intrinsics[0][0]
depth_queue = device.getOutputQueue(
    name="depth", maxSize=1, blocking=False
)
rgb_queue = device.getOutputQueue(
    name="rgb", maxSize=1, blocking=False
)

mono_left_queue = device.getOutputQueue(
    name="left", maxSize=1, blocking=False
)

mono_right_queue = device.getOutputQueue(
    name="right", maxSize=1, blocking=False
)

for i in range(0,2):
    
    frame = None
    while frame is None:
        
        frame = rgb_queue.tryGet()
    
    
    
    rgb = frame.getCvFrame()
    
    frame = None
    while frame is None:
        frame = depth_queue.tryGet()
    depth_img = frame.getFrame()
    
    
    fig, ax = plt.subplots(1, 2, figsize=(14, 6))
    ax[0].imshow(rgb)
    ax[1].imshow(np.ma.masked_array(depth_img, mask=(depth == 0)), cmap="rainbow")
    plt.show()
    
    backtorgb = cv2.cvtColor(np.ma.masked_array(depth_img, mask=(depth_img == 0)),cv2.COLOR_GRAY2RGB)
    backtorgb = (backtorgb / np.max(backtorgb) * 255).astype(np.uint8)
    dst = cv2.addWeighted(backtorgb,0.5,rgb,0.5,0)
    plt.imshow(dst)
    time.sleep(10)
                • erik replied to this.