Undistort Images Host-Side

apai25

Hi, I am trying to undistort some images, host-side, but am unable to reproduce the undistortion here: https://docs.luxonis.com/software-v3/depthai/examples/camera/camera_undistort/.

I have an Oak-D-W, and am capturing images at 640x480 (same AR as sensor itself).

Here is my code (undistortion seems to be over-correcting). However, when I just run with enableUndistortion=true, the camera returns good, undistorted images. So, I don't think calibration is the problem.

import depthai as dai
import cv2
import numpy as np

device = dai.Device()
calib = device.readCalibration()

pipeline = dai.Pipeline(device)

cam = pipeline.create(dai.node.Camera)
cam = cam.build(dai.CameraBoardSocket.CAM_A)

out_w, out_h = 640, 480

output_queue = cam.requestOutput((out_w, out_h), enableUndistortion=False,fps=30).createOutputQueue()


intrinsics = np.array(calib.getCameraIntrinsics(dai.CameraBoardSocket.CAM_A, out_w, out_h), dtype=np.float32)
dist_coeffs = np.array(calib.getDistortionCoefficients(dai.CameraBoardSocket.CAM_A), dtype=np.float32).flatten()


dist_coeffs = dist_coeffs[:14]
target_alpha = 0.0

new_intrinsics = cv2.getOptimalNewCameraMatrix(
    intrinsics, 
    dist_coeffs, 
    (out_w, out_h), 
    alpha=target_alpha, 
    newImgSize=(out_w, out_h)
)

map_x, map_y = cv2.initUndistortRectifyMap(
    intrinsics, 
    dist_coeffs, 
    None,
    new_intrinsics, 
    (out_w, out_h), 
    cv2.CV_32FC1
)

pipeline.start()

print(f"Starting capture at {out_w}x{out_h}. Press 'q' to quit.")

while True:
    packet = output_queue.get()
    frame = packet.getCvFrame()

    undistorted = cv2.remap(frame, map_x, map_y, interpolation=cv2.INTER_LINEAR)

    cv2.imshow("Original (Wide)", frame)
    cv2.imshow("Undistorted (Host-Side)", undistorted)
    
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()
device.close()

OskarSonc

Hey apai25 - likely cause is a model mismatch plus a small OpenCV bug: OAK‑D‑W uses a fisheye distortion model, but your host-side code uses the standard perspective model. That will over‑correct. Also, cv2.getOptimalNewCameraMatrix returns (matrix, roi), so new_intrinsics is a tuple right now.
Try this code:

import depthai as dai
import cv2
import numpy as np

device = dai.Device()
calib = device.readCalibration()

pipeline = dai.Pipeline(device)
cam = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)

out_w, out_h = 640, 480
output_queue = cam.requestOutput((out_w, out_h), enableUndistortion=False, fps=30).createOutputQueue()

K = np.array(calib.getCameraIntrinsics(dai.CameraBoardSocket.CAM_A, out_w, out_h), dtype=np.float64)
D = np.array(calib.getDistortionCoefficients(dai.CameraBoardSocket.CAM_A), dtype=np.float64).flatten()
model = calib.getDistortionModel(dai.CameraBoardSocket.CAM_A)

if model == dai.CameraModel.Fisheye:
    D = D[:4]
    new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(
        K, D, (out_w, out_h), np.eye(3), balance=0.0
    )
    map_x, map_y = cv2.fisheye.initUndistortRectifyMap(
        K, D, np.eye(3), new_K, (out_w, out_h), cv2.CV_32FC1
    )
else:
    new_K, _ = cv2.getOptimalNewCameraMatrix(K, D, (out_w, out_h), alpha=0.0)
    map_x, map_y = cv2.initUndistortRectifyMap(
        K, D, None, new_K, (out_w, out_h), cv2.CV_32FC1
    )

pipeline.start()

while True:
    frame = output_queue.get().getCvFrame()
    undistorted = cv2.remap(frame, map_x, map_y, interpolation=cv2.INTER_LINEAR)

    cv2.imshow("Original", frame)
    cv2.imshow("Undistorted (Host-Side)", undistorted)

    if cv2.waitKey(1) & 0xFF == ord("q"):
        break

cv2.destroyAllWindows()
device.close()

apai25

hi, thanks for the response.

i tried your code, but noticed two things:

distortion is still present
the returned model is not of type dai.CameraModel.Fisheye, so that branch of the code is not executed. even when i force it to use that type of fisheye undistortion, however, the distortion is still present
looks like over-correction, as you guessed

using regular undistortion (14 param model):

using fisheye undistortion (4 param model):

jakaskerl

apai25
Could be that the intrinsics & distortion coefficients are incorrect for the size you are requesting... Try this; this way the numbers are taken for the frame directly.

import depthai as dai
import cv2
import numpy as np

device = dai.Device()

pipeline = dai.Pipeline(device)

cam = pipeline.create(dai.node.Camera)
cam = cam.build(dai.CameraBoardSocket.CAM_A)

out_w, out_h = 640, 480

output_queue = cam.requestOutput((out_w, out_h), enableUndistortion=False, fps=30).createOutputQueue()

map_x = None
map_y = None

pipeline.start()

print(f"Starting capture at {out_w}x{out_h}. Press 'q' to quit.")

while True:
    packet = output_queue.get()
    packet: dai.ImgFrame = packet
    frame = packet.getCvFrame()
    if map_x is None or map_y is None:
        transform = packet.getTransformation()
        if transform is None:
            raise RuntimeError("Missing transformation data on ImgFrame.")
        intrinsics = np.array(transform.getIntrinsicMatrix(), dtype=np.float32)
        dist_coeffs = np.array(transform.getDistortionCoefficients(), dtype=np.float32).flatten()
        if dist_coeffs.size == 0:
            dist_coeffs = None
        target_alpha = 0.0
        new_intrinsics, _ = cv2.getOptimalNewCameraMatrix(
            intrinsics,
            dist_coeffs,
            (out_w, out_h),
            alpha=target_alpha,
            newImgSize=(out_w, out_h),
        )
        map_x, map_y = cv2.initUndistortRectifyMap(
            intrinsics,
            dist_coeffs,
            None,
            new_intrinsics,
            (out_w, out_h),
            cv2.CV_32FC1,
        )
    undistorted = cv2.remap(frame, map_x, map_y, interpolation=cv2.INTER_LINEAR)

    cv2.imshow("Original (Wide)", frame)
    cv2.imshow("Undistorted (Host-Side)", undistorted)
    
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()
device.close()

apai25

jakaskerl sorry for the late response, but yes, this was the solution. thanks!