OAK-D CM4 PoE Stereo calibration does not seems right

HumamHelfawi

Hi,

I have several OAK-D CM4 PoE cameras. I have grabbed the Rectified Left/Right images from the camera. However, I have noticed that on 3 of the cameras the Left/Right does not seem to be correctly rectified (correspondence points does not lay at the same row. There are X & Y offset for any point from the Left image to its correspondence in the right image).

However, one of them only shows reasonable rectified image (although not perfect but I can say the error is within single row at most).

What could be wrong? Isn't it supposed to be factory calibrated ?

This is my code to grab the rectified images:

#!/usr/bin/env python3

import depthai as dai
import os
import cv2

# NOTE Because we are encoding disparity values, output video will be a gray, and won't have many pixel levels - either 0..95 or 0..190
fps = 26
# Create pipeline
pipeline = dai.Pipeline()

# Create left/right mono cameras for Stereo depth
monoLeft = pipeline.create(dai.node.MonoCamera)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
monoLeft.setFps(fps)

monoRight = pipeline.create(dai.node.MonoCamera)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)
monoRight.setFps(fps)

# Create a node that will produce the depth map
stereo = pipeline.create(dai.node.StereoDepth)
stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_ACCURACY)
#stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)
stereo.setLeftRightCheck(True)
stereo.setExtendedDisparity(False)
# Subpixel disparity is of UINT16 format, which is unsupported by VideoEncoder
stereo.setSubpixel(False)

monoLeft.out.link(stereo.left)
monoRight.out.link(stereo.right)



depthVideoEnc = pipeline.create(dai.node.VideoEncoder)
depthVideoEnc.setDefaultProfilePreset(monoLeft.getFps(), dai.VideoEncoderProperties.Profile.H265_MAIN)

# rectifiedLeftVideoEnc = pipeline.create(dai.node.VideoEncoder)
# rectifiedLeftVideoEnc.setDefaultProfilePreset(monoLeft.getFps(), dai.VideoEncoderProperties.Profile.H265_MAIN)

# rectifiedRightVideoEnc = pipeline.create(dai.node.VideoEncoder)
# rectifiedRightVideoEnc.setDefaultProfilePreset(monoLeft.getFps(), dai.VideoEncoderProperties.Profile.H265_MAIN)

rectifiedLeftOut = pipeline.create(dai.node.XLinkOut)
rectifiedLeftOut.setStreamName("rectifiedLeft")
rectifiedRightOut = pipeline.create(dai.node.XLinkOut)
rectifiedRightOut.setStreamName("rectifiedRight")

stereo.disparity.link(depthVideoEnc.input)
stereo.rectifiedLeft.link(rectifiedLeftOut.input)
stereo.rectifiedRight.link(rectifiedRightOut.input)

dispOut = pipeline.create(dai.node.XLinkOut)
dispOut.setStreamName("disp")
depthVideoEnc.bitstream.link(dispOut.input)


# rectifiedLeftVideoEnc.bitstream.link(rectifiedLeftOut.input)
# rectifiedRightVideoEnc.bitstream.link(rectifiedRightOut.input)


# Connect to device and start pipeline
with dai.Device(pipeline) as device:

    # Output queue will be used to get the encoded data from the output defined above
    qDisp = device.getOutputQueue(name="disp")
    qRectifiedLeft = device.getOutputQueue(name="rectifiedLeft")
    qRectifiedRight = device.getOutputQueue(name="rectifiedRight")

    dispVideoFile = open('disparity.h265', 'wb')
    # rectifiedLeftVideoFile = open('rectifiedLeft.h265', 'wb')
    # rectifiedRightVideoFile = open('rectifiedRight.h265', 'wb')
    print("Press Ctrl+C to stop encoding...")
    try:
        while True:
            dispPacket = qDisp.get()
            rectifiedLeftPacket = qRectifiedLeft.get()
            rectifiedRightPacket = qRectifiedRight.get()
            
            dispVideoFile.write(dispPacket.getData())
            # rectifiedLeftVideoFile.write(rectifiedLeftPacket.getData())
            # rectifiedRightVideoFile.write(rectifiedRightPacket.getData())
            
            cv2.imwrite("rectifiedLeft.jpg", rectifiedLeftPacket.getCvFrame())
            cv2.imwrite("rectifiedRight.jpg", rectifiedRightPacket.getCvFrame())
            
    except KeyboardInterrupt:
        # Keyboard interrupt (Ctrl + C) detected
        pass

    print("To view the encoded data, convert the stream file (.mjpeg) into a video file (.mp4) using a command below:")
    os.system("ffmpeg -framerate " + str(fps)+ " -i disparity.h265 -c copy disparity.mp4")
    os.system("ffmpeg -framerate " + str(fps)+ " -i rectifiedLeft.h265 -c copy rectifiedLeft.mp4")
    os.system("ffmpeg -framerate " + str(fps)+ " -i rectifiedRight.h265 -c copy rectifiedRight.mp4")
    print("_____________________________________________________________________________________")