Hello,
I'm using the luxonis OAK D LR for the following use case.
- Bin picking so high accuracy required (1mm)
- Camera at 1300 mm from objects.
- Static scene (low fps is ok)
I expect the different objects in the box to be detected at least (5mm tall). The luxonis box itself is detected. (see attached images, scripts)
I've tried out a lot of different options and followed the guide on https://docs.luxonis.com/hardware/platform/depth/configuring-stereo-depth/
What I've tried:
- Filtering: Median, thresholding (on-device cannot get it to work, I do it afterward manually)
- Camera options: ISP
- Processing options: extened disparity, subpixel, lr_check
- Depth-ai V2 and v3 example scripts
The main issues I face now is a lot of noise and objects smaller than 5mm not detected. I'm just interested in what's in the grey box. Could u point me in the right direction of what version of the depth-ai framework to use and what settings? I attached the scripts i've used + output in the attachment
Version 3 script + output
#!/usr/bin/env python3
import cv2import depthai as daiimport numpy as np
pipeline = dai.Pipeline()monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C)stereo = pipeline.create(dai.node.StereoDepth)
# LinkingmonoLeftOut = monoLeft.requestOutput(size=(1280,720), type=dai.ImgFrame.Type.NV12)monoRightOut = monoRight.requestOutput(size=(1280,720), type=dai.ImgFrame.Type.NV12)
monoLeftOut.link(stereo.left)monoRightOut.link(stereo.right)
stereo.setRectification(True)stereo.setExtendedDisparity(True)stereo.setLeftRightCheck(True)
# stereo.initialConfig.postProcessing.median = dai.StereoDepthProperties.MedianFilter.KERNEL_7x7
syncedLeftQueue = stereo.syncedLeft.createOutputQueue()syncedRightQueue = stereo.syncedRight.createOutputQueue()disparityQueue = stereo.disparity.createOutputQueue()depthQueue = stereo.depth.createOutputQueue()
colorMap = cv2.applyColorMap(np.arange(256, dtype=np.uint8), cv2.COLORMAP_JET)colorMap[0] = [0, 0, 0] # to make zero-disparity pixels black
with pipeline: pipeline.start() maxDisparity = 1 minDepth, maxDepth = 1300, 1400 while pipeline.isRunning(): leftSynced = syncedLeftQueue.get() rightSynced = syncedRightQueue.get() disparity = disparityQueue.get() depth = depthQueue.get()
assert isinstance(leftSynced, dai.ImgFrame) assert isinstance(rightSynced, dai.ImgFrame) assert isinstance(disparity, dai.ImgFrame)
cv2.imshow("left", leftSynced.getCvFrame()) cv2.imshow("right", rightSynced.getCvFrame()) npDisparity = disparity.getFrame() maxDisparity = max(maxDisparity, np.max(npDisparity)) colorizedDisparity = cv2.applyColorMap(((npDisparity / maxDisparity) * 255).astype(np.uint8), colorMap) cv2.imshow("disparity", colorizedDisparity)
npDepth = depth.getFrame() mask = (npDepth >= minDepth) & (npDepth <= maxDepth) npDepthFiltered = np.where(mask, npDepth, 0).astype(np.uint8)
colorizedDisparity = cv2.applyColorMap( ((npDepthFiltered / np.max(npDepthFiltered + 1)) * 255).astype(np.uint8), cv2.COLORMAP_JET ) cv2.imshow("thresholded depth", colorizedDisparity)
key = cv2.waitKey(1) if key == ord('q'): pipeline.stop() break
Depth-ai version 2 script + output
#!/usr/bin/env python3
import cv2import depthai as daiimport numpy as np
# Closer-in minimum depth, disparity range is doubled (from 95 to 190):extended_disparity = True# Better accuracy for longer distance, fractional disparity 32-levels:subpixel = True# Better handling for occlusions:lr_check = True
enableRectified = False
# Create pipelinepipeline = dai.Pipeline()
# Define sources and outputsleft = pipeline.create(dai.node.ColorCamera)center = pipeline.create(dai.node.ColorCamera)right = pipeline.create(dai.node.ColorCamera)LC_depth = pipeline.create(dai.node.StereoDepth)LR_depth = pipeline.create(dai.node.StereoDepth)CR_depth = pipeline.create(dai.node.StereoDepth)
xout_LC = pipeline.create(dai.node.XLinkOut)xout_LR = pipeline.create(dai.node.XLinkOut)xout_CR = pipeline.create(dai.node.XLinkOut)
xout_LC.setStreamName("disparity_LC")if enableRectified: xoutl_LC = pipeline.create(dai.node.XLinkOut) xoutr_LC = pipeline.create(dai.node.XLinkOut) xoutl_LC.setStreamName("rectifiedLeft_LC") xoutr_LC.setStreamName("rectifiedRight_LC")
xout_LR.setStreamName("disparity_LR")if enableRectified: xoutl_LR = pipeline.create(dai.node.XLinkOut) xoutr_LR = pipeline.create(dai.node.XLinkOut) xoutl_LR.setStreamName("rectifiedLeft_LR") xoutr_LR.setStreamName("rectifiedRight_LR")
xout_CR.setStreamName("disparity_CR")if enableRectified: xoutl_CR = pipeline.create(dai.node.XLinkOut) xoutr_CR = pipeline.create(dai.node.XLinkOut) xoutl_CR.setStreamName("rectifiedLeft_CR") xoutr_CR.setStreamName("rectifiedRight_CR")
# Propertiesleft.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)left.setCamera("left")left.setIspScale(2, 3)
center.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)center.setBoardSocket(dai.CameraBoardSocket.CENTER)center.setIspScale(2, 3)
right.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1200_P)right.setCamera("right")right.setIspScale(2, 3)
LC_depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)LC_depth.initialConfig.setMedianFilter(dai.MedianFilter.MEDIAN_OFF)LC_depth.setLeftRightCheck(lr_check)LC_depth.setExtendedDisparity(extended_disparity)LC_depth.setSubpixel(subpixel)
LR_depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)LR_depth.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)LR_depth.setLeftRightCheck(lr_check)LR_depth.setExtendedDisparity(extended_disparity)LR_depth.setSubpixel(subpixel)
CR_depth.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)CR_depth.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7)CR_depth.setLeftRightCheck(lr_check)CR_depth.setExtendedDisparity(extended_disparity)CR_depth.setSubpixel(subpixel)
# Linking# LCleft.isp.link(LC_depth.left)center.isp.link(LC_depth.right)LC_depth.disparity.link(xout_LC.input)if enableRectified: LC_depth.rectifiedLeft.link(xoutl_LC.input) LC_depth.rectifiedRight.link(xoutr_LC.input)# LRleft.isp.link(LR_depth.left)right.isp.link(LR_depth.right)LR_depth.disparity.link(xout_LR.input)if enableRectified: LR_depth.rectifiedLeft.link(xoutl_LR.input) LR_depth.rectifiedRight.link(xoutr_LR.input)# CRcenter.isp.link(CR_depth.left)right.isp.link(CR_depth.right)CR_depth.disparity.link(xout_CR.input)if enableRectified: CR_depth.rectifiedLeft.link(xoutl_CR.input) CR_depth.rectifiedRight.link(xoutr_CR.input)
maxDisp = LC_depth.initialConfig.getMaxDisparity()
# Connect to device and start pipelinedevice_info = dai.DeviceInfo("169.254.1.222") #ip-adres
with dai.Device(pipeline, deviceInfo=device_info) as device: while not device.isClosed(): queueNames = device.getQueueEvents() for q in queueNames: message = device.getOutputQueue(q).get() # Display arrived frames if type(message) == dai.ImgFrame: frame = message.getCvFrame() if 'disparity' in q: disp = (frame * (255.0 / maxDisp)).astype(np.uint8) disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET) cv2.imshow(q, disp) else: cv2.imshow(q, frame) if cv2.waitKey(1) == ord('q'): break
