@erik This is the depthai code:
import numpy as np
import cv2
import depthai as dai
resolution = (1632,960) # 24 FPS (without visualization)
lrcheck = True # Better handling for occlusions
extended = False # Closer-in minimum depth, disparity range is doubled
subpixel = True # True # Better accuracy for longer distance, fractional disparity 32-levels
p = dai.Pipeline()
# Configure Mono Camera Properties
left = p.createMonoCamera()
left.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
left.setBoardSocket(dai.CameraBoardSocket.LEFT)
right = p.createMonoCamera()
right.setResolution(dai.MonoCameraProperties.SensorResolution.THE_800_P)
right.setBoardSocket(dai.CameraBoardSocket.RIGHT)
stereo = p.createStereoDepth()
left.out.link(stereo.left)
right.out.link(stereo.right)
# Set stereo depth options
stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
config = stereo.initialConfig.get()
config.postProcessing.speckleFilter.enable = True
config.postProcessing.speckleFilter.speckleRange = 60
config.postProcessing.temporalFilter.enable = True
config.postProcessing.spatialFilter.holeFillingRadius = 2
config.postProcessing.spatialFilter.numIterations = 1
config.postProcessing.thresholdFilter.minRange = 700 # mm
config.postProcessing.thresholdFilter.maxRange = 7000 # mm
config.censusTransform.enableMeanMode = True
config.costMatching.linearEquationParameters.alpha = 0
config.costMatching.linearEquationParameters.beta = 2
stereo.initialConfig.set(config)
stereo.setLeftRightCheck(lrcheck)
stereo.setExtendedDisparity(extended)
stereo.setSubpixel(subpixel)
stereo.setDepthAlign(dai.CameraBoardSocket.RGB)
stereo.setRectifyEdgeFillColor(0) # Black, to better see the cutout
# Depth -> Depth Diff
nn = p.createNeuralNetwork()
nn.setBlobPath("diff_images_simplified_openvino_2021.4_4shave.blob")
stereo.disparity.link(nn.inputs["input1"])
depthDiffOut = p.createXLinkOut()
depthDiffOut.setStreamName("depth_diff")
nn.out.link(depthDiffOut.input)
with dai.Device(p) as device:
qDepthDiff = device.getOutputQueue(name="depth_diff", maxSize=4, blocking=False)
while True:
depthDiff = qDepthDiff.get()
# Shape it here
floatVector = depthDiff.getFirstLayerFp16()
diff = np.array(floatVector).reshape(resolution[0], resolution[1])
colorize = cv2.normalize(diff, None, 255, 0, cv2.NORM_INF, cv2.CV_8UC1)
cv2.applyColorMap(colorize, cv2.COLORMAP_JET)
cv2.imshow("Diff", colorize)
if cv2.waitKey(1) == ord('q'):
break
This is the pytorch code. I no longer subtract from dummy frame and just pass through depth:
#! /usr/bin/env python3
from pathlib import Path
import torch
from torch import nn
import blobconverter
import onnx
from onnxsim import simplify
import sys
# Define the model
class DiffImgs(nn.Module):
def forward(self, depth):
# We will be inputting UINT16 but interprets as UINT8
# So we need to adjust to account of the 8 bit shift
depthFP16 = 256.0 * depth[:,:,:,1::2] + depth[:,:,:,::2]
return depthFP16
# depthFP16 = depthFP16.view(1, -1)
# depthFP16_shape = depthFP16.shape
# Create a dummy frame 0 frame to test if depth can be recaptured on host
# dumy_frame = torch.zeros(depthFP16_shape, dtype=torch.float16)
# return torch.sub(depthFP16, dumy_frame)
# Instantiate the model
model = DiffImgs()
# Create dummy input for the ONNX export
input1 = torch.randn(1, 1, 960, 1632 * 2, dtype=torch.float16)
input2 = torch.randn(1, 1, 960, 1632 * 2, dtype=torch.float16)
onnx_file = "diff_images.onnx"
# Export the model
torch.onnx.export(model, # model being run
(input1), # model input (or a tuple for multiple inputs)
onnx_file, # where to save the model (can be a file or file-like object)
opset_version=12, # the ONNX version to export the model to
do_constant_folding=True, # whether to execute constant folding for optimization
input_names = ['input1'], # the model's input names
output_names = ['output'])
# Simplify the model
onnx_model = onnx.load(onnx_file)
onnx_simplified, check = simplify(onnx_file)
onnx.save(onnx_simplified, "diff_images_simplified.onnx")
# Use blobconverter to convert onnx->IR->blob
blobconverter.from_onnx(
model="diff_images_simplified.onnx",
data_type="FP16",
shaves=4,
use_cache=False,
output_dir="../",
optimizer_params=[],
compile_params=['-ip U8'],
)
