HI
I am trying to get the gaze-estimation aplication to run standalone on my OAK-D POE.
Sometimes the code runs fine, other times It stops after +- 15 secs and sometimes it crashes in the beginning.
The errors i get are
[1944301001F1201300] [169.254.1.222] [9.478] [ImageManip(11)] [error] Not possible to create warp params. Error: WARP_SWCH_ERR_CACHE_TO_SMALL
[1944301001F1201300] [169.254.1.222] [9.478] [ImageManip(11)] [error] Invalid configuration or input image - skipping frame
after that i get the error
[1944301001F1201300] [169.254.1.222] [13.014] [system] [critical] Fatal error. Please report to developers. Log: 'Fatal error on MSS CPU: trap: 2A, address: 800A5ADC' '0'
I have tried to look for a solution in both the docs and through the git issues pages but without succes.
The weird thing is that some times it works without problem and other times it does not work at all.
Kind regards
Jonas
code:
import time
import depthai as dai
import depthai.node
import blobconverter
#==========[PIPELINE INIT]==========#
pipeline = dai.Pipeline()
pipeline.setOpenVINOVersion(version=dai.OpenVINO.Version.VERSION_2021_4)
openvino_version = '2021.4'
face_nn = pipeline.create(depthai.node.MobileNetDetectionNetwork)
face_nn.setBlobPath(blobconverter.from_zoo(name="face-detection-retail-0004", shaves=6, version=openvino_version))
face_nn.setConfidenceThreshold(0.7)
landmark_manip = pipeline.create(dai.node.ImageManip)
landmark_manip.initialConfig.setResize(60, 60)
landmark_manip.inputConfig.setWaitForMessage(True)
land_nn = pipeline.create(depthai.node.NeuralNetwork)
land_nn.setBlobPath(blobconverter.from_zoo(name="landmarks-regression-retail-0009", shaves=6, version=openvino_version))
headpose_manip = pipeline.create(dai.node.ImageManip)
headpose_manip.initialConfig.setResize(60, 60)
headpose_manip.inputConfig.setWaitForMessage(True)
pose_nn = pipeline.create(depthai.node.NeuralNetwork)
pose_nn.setBlobPath(blobconverter.from_zoo(name="head-pose-estimation-adas-0001", shaves=6, version=openvino_version))
#=========[CAM SETUP]===========#
VIDEO_SIZE = (1072, 1072)
camRgb = pipeline.create(dai.node.ColorCamera)
camRgb.setPreviewSize(1072, 1072)
camRgb.setVideoSize(VIDEO_SIZE)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
camRgb.setInterleaved(False)
camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
copy_manip = pipeline.create(dai.node.ImageManip)
camRgb.preview.link(copy_manip.inputImage)
copy_manip.setNumFramesPool(20)
copy_manip.setMaxOutputFrameSize(1072*1072*3)
# ImageManip that will crop the frame before sending it to the Face detection NN node
face_det_manip = pipeline.create(dai.node.ImageManip)
face_det_manip.initialConfig.setResize(300, 300)
face_det_manip.setMaxOutputFrameSize(300*300*3)
copy_manip.out.link(face_det_manip.inputImage)
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)
script = pipeline.create(dai.node.Script)
script.setProcessor(dai.ProcessorType.LEON_CSS)
#===========[LINKING]===========#
face_det_manip.out.link(face_nn.input)
copy_manip.out.link(script.inputs['preview'])
# OUTPUT FACE_NN TO SCRIPT
face_nn.out.link(script.inputs["face_nn_output"])
script.inputs["face_nn_output"].setBlocking(False)
face_nn.passthrough.link(script.inputs["face_nn_passthrough"])
script.inputs["face_nn_passthrough"].setBlocking(False)
# OUTPUT SCRIPT to INPUT HEADPOSE_MANIP
script.outputs['headpose_cfg'].link(headpose_manip.inputConfig)
script.outputs['headpose_img'].link(headpose_manip.inputImage)
# OUTPUT SRIPT TO INPUT LANDMARK_MANIP
script.outputs['landmark_cfg'].link(landmark_manip.inputConfig)
script.outputs['landmark_img'].link(landmark_manip.inputImage)
#output landmark to script
landmark_manip.out.link(land_nn.input)
land_nn.out.link(script.inputs['landmark_in'])
land_nn.passthrough.link(script.inputs['landmark_pass'])
#OUTPUT HEADPOSE TO SCRIPT
headpose_manip.out.link(pose_nn.input)
pose_nn.out.link(script.inputs['headpose_in'])
pose_nn.passthrough.link(script.inputs['headpose_pass'])
#=================[ LEFT EYE CROP ]=================
left_manip = pipeline.create(dai.node.ImageManip)
left_manip.initialConfig.setResize(128, 128)
left_manip.inputConfig.setWaitForMessage(True)
script.outputs['left_manip_img'].link(left_manip.inputImage)
script.outputs['left_manip_cfg'].link(left_manip.inputConfig)
left_manip.out.link(script.inputs['left_eye_in'])
#=================[ Right EYE CROP ]=================
right_manip = pipeline.create(dai.node.ImageManip)
right_manip.initialConfig.setResize(128, 128)
right_manip.inputConfig.setWaitForMessage(True)
script.outputs['right_manip_img'].link(right_manip.inputImage)
script.outputs['right_manip_cfg'].link(right_manip.inputConfig)
right_manip.out.link(script.inputs['right_eye_in'])
gaze_nn = pipeline.create(dai.node.NeuralNetwork)
gaze_nn.setBlobPath(blobconverter.from_zoo(
name="gaze-estimation-adas-0002",
shaves=6,
version=openvino_version
))
script.outputs['to_gaze'].link(gaze_nn.input)
gaze_nn.out.link(script.inputs['gaze_nn_out'])
script.setScript("""
import traceback
import time
sync = {} #dict of messages
def find_in_dict(target_seq, name):
# node.warn("find in dict")
if str(target_seq) in sync:
return sync[str(target_seq)][name]
def add_to_dict(det, seq, name):
# node.warn("add to dict")
sync[str(seq)][name] = det
def correct_bb(bb):
if bb.xmin < 0: bb.xmin = 0.001
if bb.ymin < 0: bb.ymin = 0.001
if bb.xmax > 1: bb.xmax = 0.999
if bb.ymax > 1: bb.ymax = 0.999
def check_gaze_est(seq):
dict = sync[str(seq)]
if "left" in dict and "right" in dict and "angles" in dict:
#node.warn("GOT ALL 3")
# 60x60x3 (x2 frames) + 6 (3 floats)
gaze_data = NNData(21606)
gaze_data.setSequenceNum(dict['left'].getSequenceNum())
gaze_data.setLayer("left_eye_image", dict['left'].getData())
gaze_data.setLayer("right_eye_image", dict['right'].getData())
gaze_data.setLayer("head_pose_angles", dict['angles'])
node.io['to_gaze'].send(gaze_data)
# Clear previous results
for i, sq in enumerate(sync):
# node.warn(f"removing seq num {i} (seq {seq})")
del sync[str(seq)]
if str(seq) == str(sq):
return
PAD = 0.05
PAD2x = PAD * 2
def get_eye_coords(x, y, det):
xdelta = (det.xmax + det.xmin) / 2
ydelta = (det.ymax + det.ymin) / 2
xmin = x - PAD
xmax = xmin + PAD2x
ymin = y - PAD
ymax = ymin + PAD2x
xmin2 = det.xmin + xdelta * xmin
xmax2 = det.xmin + xdelta * xmax
ymin2 = det.ymin + ydelta * ymin
ymax2 = det.ymin + ydelta * ymax
ret = (xmin2, ymin2, xmax2, ymax2)
# node.warn(f"Eye: {x}/{y}, Crop eyes: {ret}, det {det.xmin}, {det.ymin}, {det.xmax}, {det.ymax}")
return ret
while True:
time.sleep(0.001)
try:
preview = node.io['preview'].tryGet()
if preview is not None:
sync[str(preview.getSequenceNum())] = {
"frame": preview
}
face_dets = node.io['face_nn_output'].tryGet()
if face_dets is not None:
# node.warn("face detections not none")
passthrough = node.io['face_nn_passthrough'].tryGet()
seq = passthrough.getSequenceNum()
# No detections, carry on
if len(face_dets.detections) == 0:
del sync[str(seq)]
continue
# node.warn(f"New detection {seq}")
if len(sync) == 0:
# node.warn("nothing in sync")
continue
img = find_in_dict(seq, "frame")
if img is None:
# node.warn("img none")
continue
add_to_dict(face_dets.detections[0], seq, "detections")
for det in face_dets.detections:
# node.warn("det")
correct_bb(det)
# node.warn(str(det))
# To head post estimation model
cfg1 = ImageManipConfig()
cfg1.setCropRect(det.xmin, det.ymin, det.xmax, det.ymax)
cfg1.setResize(60, 60)
cfg1.setKeepAspectRatio(False)
# node.warn("sending headpose data")
node.io['headpose_cfg'].send(cfg1)
node.io['headpose_img'].send(img)
# To face landmark detection model
cfg2 = ImageManipConfig()
cfg2.setCropRect(det.xmin, det.ymin, det.xmax, det.ymax)
cfg2.setResize(48, 48)
cfg2.setKeepAspectRatio(False)
# node.warn("sending landmark data")
node.io['landmark_cfg'].send(cfg2)
node.io['landmark_img'].send(img)
break # Only 1 face at the time currently supported
# headpose processing
headpose = node.io['headpose_in'].tryGet()
if headpose is not None:
# node.warn("headpose")
passthrough = node.io['headpose_pass'].tryGet()
seq = passthrough.getSequenceNum()
# Face rotation in degrees
y = headpose.getLayerFp16('angle_y_fc')[0]
p = headpose.getLayerFp16('angle_p_fc')[0]
r = headpose.getLayerFp16('angle_r_fc')[0]
angles = [y,p,r]
#node.warn(f"angles {angles}")
#node.warn(str(y))
add_to_dict(angles, seq, "angles")
check_gaze_est(seq)
landmark_in = node.io['landmark_in'].tryGet()
if landmark_in is not None:
passthrough = node.io['landmark_pass'].tryGet()
seq = passthrough.getSequenceNum()
img = find_in_dict(seq, "frame")
det = find_in_dict(seq, "detections")
if img is None or det is None: continue
landmarks = landmark_in.getFirstLayerFp16()
# We need to crop left and right eye out of the face frame
left_cfg = ImageManipConfig()
left_cfg.setCropRect(*get_eye_coords(landmarks[0], landmarks[1], det))
left_cfg.setResize(60, 60)
left_cfg.setKeepAspectRatio(False)
node.io['left_manip_cfg'].send(left_cfg)
node.io['left_manip_img'].send(img)
right_cfg = ImageManipConfig()
right_cfg.setCropRect(*get_eye_coords(landmarks[2], landmarks[3], det))
right_cfg.setResize(60, 60)
right_cfg.setKeepAspectRatio(False)
node.io['right_manip_cfg'].send(right_cfg)
node.io['right_manip_img'].send(img)
left_eye = node.io['left_eye_in'].tryGet()
if left_eye is not None:
# node.warn("LEFT EYE GOT")
seq = left_eye.getSequenceNum()
add_to_dict(left_eye, seq, "left")
check_gaze_est(seq)
right_eye = node.io['right_eye_in'].tryGet()
if right_eye is not None:
# node.warn("RIGHT EYE GOT")
seq = right_eye.getSequenceNum()
add_to_dict(right_eye, seq, "right")
check_gaze_est(seq)
gaze_output = node.io['gaze_nn_out'].tryGet()
if gaze_output is not None:
node.warn("Gaze output acquired")
continue
except:
continue
""")
# bootloader = dai.DeviceBootloader("169.254.1.222", True)
# print("Connected")
# progress = lambda p: print( f'Flashing progress: { p*100 :.1f}%')
# bootloader.flash(progress, pipeline)
with dai.Device(pipeline, dai.DeviceInfo("169.254.1.222")) as device:
print("Connected")
while True:
time.sleep(0.01)`