erik No need for apologies. I see these forums and your collective expertise is in high demand.
So both of these screenshots are from the OAK-D-Lite. The 'flat tracker' image uses a modified 'object_detection.py' script while 'spatial tracker' uses a modified 'spatial_object_detection.py' script.
They use similar models too. The main difference is that I ran the openvino conversion script twice - once for 6 shaves, used with 'o_d.py' and then I converted for 5 shaves, used with 's_o_d.py'. Changing the model almost doubled my framerate from 6 to 12 but the OAK-D-Lite's running 'o_d.py' still acheives 30 fps.
from pathlib import Path
import cv2
import depthai as dai
import numpy as np
import time
import argparse
import os
fullFrameTracking = True
CWD_PATH = os.getcwd()
MODEL_NAME = "bottles_tflite0"
GRAPH_NAME = "prodlibv007_6shave.blob"
nnPath = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
labelMap = ["background", "dbaseblu0dwn", "dbaseblu0top", "petalgrn0dwn", "petalgrn0top", "petalred0dwn", "petalred0top"]
pipeline = dai.Pipeline()
camRgb = pipeline.create(dai.node.ColorCamera)
detectionNetwork = pipeline.create(dai.node.MobileNetDetectionNetwork)
objectTracker = pipeline.create(dai.node.ObjectTracker)
xlinkOut = pipeline.create(dai.node.XLinkOut)
trackerOut = pipeline.create(dai.node.XLinkOut)
xlinkOut.setStreamName("preview")
trackerOut.setStreamName("tracklets")
camRgb.setPreviewSize(300, 300)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
camRgb.setIspScale(1,3)
camRgb.setInterleaved(False)
camRgb.setPreviewKeepAspectRatio(False)
detectionNetwork.setBlobPath(nnPath)
detectionNetwork.setConfidenceThreshold(0.1)
detectionNetwork.input.setBlocking(False)
objectTracker.setTrackerType(dai.TrackerType.ZERO_TERM_COLOR_HISTOGRAM)
objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.SMALLEST_ID)
camRgb.preview.link(detectionNetwork.input)
objectTracker.passthroughTrackerFrame.link(xlinkOut.input)
if fullFrameTracking:
camRgb.video.link(objectTracker.inputTrackerFrame)
else:
detectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
detectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
detectionNetwork.out.link(objectTracker.inputDetections)
objectTracker.out.link(trackerOut.input)
found, device_info = dai.Device.getDeviceByMxId("18443010C1E8C11200") #OAK-D-Lite @ Pi
with dai.Device(pipeline, device_info) as device:
preview = device.getOutputQueue("preview", 2, False)
tracklets = device.getOutputQueue("tracklets", 2, False)
startTime = time.monotonic()
counter = 0
fps = 0
totalspeed = 0
avragspeed = 0
speedtimer = time.perf_counter()
frame = None
prevlock = []
centers = []
for i in range(100):
prevlock.append([-1,-1,0])
while(True):
imgFrame = preview.get()
track = tracklets.get()
counter+=1
current_time = time.monotonic()
if (current_time - startTime) > 1 :
fps = counter / (current_time - startTime)
counter = 0
startTime = current_time
color = (255, 0, 0)
frame = imgFrame.getCvFrame()
trackletsData = track.tracklets
tempoldvals = ""
for t in trackletsData:
roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
x1 = int(roi.topLeft().x)
y1 = int(roi.topLeft().y)
x2 = int(roi.bottomRight().x)
y2 = int(roi.bottomRight().y)
centerpoint = ((x2-x1)/2,(y2-y1)/2)
if prevlock[t.id][1] == -1:
prevlock[t.id][0] = y1
else:
prevlock[t.id][2] = prevlock[t.id][1]-y1
prevlock[t.id][0] = y1
try:
label = labelMap[t.label]
except:
label = t.label
totalspeed += prevlock[t.id][2]
cv2.putText(frame, f"{[t.id]}", (x1 + 10, y1 + 35), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (250,250,250))
cv2.rectangle(frame, (x1, y1), (x2, y2), color, cv2.FONT_HERSHEY_SIMPLEX)
longrun = len(trackletsData)
if longrun > 0:
avragspeed += totalspeed/longrun #total speed
temptimer = time.perf_counter() - speedtimer
if temptimer > 1:
if (avragspeed < 20):#2.5):
avragspeed = 0
totalspeed = 0
avragspeed = 0
speedtimer = time.perf_counter()
for i in range(100):
if prevlock[i][0] == -1:
prevlock[i][1] = -1
prevlock[i][2] = 0
else:
prevlock[i][1] = prevlock[i][0]
prevlock[i][0] = -1
cv2.putText(frame, "NN fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.6, color)
cv2.imshow("flat tracker", frame)
if cv2.waitKey(1) == ord('q'):
break
from pathlib import Path
import cv2
import depthai as dai
import numpy as np
import time
import argparse
import os
fullFrameTracking = True
CWD_PATH = os.getcwd()
MODEL_NAME = "bottles_tflite0"
GRAPH_NAME = "prodlibv007_5shave.blob"
nnPath = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
labelMap = ["background", "dbaseblu0dwn", "dbaseblu0top", "petalgrn0dwn", "petalgrn0top", "petalred0dwn", "petalred0top"]
pipeline = dai.Pipeline()
camRgb = pipeline.create(dai.node.ColorCamera)
spatialDetectionNetwork = pipeline.create(dai.node.MobileNetSpatialDetectionNetwork)
monoLeft = pipeline.create(dai.node.MonoCamera)
monoRight = pipeline.create(dai.node.MonoCamera)
stereo = pipeline.create(dai.node.StereoDepth)
objectTracker = pipeline.create(dai.node.ObjectTracker)
xoutRgb = pipeline.create(dai.node.XLinkOut)
trackerOut = pipeline.create(dai.node.XLinkOut)
xoutRgb.setStreamName("preview")
trackerOut.setStreamName("tracklets")
camRgb.setPreviewSize(300, 300)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
camRgb.setIspScale(1,3)
camRgb.setInterleaved(False)
camRgb.setPreviewKeepAspectRatio(False)
monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT)
monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT)
stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
stereo.setDepthAlign(dai.CameraBoardSocket.RGB)
spatialDetectionNetwork.setBlobPath(nnPath)
spatialDetectionNetwork.setConfidenceThreshold(0.1)
spatialDetectionNetwork.input.setBlocking(False)
spatialDetectionNetwork.setBoundingBoxScaleFactor(0.5)
spatialDetectionNetwork.setDepthLowerThreshold(100)
spatialDetectionNetwork.setDepthUpperThreshold(5000)
objectTracker.setTrackerType(dai.TrackerType.ZERO_TERM_COLOR_HISTOGRAM)
objectTracker.setTrackerIdAssignmentPolicy(dai.TrackerIdAssignmentPolicy.SMALLEST_ID)
monoLeft.out.link(stereo.left)
monoRight.out.link(stereo.right)
camRgb.preview.link(spatialDetectionNetwork.input)
objectTracker.passthroughTrackerFrame.link(xoutRgb.input)
objectTracker.out.link(trackerOut.input)
if fullFrameTracking:
camRgb.setPreviewKeepAspectRatio(False)
camRgb.video.link(objectTracker.inputTrackerFrame)
objectTracker.inputTrackerFrame.setBlocking(False)
objectTracker.inputTrackerFrame.setQueueSize(2)
else:
spatialDetectionNetwork.passthrough.link(objectTracker.inputTrackerFrame)
spatialDetectionNetwork.passthrough.link(objectTracker.inputDetectionFrame)
spatialDetectionNetwork.out.link(objectTracker.inputDetections)
stereo.depth.link(spatialDetectionNetwork.inputDepth)
found, device_info = dai.Device.getDeviceByMxId("18443010C1E8C11200") #OAK-D-Lite @ Pi
with dai.Device(pipeline, device_info) as device:
preview = device.getOutputQueue("preview", 2, False)
tracklets = device.getOutputQueue("tracklets", 2, False)
startTime = time.monotonic()
counter = 0
fps = 0
color = (255, 0, 0)
while(True):
imgFrame = preview.get()
track = tracklets.get()
counter+=1
current_time = time.monotonic()
if (current_time - startTime) > 1 :
fps = counter / (current_time - startTime)
counter = 0
startTime = current_time
frame = imgFrame.getCvFrame()
trackletsData = track.tracklets
for t in trackletsData:
roi = t.roi.denormalize(frame.shape[1], frame.shape[0])
x1 = int(roi.topLeft().x)
y1 = int(roi.topLeft().y)
x2 = int(roi.bottomRight().x)
y2 = int(roi.bottomRight().y)
centerpoint = ((x2-x1)/2,(y2-y1)/2)
try:
label = labelMap[t.label]
except:
label = t.label
cv2.putText(frame, f"ID: {[t.id]}", (x1 + 10, y1 + 35), cv2.FONT_HERSHEY_TRIPLEX, 0.5, (250,250,250))
cv2.rectangle(frame, (x1, y1), (x2, y2), color, cv2.FONT_HERSHEY_SIMPLEX)
cv2.putText(frame, "NN fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.6, color)
cv2.imshow("spatial tracker", frame)
if cv2.waitKey(1) == ord('q'):
break
`
--------------------------------------------------------------------------
--------------------------------------------------------------------------