Neural Network Node (multiple inputs to one node)

erik Thanks Erik. This is my first project with oak-d and depthai that is why i am asking a lot of basic questions. I pasted my code below can you suggest what is wrong in it. As my pipeline gets blocked or i do not know what happens. It keep running or it seems so because it keeps printing warnings that size of input and network does not match but it does not print the other lines that i am using for debugging like print statement in line 67 and 68. I think i gets stuck in script node second time it goes through it. Can you help me here? Last one is pic for what see on command line.

erik

import cv2
import depthai as dai
import numpy as np
time_bb_cord=[200,400,100,350]
local_bb_cord=[300,500,200,350]
visit_bb_cord=[300,500,200,350]
# Create pipeline
def Manip_Frame(pipeline,region_bb_cord):
	manip_time = pipeline.create(dai.node.ImageManip)
	rgbRr_time = dai.RotatedRect()
	rgbRr_time.center.x, rgbRr_time.center.y = (region_bb_cord[0]+region_bb_cord[1]) // 2, (region_bb_cord[2]+region_bb_cord[3]) // 2
	rgbRr_time.size.width, rgbRr_time.size.height = region_bb_cord[1]-region_bb_cord[0], region_bb_cord[3]-region_bb_cord[2]
	rgbRr_time.angle = 0
	manip_time.initialConfig.setCropRotatedRect(rgbRr_time, False)
	manip_time.initialConfig.setResize(32,100)
	return manip_time
def NN_node(pipeline,path):
	nn = pipeline.create(dai.node.NeuralNetwork)
	nn.setBlobPath(path)
	return nn

pipeline = dai.Pipeline()
path_1='model/model_norm_gray.blob'
path_rec='crnn.blob'

cam = pipeline.create(dai.node.ColorCamera)
cam.setBoardSocket(dai.CameraBoardSocket.RGB)
cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
cam.setInterleaved(False)
cam.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)

manip_time=Manip_Frame(pipeline,time_bb_cord)
cam.preview.link(manip_time.inputImage)

detection = pipeline.createNeuralNetwork()
detection.setBlobPath(path_1)
manip_time.out.link(detection.input)

pass_out = pipeline.createXLinkOut()
pass_out.setStreamName('pass_through')
detection.passthrough.link(pass_out.input)

manip_script = pipeline.create(dai.node.Script)
manip_script.inputs['nn_in'].setBlocking(False)
manip_script.inputs['nn_in'].setQueueSize(5)
# manip_time.out.link(manip_script.inputs['nn_in'])
detection.out.link(manip_script.inputs['nn_in'])
manip_script.setScript(""" 
frame=ImgFrame(6400)
data=node.io['nn_in'].get()
node.warn(f"Type of data: {len(data.getData())}")
frame.setData(data.getData())
frame.setWidth(100)
frame.setHeight(32)
node.io['script_out'].send(frame)
""")

x_out = pipeline.createXLinkOut()
x_out.setStreamName('custom')
manip_script.outputs['script_out'].link(x_out.input)
with dai.Device(pipeline) as device:
	nn_queue = device.getOutputQueue(name="custom", maxSize=5, blocking=False)
	pass_queue = device.getOutputQueue(name="pass_through", maxSize=5, blocking=False)
	i=0
	while True:
		pass_frame=pass_queue.get()
		print('pass through ',pass_frame.getSequenceNum())
		print(i)
		frame=nn_queue.get()
		if frame is None:
			print('here')
		else:	
			print('size ',frame.getHeight(),frame.getWidth())
		# print(type(time_frame))
		i=i+1
		print(i)
		if i>=500:
			break`

erik Hi Erik, this is code that i am using for the model. It converts image to grayscale and normalizes it

from typing import Tuple

import torch
import torch.nn as nn

class Grayscale(nn.Module):
    def __init__(self, shape: Tuple[int, int, int, int], dtype=torch.float):
        super(Grayscale, self).__init__()
        self.shape = shape
        self.dtype = dtype

    def forward(self, x):
        y_r = x[ :, :, 0]
        y_g = x[ :, :, 1]
        y_b = x[ :, :, 2]
        g_ = 0.3 * y_r + 0.59 * y_g + 0.11 * y_b
        g_=torch.sub(torch.div(g_,127.5),1)
        return g_
def export_onnx():

    shape = (32, 100, 3)
    model = Grayscale(shape=shape, dtype=torch.float)
    X = torch.ones(shape, dtype=torch.float)
    torch.onnx.export(
        model,
        X,
        'model2.onnx',
        opset_version=12,
        do_constant_folding=True
    )

export_onnx()
print('done')`

erik Thanks erik for your patience and guidance. I changed the size of input to [3,100,32] as you suggested now the output looks as in first pic below. It did not solve the warning but warning just changes to [warning] Input image (100x32) does not match NN (32x100). In this case the program at least keeps running but it just keeps printing the warning it does not show any thing else that i am printing to console.

Then i changed the input shape to [3,32,100] and the warning resolved but pipeline gets stuck after second passthrough as you can see in second pic. ImageManip node is same for both cases

erik one more thing I need a high quality gray scale image. How can I convert ColorCamera Image to gray scale? without using any neural network. Are there any functions inside depthai API that can do this?