Getting this issue "possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'nn' (X_LINK_ERROR)'" while running the camera module with two oakd devices. The cameras are connected to a laptop on usb 3 ports. Tried to save the crash dump as well after the crash but it always says no crash dump found i am using depthai version 2.24 Adding full MRE for proper analysis. ``` #include #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" using namespace std; static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; // function to get camera pipeline std::shared_ptr getPipeline(std::string nnPath) { auto pipeline = std::make_shared(); auto camRgb = pipeline->create(); camRgb->setPreviewSize(300, 300); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_4_K); camRgb->setInterleaved(false); camRgb->setIspScale(1, 3); camRgb->setFps(20); camRgb->setPreviewKeepAspectRatio(false); camRgb->initialControl.setManualFocus(100); auto xoutFrames = pipeline->create(); xoutFrames->setStreamName("frames"); camRgb->video.link(xoutFrames->input); auto spatialDetectionNetwork = pipeline->create(); spatialDetectionNetwork->setBlobPath(nnPath); spatialDetectionNetwork->setConfidenceThreshold(0.5f); spatialDetectionNetwork->input.setBlocking(false); spatialDetectionNetwork->setBoundingBoxScaleFactor(0.5); spatialDetectionNetwork->setDepthLowerThreshold(100); spatialDetectionNetwork->setDepthUpperThreshold(5000); camRgb->preview.link(spatialDetectionNetwork->input); auto monoLeft = pipeline->create(); auto monoRight = pipeline->create(); auto stereo = pipeline->create(); stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::HIGH_DENSITY); stereo->setDepthAlign(dai::CameraBoardSocket::RGB); stereo->setOutputSize(monoLeft->getResolutionWidth(), monoLeft->getResolutionHeight()); monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT); monoLeft->out.link(stereo->left); monoRight->out.link(stereo->right); auto xoutRgb = pipeline->create(); xoutRgb->setStreamName("rgb"); spatialDetectionNetwork->passthrough.link(xoutRgb->input); auto xoutNN = pipeline->create(); xoutNN->setStreamName("nn"); spatialDetectionNetwork->out.link(xoutNN->input); auto xoutDepth = pipeline->create(); xoutDepth->setStreamName("depth"); stereo->depth.link(spatialDetectionNetwork->inputDepth); spatialDetectionNetwork->passthroughDepth.link(xoutDepth->input); return pipeline; } std::vector getDevicesPipeline(string nnPath, int &Camera_Status_API_x, std::vector &devices) { auto device_infos = dai::Device::getAllAvailableDevices(); auto usb2_mode = dai::UsbSpeed::SUPER; auto openvino_version = dai::OpenVINO::Version::VERSION_2021_4; auto color = cv::Scalar(255, 255, 255); std::vector qRgbMap; // LOG camera status int cameraStatus = 0; if (device_infos.size() == 0) { cameraStatus = 0; } if (device_infos.size() == 1) { cameraStatus = 1; auto device = std::make_shared(openvino_version, device_infos[0], usb2_mode); } if (device_infos.size() == 2) { cameraStatus = 2; } Camera_Status_API_x = 2 - cameraStatus; if (Camera_Status_API_x == 0) { // Getting pipleline connected for multiple cameras for (auto &device_info : device_infos) { auto device = std::make_shared(openvino_version, device_info, usb2_mode); devices.push_back(device); auto mxId = device->getMxId(); auto usbSpeed = device->getUsbSpeed(); auto cameraControl = std::make_shared(); cameraControl->setAutoFocusMode(dai::RawCameraControl::AutoFocusMode::OFF); auto pipeline = getPipeline(nnPath); device->startPipeline(*pipeline); auto qrgb = device->getOutputQueue("rgb", 4, false); auto qnn = device->getOutputQueue("nn", 4, false); auto qframes = device->getOutputQueue("frames", 4, false); auto qdepth = device->getOutputQueue("depth", 4, false); std::string streamName = mxId; auto qcmap = std::make_pair(qnn, qframes); auto qsdmap = std::make_pair(qdepth, qcmap); auto last = std::make_pair(streamName, qsdmap); qRgbMap.push_back(last); } if(qRgbMap.size()==2) { std::vector camIDs; for(auto t : qRgbMap) camIDs.push_back(t.first); sort(camIDs.begin(),camIDs.end()); if(qRgbMap[0].first!=camIDs[0]) reverse(qRgbMap.begin(),qRgbMap.end()); } } return qRgbMap; } // function to get the frames from the pipeline void getCameraFrames(string nnPath) { int Camera_Status_API_x; std::vector devices; std::vector qRgbMap = getDevicesPipeline(nnPath, Camera_Status_API_x, devices); auto color = cv::Scalar(255, 255, 255); while (true) { try { for (int i = 0; i tryGet(); auto inDet = inDets->get(); auto inDepth = qDepth->tryGet(); cv::Mat zeroframe = cv::Mat::zeros(cv::Size(1280,720),CV_8UC1); if (inRgb != nullptr && inDet != nullptr && inDepth != nullptr) { cv::Mat oframe = inRgb->getCvFrame(); cv::Mat depthFrame = inDepth->getFrame(); auto detections = inDet->detections; for (const auto &detection : detections) { uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if (labelIndex < labelMap.size()) labelStr = labelMap[labelIndex]; int x1 = std::max((int)(detection.xmin * oframe.cols), 0); int y1 = std::max((int)(detection.ymin * oframe.rows), 0); int x2 = std::min((int)(detection.xmax * oframe.cols), oframe.cols); int y2 = std::min((int)(detection.ymax * oframe.rows), oframe.rows); cv::putText(oframe, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); std::stringstream confStr; confStr

Hi @"abhay"#2254 Does this only happen when multiple devices are connected? Can you make a MRE so we can send it to dev team? It looks to me like some queue gets saturated and blocks the pipeline from processing more frames, which in turn crashes the device. Thanks, Jaka

Hi @"jakaskerl"#1112 Its completely random, when we are testing the devices internally we are seeing the error very rarely but when we are deploying them to a location their we are experiencing this issue very frequently. I have already added the complete MRE to run the complete pipeline in the question itself adding it here as well for visibility. ``` #include #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" using namespace std; static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; // function to get camera pipeline std::shared_ptr getPipeline(std::string nnPath) { auto pipeline = std::make_shared(); auto camRgb = pipeline->create(); camRgb->setPreviewSize(300, 300); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_4_K); camRgb->setInterleaved(false); camRgb->setIspScale(1, 3); camRgb->setFps(20); camRgb->setPreviewKeepAspectRatio(false); camRgb->initialControl.setManualFocus(100); auto xoutFrames = pipeline->create(); xoutFrames->setStreamName("frames"); camRgb->video.link(xoutFrames->input); auto spatialDetectionNetwork = pipeline->create(); spatialDetectionNetwork->setBlobPath(nnPath); spatialDetectionNetwork->setConfidenceThreshold(0.5f); spatialDetectionNetwork->input.setBlocking(false); spatialDetectionNetwork->setBoundingBoxScaleFactor(0.5); spatialDetectionNetwork->setDepthLowerThreshold(100); spatialDetectionNetwork->setDepthUpperThreshold(5000); camRgb->preview.link(spatialDetectionNetwork->input); auto monoLeft = pipeline->create(); auto monoRight = pipeline->create(); auto stereo = pipeline->create(); stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::HIGH_DENSITY); stereo->setDepthAlign(dai::CameraBoardSocket::RGB); stereo->setOutputSize(monoLeft->getResolutionWidth(), monoLeft->getResolutionHeight()); monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT); monoLeft->out.link(stereo->left); monoRight->out.link(stereo->right); auto xoutRgb = pipeline->create(); xoutRgb->setStreamName("rgb"); spatialDetectionNetwork->passthrough.link(xoutRgb->input); auto xoutNN = pipeline->create(); xoutNN->setStreamName("nn"); spatialDetectionNetwork->out.link(xoutNN->input); auto xoutDepth = pipeline->create(); xoutDepth->setStreamName("depth"); stereo->depth.link(spatialDetectionNetwork->inputDepth); spatialDetectionNetwork->passthroughDepth.link(xoutDepth->input); return pipeline; } std::vector getDevicesPipeline(string nnPath, int &Camera_Status_API_x, std::vector &devices) { auto device_infos = dai::Device::getAllAvailableDevices(); auto usb2_mode = dai::UsbSpeed::SUPER; auto openvino_version = dai::OpenVINO::Version::VERSION_2021_4; auto color = cv::Scalar(255, 255, 255); std::vector qRgbMap; // LOG camera status int cameraStatus = 0; if (device_infos.size() == 0) { cameraStatus = 0; } if (device_infos.size() == 1) { cameraStatus = 1; auto device = std::make_shared(openvino_version, device_infos[0], usb2_mode); } if (device_infos.size() == 2) { cameraStatus = 2; } Camera_Status_API_x = 2 - cameraStatus; if (Camera_Status_API_x == 0) { // Getting pipleline connected for multiple cameras for (auto &device_info : device_infos) { auto device = std::make_shared(openvino_version, device_info, usb2_mode); devices.push_back(device); auto mxId = device->getMxId(); auto usbSpeed = device->getUsbSpeed(); auto cameraControl = std::make_shared(); cameraControl->setAutoFocusMode(dai::RawCameraControl::AutoFocusMode::OFF); auto pipeline = getPipeline(nnPath); device->startPipeline(*pipeline); auto qrgb = device->getOutputQueue("rgb", 4, false); auto qnn = device->getOutputQueue("nn", 4, false); auto qframes = device->getOutputQueue("frames", 4, false); auto qdepth = device->getOutputQueue("depth", 4, false); std::string streamName = mxId; auto qcmap = std::make_pair(qnn, qframes); auto qsdmap = std::make_pair(qdepth, qcmap); auto last = std::make_pair(streamName, qsdmap); qRgbMap.push_back(last); } if(qRgbMap.size()==2) { std::vector camIDs; for(auto t : qRgbMap) camIDs.push_back(t.first); sort(camIDs.begin(),camIDs.end()); if(qRgbMap[0].first!=camIDs[0]) reverse(qRgbMap.begin(),qRgbMap.end()); } } return qRgbMap; } // function to get the frames from the pipeline void getCameraFrames(string nnPath) { int Camera_Status_API_x; std::vector devices; std::vector qRgbMap = getDevicesPipeline(nnPath, Camera_Status_API_x, devices); auto color = cv::Scalar(255, 255, 255); while (true) { try { for (int i = 0; i tryGet(); auto inDet = inDets->get(); auto inDepth = qDepth->tryGet(); cv::Mat zeroframe = cv::Mat::zeros(cv::Size(1280,720),CV_8UC1); if (inRgb != nullptr && inDet != nullptr && inDepth != nullptr) { cv::Mat oframe = inRgb->getCvFrame(); cv::Mat depthFrame = inDepth->getFrame(); auto detections = inDet->detections; for (const auto &detection : detections) { uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if (labelIndex < labelMap.size()) labelStr = labelMap[labelIndex]; int x1 = std::max((int)(detection.xmin * oframe.cols), 0); int y1 = std::max((int)(detection.ymin * oframe.rows), 0); int x2 = std::min((int)(detection.xmax * oframe.cols), oframe.cols); int y2 = std::min((int)(detection.ymax * oframe.rows), oframe.rows); cv::putText(oframe, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); std::stringstream confStr; confStr

Hi @"abhay"#2254 What are the differences between the environment "internally" vs the "deployed"? Temperature variations, longer USB cables, different host OS...? Thanks, Jaka

Hi @"jakaskerl"#1112 , Environment are the same as we are using docker environment for software deployment, the average temperature coming up internally is 60 degree and when deploying its coming up 64 degree inside of shadow box with good ventilation, cables we are using comes with the oak d devices itself. OS is same but i think that should not make much difference as we are deploying via docker. Hope it answered your questions Thanks, Abhay

Hi @"abhay"#2254 Can you run the device with `DEPTHAI_DEBUG=1` environment variable and pipe the contents out to a log file? Then paste the log file here when the crash occurs. It should give us some more insight on the issue. Thanks, Jaka

Hi @"jakaskerl"#1112 We have generated the logs as mentioned by you i am not able to attach the file here directly so adding the link you can see in the logs the camera crashed randomly with error **_X_LINK_ERROR_** on lines 41928 and 50616 [Log_File](https://kritikalvision365-my.sharepoint.com/:t:/g/personal/rajaram_tripathy_kritikalvision_ai/EdFFXelre4FGpFuLFFOCJuUBIKLKbeLeQwBbBmRhjFYepg?e=u9RR4e) Thanks, Abhay

Hi @"abhay"#2254 Try wrapping your code in try..catch block to force depthai to create a crash dump in case of a crash. Current logs give me no additional info on why the device crashes. Thanks, Jaka

Hi @"jakaskerl"#1112 , The code is already running inside the try catch itself that why we were able to capture the error "possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'nn' (X_LINK_ERROR)" and restarted the camera. Let me know if their is anything else we can do. Thanks, Abhay

Hi @"abhay"#2254 Then I think you should have received a crash dump. Is there any indication of that? https://docs.luxonis.com/projects/api/en/latest/samples/crash_report/crash_report/ Thanks, Jaka

Hi @"jakaskerl"#1112 We have checked it already it says no crash dump found we tired multiple times. Thanks, Abhay [upl-image-preview url=https://discuss.luxonis.com/assets/files/2024-03-04/1709529905-224338-screenshot-from-2023-12-09-11-30-17.png]

OAKD Camera crashing randomly

erik

@CrisLuce @CliffAtSmartSort we'd love to help you guys and resolve this issue, but the link to the computer unfortunately doesn't help us much.
Ideally, the whole setup would be shipped to our local office (in EU) so we can repro, or you could first measure the voltage with oscilloscope and measure if there was any voltage dip when the crash happens.
Thanks, Erik

CrisLuce

Hi erik Lets focus on getting the crash dump data (all data) so we can further inspect and go from there.

erik

Hi @CrisLuce , If you already tried getting the crash dump (and there was none found), then it (very likely) wasn't FW issue, and we can't do much about it. So next option would be to look into power/connectivity.

CrisLuce

erik Even on the systems that were double counting/crashing/causing us a problem, there was no crash dump data, so, outside of power / connectivity, what could it be?

It sounds like FW crash is out of the question, we know the cables are secure, we know the power is good, but we'll continue to inspect the power side.

erik

@CrisLuce let's first check power side if that is stable. If it is, then it would be best to send the whole system over, so we can debug locally with a JTAG debugger, and actually see what is going on.

CrisLuce

Hi erik That is a very expensive and lengthy situation. If all units experienced the same issue I'd be much more likely to jump to that. We have proprietary code and components, this would require a several legal agreements due to multi-national patents, including EU. I'm assuming there is no way to perform a remote JTAG debugger?

Are you unable to suggest other areas of potential issue. For example, we have a custom docker, any potential conversion issues, ques getting jammed up with sensor data from our vision app.

I appreciate the attempt to support within this forum, I think a quick call might be best. I'll connect with Jim. Please let me know if you have any other thoughts - many thanks!

erik

Hi @CrisLuce ,

JTAG: I don't think so, as we need toolchains that are under NDA to debug that. So same issue as you have on your side.
Docker: we haven't heard about that, but yes, I'd try without docker container as well.
Call: sure

« Previous Page