Getting this issue "possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'nn' (X_LINK_ERROR)'" while running the camera module with two oakd devices. The cameras are connected to a laptop on usb 3 ports. Tried to save the crash dump as well after the crash but it always says no crash dump found i am using depthai version 2.24 Adding full MRE for proper analysis. ``` #include #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" using namespace std; static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; // function to get camera pipeline std::shared_ptr getPipeline(std::string nnPath) { auto pipeline = std::make_shared(); auto camRgb = pipeline->create(); camRgb->setPreviewSize(300, 300); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_4_K); camRgb->setInterleaved(false); camRgb->setIspScale(1, 3); camRgb->setFps(20); camRgb->setPreviewKeepAspectRatio(false); camRgb->initialControl.setManualFocus(100); auto xoutFrames = pipeline->create(); xoutFrames->setStreamName("frames"); camRgb->video.link(xoutFrames->input); auto spatialDetectionNetwork = pipeline->create(); spatialDetectionNetwork->setBlobPath(nnPath); spatialDetectionNetwork->setConfidenceThreshold(0.5f); spatialDetectionNetwork->input.setBlocking(false); spatialDetectionNetwork->setBoundingBoxScaleFactor(0.5); spatialDetectionNetwork->setDepthLowerThreshold(100); spatialDetectionNetwork->setDepthUpperThreshold(5000); camRgb->preview.link(spatialDetectionNetwork->input); auto monoLeft = pipeline->create(); auto monoRight = pipeline->create(); auto stereo = pipeline->create(); stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::HIGH_DENSITY); stereo->setDepthAlign(dai::CameraBoardSocket::RGB); stereo->setOutputSize(monoLeft->getResolutionWidth(), monoLeft->getResolutionHeight()); monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT); monoLeft->out.link(stereo->left); monoRight->out.link(stereo->right); auto xoutRgb = pipeline->create(); xoutRgb->setStreamName("rgb"); spatialDetectionNetwork->passthrough.link(xoutRgb->input); auto xoutNN = pipeline->create(); xoutNN->setStreamName("nn"); spatialDetectionNetwork->out.link(xoutNN->input); auto xoutDepth = pipeline->create(); xoutDepth->setStreamName("depth"); stereo->depth.link(spatialDetectionNetwork->inputDepth); spatialDetectionNetwork->passthroughDepth.link(xoutDepth->input); return pipeline; } std::vector getDevicesPipeline(string nnPath, int &Camera_Status_API_x, std::vector &devices) { auto device_infos = dai::Device::getAllAvailableDevices(); auto usb2_mode = dai::UsbSpeed::SUPER; auto openvino_version = dai::OpenVINO::Version::VERSION_2021_4; auto color = cv::Scalar(255, 255, 255); std::vector qRgbMap; // LOG camera status int cameraStatus = 0; if (device_infos.size() == 0) { cameraStatus = 0; } if (device_infos.size() == 1) { cameraStatus = 1; auto device = std::make_shared(openvino_version, device_infos[0], usb2_mode); } if (device_infos.size() == 2) { cameraStatus = 2; } Camera_Status_API_x = 2 - cameraStatus; if (Camera_Status_API_x == 0) { // Getting pipleline connected for multiple cameras for (auto &device_info : device_infos) { auto device = std::make_shared(openvino_version, device_info, usb2_mode); devices.push_back(device); auto mxId = device->getMxId(); auto usbSpeed = device->getUsbSpeed(); auto cameraControl = std::make_shared(); cameraControl->setAutoFocusMode(dai::RawCameraControl::AutoFocusMode::OFF); auto pipeline = getPipeline(nnPath); device->startPipeline(*pipeline); auto qrgb = device->getOutputQueue("rgb", 4, false); auto qnn = device->getOutputQueue("nn", 4, false); auto qframes = device->getOutputQueue("frames", 4, false); auto qdepth = device->getOutputQueue("depth", 4, false); std::string streamName = mxId; auto qcmap = std::make_pair(qnn, qframes); auto qsdmap = std::make_pair(qdepth, qcmap); auto last = std::make_pair(streamName, qsdmap); qRgbMap.push_back(last); } if(qRgbMap.size()==2) { std::vector camIDs; for(auto t : qRgbMap) camIDs.push_back(t.first); sort(camIDs.begin(),camIDs.end()); if(qRgbMap[0].first!=camIDs[0]) reverse(qRgbMap.begin(),qRgbMap.end()); } } return qRgbMap; } // function to get the frames from the pipeline void getCameraFrames(string nnPath) { int Camera_Status_API_x; std::vector devices; std::vector qRgbMap = getDevicesPipeline(nnPath, Camera_Status_API_x, devices); auto color = cv::Scalar(255, 255, 255); while (true) { try { for (int i = 0; i tryGet(); auto inDet = inDets->get(); auto inDepth = qDepth->tryGet(); cv::Mat zeroframe = cv::Mat::zeros(cv::Size(1280,720),CV_8UC1); if (inRgb != nullptr && inDet != nullptr && inDepth != nullptr) { cv::Mat oframe = inRgb->getCvFrame(); cv::Mat depthFrame = inDepth->getFrame(); auto detections = inDet->detections; for (const auto &detection : detections) { uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if (labelIndex < labelMap.size()) labelStr = labelMap[labelIndex]; int x1 = std::max((int)(detection.xmin * oframe.cols), 0); int y1 = std::max((int)(detection.ymin * oframe.rows), 0); int x2 = std::min((int)(detection.xmax * oframe.cols), oframe.cols); int y2 = std::min((int)(detection.ymax * oframe.rows), oframe.rows); cv::putText(oframe, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); std::stringstream confStr; confStr

Hi @"abhay"#2254 Does this only happen when multiple devices are connected? Can you make a MRE so we can send it to dev team? It looks to me like some queue gets saturated and blocks the pipeline from processing more frames, which in turn crashes the device. Thanks, Jaka

Hi @"jakaskerl"#1112 Its completely random, when we are testing the devices internally we are seeing the error very rarely but when we are deploying them to a location their we are experiencing this issue very frequently. I have already added the complete MRE to run the complete pipeline in the question itself adding it here as well for visibility. ``` #include #include // Includes common necessary includes for development using depthai library #include "depthai/depthai.hpp" using namespace std; static const std::vector labelMap = {"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"}; // function to get camera pipeline std::shared_ptr getPipeline(std::string nnPath) { auto pipeline = std::make_shared(); auto camRgb = pipeline->create(); camRgb->setPreviewSize(300, 300); camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_4_K); camRgb->setInterleaved(false); camRgb->setIspScale(1, 3); camRgb->setFps(20); camRgb->setPreviewKeepAspectRatio(false); camRgb->initialControl.setManualFocus(100); auto xoutFrames = pipeline->create(); xoutFrames->setStreamName("frames"); camRgb->video.link(xoutFrames->input); auto spatialDetectionNetwork = pipeline->create(); spatialDetectionNetwork->setBlobPath(nnPath); spatialDetectionNetwork->setConfidenceThreshold(0.5f); spatialDetectionNetwork->input.setBlocking(false); spatialDetectionNetwork->setBoundingBoxScaleFactor(0.5); spatialDetectionNetwork->setDepthLowerThreshold(100); spatialDetectionNetwork->setDepthUpperThreshold(5000); camRgb->preview.link(spatialDetectionNetwork->input); auto monoLeft = pipeline->create(); auto monoRight = pipeline->create(); auto stereo = pipeline->create(); stereo->setDefaultProfilePreset(dai::node::StereoDepth::PresetMode::HIGH_DENSITY); stereo->setDepthAlign(dai::CameraBoardSocket::RGB); stereo->setOutputSize(monoLeft->getResolutionWidth(), monoLeft->getResolutionHeight()); monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoLeft->setBoardSocket(dai::CameraBoardSocket::LEFT); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT); monoLeft->out.link(stereo->left); monoRight->out.link(stereo->right); auto xoutRgb = pipeline->create(); xoutRgb->setStreamName("rgb"); spatialDetectionNetwork->passthrough.link(xoutRgb->input); auto xoutNN = pipeline->create(); xoutNN->setStreamName("nn"); spatialDetectionNetwork->out.link(xoutNN->input); auto xoutDepth = pipeline->create(); xoutDepth->setStreamName("depth"); stereo->depth.link(spatialDetectionNetwork->inputDepth); spatialDetectionNetwork->passthroughDepth.link(xoutDepth->input); return pipeline; } std::vector getDevicesPipeline(string nnPath, int &Camera_Status_API_x, std::vector &devices) { auto device_infos = dai::Device::getAllAvailableDevices(); auto usb2_mode = dai::UsbSpeed::SUPER; auto openvino_version = dai::OpenVINO::Version::VERSION_2021_4; auto color = cv::Scalar(255, 255, 255); std::vector qRgbMap; // LOG camera status int cameraStatus = 0; if (device_infos.size() == 0) { cameraStatus = 0; } if (device_infos.size() == 1) { cameraStatus = 1; auto device = std::make_shared(openvino_version, device_infos[0], usb2_mode); } if (device_infos.size() == 2) { cameraStatus = 2; } Camera_Status_API_x = 2 - cameraStatus; if (Camera_Status_API_x == 0) { // Getting pipleline connected for multiple cameras for (auto &device_info : device_infos) { auto device = std::make_shared(openvino_version, device_info, usb2_mode); devices.push_back(device); auto mxId = device->getMxId(); auto usbSpeed = device->getUsbSpeed(); auto cameraControl = std::make_shared(); cameraControl->setAutoFocusMode(dai::RawCameraControl::AutoFocusMode::OFF); auto pipeline = getPipeline(nnPath); device->startPipeline(*pipeline); auto qrgb = device->getOutputQueue("rgb", 4, false); auto qnn = device->getOutputQueue("nn", 4, false); auto qframes = device->getOutputQueue("frames", 4, false); auto qdepth = device->getOutputQueue("depth", 4, false); std::string streamName = mxId; auto qcmap = std::make_pair(qnn, qframes); auto qsdmap = std::make_pair(qdepth, qcmap); auto last = std::make_pair(streamName, qsdmap); qRgbMap.push_back(last); } if(qRgbMap.size()==2) { std::vector camIDs; for(auto t : qRgbMap) camIDs.push_back(t.first); sort(camIDs.begin(),camIDs.end()); if(qRgbMap[0].first!=camIDs[0]) reverse(qRgbMap.begin(),qRgbMap.end()); } } return qRgbMap; } // function to get the frames from the pipeline void getCameraFrames(string nnPath) { int Camera_Status_API_x; std::vector devices; std::vector qRgbMap = getDevicesPipeline(nnPath, Camera_Status_API_x, devices); auto color = cv::Scalar(255, 255, 255); while (true) { try { for (int i = 0; i tryGet(); auto inDet = inDets->get(); auto inDepth = qDepth->tryGet(); cv::Mat zeroframe = cv::Mat::zeros(cv::Size(1280,720),CV_8UC1); if (inRgb != nullptr && inDet != nullptr && inDepth != nullptr) { cv::Mat oframe = inRgb->getCvFrame(); cv::Mat depthFrame = inDepth->getFrame(); auto detections = inDet->detections; for (const auto &detection : detections) { uint32_t labelIndex = detection.label; std::string labelStr = to_string(labelIndex); if (labelIndex < labelMap.size()) labelStr = labelMap[labelIndex]; int x1 = std::max((int)(detection.xmin * oframe.cols), 0); int y1 = std::max((int)(detection.ymin * oframe.rows), 0); int x2 = std::min((int)(detection.xmax * oframe.cols), oframe.cols); int y2 = std::min((int)(detection.ymax * oframe.rows), oframe.rows); cv::putText(oframe, labelStr, cv::Point(x1 + 10, y1 + 20), cv::FONT_HERSHEY_TRIPLEX, 0.5, 255); std::stringstream confStr; confStr

Hi @"abhay"#2254 What are the differences between the environment "internally" vs the "deployed"? Temperature variations, longer USB cables, different host OS...? Thanks, Jaka

Hi @"jakaskerl"#1112 We have checked it already it says no crash dump found we tired multiple times. Thanks, Abhay [upl-image-preview url=https://discuss.luxonis.com/assets/files/2024-03-04/1709529905-224338-screenshot-from-2023-12-09-11-30-17.png]

OAKD Camera crashing randomly

abhay

Hi @jakaskerl ,

Environment are the same as we are using docker environment for software deployment,
the average temperature coming up internally is 60 degree and when deploying its coming up 64 degree inside of shadow box with good ventilation,
cables we are using comes with the oak d devices itself.
OS is same but i think that should not make much difference as we are deploying via docker.
Hope it answered your questions

Thanks,
Abhay

jakaskerl

Hi @abhay
Can you run the device with DEPTHAI_DEBUG=1 environment variable and pipe the contents out to a log file? Then paste the log file here when the crash occurs. It should give us some more insight on the issue.

Thanks,
Jaka

abhay

Hi @jakaskerl

We have generated the logs as mentioned by you i am not able to attach the file here directly so adding the link you can see in the logs the camera crashed randomly with error X_LINK_ERROR on lines 41928 and 50616
Log_File

Thanks,
Abhay

jakaskerl

Hi @abhay
Try wrapping your code in try..catch block to force depthai to create a crash dump in case of a crash. Current logs give me no additional info on why the device crashes.

Thanks,
Jaka

abhay

Hi @jakaskerl ,

The code is already running inside the try catch itself that why we were able to capture the error "possible device error/misconfiguration. Original message 'Couldn't read data from stream: 'nn' (X_LINK_ERROR)" and restarted the camera. Let me know if their is anything else we can do.

Thanks,
Abhay

jakaskerl

Hi @abhay
Then I think you should have received a crash dump. Is there any indication of that?
https://docs.luxonis.com/projects/api/en/latest/samples/crash_report/crash_report/

Thanks,
Jaka

abhay

Hi @jakaskerl

We have checked it already it says no crash dump found we tired multiple times.
Thanks,
Abhay

jakaskerl

Hi @abhay
I'll see if I can reproduce the issue you are having. Could you give me some more info on the system you are using (so base OS, architecture and docker image used). Also these are OAK-D or OAK-D S2 or ..?

Thanks for the patience,
Jaka

abhay

Hi jakaskerl

Thanks for the response below are the information you asked:
Base OS = 22.04 ubuntu
Architecture = x86_64
Docker image = Custom docker image build with 2.24 depthai version installed for both cpp and python
camera = OAKD W

Thanks,
Abhay

erik

The cameras are connected to a laptop on usb 3 ports.

Also, how are you powering the 3 devices? Directly from the laptop? If that's the case, could you try with powered USB hub or Y-adapter?
It might be that laptop can't provide power for 3 devices, and they randomly crash when there's a power spike (power spikes from one device can be up to 2W of average power consumption).

abhay

Hi erik ,

We are using by connecting it by laptop but we have many of them and it is only happening for few laptops that to after deployment is their any way we can get the log of power consumption by the devices that would be very helpful.

Thanks,
Abhay

erik

Hi @abhay ,
Not really, you could add USB power meter, but they aren't that precise and can't detect short power spikes. I'd suggest trying with a powered USB hub first.

CrisLuce

erik We're using Dell's GL15 gaming laptop deployed only inside the US, with very stable power, with a very advanced and capable power brick. We've used the Y-Adapter P/N 00513 with the same inconsistency as without. We've eliminated camera's damaged during transit by replacing the camera's onsite before powering up the system with the same double-count effect. We've had camera's which double counted onsite come back to the production facility and now have been operating over 1 month, with no double counting. We don't mind adding the Y-Adapter to our bill of materials but….this problem is based elsewhere, I believe.

erik

@CrisLuce so either it's (from more to least likely imo):

Power issue - power dips below 5v and device brownouts, you can use oscilloscope to measure that)
Mechanical/connectivity issue - eg. cable moves around and isn't screwed via USB-C screw holes, disconnects the power/comms lines for a short time and connection drops. Here you could use USB-C with screws for a better connection
Hardware issue - device/cable has an issue.Not sure if you already tried different devices/cables. Here we can replace the units, or even ship units+laptop to our office to try to repro.
FW issue, but seems unlikely, as it only happens at the customer, not at your place. Would need to repro as well to come to the culprit of the issue.

Thoughts?

CrisLuce

Hi erik Thank you for the detailed thoughts. We randomly have the same issue at our production facility as well. We've replace the USB A / USB C cables that come with the camera as well, with the same random results. We are not receiving a crash dump report from the devices - we are experiencing inconsistent operating outcomes - I would like to figure out how we can get a crash dump report from you guys - all the data from you guys - so we can use that data to pinpoint a direction to resolve these random issue. Many thanks for your attention and interest in helping us resolve the issues.

CliffAtSmartSort

This is a link to the laptop we use: G15 Gaming Laptop – Gaming Laptop Computers | Dell USA

All the systems are setup the same with the same power & connected devices, etc. Some systems will work with no issues and others will have crashing issues.

The only accessories being powered by the laptop are the two Oak-D cameras.

We have tried it with the cables that come with the cameras and with higher end aftermarket cables with the same issues.

We'll get with our Electrical guy and see if there's a way for us to test the voltage coming from the port.

Thanks!

erik

@CrisLuce @CliffAtSmartSort we'd love to help you guys and resolve this issue, but the link to the computer unfortunately doesn't help us much.
Ideally, the whole setup would be shipped to our local office (in EU) so we can repro, or you could first measure the voltage with oscilloscope and measure if there was any voltage dip when the crash happens.
Thanks, Erik

CrisLuce

Hi erik Lets focus on getting the crash dump data (all data) so we can further inspect and go from there.

erik

Hi @CrisLuce , If you already tried getting the crash dump (and there was none found), then it (very likely) wasn't FW issue, and we can't do much about it. So next option would be to look into power/connectivity.

CrisLuce

erik Even on the systems that were double counting/crashing/causing us a problem, there was no crash dump data, so, outside of power / connectivity, what could it be?

It sounds like FW crash is out of the question, we know the cables are secure, we know the power is good, but we'll continue to inspect the power side.