diff --git a/.gitignore b/.gitignore index f4f8dfa..a322ac4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build release debug .qtc_clangd +.idea diff --git a/camera.cpp b/camera.cpp index 6ff72b4..c538041 100644 --- a/camera.cpp +++ b/camera.cpp @@ -1,5 +1,4 @@ #include "camera.h" -#include "globals.h" #include @@ -47,6 +46,7 @@ ONNXRunner runner; std::map params; +int dual_cam_offset_y = 89; // 双相机之间的上下偏移值 int widthBlocks = 20; // 输出的喷阀通道数 int heightBlocks = 512; // 输出的Mask高度 int sizeThreshold = 20; // 转化为喷阀的每块要求像素个数 @@ -58,6 +58,9 @@ int ignoreSide = 0; // 左右两侧的忽略的喷阀通道数量 int skipLeftCols = 0; // 生成mask时跳过左侧的像素数量 int skipRightCols = 0; // 生成mask时跳过右侧的像素数量 +static std::vector> mask_0(0); +static std::vector> mask_1(0); + static std::vector> tail_0(0); static std::vector> tail_1(0); @@ -167,76 +170,74 @@ MIL_INT ProcessingFunction0(MIL_INT HookType, MIL_ID HookId, void *HookDataPtr) #if(GlobalDebug) qDebug()<<"回调1"; #endif - //拷贝艳丽色检测图像 -// MbufCopy(ModifiedBufferId0,MilImage_Color0); -// // //拷贝onnx检测图像 -// // MbufCopy(ModifiedBufferId1,MilImage_Onnx1); + // 拷贝艳丽色检测图像 + MbufCopy(ModifiedBufferId0,MilImage_Color0); + MIL_UNIQUE_BUF_ID MimResizedestination = MbufAllocColor(MilSystem, 3, 2048, 512, 8+M_UNSIGNED, M_IMAGE+M_PROC+M_DISP, M_UNIQUE_ID); + MbufClear(MimResizedestination, M_COLOR_BLACK); + MimResize(MilImage_Color0, MimResizedestination, 0.5, 1 , M_DEFAULT); -// MIL_UNIQUE_BUF_ID MimResizedestination = MbufAllocColor(MilSystem, 3, 2048, 512, 8+M_UNSIGNED, M_IMAGE+M_PROC+M_DISP, M_UNIQUE_ID); -// MbufClear(MimResizedestination, M_COLOR_BLACK); -// MimResize(MilImage_Color0, MimResizedestination, 0.5, 1 , M_DEFAULT); + //图片镜像翻转 + MIL_UNIQUE_BUF_ID MimFlipDedtination = MbufClone(MimResizedestination, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_UNIQUE_ID); + MbufClear(MimFlipDedtination, M_COLOR_BLACK); + MimFlip(MimResizedestination, MimFlipDedtination, M_FLIP_HORIZONTAL, M_DEFAULT); -// MIL_UNIQUE_BUF_ID MimFlipDedtination = MbufClone(MimResizedestination, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_UNIQUE_ID); -// MbufClear(MimFlipDedtination, M_COLOR_BLACK); -// MimFlip(MimResizedestination, MimFlipDedtination, M_FLIP_HORIZONTAL, M_DEFAULT); + //艳丽检测mask + high_sat_detect(MimFlipDedtination, detection_result0, params); -// #if(GlobalDebug && DebugDetectionTime) -// Timer timer2; -// #endif -// //艳丽检测mask -// high_sat_detect(MimFlipDedtination, detection_result0, params); + // Update the current Img MIl id + { + QMutexLocker locker(&gMaskMutex0); + gMask0 = detection_result0; + } -// #if(GlobalDebug && DebugDetectionTime) -// timer2.printElapsedTime("Algorithm Spent: "); -// #endif - -// #if(GlobalDebug && DebugDetection) -// MbufSave(SAVE_PATH_flip,MimFlipDedtination); -// // MbufSave(SAVE_PATH_resize,MimResizedestination); -// MbufSave(SAVE_PATH_raw,MilImage_Color0); -// MbufSave(SAVE_PATH_result,detection_result0); -// #endif - -// //Onnx检测mask -// // cv::Mat image = mil2mat(MilImage_Onnx1); -// // cv::Mat Img_Onnx; -// // std::vector result = runner.predict(image); -// // Img_Onnx = runner.postProcess(result, image); - -// // std::vector> mask_Onnx1 = generateMaskFromImage2(Img_Onnx, widthBlocks, heightBlocks, sizeThreshold); -// // timer1.printElapsedTime("onnx sat finished"); - -// auto [mask_1, newTail] = generateMaskWithTail(detection_result0, tail_1, widthBlocks, heightBlocks, sizeThreshold, rowRange, skipLeftCols, skipRightCols); - -// tail_1 = newTail; - -// PadColumns(mask_1,padLeft,padRight,0); -// std::vector> mask_Total = expandArray(mask_1,64); - -// // save masks -// #if(GlobalDebug && DebugDetection) -// VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask" + std::to_string(FuncCount1) + ".bmp"); -// VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask_ignored" + std::to_string(FuncCount1) + ".bmp"); -// VectorToImg(mask_Total,"C:/Users/Pc/Desktop/img/mask_expended" + std::to_string(FuncCount1) + ".bmp"); -// #endif -// bool result_Low = get_valve_data(mask_Total); -// if(!result_Low) -// { -// qWarning()<<"下位机发送失败"; -// } + auto [mask_tmp, newTail] = generateMaskWithTail(detection_result0, tail_0, widthBlocks, heightBlocks, sizeThreshold, rowRange, skipLeftCols, skipRightCols); + tail_0 = newTail; + mask_0 = mask_tmp; -// #if(GlobalDebug && DebugDetectionTime) -// timer_detection_time.printElapsedTime("Time of Processing From Get into CallBack to Sent to Lower Mac"); -// #endif + bool isReady; + vector> merged_mask; + vector> mask_tail; + + { + QMutexLocker locker_self(&imgDetectionReady0Mutex); + imgDetectionReady1 = true; + QMutexLocker locker_other(&imgDetectionReady1Mutex); + isReady = imgDetectionReady0 & imgDetectionReady1; + if(isReady) + { + merged_mask = ImageUtils::mergeMasks(mask_0, mask_1, dual_cam_offset_y); + std::tie(merged_mask, mask_tail) = ImageUtils::extractROI(merged_mask, 0, 0, widthBlocks, heightBlocks); + } + } + // 没有两个都完成就不发送 + if(!isReady) + return 0; + + // 发送前的处理 + // 将每个结果左右扩展，扩展半径默认为1 + auto mask_expaned = expandMaskHorizontally(merged_mask, expansionRaidus); + // 将结果的左右补充上0，让物体大小符合要求 + PadColumns(mask_expaned,padLeft,padRight,0); + //将mask扩展到合适的大小 + std::vector> mask_Total = expandArray(mask_expaned,64); + // save masks +#if(GlobalDebug && DebugDetection) + VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask" + std::to_string(FuncCount1) + ".bmp"); + VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask_ignored" + std::to_string(FuncCount1) + ".bmp"); + VectorToImg(mask_Total,"C:/Users/Pc/Desktop/img/mask_expended" + std::to_string(FuncCount1) + ".bmp"); +#endif + // 发送到下位机 + bool result_Low = get_valve_data(mask_Total); + if(!result_Low) + { + qWarning()<<"下位机发送失败"; + } + return 0; } -Timer timer2; - - - MIL_INT ProcessingFunction1(MIL_INT HookType, MIL_ID HookId, void *HookDataPtr) { // CallBackTimer1.printElapsedTime("Call Back 1 reached"); @@ -252,8 +253,8 @@ MIL_INT ProcessingFunction1(MIL_INT HookType, MIL_ID HookId, void *HookDataPtr) // Update the current Img MIl id { - QMutexLocker locker(&gDispPicMutex1); - gDispCurrentPicId1 = ModifiedBufferId1; + QMutexLocker locker(&gDispPicMutex1); + gDispCurrentPicId1 = ModifiedBufferId1; } if(SaveImg_Flag) @@ -282,9 +283,7 @@ MIL_INT ProcessingFunction1(MIL_INT HookType, MIL_ID HookId, void *HookDataPtr) MbufClear(MimResizedestination, M_COLOR_BLACK); MimResize(MilImage_Color1, MimResizedestination, 0.5, 1 , M_DEFAULT); - // MIL_UNIQUE_BUF_ID MimFlipDedtination = MbufClone(MimResizedestination, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_DEFAULT, M_UNIQUE_ID); - // MbufClear(MimFlipDedtination, M_COLOR_BLACK); - // MimFlip(MimResizedestination, MimFlipDedtination, M_FLIP_HORIZONTAL, M_DEFAULT); + #if(GlobalDebug && DebugDetectionTime) Timer timer2; @@ -302,37 +301,56 @@ MIL_INT ProcessingFunction1(MIL_INT HookType, MIL_ID HookId, void *HookDataPtr) MbufSave(SAVE_PATH_raw,MilImage_Color0); MbufSave(SAVE_PATH_result,detection_result0); #endif + // Update the current Img MIl id + { + QMutexLocker locker(&gMaskMutex1); + gMask1 = detection_result1; + } - //Onnx检测mask - // cv::Mat image = mil2mat(MilImage_Onnx1); - // cv::Mat Img_Onnx; - // std::vector result = runner.predict(image); - // Img_Onnx = runner.postProcess(result, image); - - // std::vector> mask_Onnx1 = generateMaskFromImage2(Img_Onnx, widthBlocks, heightBlocks, sizeThreshold); - // timer1.printElapsedTime("onnx sat finished"); - - auto [mask_1, newTail] = generateMaskWithTail(detection_result1, tail_1, widthBlocks, heightBlocks, sizeThreshold, rowRange, skipLeftCols, skipRightCols); + auto [mask_tmp, newTail] = generateMaskWithTail(detection_result1, tail_1, widthBlocks, heightBlocks, sizeThreshold, rowRange, skipLeftCols, skipRightCols); tail_1 = newTail; - auto mask_expaned = expandMaskHorizontally(mask_1, expansionRaidus); + mask_1 = mask_tmp; + bool isReady; + vector> merged_mask; + vector> mask_tail; + { + QMutexLocker locker_self(&imgDetectionReady1Mutex); + imgDetectionReady1 = true; + QMutexLocker locker_other(&imgDetectionReady0Mutex); + + isReady = imgDetectionReady0 & imgDetectionReady1; + if(isReady) + { + merged_mask = ImageUtils::mergeMasks(mask_0, mask_1, dual_cam_offset_y); + std::tie(merged_mask, mask_tail) = ImageUtils::extractROI(merged_mask, 0, 0, widthBlocks, heightBlocks); + } + } + // 没有两个都完成就不发送 + if(!isReady) + return 0; + + // 发送前的处理 + // 将每个结果左右扩展，扩展半径默认为1 + auto mask_expaned = expandMaskHorizontally(merged_mask, expansionRaidus); + // 将结果的左右补充上0，让物体大小符合要求 PadColumns(mask_expaned,padLeft,padRight,0); + //将mask扩展到合适的大小 std::vector> mask_Total = expandArray(mask_expaned,64); - -// save masks + // save masks #if(GlobalDebug && DebugDetection) VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask" + std::to_string(FuncCount1) + ".bmp"); VectorToImg(mask_1,"C:/Users/Pc/Desktop/img/mask_ignored" + std::to_string(FuncCount1) + ".bmp"); VectorToImg(mask_Total,"C:/Users/Pc/Desktop/img/mask_expended" + std::to_string(FuncCount1) + ".bmp"); #endif + // 发送到下位机 bool result_Low = get_valve_data(mask_Total); if(!result_Low) { qWarning()<<"下位机发送失败"; } - #if(GlobalDebug && DebugDetectionTime) timer_detection_time.printElapsedTime("Time of Processing From Get into Ca" "..0llBack to Sent to Lower Mac"); @@ -357,7 +375,6 @@ bool DestoryCamera() MbufFree(MilGrabBufferList1[i]); } - MbufFree(MilImage0); MbufFree(MilImage1); MbufFree(MilImage_Color0); diff --git a/camera.h b/camera.h index 1d6f6c9..80fcc28 100644 --- a/camera.h +++ b/camera.h @@ -10,8 +10,6 @@ #include #include - - #include #include #include @@ -23,6 +21,9 @@ #include #include +#include "globals.h" +#include "img_utils.h" + extern MIL_ID MilApplication; extern MIL_ID MilSystem; diff --git a/cotton_double2.pro b/cotton_double2.pro index cb73455..ed0ea51 100644 --- a/cotton_double2.pro +++ b/cotton_double2.pro @@ -36,5 +36,5 @@ DEPENDPATH += D:/desktop/Cotton/cotton/Include LIBS += -LD:/desktop/Cotton/cotton/LIB -lmil LIBS += -LD:/desktop/Cotton/cotton/LIB -lMilim -INCLUDEPATH += F:/matrox/opencv/build/include +INCLUDEPATH += D:/desktop/Cotton/opencv/Withcontrib/include LIBS += F:/matrox/opencv/build/x64/vc16/lib/opencv*.lib diff --git a/globals.cpp b/globals.cpp index 0a38724..ffdae15 100644 --- a/globals.cpp +++ b/globals.cpp @@ -1,10 +1,20 @@ // globals.cpp #include "globals.h" -// 图片显示0 +// 图片显示 QMutex gDispPicMutex0; MIL_ID gDispCurrentPicId0 = 0; - -// 图片显示1 QMutex gDispPicMutex1; MIL_ID gDispCurrentPicId1 = 0; + +// 检测结果 +QMutex gMaskMutex0; +MIL_ID gMask0 = 0; +QMutex gMaskMutex1; +MIL_ID gMask1 = 0; + +// 双相机结果同步 +QMutex imgDetectionReady0Mutex; +QMutex imgDetectionReady1Mutex; +bool imgDetectionReady0 = false; +bool imgDetectionReady1 = false; diff --git a/globals.h b/globals.h index 2de9eab..6371a7d 100644 --- a/globals.h +++ b/globals.h @@ -4,6 +4,7 @@ #include #include +#include // 图片显示0 extern QMutex gDispPicMutex0; @@ -13,4 +14,15 @@ extern MIL_ID gDispCurrentPicId0; extern QMutex gDispPicMutex1; extern MIL_ID gDispCurrentPicId1; +extern QMutex gMaskMutex0; +extern MIL_ID gMask0; + +extern QMutex gMaskMutex1; +extern MIL_ID gMask1; + +extern QMutex imgDetectionReady0Mutex; +extern QMutex imgDetectionReady1Mutex; +extern bool imgDetectionReady0; +extern bool imgDetectionReady1; + #endif // GLOBALS_H diff --git a/img_utils.cpp b/img_utils.cpp index 40fb322..09a9f10 100644 --- a/img_utils.cpp +++ b/img_utils.cpp @@ -115,3 +115,153 @@ void ImageUtils::convert_to_uint8(const MIL_ID& input_img, MIL_ID& output_img) { std::cout << "Unsupported channel number!" << std::endl; } } + +cv::Mat overlayResultOnInput(const cv::Mat& cv_input, const cv::Mat& total_result, double alpha = 0.5, int colormap = cv::COLORMAP_JET) { + // 1. 确保 total_result 的尺寸与 cv_input 一致 + cv::Mat resized_total_result; + if (total_result.size() != cv_input.size()) { + cv::resize(total_result, resized_total_result, cv_input.size(), 0, 0, cv::INTER_LINEAR); + } else { + resized_total_result = total_result; + } + + // 2. 将 resized_total_result 转换为伪彩色图像 + cv::Mat total_result_color; + // 确保 total_result 是单通道图像以应用色图 + if (resized_total_result.channels() == 1) { + cv::applyColorMap(resized_total_result, total_result_color, colormap); + } else { + // 如果已经是多通道图像，可以选择直接使用或根据需要处理 + total_result_color = resized_total_result.clone(); + } + + // 3. 确保 cv_input 是三通道图像（如果是灰度图像，则转换为 BGR） + cv::Mat cv_input_rgb; + if (cv_input.channels() == 1) { + cv::cvtColor(cv_input, cv_input_rgb, cv::COLOR_GRAY2BGR); + } else { + cv_input_rgb = cv_input.clone(); // 保证不修改原始图像 + } + + // 4. 设置叠加透明度（alpha: 0.0-1.0） + double beta = 1.0 - alpha; + + // 5. 使用加权和将 total_result_color 叠加到 cv_input_rgb 上 + cv::Mat overlay; + cv::addWeighted(cv_input_rgb, alpha, total_result_color, beta, 0.0, overlay); + + // 6. 返回叠加后的图像 + return overlay; +} + + +// 合并两个二值掩码，通过按位或操作，并考虑垂直偏移 offset_y +std::vector> mergeMasks( + const std::vector>& mask1, + const std::vector>& mask2, + int offset_y = 0 +) { + if (mask1.empty() || mask1[0].empty() || mask2.empty() || mask2[0].empty()) { + throw std::invalid_argument("输入的掩码不能为空"); + } + + // 假设两个掩码的宽度相同 + size_t width = mask1[0].size(); + for (const auto& row : mask1) + { + if (row.size() != width) + { + throw std::invalid_argument("mask1 的所有行必须具有相同的宽度"); + } + } + for (const auto& row : mask2) + { + if (row.size() != width) + { + throw std::invalid_argument("mask2 的所有行必须具有相同的宽度，并且与 mask1 的宽度一致"); + } + } + + size_t height1 = mask1.size(); + size_t height2 = mask2.size(); + + // 计算合并后的高度 + int merged_top = std::min(0, offset_y); + int merged_bottom = std::max(static_cast(height1), static_cast(height2) + offset_y); + size_t merged_height = merged_bottom - merged_top; + + // 初始化合并后的掩码为全 0 + std::vector> merged_mask(merged_height, std::vector(width, 0)); + + // 将 mask1 放入 merged_mask + for (size_t y = 0; y < height1; ++y) + { + int merged_y = y - merged_top; + for (size_t x = 0; x < width; ++x) + { + merged_mask[merged_y][x] = mask1[y][x] ? 1 : 0; + } + } + + // 将 mask2 放入 merged_mask，考虑 offset_y + for (size_t y = 0; y < height2; ++y) + { + int merged_y = y + offset_y - merged_top; + if (merged_y < 0 || merged_y >= static_cast(merged_height)) + { + // 超出合并掩码的范围，跳过 + continue; + } + for (size_t x = 0; x < width; ++x) + { + merged_mask[merged_y][x] = (merged_mask[merged_y][x] || mask2[y][x]) ? 1 : 0; + } + } + + return merged_mask; +} + +// 提取 ROI 并返回剩余部分 +std::pair>, std::vector>> +extractROI( + const std::vector>& merged_mask, + int roi_x, + int roi_y, + int roi_width, + int roi_height +) { + if (merged_mask.empty() || merged_mask[0].empty()) { + throw std::invalid_argument("merged_mask 不能为空"); + } + + size_t mask_height = merged_mask.size(); + size_t mask_width = merged_mask[0].size(); + + // 校正 ROI 的边界 + int x_start = std::max(0, roi_x); + int y_start = std::max(0, roi_y); + int x_end = std::min(static_cast(mask_width), roi_x + roi_width); + int y_end = std::min(static_cast(mask_height), roi_y + roi_height); + + // 初始化 ROI 和剩余部分的掩码 + std::vector> roi_mask(roi_height, std::vector(roi_width, 0)); + std::vector> remaining_mask(mask_height, std::vector(mask_width, 0)); + + for (size_t y = 0; y < mask_height; ++y) { + for (size_t x = 0; x < mask_width; ++x) { + if (x >= static_cast(roi_x) && x < static_cast(roi_x + roi_width) && + y >= static_cast(roi_y) && y < static_cast(roi_y + roi_height)) { + // 属于 ROI + if (x >= static_cast(x_start) && y >= static_cast(y_start) && + x < static_cast(x_end) && y < static_cast(y_end)) { + roi_mask[y - roi_y][x - roi_x] = merged_mask[y][x]; + } + } else { + // 属于剩余部分 + remaining_mask[y][x] = merged_mask[y][x]; + } + } + } + + return { roi_mask, remaining_mask }; +} diff --git a/img_utils.h b/img_utils.h index ab9babe..a119f59 100644 --- a/img_utils.h +++ b/img_utils.h @@ -7,6 +7,7 @@ #include #include #include +#include extern MIL_ID MilSystem; @@ -22,6 +23,23 @@ public: static cv::Mat mil2Mat(const MIL_ID mil_img); static void convert_to_uint8(const MIL_ID& input_img, MIL_ID& output_img); + + static cv::Mat overlayResultOnInput(const cv::Mat& cv_input, const cv::Mat& total_result, double alpha = 0.5, int colormap = cv::COLORMAP_JET); + + static std::vector> mergeMasks( + const std::vector>& mask1, + const std::vector>& mask2, + int offset_y = 0 + ); + + static std::pair>, std::vector>> + extractROI( + const std::vector>& merged_mask, + int roi_x, + int roi_y, + int roi_width, + int roi_height + ); }; #endif // IMG_UTILS_H diff --git a/widget.cpp b/widget.cpp index 629b6c2..b49fe5e 100644 --- a/widget.cpp +++ b/widget.cpp @@ -51,25 +51,36 @@ void Widget::refreshImage() refreshSingleImage(1); } -void Widget::refreshSingleImage(int camera_id) +void Widget::refreshSingleImage(int camera_id, bool overlay_result) { // 更新当前的图片0 MIL_ID current_id = 0; if (camera_id == 0) - { + { // 获取当前图片 { QMutexLocker locker(&gDispPicMutex0); current_id = gDispCurrentPicId0; } if (current_id == 0) return; - // 获取 QPixmap 并设置到 QLabel - QPixmap pixmap0 = ImageUtils::mat2QPixmap(ImageUtils::mil2Mat(current_id)); + cv::Mat img = ImageUtils::mil2Mat(current_id); + //进行结果的转化叠加 + if (overlay_result) + { + MIL_ID mask_id = 0; + { + QMutexLocker locker(&gMaskMutex0); + mask_id = gMask0; + } + cv::Mat mask = ImageUtils::mil2Mat(mask_id); + img = ImageUtils::overlayResultOnInput(img,mask,0.5); + } + //刷新显示 + QPixmap pixmap0 = ImageUtils::mat2QPixmap(img); if (!pixmap0.isNull()) { ui->camera_0_img->setPixmap(pixmap0); } - } // 更新当前的图片1 else if (camera_id == 1) @@ -81,8 +92,20 @@ void Widget::refreshSingleImage(int camera_id) if (current_id == 0) return; - // 获取 QPixmap 并设置到 QLabel - QPixmap pixmap1 = ImageUtils::mat2QPixmap(ImageUtils::mil2Mat(current_id)); + cv::Mat img = ImageUtils::mil2Mat(current_id); + //进行结果的转化叠加 + if (overlay_result) + { + MIL_ID mask_id = 0; + { + QMutexLocker locker(&gMaskMutex1); + mask_id = gMask1; + } + cv::Mat mask = ImageUtils::mil2Mat(mask_id); + img = ImageUtils::overlayResultOnInput(img,mask,0.5); + } + //刷新显示 + QPixmap pixmap1 = ImageUtils::mat2QPixmap(img); if (!pixmap1.isNull()) { ui->camera_1_img->setPixmap(pixmap1); diff --git a/widget.h b/widget.h index f552fcb..5023732 100644 --- a/widget.h +++ b/widget.h @@ -26,7 +26,7 @@ public slots: private slots: - void refreshSingleImage(int camera_id); + void refreshSingleImage(int camera_id, bool overlay_result = true); void on_pushButton_clicked();