diff --git a/opencv_onnx.cpp b/opencv_onnx.cpp index c53ab36..2dd258a 100644 --- a/opencv_onnx.cpp +++ b/opencv_onnx.cpp @@ -124,7 +124,7 @@ int main() { timer1.printElapsedTime("Time to preprocess image"); - for (int j = 0; j < 1; j++) { + for (int j = 0; j < 10; j++) { // Run inference cv::Mat output = net.forward(); @@ -187,8 +187,8 @@ int main() { cv::imshow("Detection Mask", detectionMask); // Save the result as an image - std::string savepath = "C:\\Users\\zjc\\Desktop\\suspect_mask.png"; - cv::imwrite(savepath, detectionMask); + // std::string savepath = "C:\\Users\\zjc\\Desktop\\suspect_mask.png"; + // cv::imwrite(savepath, detectionMask); timer1.printElapsedTime("Time to run inference"); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c332fe0..68c580a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,7 +11,6 @@ target_include_directories(Matrox PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(Matrox PUBLIC Qt6::Widgets ${OpenCV_LIBS} ${MIL_LIBS}) - #定义CVDL模块的库 add_library(CVDL CVDL/OnnxRunner.cpp @@ -19,4 +18,4 @@ add_library(CVDL # 头文件路径 target_include_directories(CVDL PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) # 链接依赖库 -target_link_libraries(CVDL PUBLIC Qt6::Widgets ${OpenCV_LIBS} ${MIL_LIBS}) \ No newline at end of file +target_link_libraries(CVDL PUBLIC Qt6::Widgets Matrox ${OpenCV_LIBS} ${MIL_LIBS}) \ No newline at end of file diff --git a/src/CVDL/OnnxRunner.cpp b/src/CVDL/OnnxRunner.cpp index 01475c2..20a04f6 100644 --- a/src/CVDL/OnnxRunner.cpp +++ b/src/CVDL/OnnxRunner.cpp @@ -15,7 +15,7 @@ void Timer::printElapsedTime(const std::string& message) { } // Resize and pad input image -cv::Mat resizeAndPad(const cv::Mat& image, int targetWidth, int targetHeight, int& padTop, int& padLeft, float& scale, const cv::Scalar& padColor) { +cv::Mat ONNXRunner::resizeAndPad(const cv::Mat& image, int targetWidth, int targetHeight, int& padTop, int& padLeft, float& scale, const cv::Scalar& padColor) { int originalWidth = image.cols; int originalHeight = image.rows; @@ -38,7 +38,7 @@ cv::Mat resizeAndPad(const cv::Mat& image, int targetWidth, int targetHeight, in } // Create detection mask -cv::Mat createDetectionMask(const cv::Mat& originalImage, const std::vector& detections, float scale, int padTop, int padLeft) { +cv::Mat ONNXRunner::createDetectionMask(const cv::Mat& originalImage, const std::vector& detections, float scale, int padTop, int padLeft) { cv::Mat mask = cv::Mat::zeros(originalImage.size(), CV_8UC1); // Single channel mask for (const auto& detection : detections) { @@ -59,7 +59,7 @@ cv::Mat createDetectionMask(const cv::Mat& originalImage, const std::vector performInference(cv::dnn::Net& net, const cv::Mat& inputImage) { +std::vector ONNXRunner::performInference(cv::dnn::Net& net, const cv::Mat& inputImage) const { std::vector detections; cv::Mat output = net.forward(); float* data = (float*)output.data; @@ -112,7 +112,7 @@ std::vector performInference(cv::dnn::Net& net, const cv::Mat& inputI } // Apply Non-Maximum Suppression -std::vector applyNMS(std::vector& detections) { +std::vector ONNXRunner::applyNMS(std::vector& detections) const { std::vector indices; std::vector boxes; std::vector scores; @@ -130,3 +130,31 @@ std::vector applyNMS(std::vector& detections) { return finalDetections; } + + +void ONNXRunner::load(const std::string& modelPath) { + this->net = ONNXRunner::loadModel(modelPath); + // 创建一个空的输入矩阵作为预热数据(假定模型输入是 RGB 图像) + cv::Mat dummyInput = cv::Mat::zeros(INPUT_HEIGHT, INPUT_WIDTH, CV_8UC3); // 640x640 的全零矩阵 + + // 调用 predict 方法进行预热 + this->predict(dummyInput); +}; // Load the model + +std::vector ONNXRunner::predict(const cv::Mat &image) { + cv::dnn::Net net = this->net; + + cv::Mat inputImage = preprocessImage(image, net, this->pad_top, this->pad_left, this->scale); + + // Perform inference + std::vector detections = performInference(net, inputImage); + // Apply Non-Maximum Suppression + std::vector finalDetections = applyNMS(detections); + return finalDetections; +} + +cv::Mat ONNXRunner::postProcess(const std::vector& detections, const cv::Mat& image) const { + // Create and show the detection mask + cv::Mat detectionMask = ONNXRunner::createDetectionMask(image, detections, this->scale, this->pad_top, this->pad_left); + return detectionMask; +} \ No newline at end of file diff --git a/src/CVDL/OnnxRunner.h b/src/CVDL/OnnxRunner.h index 865429b..9811192 100644 --- a/src/CVDL/OnnxRunner.h +++ b/src/CVDL/OnnxRunner.h @@ -27,13 +27,31 @@ public: private: std::chrono::high_resolution_clock::time_point start_time; }; +class ONNXRunner { +public: + ONNXRunner() = default; // Default constructor + void load(const std::string& modelPath); // Load the model + std::vector predict(const cv::Mat& image); // Predict and generate results + cv::Mat postProcess(const std::vector &detections, const cv::Mat &image) const; + +private: + cv::dnn::Net net; + int pad_top, pad_left; + float scale; + + // Function prototypes + static cv::Mat resizeAndPad(const cv::Mat& image, int targetWidth, int targetHeight, int& padTop, int& padLeft, float& scale, const cv::Scalar& padColor); + static cv::Mat createDetectionMask(const cv::Mat& originalImage, const std::vector& detections, float scale, int padTop, int padLeft); + static cv::dnn::Net loadModel(const std::string& modelPath); + cv::Mat preprocessImage(const cv::Mat& image, cv::dnn::Net& net, int& padTop, int& padLeft, float& scale) const; + std::vector performInference(cv::dnn::Net& net, const cv::Mat& inputImage) const; + std::vector applyNMS(std::vector& detections) const; + // Constants + const float CONFIDENCE_THRESHOLD = 0.2f; + const float NMS_THRESHOLD = 0.2f; + const int INPUT_WIDTH = 640; + const int INPUT_HEIGHT = 640; +}; -// Function prototypes -cv::Mat resizeAndPad(const cv::Mat& image, int targetWidth, int targetHeight, int& padTop, int& padLeft, float& scale, const cv::Scalar& padColor); -cv::Mat createDetectionMask(const cv::Mat& originalImage, const std::vector& detections, float scale, int padTop, int padLeft); -cv::dnn::Net loadModel(const std::string& modelPath); -cv::Mat preprocessImage(const cv::Mat& image, cv::dnn::Net& net, int& padTop, int& padLeft, float& scale); -std::vector performInference(cv::dnn::Net& net, const cv::Mat& inputImage); -std::vector applyNMS(std::vector& detections); #endif // ONNXRUNNER_H diff --git a/src/Matrox/template_matching.cpp b/src/Matrox/template_matching.cpp index b279084..e70a586 100644 --- a/src/Matrox/template_matching.cpp +++ b/src/Matrox/template_matching.cpp @@ -119,8 +119,9 @@ void TemplateMatcher::findModels(const MIL_ID& inputImage,MIL_ID& outputImage) return; } MIL_ID input_image_uint8 = convert_to_uint8(inputImage); - - MdispSelect(MilDisplay, input_image_uint8); + if(this->param["isdisplay"] == 1) { + MdispSelect(MilDisplay, input_image_uint8); + } // Clear previous annotations MgraClear(M_DEFAULT, GraphicList); @@ -155,13 +156,16 @@ void TemplateMatcher::findModels(const MIL_ID& inputImage,MIL_ID& outputImage) MbufClear(outputImage, 0); // Display results - std::cout << "Found " << NumResults << " model(s) in " << Time * 1000.0 << " ms:\n"; - std::cout << "Result Model X Position Y Position Angle Scale Score\n"; + if(this->param["debug_mode"] == 1) { + std::cout << "Found " << NumResults << " model(s) in " << Time * 1000.0 << " ms:\n"; + std::cout << "Result Model X Position Y Position Angle Scale Score\n"; + } for (MIL_INT i = 0; i < NumResults; ++i) { - std::cout << i << " " << Models[i] << " " << XPosition[i] << " " - << YPosition[i] << " " << Angle[i] << " " << Scale[i] - << " " << Score[i] << "%\n"; - + if (this->param["debug_mode"] == 1) { + std::cout << i << " " << Models[i] << " " << XPosition[i] << " " + << YPosition[i] << " " << Angle[i] << " " << Scale[i] + << " " << Score[i] << "%\n"; + } // Draw results onto the binary image MgraColor(M_DEFAULT, 255); // White color for binary image MmodDraw(M_DEFAULT, MilResult, outputImage, M_DRAW_EDGES + M_DRAW_POSITION, i, M_DEFAULT); @@ -172,15 +176,11 @@ void TemplateMatcher::findModels(const MIL_ID& inputImage,MIL_ID& outputImage) M_DRAW_EDGES + M_DRAW_POSITION, i, M_DEFAULT); } - // Display or save the binary image - - MbufSave(SAVE_PATH2, outputImage); - } else { std::cout << "No models found.\n"; } - MosPrintf(MIL_TEXT("Press to EXIT.\n\n")); - MosGetch(); + // MosPrintf(MIL_TEXT("Press to EXIT.\n\n")); + // MosGetch(); MbufFree(input_image_uint8); } @@ -237,15 +237,14 @@ void TemplateMatcher::loadConfig(const std::string& filename, file.close(); } -void TemplateMatcher::LoadTemplate(std::map& params) +void TemplateMatcher::LoadConfig(const string& config_path) { std::vector template_paths; std::vector offsetX, offsetY, sizeX, sizeY; std::vector drawColor; // 调用 loadConfig 并加载配置 - loadConfig("C:\\Users\\zjc\\Desktop\\config\\template_config.txt", - template_paths, offsetX, offsetY, sizeX, sizeY, drawColor); + loadConfig(config_path, template_paths, offsetX, offsetY, sizeX, sizeY, drawColor); // 调用 matcher 的 loadTemplates 方法 this->loadTemplates(template_paths, offsetX, offsetY, sizeX, sizeY, drawColor); @@ -256,12 +255,24 @@ void TemplateMatcher::FindTemplates( const MIL_ID& inputImage, MIL_ID& outputIma { // Perform template matching this -> findModels(inputImage,outputImage); - - // Notify user that matching is complete - cout << "Template matching completed.\n"; } - +void TemplateMatcher::predict(const MIL_ID &inputImage, MIL_ID &outputImage, const std::map ¶ms) { + MIL_ID detection_result, detection_resize, output_image_unresize; + pre_process(inputImage, detection_result, params); + MbufAlloc2d(MilSystem, MbufInquire(detection_result, M_SIZE_X, M_NULL)/2, + MbufInquire(detection_result, M_SIZE_Y, M_NULL)/2, 1 + M_UNSIGNED, + M_IMAGE + M_PROC, &detection_resize); + MimResize(detection_result,detection_resize,0.5,0.5,M_DEFAULT); + this->FindTemplates(detection_resize,output_image_unresize, params); + MbufAlloc2d(MilSystem, MbufInquire(detection_result, M_SIZE_X, M_NULL), + MbufInquire(detection_result, M_SIZE_Y, M_NULL), 1 + M_UNSIGNED, + M_IMAGE + M_PROC, &outputImage); + MimResize(output_image_unresize,outputImage,2,2,M_DEFAULT); + MbufFree(detection_result); + MbufFree(detection_resize); + MbufFree(output_image_unresize); +} // TODO: Opencv ONNX runner, 已 diff --git a/src/Matrox/template_matching.h b/src/Matrox/template_matching.h index 00f8e88..93be1ce 100644 --- a/src/Matrox/template_matching.h +++ b/src/Matrox/template_matching.h @@ -21,7 +21,6 @@ private: MIL_ID MilSearchContext; MIL_ID MilResult; MIL_ID GraphicList; -#define SAVE_PATH2 MIL_TEXT("C:\\Users\\zjc\\Desktop\\detection.png") std::vector ModelImgPaths; std::vector ModelsOffsetX; @@ -38,7 +37,7 @@ public: // Constructor TemplateMatcher(MIL_ID system, MIL_ID display, std::map& param); - + void LoadConfig(const std::string& config_path); // Load template models void loadTemplates(const std::vector& template_paths, @@ -49,11 +48,13 @@ public: // Search for models in the input image void findModels(const MIL_ID& inputImage,MIL_ID& outputImage); - void LoadTemplate(std::map ¶ms); + void FindTemplates(const MIL_ID &inputImage, MIL_ID &outputImage,const std::map ¶ms); - void loadConfig(const std::string& filename, + void predict(const MIL_ID& inputImage, MIL_ID& outputImage, const std::map ¶ms); + + static void loadConfig(const std::string& filename, std::vector& template_paths, std::vector& offsetX, std::vector& offsetY, diff --git a/tests/test_onnx.cpp b/tests/test_onnx.cpp index 5964892..d01c144 100644 --- a/tests/test_onnx.cpp +++ b/tests/test_onnx.cpp @@ -7,7 +7,8 @@ int main() { Timer timer1; // Load the model - cv::dnn::Net net = loadModel(modelPath); + ONNXRunner runner; + runner.load(modelPath); timer1.printElapsedTime("Time to load the model"); // Read the input image @@ -16,30 +17,19 @@ int main() { std::cerr << "Could not read the image: " << imagePath << std::endl; return -1; } - - // Preprocess image - int padTop, padLeft; - float scale; - cv::Mat inputImage = preprocessImage(image, net, padTop, padLeft, scale); - timer1.printElapsedTime("Time to preprocess image"); - - // Perform inference - std::vector detections = performInference(net, inputImage); - - // Apply Non-Maximum Suppression - std::vector finalDetections = applyNMS(detections); - std::cout << "Number of detections after NMS: " << finalDetections.size() << std::endl; - - // Create and show the detection mask - cv::Mat detectionMask = createDetectionMask(image, finalDetections, scale, padTop, padLeft); - - // cv::imshow("Detection Mask", detectionMask); - + cv::Mat mask; + timer1.printElapsedTime("Time to load image."); + for(int i = 0; i < 10; i++) { + std:: cout << "Run time: " << i << std::endl; + std::vector result = runner.predict(image); + timer1.printElapsedTime("Time to predict result"); + mask = runner.postProcess(result, image); + timer1.printElapsedTime("Time to predict result"); + } // Save the result - std::string savepath = "C:\\Users\\zjc\\Desktop\\suspect_mask.png"; - cv::imwrite(savepath, detectionMask); - timer1.printElapsedTime("Time to run inference"); + cv::imwrite(savepath, mask); + timer1.printElapsedTime("Time to save image"); cv::waitKey(0); return 0; diff --git a/tests/test_template_matching.cpp b/tests/test_template_matching.cpp index 5ff24dc..c09b1ad 100644 --- a/tests/test_template_matching.cpp +++ b/tests/test_template_matching.cpp @@ -37,7 +37,7 @@ int main() { MIL_ID output_Image= M_NULL; TemplateMatcher matcher(MilSystem, MilDisplay, params); - + matcher.LoadConfig("C:\\Users\\zjc\\Desktop\\config\\template_config.txt"); // Measure execution time measure_execution_time([&]() { @@ -46,8 +46,6 @@ int main() { MbufInquire(detection_result, M_SIZE_Y, M_NULL)/2, 1 + M_UNSIGNED, M_IMAGE + M_PROC, &detection_resize); MimResize(detection_result,detection_resize,0.5,0.5,M_DEFAULT); - - matcher.LoadTemplate(params); matcher.FindTemplates(detection_resize,output_Image,params); }); diff --git a/tests/test_total.cpp b/tests/test_total.cpp index ff83f03..57d3026 100644 --- a/tests/test_total.cpp +++ b/tests/test_total.cpp @@ -2,6 +2,8 @@ #include #include #include +#include + #include "Mil.h" #include "Matrox/utils.h" #include "Matrox/color_range.h" @@ -10,101 +12,74 @@ // 宏定义 #define SAVE_PATH3 MIL_TEXT("C:\\Users\\zjc\\Desktop\\suspect.png") #define SAVE_PATH4 MIL_TEXT("C:\\Users\\zjc\\Desktop\\suspect2.png") -#define IMAGE_PATH MIL_TEXT("C:\\Users\\zjc\\Desktop\\cotton_image_new\\357.bmp") +#define IMAGE_PATH MIL_TEXT(".\\test_imgs\\357.bmp") +std::string imagePath = "C:\\Users\\zjc\\Desktop\\dimo.bmp"; MIL_ID MilApplication, MilSystem, MilDisplay; std::map params; -void run_high_sat_detect(MIL_ID MilImage, std::map& params) { - MIL_ID detection_result = M_NULL; - read_params_from_file("C:\\Users\\zjc\\Desktop\\config\\color_range_config.txt", params); - - measure_execution_time([&]() { - high_sat_detect(MilImage, detection_result, params); - }); - - MbufSave(SAVE_PATH3, detection_result); - MbufFree(detection_result); -} - -void run_pre_process(MIL_ID MilImage, std::map& params) { - MIL_ID detection_result = M_NULL; - MIL_ID detection_resize = M_NULL; - MIL_ID output_Image = M_NULL; - - read_params_from_file("C:\\Users\\zjc\\Desktop\\config\\template_color_config.txt", params); - TemplateMatcher matcher(MilSystem, MilDisplay, params); - - measure_execution_time([&]() { - pre_process(MilImage, detection_result, params); - MbufAlloc2d(MilSystem, MbufInquire(detection_result, M_SIZE_X, M_NULL) / 2, - MbufInquire(detection_result, M_SIZE_Y, M_NULL) / 2, 1 + M_UNSIGNED, - M_IMAGE + M_PROC, &detection_resize); - MimResize(detection_result, detection_resize, 0.5, 0.5, M_DEFAULT); - - matcher.LoadTemplate(params); - matcher.FindTemplates(detection_resize, output_Image, params); - }); - - MbufSave(SAVE_PATH4, detection_result); - MbufFree(detection_resize); - MbufFree(output_Image); - MbufFree(detection_result); -} - +using namespace std; int main() { // 初始化 MIL 应用 - MappAllocDefault(M_DEFAULT, &MilApplication, &MilSystem, &MilDisplay, M_NULL, M_NULL) ; // 读取图片 - MIL_ID MilImage = M_NULL; + MIL_ID MilImage=M_NULL, MilHighSatResult = M_NULL, MilTemplateMatchingResult = M_NULL; MbufRestore(IMAGE_PATH, MilSystem, &MilImage); if (MilImage == M_NULL) { std::cerr << "Error: Failed to restore image from " << IMAGE_PATH << std::endl; MappFreeDefault(MilApplication, MilSystem, MilDisplay, M_NULL, M_NULL); return -1; } + Timer timer1, timer2; + std::map params; + read_params_from_file("..\\config\\color_range_config.txt", params); + read_params_from_file("..\\config\\template_color_config.txt", params); + TemplateMatcher matcher(MilSystem, MilDisplay, params); + matcher.LoadConfig("..\\config\\template_config.txt"); + ONNXRunner runner; + runner.load("C:\\Users\\zjc\\Desktop\\dimo_11.14.onnx"); + timer1.printElapsedTime("Load config and templates and models"); + cv::Mat deep_result, high_sat_result, template_result, total_result; + cout << "Sequence running start:" << endl; + for(int i = 0; i < 10; i++) { + cout << i << endl; + timer1.restart(); + timer2.restart(); + // 艳丽色彩检测 + high_sat_detect(MilImage, MilHighSatResult, params); + timer1.printElapsedTime("High Sat Predict finished"); + // 模板匹配检测 + matcher.predict(MilImage, MilTemplateMatchingResult, params); + timer1.printElapsedTime("Template Matching Predict finished"); + // 深度学习检测 + cv::Mat cv_input = mil2mat(MilImage); + std::vector result = runner.predict(cv_input); + deep_result = runner.postProcess(result, cv_input); + std::string savepath = "C:\\Users\\zjc\\Desktop\\suspect_mask2.png"; + cv::imwrite(savepath, deep_result); + timer1.printElapsedTime("Deep Learning Predict finished"); + high_sat_result = mil2mat(MilHighSatResult); + template_result = mil2mat(MilTemplateMatchingResult); - // 加载参数 - // 创建两个子进程 - PROCESS_INFORMATION processInfo1, processInfo2; - STARTUPINFO startupInfo1, startupInfo2; + if (!deep_result.empty() && !high_sat_result.empty() && !template_result.empty()) { + cv::bitwise_and(deep_result, high_sat_result, total_result); + cv::bitwise_and(total_result, template_result, total_result); + } else { + cerr << "Error: One or more detection results are empty!" << endl; + } + timer2.printElapsedTime("Prediction finished Total"); - ZeroMemory(&startupInfo1, sizeof(startupInfo1)); - ZeroMemory(&startupInfo2, sizeof(startupInfo2)); - ZeroMemory(&processInfo1, sizeof(processInfo1)); - ZeroMemory(&processInfo2, sizeof(processInfo2)); + cv::imshow("Result", total_result); + cv::waitKey(0); - startupInfo1.cb = sizeof(startupInfo1); - startupInfo2.cb = sizeof(startupInfo2); - - // 子进程1 - if (!CreateProcess(NULL, LPWSTR((LPSTR) "ChildProcess1.exe"), NULL, NULL, FALSE, 0, NULL, NULL, &startupInfo1, &processInfo1)) { - std::cerr << "Error: Failed to create process 1. Error code: " << GetLastError() << std::endl; - MbufFree(MilImage); - MappFreeDefault(MilApplication, MilSystem, MilDisplay, M_NULL, M_NULL); - return -1; } - // 子进程2 - if (!CreateProcess(NULL, LPWSTR((LPSTR) "ChildProcess2.exe"), NULL, NULL, FALSE, 0, NULL, NULL, &startupInfo2, &processInfo2)) { - std::cerr << "Error: Failed to create process 2. Error code: " << GetLastError() << std::endl; - MbufFree(MilImage); - MappFreeDefault(MilApplication, MilSystem, MilDisplay, M_NULL, M_NULL); - return -1; - } - // 等待两个子进程完成 - WaitForSingleObject(processInfo1.hProcess, INFINITE); - WaitForSingleObject(processInfo2.hProcess, INFINITE); - - CloseHandle(processInfo1.hProcess); - CloseHandle(processInfo1.hThread); - CloseHandle(processInfo2.hProcess); - CloseHandle(processInfo2.hThread); // 释放资源 MbufFree(MilImage); + MbufFree(MilHighSatResult); + MbufFree(MilTemplateMatchingResult); MappFreeDefault(MilApplication, MilSystem, MilDisplay, M_NULL, M_NULL); std::cout << "所有模块检测已完成!按 退出。" << std::endl;