From 09bdd8007d074b64947533e332b427ea56075d33 Mon Sep 17 00:00:00 2001 From: wdp <544209413@qq.com> Date: Sat, 18 Jan 2025 21:12:51 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E6=9B=B4=E6=96=B0=E3=80=91center-face?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E5=8A=9F=E8=83=BD=E8=B0=83=E8=AF=95=E5=AE=8C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/c_cpp_properties.json | 1 + .vscode/settings.json | 62 +++++++++ .vscode/tasks.json | 6 +- CMakeLists.txt | 12 +- app/main.cpp | 23 +++- build-win-app.ps1 | 2 +- src/CenterFaceMnn.cpp | 235 ++++++++++++++++++++++++++++++++++ src/CenterFaceMnn.h | 45 +++++++ src/TypeInfo.cpp | 1 + src/TypeInfo.h | 32 +++++ src/util/TimeCount.cpp | 24 ++-- src/util/TimeCount.h | 14 +- 12 files changed, 435 insertions(+), 22 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 src/CenterFaceMnn.cpp create mode 100644 src/CenterFaceMnn.h create mode 100644 src/TypeInfo.cpp create mode 100644 src/TypeInfo.h diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 8ae0264..a71f4a0 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -5,6 +5,7 @@ "includePath": [ "${workspaceFolder}/**", "${workspaceFolder}/3rd/**", + "${workspaceFolder}/models/**", ], "defines": [ "_DEBUG", diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..40830d7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,62 @@ +{ + "files.associations": { + "*.inc": "c", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "unordered_map": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "typeinfo": "cpp" + } +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 3030132..af40270 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -11,13 +11,13 @@ "-build_type", "Release", "-build_and_run", - "false", + "true", "-platform", "win" ], "group": { "kind": "build", - "isDefault": false + "isDefault": true }, "presentation": { "reveal": "silent" @@ -40,7 +40,7 @@ ], "group": { "kind": "build", - "isDefault": true + "isDefault": false }, "presentation": { "reveal": "silent" diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ae4476..8b695eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,10 @@ endif() # 根据 platform 值设置 CMake 变量 if(${CMAKE_BUILD_PLATFORM} STREQUAL "win") link_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/opencv/x64/vc15/lib) + link_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/mnn/x64/) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/opencv/include) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/mnn/include) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/models) else() message(FATAL_ERROR "不支持的平台: ${CMAKE_BUILD_PLATFORM}") endif() @@ -19,13 +22,18 @@ endif() include_directories(src) aux_source_directory(./src DIR_SRCS) +aux_source_directory(./src/util DIR_SRCS) aux_source_directory(./app DIR_SRCS) add_executable(Demo ${DIR_SRCS}) if(CMAKE_BUILD_TYPE STREQUAL "Debug") - target_link_libraries(${PROJECT_NAME} opencv_world430d) + target_compile_options(${PROJECT_NAME} PRIVATE /MTd) + target_link_libraries(${PROJECT_NAME} opencv_world430d) + target_link_libraries(${PROJECT_NAME} MNN) else() - target_link_libraries(${PROJECT_NAME} opencv_world430) + target_compile_options(${PROJECT_NAME} PRIVATE /MT) + target_link_libraries(${PROJECT_NAME} opencv_world430) + target_link_libraries(${PROJECT_NAME} MNN) endif() diff --git a/app/main.cpp b/app/main.cpp index dcdca77..20054b7 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -1,6 +1,27 @@ #include +#include +#include "util/TimeCount.h" +#include "CenterFaceMnn.h" int32_t main(int32_t argc, char** argv) { - std::cout << "Hello, World!" << std::endl; + CenterFaceMnn::GetInstance(); + std::vector faces; + cv::Mat img = cv::imread("F:/33.jpg"); +int32_t count = 0; +re_test: + count++; + { + USE_TIME t(USE_TIME_US, "face_detect: "); + CenterFaceMnn::GetInstance()->Detect(img, faces, 1); + } + if (count < 10) { + goto re_test; + } + + for(auto& face : faces) { + cv::rectangle(img, cv::Rect(face.x1, face.y1, face.x2 - face.x1, face.y2 - face.y1), cv::Scalar(0, 0, 255), 2); + } + cv::imshow("test", img); + cv::waitKey(0); return 0; } \ No newline at end of file diff --git a/build-win-app.ps1 b/build-win-app.ps1 index 0ad12e6..70d7ce6 100644 --- a/build-win-app.ps1 +++ b/build-win-app.ps1 @@ -42,6 +42,6 @@ Write-Host "build done." -ForegroundColor Darkgreen if($build_and_run -eq "true"){ Write-Host "################## APP Run ##################" -ForegroundColor Blue - $env:PATH += ";3rd\win\opencv\x64\vc15\bin" + $env:PATH += ";3rd\win\opencv\x64\vc15\bin;3rd\win\mnn\x64\" Start-Process -FilePath "build\\${build_type}\\Demo.exe" } diff --git a/src/CenterFaceMnn.cpp b/src/CenterFaceMnn.cpp new file mode 100644 index 0000000..b6f338b --- /dev/null +++ b/src/CenterFaceMnn.cpp @@ -0,0 +1,235 @@ +#include "CenterFaceMnn.h" + +CenterFaceMnn* CenterFaceMnn::m_hInstance = nullptr; + +static const unsigned char centerface_model[] = { + #include "centerface_small_mnn.dat" +}; + +CenterFaceMnn* CenterFaceMnn::GetInstance(){ + if (!m_hInstance){ + m_hInstance = new CenterFaceMnn(); + } + return m_hInstance; +} + +CenterFaceMnn::CenterFaceMnn(){ + m_detector = std::shared_ptr(MNN::Interpreter::createFromBuffer(centerface_model, sizeof(centerface_model))); + + MNN::ScheduleConfig config; + MNN::BackendConfig backendConfig; + backendConfig.precision = MNN::BackendConfig::Precision_High; + backendConfig.power = MNN::BackendConfig::Power_High; + backendConfig.memory = MNN::BackendConfig::Memory_High; + config.backendConfig = &backendConfig; + config.type = MNN_FORWARD_CPU; + config.numThread = 4; + m_session = m_detector->createSession(config); + + const float mean_vals[3] = { 127.5f, 127.5f, 127.5f }; + const float norm_vals[3] = { 0.0078431373f, 0.0078431373f, 0.0078431373f }; + ::memcpy(m_img_config.mean, mean_vals, sizeof(mean_vals)); + ::memcpy(m_img_config.normal, norm_vals, sizeof(norm_vals)); + m_img_config.sourceFormat = (MNN::CV::ImageFormat)2; + m_img_config.destFormat = (MNN::CV::ImageFormat)1; + m_img_config.filterType = (MNN::CV::Filter)(1); + m_img_config.wrap = (MNN::CV::Wrap)(1); + + input_tensor = m_detector->getSessionInput(m_session, NULL); + hm_tensor = m_detector->getSessionOutput(m_session, "hm"); + wh_tensor = m_detector->getSessionOutput(m_session, "wh"); + reg_tensor = m_detector->getSessionOutput(m_session, "reg"); + lm_tensor = m_detector->getSessionOutput(m_session, "lm"); +} + +CenterFaceMnn::~CenterFaceMnn(){ + m_detector->releaseSession(m_session); +} + +int CenterFaceMnn::Detect(const cv::Mat& img, std::vector& faces, float scale){ + std::lock_guard lock(m_mt); + faces.clear(); + if (img.cols * img.rows == 0){ + return -1; + } + + if (scale <= 0 || scale > 1) { + scale = img.cols > img.rows ? 160.0f / img.cols : 160.0f / img.rows; + } + int resize_w = (int)(img.cols * scale) / 32 * 32; + int resize_h = (int)(img.rows * scale) / 32 * 32; + float scale_w = (float)img.cols / (float)resize_w; + float scale_h = (float)img.rows / (float)resize_h; + + + cv::Mat input; + cv::resize(img, input, cv::Size(resize_w, resize_h)); + m_detector->resizeTensor(input_tensor, 1, 3, resize_h, resize_w); + m_detector->resizeSession(m_session); + //prepare data + std::shared_ptr pretreat(MNN::CV::ImageProcess::create(m_img_config)); + pretreat->convert(input.data, resize_w, resize_h, input.step[0], input_tensor); + m_detector->runSession(m_session); + + MNN::Tensor tensor_hm(hm_tensor, MNN::Tensor::CAFFE); + MNN::Tensor tensor_wh(wh_tensor, MNN::Tensor::CAFFE); + MNN::Tensor tensor_reg(reg_tensor, MNN::Tensor::CAFFE); + MNN::Tensor tensor_lm(lm_tensor, MNN::Tensor::CAFFE); + + hm_tensor->copyToHostTensor(&tensor_hm); + wh_tensor->copyToHostTensor(&tensor_wh); + reg_tensor->copyToHostTensor(&tensor_reg); + lm_tensor->copyToHostTensor(&tensor_lm); + + std::vector hm_shape = tensor_hm.shape(); + + float* heatmap = tensor_hm.host(); + float* wh = tensor_wh.host(); + float* reg = tensor_reg.host(); + float* lm = tensor_lm.host(); + + Decode(heatmap, wh, reg, lm, hm_shape[2], hm_shape[3], faces, 0.4, 0.3); + for (int i = 0; i < faces.size(); i++){ + FaceInfo& tmp = faces[i]; + tmp.x1 *= scale_w; + tmp.x2 *= scale_w; + tmp.y1 *= scale_h; + tmp.y2 *= scale_h; + + for (int j = 0; j < 5; j++){ + tmp.landmarks[j * 2] *= scale_w; + tmp.landmarks[j * 2 + 1] *= scale_h; + } + } + return faces.size(); +} +#include + +void CenterFaceMnn::Decode(float* heatmap, float* scale, float* offset, float* landmarks, int h, int w, std::vector& faces, float scoreThresh, float nmsThresh){ + int fea_h = h; + int fea_w = w; + int d_w = w * 4; + int d_h = h * 4; + int spacial_size = fea_w * fea_h; + + float *scale1 = (float*)(scale); + float *scale0 = scale1 + spacial_size; + + float *offset1 = (float*)(offset); + float *offset0 = offset1 + spacial_size; + + std::vector ids = GetIds(heatmap, fea_h, fea_w, scoreThresh); + + std::vector faces_tmp; + #pragma omp parallel for + for (int i = 0; i < ids.size() / 2; i++) { + int id_h = ids[2 * i]; + int id_w = ids[2 * i + 1]; + int index = id_h * fea_w + id_w; + + float s0 = std::exp(scale0[index]) * 4; + float s1 = std::exp(scale1[index]) * 4; + float o0 = offset0[index]; + float o1 = offset1[index]; + + float x1 = std::max(0., (id_w + o1 + 0.5) * 4 - s1 / 2); + float y1 = std::max(0., (id_h + o0 + 0.5) * 4 - s0 / 2); + float x2 = std::max(0., (id_w + o1 + 0.5) * 4 + s1 / 2); + float y2 = std::max(0., (id_h + o0 + 0.5) * 4 + s0 / 2); + x1 = std::min(x1, (float)d_w); + y1 = std::min(y1, (float)d_h); + x2 = std::min(x2, (float)d_w); + y2 = std::min(y2, (float)d_h); + + FaceInfo facebox; + facebox.x1 = x1; + facebox.y1 = y1; + facebox.x2 = x2; + facebox.y2 = y2; + facebox.score = heatmap[index]; + + float box_w = x2 - x1; + float box_h = y2 - y1; + + for (int j = 0; j < 5; j++) { + float *xmap = (float*)landmarks + (2 * j) * spacial_size; + float *ymap = (float*)landmarks + (2 * j + 1) * spacial_size; + facebox.landmarks[2 * j] = x1 + xmap[index] * s1; + facebox.landmarks[2 * j + 1] = y1 + ymap[index] * s0; + } + #pragma omp critical + { + faces_tmp.push_back(facebox); + } + } + NMS(faces_tmp, faces, nmsThresh); +} + +void CenterFaceMnn::NMS(std::vector& input, std::vector& output, float nmsthreshold) { + std::sort(input.begin(), input.end(), + [](const FaceInfo& a, const FaceInfo& b) { + return a.score > b.score; + }); + + int box_num = input.size(); + + std::vector merged(box_num, 0); + + #pragma omp parallel for + for (int i = 0; i < box_num; i++) { + if (merged[i]) + continue; + + output.push_back(input[i]); + + float h0 = input[i].y2 - input[i].y1 + 1; + float w0 = input[i].x2 - input[i].x1 + 1; + + float area0 = h0 * w0; + + for (int j = i + 1; j < box_num; j++) { + if (merged[j]) + continue; + + float inner_x0 = std::max(input[i].x1, input[j].x1); + float inner_y0 = std::max(input[i].y1, input[j].y1); + + float inner_x1 = std::min(input[i].x2, input[j].x2); + float inner_y1 = std::min(input[i].y2, input[j].y2); + + float inner_h = inner_y1 - inner_y0 + 1; + float inner_w = inner_x1 - inner_x0 + 1; + + if (inner_h <= 0 || inner_w <= 0) + continue; + + float inner_area = inner_h * inner_w; + + float h1 = input[j].y2 - input[j].y1 + 1; + float w1 = input[j].x2 - input[j].x1 + 1; + + float area1 = h1 * w1; + + float score = inner_area / (area0 + area1 - inner_area); + + if (score > nmsthreshold) + merged[j] = 1; + } + } +} +std::vector CenterFaceMnn::GetIds(float *heatmap, int h, int w, float thresh){ + std::vector ids; + for (int i = 0; i < h; i++) { + for (int j = 0; j < w; j++) { + if (heatmap[i*w + j] > thresh) { + ids.push_back(i); + ids.push_back(j); + } + } + } + return ids; +} + +const std::string CenterFaceMnn::GetModelVer(){ + return m_model_ver; +} \ No newline at end of file diff --git a/src/CenterFaceMnn.h b/src/CenterFaceMnn.h new file mode 100644 index 0000000..35c76e4 --- /dev/null +++ b/src/CenterFaceMnn.h @@ -0,0 +1,45 @@ +#ifndef CENTERFACE_H +#define CENTERFACE_H +#include +#include +#include +#include +#include +#include +#include +#include +#include "TypeInfo.h" + +class CenterFaceMnn{ +public: + static CenterFaceMnn* GetInstance(); + + int Detect(const cv::Mat& img, std::vector& faces, float scale = 0); + + const std::string GetModelVer(); +private: + CenterFaceMnn(); + ~CenterFaceMnn(); + void Decode(float* heatmap, float* scale, float* offset, float* landmarks, int h, int w, std::vector& faces, float scoreThresh, float nmsThresh); + void NMS(std::vector& input, std::vector& output, float nmsthreshold); + std::vector GetIds(float *heatmap, int h, int w, float thresh); + +private: + static CenterFaceMnn* m_hInstance; + std::mutex m_mt; + + std::shared_ptr m_detector; + + MNN::Tensor* input_tensor; + MNN::Tensor* hm_tensor; + MNN::Tensor* wh_tensor; + MNN::Tensor* reg_tensor; + MNN::Tensor* lm_tensor; + + MNN::Session* m_session; + MNN::CV::ImageProcess::Config m_img_config; + + const std::string m_model_ver = ""; +}; + +#endif \ No newline at end of file diff --git a/src/TypeInfo.cpp b/src/TypeInfo.cpp new file mode 100644 index 0000000..29aed55 --- /dev/null +++ b/src/TypeInfo.cpp @@ -0,0 +1 @@ +#include "TypeInfo.h" diff --git a/src/TypeInfo.h b/src/TypeInfo.h new file mode 100644 index 0000000..33094e7 --- /dev/null +++ b/src/TypeInfo.h @@ -0,0 +1,32 @@ +#ifndef TYPE_INFO_H +#define TYPE_INFO_H + +typedef enum _ret_t { + RET_OK = 0, // 操作成功 + RET_AUTHOR, // 授权失败 + RET_IMG, // 输入图片错误 + RET_INVALID_FEATURE, // 人脸特征数据无效 + RET_INVALID_LANDMARK, // 人脸Landmark数据无效 +} ret_t; + +// #define ERR_FACE_RECT -10 //人脸框无效 +// #define ERR_INDEX_INVALID -20 //索引无效 + +struct FaceInfo { + float x1; //人脸框 左上x坐标 + float y1; //人脸框 左上y坐标 + float x2; //人脸框 右下x坐标 + float y2; //人脸框 右下y坐标 + float score; //人脸框 置信度 + float landmarks[10]; //人脸框 关键点坐标 + + int GetWidth() { + return static_cast(x2 - x1); + } + + int GetHeight() { + return static_cast(y2 - y1); + } +}; + +#endif \ No newline at end of file diff --git a/src/util/TimeCount.cpp b/src/util/TimeCount.cpp index c557680..7603964 100644 --- a/src/util/TimeCount.cpp +++ b/src/util/TimeCount.cpp @@ -1,20 +1,22 @@ #include "TimeCount.h" #include -USE_TIME::USE_TIME(std::string str) : - strTmp(str) -{ - start = std::chrono::system_clock::now(); +USE_TIME::USE_TIME(USE_TIME_UNIT unit, std::string str) : + strTmp(str), + unit(unit){ + start = std::chrono::system_clock::now(); } -USE_TIME::~USE_TIME() -{ +USE_TIME::~USE_TIME(){ end = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::milliseconds::period::num << "ms \t\n"; - - //auto duration = std::chrono::duration_cast(end - start); - //std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::microseconds::period::num << "ms \t\n"; + if (unit == USE_TIME_MS) { + auto duration = std::chrono::duration_cast(end - start); + std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::milliseconds::period::num << "ms \t\n"; + } + else { + auto duration = std::chrono::duration_cast(end - start); + std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::microseconds::period::num << "us \t\n"; + } } diff --git a/src/util/TimeCount.h b/src/util/TimeCount.h index 18babcb..ca020b7 100644 --- a/src/util/TimeCount.h +++ b/src/util/TimeCount.h @@ -4,14 +4,20 @@ #include #include +typedef enum { + USE_TIME_MS = 0, + USE_TIME_US = 1, +}USE_TIME_UNIT; + class USE_TIME{ public: - USE_TIME(std::string str = ""); + USE_TIME(USE_TIME_UNIT unit, std::string str = ""); ~USE_TIME(); - + private: - std::string strTmp; - std::chrono::system_clock::time_point start; + std::string strTmp; + USE_TIME_UNIT unit; + std::chrono::system_clock::time_point start; std::chrono::system_clock::time_point end; };