【更新】center-face检测功能调试完成

.vscode/c_cpp_properties.json

@@ -5,6 +5,7 @@
             "includePath": [
                 "${workspaceFolder}/**",
                 "${workspaceFolder}/3rd/**",
+                "${workspaceFolder}/models/**",
             ],
             "defines": [
                 "_DEBUG",

.vscode/settings.json

@@ -0,0 +1,62 @@
+{
+    "files.associations": {
+        "*.inc": "c",
+        "array": "cpp",
+        "atomic": "cpp",
+        "bit": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "complex": "cpp",
+        "condition_variable": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "list": "cpp",
+        "map": "cpp",
+        "set": "cpp",
+        "unordered_map": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "functional": "cpp",
+        "iterator": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "random": "cpp",
+        "ratio": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "fstream": "cpp",
+        "future": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iosfwd": "cpp",
+        "iostream": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "ostream": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "thread": "cpp",
+        "typeinfo": "cpp"
+    }
+}

.vscode/tasks.json

@@ -11,13 +11,13 @@
                 "-build_type",
                 "Release",
                 "-build_and_run",
-                "false",
+                "true",
                 "-platform",
                 "win"
             ],
             "group": {
                 "kind": "build",
-                "isDefault": false
+                "isDefault": true
             },
             "presentation": {
                 "reveal": "silent"
@@ -40,7 +40,7 @@
             ],
             "group": {
                 "kind": "build",
-                "isDefault": true
+                "isDefault": false
             },
             "presentation": {
                 "reveal": "silent"

CMakeLists.txt

@@ -11,7 +11,10 @@ endif()
 # 根据 platform 值设置 CMake 变量
 if(${CMAKE_BUILD_PLATFORM} STREQUAL "win")
     link_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/opencv/x64/vc15/lib)
+    link_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/mnn/x64/)
     include_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/opencv/include)
+    include_directories(${CMAKE_CURRENT_SOURCE_DIR}/3rd/win/mnn/include)
+    include_directories(${CMAKE_CURRENT_SOURCE_DIR}/models)
 else()
     message(FATAL_ERROR "不支持的平台: ${CMAKE_BUILD_PLATFORM}")
 endif()
@@ -19,13 +22,18 @@ endif()
 include_directories(src)
 
 aux_source_directory(./src DIR_SRCS)
+aux_source_directory(./src/util DIR_SRCS)
 aux_source_directory(./app DIR_SRCS)
 
 
 add_executable(Demo ${DIR_SRCS})
 
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  target_link_libraries(${PROJECT_NAME} opencv_world430d)
+    target_compile_options(${PROJECT_NAME} PRIVATE /MTd)
+    target_link_libraries(${PROJECT_NAME} opencv_world430d)
+    target_link_libraries(${PROJECT_NAME} MNN)
 else()
-  target_link_libraries(${PROJECT_NAME} opencv_world430)
+    target_compile_options(${PROJECT_NAME} PRIVATE /MT)
+    target_link_libraries(${PROJECT_NAME} opencv_world430)
+    target_link_libraries(${PROJECT_NAME} MNN)
 endif()

app/main.cpp

@@ -1,6 +1,27 @@
 #include <iostream>
+#include <opencv2/opencv.hpp>
+#include "util/TimeCount.h"
+#include "CenterFaceMnn.h"
 
 int32_t main(int32_t argc, char** argv) {
-    std::cout << "Hello, World!" << std::endl;
+    CenterFaceMnn::GetInstance();
+    std::vector<FaceInfo> faces;
+    cv::Mat img = cv::imread("F:/33.jpg");
+int32_t count = 0;
+re_test:
+    count++;
+    {
+        USE_TIME t(USE_TIME_US, "face_detect: ");
+        CenterFaceMnn::GetInstance()->Detect(img, faces, 1);
+    }
+    if (count < 10) {
+        goto re_test;
+    }
+    
+    for(auto& face : faces) {
+        cv::rectangle(img, cv::Rect(face.x1, face.y1, face.x2 - face.x1, face.y2 - face.y1), cv::Scalar(0, 0, 255), 2);
+    }
+    cv::imshow("test", img);
+    cv::waitKey(0);
     return 0;
 }

build-win-app.ps1

@@ -42,6 +42,6 @@ Write-Host "build done." -ForegroundColor Darkgreen
 
 if($build_and_run -eq "true"){
     Write-Host "################## APP Run ##################" -ForegroundColor Blue
-    $env:PATH += ";3rd\win\opencv\x64\vc15\bin"
+    $env:PATH += ";3rd\win\opencv\x64\vc15\bin;3rd\win\mnn\x64\"
     Start-Process -FilePath "build\\${build_type}\\Demo.exe"
 }

src/CenterFaceMnn.cpp

@@ -0,0 +1,235 @@
+#include "CenterFaceMnn.h"
+
+CenterFaceMnn* CenterFaceMnn::m_hInstance = nullptr;
+
+static const unsigned char centerface_model[] = {
+    #include "centerface_small_mnn.dat"
+};
+
+CenterFaceMnn* CenterFaceMnn::GetInstance(){
+	if (!m_hInstance){
+		m_hInstance = new CenterFaceMnn();
+	}
+	return m_hInstance;
+}
+
+CenterFaceMnn::CenterFaceMnn(){
+	m_detector = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromBuffer(centerface_model, sizeof(centerface_model)));
+
+	MNN::ScheduleConfig config;
+	MNN::BackendConfig backendConfig;
+	backendConfig.precision = MNN::BackendConfig::Precision_High;
+	backendConfig.power = MNN::BackendConfig::Power_High;
+	backendConfig.memory = MNN::BackendConfig::Memory_High;
+	config.backendConfig = &backendConfig;
+	config.type = MNN_FORWARD_CPU;
+	config.numThread = 4;
+	m_session = m_detector->createSession(config);
+
+	const float mean_vals[3] = { 127.5f, 127.5f, 127.5f };
+	const float norm_vals[3] = { 0.0078431373f, 0.0078431373f, 0.0078431373f };
+	::memcpy(m_img_config.mean, mean_vals, sizeof(mean_vals));
+	::memcpy(m_img_config.normal, norm_vals, sizeof(norm_vals));
+	m_img_config.sourceFormat = (MNN::CV::ImageFormat)2;
+	m_img_config.destFormat = (MNN::CV::ImageFormat)1;
+	m_img_config.filterType = (MNN::CV::Filter)(1);
+	m_img_config.wrap = (MNN::CV::Wrap)(1);
+
+	input_tensor = m_detector->getSessionInput(m_session, NULL);
+	hm_tensor = m_detector->getSessionOutput(m_session, "hm");
+	wh_tensor = m_detector->getSessionOutput(m_session, "wh");
+	reg_tensor = m_detector->getSessionOutput(m_session, "reg");
+	lm_tensor = m_detector->getSessionOutput(m_session, "lm");
+}
+
+CenterFaceMnn::~CenterFaceMnn(){
+	m_detector->releaseSession(m_session);
+}
+
+int CenterFaceMnn::Detect(const cv::Mat& img, std::vector<FaceInfo>& faces, float scale){
+	std::lock_guard<std::mutex> lock(m_mt);
+	faces.clear();
+	if (img.cols * img.rows == 0){
+		return -1;
+	}
+
+	if (scale <= 0 || scale > 1) {
+		scale = img.cols > img.rows ? 160.0f / img.cols : 160.0f / img.rows;
+	}
+	int resize_w = (int)(img.cols * scale) / 32 * 32;
+	int resize_h = (int)(img.rows * scale) / 32 * 32;
+	float scale_w = (float)img.cols / (float)resize_w;
+	float scale_h = (float)img.rows / (float)resize_h;
+
+
+	cv::Mat input;
+	cv::resize(img, input, cv::Size(resize_w, resize_h));
+	m_detector->resizeTensor(input_tensor, 1, 3, resize_h, resize_w);
+	m_detector->resizeSession(m_session);
+	//prepare data
+	std::shared_ptr<MNN::CV::ImageProcess> pretreat(MNN::CV::ImageProcess::create(m_img_config));
+	pretreat->convert(input.data, resize_w, resize_h, input.step[0], input_tensor);
+	m_detector->runSession(m_session);
+
+	MNN::Tensor tensor_hm(hm_tensor, MNN::Tensor::CAFFE);
+	MNN::Tensor tensor_wh(wh_tensor, MNN::Tensor::CAFFE);
+	MNN::Tensor tensor_reg(reg_tensor, MNN::Tensor::CAFFE);
+	MNN::Tensor tensor_lm(lm_tensor, MNN::Tensor::CAFFE);
+
+	hm_tensor->copyToHostTensor(&tensor_hm);
+	wh_tensor->copyToHostTensor(&tensor_wh);
+	reg_tensor->copyToHostTensor(&tensor_reg);
+	lm_tensor->copyToHostTensor(&tensor_lm);
+
+	std::vector<int> hm_shape = tensor_hm.shape();
+
+	float* heatmap = tensor_hm.host<float>();
+	float* wh = tensor_wh.host<float>();
+	float* reg = tensor_reg.host<float>();
+	float* lm = tensor_lm.host<float>();
+
+	Decode(heatmap, wh, reg, lm, hm_shape[2], hm_shape[3], faces, 0.4, 0.3);
+	for (int i = 0; i < faces.size(); i++){
+		FaceInfo& tmp = faces[i];
+		tmp.x1 *= scale_w;
+		tmp.x2 *= scale_w;
+		tmp.y1 *= scale_h;
+		tmp.y2 *= scale_h;
+
+		for (int j = 0; j < 5; j++){
+			tmp.landmarks[j * 2] *= scale_w;
+			tmp.landmarks[j * 2 + 1] *= scale_h;
+		}
+	}
+	return faces.size();
+}
+#include <omp.h>
+
+void CenterFaceMnn::Decode(float* heatmap, float* scale, float* offset, float* landmarks, int h, int w, std::vector<FaceInfo>& faces, float scoreThresh, float nmsThresh){
+	int fea_h = h;
+	int fea_w = w;
+	int d_w = w * 4;
+	int d_h = h * 4;
+	int spacial_size = fea_w * fea_h;
+
+	float *scale1 = (float*)(scale);
+	float *scale0 = scale1 + spacial_size;
+
+	float *offset1 = (float*)(offset);
+	float *offset0 = offset1 + spacial_size;
+
+	std::vector<int> ids = GetIds(heatmap, fea_h, fea_w, scoreThresh);
+
+	std::vector<FaceInfo> faces_tmp;
+	#pragma omp parallel for
+	for (int i = 0; i < ids.size() / 2; i++) {
+		int id_h = ids[2 * i];
+		int id_w = ids[2 * i + 1];
+		int index = id_h * fea_w + id_w;
+
+		float s0 = std::exp(scale0[index]) * 4;
+		float s1 = std::exp(scale1[index]) * 4;
+		float o0 = offset0[index];
+		float o1 = offset1[index];
+
+		float x1 = std::max(0., (id_w + o1 + 0.5) * 4 - s1 / 2);
+		float y1 = std::max(0., (id_h + o0 + 0.5) * 4 - s0 / 2);
+		float x2 = std::max(0., (id_w + o1 + 0.5) * 4 + s1 / 2);
+		float y2 = std::max(0., (id_h + o0 + 0.5) * 4 + s0 / 2);
+		x1 = std::min(x1, (float)d_w);
+		y1 = std::min(y1, (float)d_h);
+		x2 = std::min(x2, (float)d_w);
+		y2 = std::min(y2, (float)d_h);
+
+		FaceInfo facebox;
+		facebox.x1 = x1;
+		facebox.y1 = y1;
+		facebox.x2 = x2;
+		facebox.y2 = y2;
+		facebox.score = heatmap[index];
+
+		float box_w = x2 - x1;
+		float box_h = y2 - y1;
+
+		for (int j = 0; j < 5; j++) {
+			float *xmap = (float*)landmarks + (2 * j) * spacial_size;
+			float *ymap = (float*)landmarks + (2 * j + 1) * spacial_size;
+			facebox.landmarks[2 * j] = x1 + xmap[index] * s1;
+			facebox.landmarks[2 * j + 1] = y1 + ymap[index] * s0;
+		}
+		#pragma omp critical
+		{
+			faces_tmp.push_back(facebox);
+		}
+	}
+	NMS(faces_tmp, faces, nmsThresh);
+}
+
+void CenterFaceMnn::NMS(std::vector<FaceInfo>& input, std::vector<FaceInfo>& output, float nmsthreshold) {
+    std::sort(input.begin(), input.end(),
+        [](const FaceInfo& a, const FaceInfo& b) {
+            return a.score > b.score;
+        });
+
+    int box_num = input.size();
+
+    std::vector<int> merged(box_num, 0);
+
+    #pragma omp parallel for
+    for (int i = 0; i < box_num; i++) {
+        if (merged[i])
+            continue;
+
+        output.push_back(input[i]);
+
+        float h0 = input[i].y2 - input[i].y1 + 1;
+        float w0 = input[i].x2 - input[i].x1 + 1;
+
+        float area0 = h0 * w0;
+
+        for (int j = i + 1; j < box_num; j++) {
+            if (merged[j])
+                continue;
+
+            float inner_x0 = std::max(input[i].x1, input[j].x1);
+            float inner_y0 = std::max(input[i].y1, input[j].y1);
+
+            float inner_x1 = std::min(input[i].x2, input[j].x2);
+            float inner_y1 = std::min(input[i].y2, input[j].y2);
+
+            float inner_h = inner_y1 - inner_y0 + 1;
+            float inner_w = inner_x1 - inner_x0 + 1;
+
+            if (inner_h <= 0 || inner_w <= 0)
+                continue;
+
+            float inner_area = inner_h * inner_w;
+
+            float h1 = input[j].y2 - input[j].y1 + 1;
+            float w1 = input[j].x2 - input[j].x1 + 1;
+
+            float area1 = h1 * w1;
+
+            float score = inner_area / (area0 + area1 - inner_area);
+
+            if (score > nmsthreshold)
+                merged[j] = 1;
+        }
+    }
+}
+std::vector<int> CenterFaceMnn::GetIds(float *heatmap, int  h, int w, float thresh){
+	std::vector<int> ids;
+	for (int i = 0; i < h; i++) {
+		for (int j = 0; j < w; j++) {
+			if (heatmap[i*w + j] > thresh) {
+				ids.push_back(i);
+				ids.push_back(j);
+			}
+		}
+	}
+	return ids;
+}
+
+const std::string CenterFaceMnn::GetModelVer(){
+	return m_model_ver;
+}

src/CenterFaceMnn.h

@@ -0,0 +1,45 @@
+#ifndef CENTERFACE_H
+#define CENTERFACE_H
+#include <opencv2/opencv.hpp>
+#include <vector>
+#include <memory>
+#include <mutex>
+#include <MNN/Interpreter.hpp>
+#include <MNN/MNNDefine.h>
+#include <MNN/ImageProcess.hpp>
+#include <MNN/Tensor.hpp>
+#include "TypeInfo.h"
+
+class CenterFaceMnn{
+public:
+	static CenterFaceMnn* GetInstance();
+
+	int Detect(const cv::Mat& img, std::vector<FaceInfo>& faces, float scale = 0);
+
+	const std::string GetModelVer();
+private:
+	CenterFaceMnn();
+	~CenterFaceMnn();
+	void Decode(float* heatmap, float* scale, float* offset, float* landmarks, int h, int w, std::vector<FaceInfo>& faces, float scoreThresh, float nmsThresh);
+	void NMS(std::vector<FaceInfo>& input, std::vector<FaceInfo>& output, float nmsthreshold);
+	std::vector<int> GetIds(float *heatmap, int  h, int w, float thresh);
+
+private:
+	static CenterFaceMnn* m_hInstance;
+	std::mutex m_mt;
+
+	std::shared_ptr<MNN::Interpreter> m_detector;
+
+	MNN::Tensor* input_tensor;
+	MNN::Tensor* hm_tensor;
+	MNN::Tensor* wh_tensor;
+	MNN::Tensor* reg_tensor;
+	MNN::Tensor* lm_tensor;
+
+	MNN::Session* m_session;
+	MNN::CV::ImageProcess::Config m_img_config;
+
+	const std::string m_model_ver = "";
+};
+
+#endif

src/TypeInfo.cpp

@@ -0,0 +1 @@
+#include "TypeInfo.h"

src/TypeInfo.h

@@ -0,0 +1,32 @@
+#ifndef TYPE_INFO_H
+#define TYPE_INFO_H
+
+typedef enum _ret_t {
+    RET_OK = 0,                     // 操作成功
+    RET_AUTHOR,                     // 授权失败
+    RET_IMG,                        // 输入图片错误
+    RET_INVALID_FEATURE,            // 人脸特征数据无效
+    RET_INVALID_LANDMARK,           // 人脸Landmark数据无效
+} ret_t;
+
+// #define ERR_FACE_RECT			-10					//人脸框无效
+// #define ERR_INDEX_INVALID		-20					//索引无效
+
+struct FaceInfo {
+	float x1;                           //人脸框 左上x坐标
+	float y1;                           //人脸框 左上y坐标
+	float x2;                           //人脸框 右下x坐标
+	float y2;                           //人脸框 右下y坐标
+	float score;                        //人脸框 置信度
+	float landmarks[10];                //人脸框 关键点坐标
+
+	int GetWidth() {
+		return static_cast<int>(x2 - x1);
+	}
+
+	int GetHeight() {
+		return static_cast<int>(y2 - y1);
+	}
+};
+
+#endif  

src/util/TimeCount.cpp

@@ -1,20 +1,22 @@
 #include "TimeCount.h"
 #include <iostream>
 
-USE_TIME::USE_TIME(std::string str) :
+USE_TIME::USE_TIME(USE_TIME_UNIT unit, std::string str) :
-	strTmp(str)
+    strTmp(str),
-{
+    unit(unit){
-	start = std::chrono::system_clock::now();
+    start = std::chrono::system_clock::now();
 }
 
-USE_TIME::~USE_TIME()
+USE_TIME::~USE_TIME(){
-{
 	end = std::chrono::system_clock::now();
 
-	auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    if (unit == USE_TIME_MS) {
-	std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::milliseconds::period::num << "ms \t\n";
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-
+	    std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::milliseconds::period::num << "ms \t\n";
-	//auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+    }
-	//std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::microseconds::period::num << "ms \t\n";
+    else {
+        auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+	    std::cout << strTmp << "\t" << float(duration.count()*1.0) * std::chrono::microseconds::period::num << "us \t\n";
+    }
 }
 

src/util/TimeCount.h

@@ -4,14 +4,20 @@
 #include <string>
 #include <chrono>
 
+typedef enum {
+	USE_TIME_MS = 0,
+	USE_TIME_US = 1,
+}USE_TIME_UNIT;
+
 class USE_TIME{
 public:
-	USE_TIME(std::string str = "");
+	USE_TIME(USE_TIME_UNIT unit, std::string str = "");
 	~USE_TIME();
-
+    
 private:
-	std::string strTmp;
+    std::string strTmp;
-	std::chrono::system_clock::time_point start;
+	USE_TIME_UNIT unit;
+    std::chrono::system_clock::time_point start;
 	std::chrono::system_clock::time_point end;
 };