mirror of
https://github.com/opencv/opencv.git
synced 2025-06-10 11:03:03 +08:00
modify the detect and decode part for WeChat QRCodeDetector
This commit is contained in:
cswccc
parent
fee096f4e8
commit
f5ecad6dd4
@ -17,12 +17,3 @@ if(HAVE_QUIRC)
|
||||
ocv_include_directories(${QUIRC_INCLUDE})
|
||||
ocv_target_link_libraries(${the_module} quirc)
|
||||
endif()
|
||||
|
||||
if(CMAKE_VERSION VERSION_GREATER "3.11")
|
||||
find_package(Iconv QUIET)
|
||||
if(Iconv_FOUND)
|
||||
ocv_target_link_libraries(${the_module} Iconv::Iconv)
|
||||
else()
|
||||
ocv_target_compile_definitions(${the_module} PRIVATE "NO_ICONV=1")
|
||||
endif()
|
||||
endif()
|
||||
@ -868,15 +868,15 @@ public:
|
||||
/** @brief Initialize the QRCodeDetectorWeChat.
|
||||
*
|
||||
* Parameters allow to load _optional_ Detection and Super Resolution DNN model for better quality.
|
||||
* @param detection_model_path_ model file path for the detection model
|
||||
* @param super_resolution_model_path_ model file path for the super resolution model
|
||||
* @param detection_model_path model file path for the detection model
|
||||
* @param super_resolution_model_path model file path for the super resolution model
|
||||
* @param graphical_detector detector to be optimized
|
||||
* @param detector_iou_thres nms iou threshold for detection part
|
||||
* @param score_thres score threshold for detection part
|
||||
* @param reference_size the length of the image to align during pre-processing before detection
|
||||
*/
|
||||
CV_WRAP QRCodeDetectorWeChat(const std::string& detection_model_path_ = "",
|
||||
const std::string& super_resolution_model_path_ = "",
|
||||
CV_WRAP QRCodeDetectorWeChat(const std::string& detection_model_path = "",
|
||||
const std::string& super_resolution_model_path = "",
|
||||
Ptr<GraphicalCodeDetector> graphical_detector = cv::makePtr<QRCodeDetectorAruco>(),
|
||||
const float detector_iou_thres = 0.6,
|
||||
const float score_thres = 0.3,
|
||||
|
||||
@ -1019,7 +1019,7 @@ class QRDecode
|
||||
{
|
||||
public:
|
||||
QRDecode(bool useAlignmentMarkers);
|
||||
void init(const Mat &src, const vector<Point2f> &points);
|
||||
void init(const Mat &src, const vector<Point2f> &points, float sr_scale_=1.f);
|
||||
Mat getIntermediateBarcode() { return intermediate; }
|
||||
Mat getStraightBarcode() { return straight; }
|
||||
size_t getVersion() { return version; }
|
||||
@ -1028,6 +1028,7 @@ public:
|
||||
bool curvedDecodingProcess();
|
||||
vector<Point2f> alignment_coords;
|
||||
float coeff_expansion = 1.f;
|
||||
float sr_scale = 1.f;
|
||||
vector<Point2f> getOriginalPoints() {return original_points;}
|
||||
bool useAlignmentMarkers;
|
||||
|
||||
@ -1137,7 +1138,7 @@ float static getMinSideLen(const vector<Point2f> &points) {
|
||||
}
|
||||
|
||||
|
||||
void QRDecode::init(const Mat &src, const vector<Point2f> &points)
|
||||
void QRDecode::init(const Mat &src, const vector<Point2f> &points, float sr_scale_)
|
||||
{
|
||||
CV_TRACE_FUNCTION();
|
||||
vector<Point2f> bbox = points;
|
||||
@ -1150,6 +1151,7 @@ void QRDecode::init(const Mat &src, const vector<Point2f> &points)
|
||||
version_size = 0;
|
||||
test_perspective_size = max(getMinSideLen(points)+1.f, 251.f);
|
||||
result_info = "";
|
||||
sr_scale = sr_scale_;
|
||||
}
|
||||
|
||||
inline double QRDecode::pointPosition(Point2f a, Point2f b , Point2f c)
|
||||
@ -3625,6 +3627,10 @@ void QRDetectMulti::findQRCodeContours(vector<Point2f>& tmp_localization_points,
|
||||
int count_contours = num_qrcodes;
|
||||
if (all_contours_points.size() < size_t(num_qrcodes))
|
||||
count_contours = (int)all_contours_points.size();
|
||||
|
||||
// If the contours cannot be found, return. Otherwise kmeans will get error.
|
||||
if (all_contours_points.size() == 0)
|
||||
return;
|
||||
kmeans(all_contours_points, count_contours, qrcode_labels,
|
||||
TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 10, 0.1),
|
||||
count_contours, KMEANS_PP_CENTERS, clustered_localization_points);
|
||||
@ -4008,9 +4014,9 @@ class ParallelDecodeProcess : public ParallelLoopBody
|
||||
{
|
||||
public:
|
||||
ParallelDecodeProcess(Mat& inarr_, vector<QRDecode>& qrdec_, vector<std::string>& decoded_info_,
|
||||
vector<Mat>& straight_barcode_, vector< vector< Point2f > >& src_points_, std::shared_ptr<SuperScale> sr_ = nullptr)
|
||||
vector<Mat>& straight_barcode_, vector< vector< Point2f > >& src_points_, std::shared_ptr<SuperScale> sr_ = nullptr, bool use_sr_model_ = false)
|
||||
: inarr(inarr_), qrdec(qrdec_), decoded_info(decoded_info_)
|
||||
, straight_barcode(straight_barcode_), src_points(src_points_), sr_(sr_)
|
||||
, straight_barcode(straight_barcode_), src_points(src_points_), sr(sr_), use_sr_model(use_sr_model_)
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
@ -4018,13 +4024,13 @@ public:
|
||||
{
|
||||
for (int i = range.start; i < range.end; i++)
|
||||
{
|
||||
if (sr_ != nullptr) {
|
||||
// Modified the input image for decoding,
|
||||
// by extracting the part containing the QR code and adjusting the resolution.
|
||||
// Only QRCodeDetectorWeChat() will call this decoding attempt.
|
||||
if (sr != nullptr) {
|
||||
int width = inarr.size().width, height = inarr.size().height;
|
||||
|
||||
int min_x = src_points[i][0].x;
|
||||
int min_y = src_points[i][0].y;
|
||||
int max_x = src_points[i][0].x;
|
||||
int max_y = src_points[i][0].y;
|
||||
int min_x = src_points[i][0].x, min_y = src_points[i][0].y;
|
||||
int max_x = src_points[i][0].x, max_y = src_points[i][0].y;
|
||||
for (const auto& point : src_points[i]) {
|
||||
min_x = min_x > point.x ? point.x : min_x;
|
||||
min_y = min_y > point.y ? point.y : min_y;
|
||||
@ -4035,30 +4041,33 @@ public:
|
||||
max_x = min(max(0, max_x), width);
|
||||
min_y = min(max(0, min_y), height);
|
||||
max_y = min(max(0, max_y), height);
|
||||
|
||||
|
||||
cv::Rect cropRect(min_x, min_y, max_x - min_x, max_y - min_y);
|
||||
|
||||
auto scale_list = sr_->getScaleList(max_y - min_y, max_x - min_x);
|
||||
auto scale_list = sr->getScaleList(max_x - min_x, max_y - min_y);
|
||||
Mat crop_image = inarr(cropRect).clone();
|
||||
for (auto cur_scale : scale_list) {
|
||||
Mat scaled_img;
|
||||
// If a super-resolution model is loaded, parallelism cannot be performed
|
||||
if (use_sr_model)
|
||||
std::lock_guard<std::mutex> lock(sr_mutex);
|
||||
Mat scaled_img = sr_->ProcessImageScale(crop_image, cur_scale, true);
|
||||
sr->processImageScale(crop_image, scaled_img, cur_scale, use_sr_model);
|
||||
vector<Point2f> points;
|
||||
for (const auto& point : src_points[i])
|
||||
points.push_back(Point2f((point.x - min_x) * cur_scale, (point.y - min_y) * cur_scale));
|
||||
|
||||
qrdec[i].init(scaled_img, points);
|
||||
qrdec[i].init(scaled_img, points, cur_scale);
|
||||
bool ok = qrdec[i].straightDecodingProcess();
|
||||
if (ok)
|
||||
{
|
||||
decoded_info[i] = qrdec[i].getDecodeInformation();
|
||||
straight_barcode[i] = qrdec[i].getStraightBarcode();
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (decoded_info[i].empty())
|
||||
decoded_info[i] = "";
|
||||
}
|
||||
}
|
||||
// The old decoding attempt.
|
||||
else {
|
||||
qrdec[i].init(inarr, src_points[i]);
|
||||
bool ok = qrdec[i].straightDecodingProcess();
|
||||
@ -4102,7 +4111,8 @@ private:
|
||||
vector<std::string>& decoded_info;
|
||||
vector<Mat>& straight_barcode;
|
||||
vector< vector< Point2f > >& src_points;
|
||||
std::shared_ptr<SuperScale> sr_;
|
||||
std::shared_ptr<SuperScale> sr;
|
||||
bool use_sr_model;
|
||||
mutable std::mutex sr_mutex;
|
||||
};
|
||||
|
||||
@ -4198,8 +4208,20 @@ bool ImplContour::decodeMulti(
|
||||
updateQrCorners.resize(src_points.size()*4ull);
|
||||
for (size_t i = 0ull; i < src_points.size(); i++) {
|
||||
alignmentMarkers[i] = qrdec[i].alignment_coords;
|
||||
for (size_t j = 0ull; j < 4ull; j++)
|
||||
updateQrCorners[i*4ull+j] = qrdec[i].getOriginalPoints()[j] * qrdec[i].coeff_expansion;
|
||||
for (size_t j = 0ull; j < 4ull; j++) {
|
||||
updateQrCorners[i*4ull+j] = qrdec[i].getOriginalPoints()[j] * qrdec[i].coeff_expansion / qrdec[i].sr_scale;
|
||||
if (sr_ != nullptr) {
|
||||
int min_x = src_points[i][0].x, min_y = src_points[i][0].y;
|
||||
for (const auto& point : src_points[i]) {
|
||||
min_x = min_x > point.x ? point.x : min_x;
|
||||
min_y = min_y > point.y ? point.y : min_y;
|
||||
}
|
||||
min_x = min(max(0, min_x), inarr.size().width);
|
||||
min_y = min(max(0, min_y), inarr.size().height);
|
||||
updateQrCorners[i*4ull+j].x += min_x;
|
||||
updateQrCorners[i*4ull+j].y += min_y;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!decoded_info.empty())
|
||||
return true;
|
||||
@ -4761,13 +4783,11 @@ public:
|
||||
|
||||
PimplQRWeChat(std::shared_ptr<GraphicalCodeDetector> graphical_detector)
|
||||
: graphical_detector_(std::move(graphical_detector)) {
|
||||
sr_ = std::make_shared<SuperScale>();
|
||||
detector_ = std::make_shared<Detector>();
|
||||
sr_ = std::make_shared<SuperScale>();
|
||||
}
|
||||
|
||||
bool detectMulti(InputArray img, OutputArray points) const override;
|
||||
bool detectAndDecodeMulti(InputArray img, std::vector<cv::String>& decoded_info, OutputArray points,
|
||||
OutputArrayOfArrays straight_qrcode) const override;
|
||||
};
|
||||
|
||||
bool PimplQRWeChat::detectMulti(InputArray img, OutputArray points) const
|
||||
@ -4778,70 +4798,107 @@ bool PimplQRWeChat::detectMulti(InputArray img, OutputArray points) const
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<DetectInfo> _detect_results;
|
||||
|
||||
if (use_det_model_) {
|
||||
detector_->detect(gray, _detect_results);
|
||||
std::vector<DetectInfo> detect_results;
|
||||
std::vector<Rect> crop_rects;
|
||||
detector_->detect(gray, detect_results);
|
||||
|
||||
vector<Point2f> results;
|
||||
for (size_t k = 0; k < _detect_results.size(); k++) {
|
||||
int x0 = _detect_results[k].x, y0 = _detect_results[k].y;
|
||||
int width = _detect_results[k].width, height = _detect_results[k].height;
|
||||
for (size_t k = 0; k < detect_results.size(); k++) {
|
||||
int x0 = detect_results[k].x, y0 = detect_results[k].y;
|
||||
int width = detect_results[k].width, height = detect_results[k].height;
|
||||
int x2 = x0 + width - 1, y2 = y0 + width -1;
|
||||
|
||||
int padx = max(0.5f * width, static_cast<float>(20));
|
||||
int pady = max(0.5f * height, static_cast<float>(20));
|
||||
int crop_x_ = max(x0 - padx, 0);
|
||||
int crop_y_ = max(y0 - pady, 0);
|
||||
int end_x = min(x2 + padx, gray.cols);
|
||||
int end_y = min(y2 + pady, gray.rows);
|
||||
cv::Rect cropRect(crop_x_, crop_y_, end_x - crop_x_, end_y - crop_y_);
|
||||
int crop_x_ = max(x0 - padx, 0), crop_y_ = max(y0 - pady, 0);
|
||||
int end_x = min(x2 + padx, gray.cols), end_y = min(y2 + pady, gray.rows);
|
||||
crop_rects.push_back(Rect(crop_x_, crop_y_, end_x - crop_x_, end_y - crop_y_));
|
||||
}
|
||||
|
||||
vector<vector<Point2f> > crop_images_corners;
|
||||
vector<Point2f> results;
|
||||
for (auto& rect : crop_rects) {
|
||||
Mat crop_image = gray(rect).clone();
|
||||
int width = rect.width, height = rect.height;
|
||||
auto scale_lists = sr_->getScaleList(width, height);
|
||||
bool flag = false;
|
||||
for (auto cur_scale : scale_lists) {
|
||||
Mat scaled_image;
|
||||
sr_->processImageScale(crop_image, scaled_image, cur_scale, use_sr_model_);
|
||||
vector<Point2f> corners;
|
||||
|
||||
auto scale_list = sr_->getScaleList(end_y - crop_y_, end_x - crop_x_);
|
||||
Mat crop_image = gray(cropRect).clone();
|
||||
for (auto cur_scale : scale_list) {
|
||||
Mat scaled_img = sr_->ProcessImageScale(crop_image, cur_scale, use_sr_model_);
|
||||
if (graphical_detector_->detectMulti(scaled_img, corners)) {
|
||||
for (size_t i = 0; i < corners.size(); i += 4) {
|
||||
bool flag = true;
|
||||
std::vector<Point2f> pts_i, pts_j;
|
||||
for (size_t p = 0; p < 4; p++) {
|
||||
int x = corners[i+p].x/cur_scale + crop_x_, y = corners[i+p].y/cur_scale + crop_y_;
|
||||
pts_i.push_back(Point2f(min(max(x, 0), gray.cols-1), min(max(y, 0), gray.rows-1)));
|
||||
if (graphical_detector_->detectMulti(scaled_image, corners)) {
|
||||
if (corners.size() % 4 != 0)
|
||||
continue;
|
||||
for (size_t i = 0; i < corners.size(); i++) {
|
||||
corners[i].x = corners[i].x/cur_scale + rect.x;
|
||||
corners[i].x = min(max(corners[i].x, 0.0f), gray.cols * 1.0f);
|
||||
corners[i].y = corners[i].y/cur_scale + rect.y;
|
||||
corners[i].y = min(max(corners[i].y, 0.0f), gray.rows * 1.0f);
|
||||
}
|
||||
float area1 = cv::contourArea(pts_i);
|
||||
|
||||
for (size_t j = 0; j < results.size(); j+= 4) {
|
||||
pts_j.clear();
|
||||
for (size_t p = 0; p < 4; p++)
|
||||
pts_j.push_back(Point2f(results[j+p].x, results[j+p].y));
|
||||
|
||||
float area2 = cv::contourArea(pts_j);
|
||||
float intersectionArea = 0.0;
|
||||
std::vector<cv::Point2f> intersection;
|
||||
cv::rotatedRectangleIntersection(cv::minAreaRect(pts_i), cv::minAreaRect(pts_j), intersection);
|
||||
|
||||
if (!intersection.empty())
|
||||
intersectionArea = cv::contourArea(intersection);
|
||||
|
||||
double iou = intersectionArea / (area1 + area2 - intersectionArea);
|
||||
double cover = intersectionArea / min(area1, area2);
|
||||
if (iou > 0.7 || cover > 0.96) {
|
||||
flag = false;
|
||||
crop_images_corners.push_back(corners);
|
||||
flag = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (flag) {
|
||||
for (auto p : pts_i)
|
||||
if (!flag) {
|
||||
crop_images_corners.push_back(vector<Point2f>());
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < crop_images_corners.size(); j++) {
|
||||
auto& corners = crop_images_corners[j];
|
||||
if (corners.size() != 0) {
|
||||
for (size_t p_1 = 0; p_1 < corners.size(); p_1 += 4) {
|
||||
if (corners[p_1].x < 0) continue;
|
||||
Point2f topLeft, bottomRight;
|
||||
topLeft.x = min(min(corners[p_1].x, corners[p_1+1].x), min(corners[p_1+2].x, corners[p_1+3].x));
|
||||
topLeft.y = min(min(corners[p_1].y, corners[p_1+1].y), min(corners[p_1+2].y, corners[p_1+3].y));
|
||||
bottomRight.x = max(max(corners[p_1].x, corners[p_1+1].x), max(corners[p_1+2].x, corners[p_1+3].x));
|
||||
bottomRight.y = max(max(corners[p_1].y, corners[p_1+1].y), max(corners[p_1+2].y, corners[p_1+3].y));
|
||||
float area1 = (bottomRight.x - topLeft.x) * (bottomRight.y - topLeft.y);
|
||||
for (size_t i = j+1; i < crop_images_corners.size(); i++) {
|
||||
vector<Point2f>& corners_i = crop_images_corners[i];
|
||||
for (size_t p_i = 0; p_i < corners_i.size(); p_i += 4) {
|
||||
if (corners_i[p_i].x < 0) continue;
|
||||
Point2f topLeft_i, bottomRight_i;
|
||||
topLeft_i.x = min(min(corners_i[p_i].x, corners_i[p_i+1].x), min(corners_i[p_i+2].x, corners_i[p_i+3].x));
|
||||
topLeft_i.y = min(min(corners_i[p_i].y, corners_i[p_i+1].y), min(corners_i[p_i+2].y, corners_i[p_i+3].y));
|
||||
bottomRight_i.x = max(max(corners_i[p_i].x, corners_i[p_i+1].x), max(corners_i[p_i+2].x, corners_i[p_i+3].x));
|
||||
bottomRight_i.y = max(max(corners_i[p_i].y, corners_i[p_i+1].y), max(corners_i[p_i+2].y, corners_i[p_i+3].y));
|
||||
|
||||
float interLeft = std::max(topLeft.x, topLeft_i.x), interTop = std::max(topLeft.y, topLeft_i.y);
|
||||
float interRight = std::min(bottomRight.x, bottomRight_i.x), interBottom = std::min(bottomRight.y, bottomRight_i.y);
|
||||
float interWidth = max(interRight - interLeft, 0.0f), interHeight = max(interBottom - interTop, 0.0f);
|
||||
float interArea = interWidth * interHeight;
|
||||
float area2 = (bottomRight_i.x - topLeft_i.x) * (bottomRight_i.y - topLeft_i.y);
|
||||
|
||||
float iou = interArea / (area1 + area2 - interArea);
|
||||
if (iou > 0.5) {
|
||||
if (area2 < area1) {
|
||||
topLeft = topLeft_i;
|
||||
bottomRight = bottomRight_i;
|
||||
area1 = area2;
|
||||
corners[p_1] = corners_i[p_i];
|
||||
corners[p_1+1] = corners_i[p_i+1];
|
||||
corners[p_1+2] = corners_i[p_i+2];
|
||||
corners[p_1+3] = corners_i[p_i+3];
|
||||
}
|
||||
corners_i[p_i].x = -1;
|
||||
corners_i[p_i+1].x = -1;
|
||||
corners_i[p_i+2].x = -1;
|
||||
corners_i[p_i+3].x = -1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto p : corners)
|
||||
if (p.x >= 0) {
|
||||
results.push_back(p);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.size() >= 4) {
|
||||
updatePointsResult(points, results);
|
||||
@ -4855,22 +4912,8 @@ bool PimplQRWeChat::detectMulti(InputArray img, OutputArray points) const
|
||||
}
|
||||
}
|
||||
|
||||
bool PimplQRWeChat::detectAndDecodeMulti(
|
||||
InputArray img,
|
||||
CV_OUT std::vector<cv::String>& decoded_info,
|
||||
OutputArray points_,
|
||||
OutputArrayOfArrays straight_qrcode
|
||||
) const {
|
||||
bool ok = detectMulti(img, points_);
|
||||
|
||||
if (ok)
|
||||
return decodeMulti(img, points_, decoded_info, straight_qrcode);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
QRCodeDetectorWeChat::QRCodeDetectorWeChat(const std::string& detection_model_path_,
|
||||
const std::string& super_resolution_model_path_,
|
||||
QRCodeDetectorWeChat::QRCodeDetectorWeChat(const std::string& detection_model_path,
|
||||
const std::string& super_resolution_model_path,
|
||||
Ptr<GraphicalCodeDetector> graphical_detector,
|
||||
const float detector_iou_thres,
|
||||
const float score_thres,
|
||||
@ -4878,15 +4921,15 @@ QRCodeDetectorWeChat::QRCodeDetectorWeChat(const std::string& detection_model_pa
|
||||
Ptr<PimplQRWeChat> p_ = std::make_shared<PimplQRWeChat>(std::move(graphical_detector));
|
||||
p = p_;
|
||||
|
||||
if (!super_resolution_model_path_.empty()) {
|
||||
CV_Assert(utils::fs::exists(super_resolution_model_path_));
|
||||
int res = p_->sr_->init(super_resolution_model_path_);
|
||||
if (!super_resolution_model_path.empty()) {
|
||||
CV_Assert(utils::fs::exists(super_resolution_model_path));
|
||||
int res = p_->sr_->init(super_resolution_model_path);
|
||||
CV_Assert(res == 0);
|
||||
p_->use_sr_model_ = true;
|
||||
}
|
||||
if (!detection_model_path_.empty()) {
|
||||
CV_Assert(utils::fs::exists(detection_model_path_));
|
||||
int res = p_->detector_->init(detection_model_path_);
|
||||
if (!detection_model_path.empty()) {
|
||||
CV_Assert(utils::fs::exists(detection_model_path));
|
||||
int res = p_->detector_->init(detection_model_path);
|
||||
CV_Assert(res == 0);
|
||||
p_->use_det_model_ = true;
|
||||
}
|
||||
|
||||
@ -10,37 +10,24 @@
|
||||
#define CLIP(x, x1, x2) (std::fmax<float>)(x1, (std::fmin<float>)(x, x2))
|
||||
|
||||
namespace cv {
|
||||
int Detector::init(const std::string &det_path)
|
||||
{
|
||||
try
|
||||
{
|
||||
int Detector::init(const std::string &det_path)
|
||||
{
|
||||
dnn::Net network = dnn::readNetFromONNX(det_path);
|
||||
if(network.empty())
|
||||
{
|
||||
return -101;
|
||||
}
|
||||
this->qbar_detector = std::make_shared<dnn::Net>(network);
|
||||
}
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
printf("%s", e.what());
|
||||
return -3;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int Detector::detect(const Mat &image,std::vector<DetectInfo> &bboxes)
|
||||
{
|
||||
bool Detector::detect(const Mat &image,std::vector<DetectInfo> &bboxes)
|
||||
{
|
||||
Mat input_blob;
|
||||
int ret = this->pre_process_det(image,input_blob);
|
||||
bool ret = this->pre_process_det(image,input_blob);
|
||||
|
||||
input_blob = dnn::blobFromImage(input_blob);
|
||||
|
||||
std::vector<Mat> outputs;
|
||||
|
||||
//forward
|
||||
|
||||
this->qbar_detector->setInput(input_blob, "input");
|
||||
|
||||
std::vector<std::string> output_names;
|
||||
@ -50,18 +37,15 @@ namespace cv {
|
||||
output_names.push_back("dis_pred_stride_16");
|
||||
output_names.push_back("cls_pred_stride_32");
|
||||
output_names.push_back("dis_pred_stride_32");
|
||||
|
||||
this->qbar_detector->forward(outputs, output_names);
|
||||
|
||||
if(outputs.size()==0)
|
||||
{
|
||||
return -103;
|
||||
}
|
||||
|
||||
bboxes.clear();
|
||||
if (outputs.size() == 0)
|
||||
return false;
|
||||
|
||||
std::vector<BoxInfo> det_bboxes;
|
||||
ret = this->post_process_det(outputs,input_blob.size[3],input_blob.size[2],det_bboxes);
|
||||
if (!ret)
|
||||
if (ret)
|
||||
{
|
||||
for (size_t i = 0; i < det_bboxes.size(); i++)
|
||||
{
|
||||
@ -77,30 +61,27 @@ namespace cv {
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
int Detector::pre_process_det(const Mat &image,Mat &out_blob)
|
||||
{
|
||||
int reference_size = this->reference_size;
|
||||
bool Detector::pre_process_det(const Mat &image,Mat &out_blob)
|
||||
{
|
||||
int set_size = this->reference_size;
|
||||
int setWidth, setHeight;
|
||||
|
||||
if (image.cols <= reference_size && image.rows <= reference_size) {
|
||||
if (image.cols >= image.rows)
|
||||
{
|
||||
setWidth = reference_size;
|
||||
setHeight = std::ceil(image.rows * 1.0 * reference_size / image.cols);
|
||||
if (image.cols <= set_size && image.rows <= set_size) {
|
||||
if (image.cols >= image.rows) {
|
||||
setWidth = set_size;
|
||||
setHeight = std::ceil(image.rows * 1.0 * set_size / image.cols);
|
||||
}
|
||||
else
|
||||
{
|
||||
setHeight = reference_size;
|
||||
setWidth = std::ceil(image.cols * 1.0 * reference_size / image.rows);
|
||||
else {
|
||||
setHeight = set_size;
|
||||
setWidth = std::ceil(image.cols * 1.0 * set_size / image.rows);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
float resizeRatio = sqrt(image.cols * image.rows * 1.0 / (reference_size * reference_size));
|
||||
else {
|
||||
float resizeRatio = sqrt(image.cols * image.rows * 1.0 / (set_size * set_size));
|
||||
setWidth = image.cols / resizeRatio;
|
||||
setHeight = image.rows / resizeRatio;
|
||||
}
|
||||
@ -111,29 +92,25 @@ namespace cv {
|
||||
resize(image,out_blob,Size(setWidth, setHeight));
|
||||
out_blob.convertTo(out_blob,CV_32FC1,1.0f/128.0f);
|
||||
out_blob = out_blob - Scalar(1.0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
int Detector::post_process_det(std::vector<Mat> outputs,float inputWidth,float inputHeight,std::vector<BoxInfo>& dets)
|
||||
{
|
||||
bool Detector::post_process_det(std::vector<Mat> outputs,float inputWidth,float inputHeight,std::vector<BoxInfo>& dets)
|
||||
{
|
||||
// step 1: extract
|
||||
std::vector<std::vector<int>> outShape(6);
|
||||
float *outPtr[6];
|
||||
for (int i = 0; i < 6; i++)
|
||||
{
|
||||
|
||||
outPtr[i] = reinterpret_cast<float *>(outputs[i].data);
|
||||
if (outPtr[i] == NULL)
|
||||
return -1;
|
||||
return false;
|
||||
|
||||
for(int j = 0;j<outputs[i].dims;j++)
|
||||
{
|
||||
outShape[i].push_back(outputs[i].size[j]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// step2: decode s8\s16\s32
|
||||
std::vector<std::vector<BoxInfo>> results;
|
||||
int numClasses = outShape[0][2];
|
||||
@ -143,11 +120,10 @@ namespace cv {
|
||||
this->decode_infer(outPtr[2], outPtr[3], 16, results, outShape[2], outShape[3], score_thres,inputWidth, inputHeight);
|
||||
this->decode_infer(outPtr[4], outPtr[5], 32, results, outShape[4], outShape[5], score_thres,inputWidth, inputHeight);
|
||||
// step3: nms
|
||||
|
||||
std::vector<BoxInfo> rets;
|
||||
for (size_t i = 0; i < results.size(); i++)
|
||||
{
|
||||
this->nms(results[i], iou_thres); // 0.5
|
||||
this->nms(results[i], iou_thres);
|
||||
for (auto & box : results[i])
|
||||
{
|
||||
if (box.score > score_thres)
|
||||
@ -158,24 +134,22 @@ namespace cv {
|
||||
}
|
||||
// step4: multi-class nms to gen BoxMultiInfo results
|
||||
this->multiclass_nms(rets, dets, iou_thres, inputWidth, inputHeight);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void Detector::multiclass_nms(std::vector<BoxInfo> &input_boxes, std::vector<BoxInfo> &output_boxes, float thr, int inputWidth, int inputHeight)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
void Detector::multiclass_nms(std::vector<BoxInfo> &input_boxes, std::vector<BoxInfo> &output_boxes, float thr, int inputWidth, int inputHeight)
|
||||
{
|
||||
if (input_boxes.size() <= 0) return;
|
||||
output_boxes.clear();
|
||||
|
||||
std::vector<bool> skip(input_boxes.size());
|
||||
for (size_t i = 0; i < input_boxes.size(); ++i)
|
||||
{
|
||||
skip[i] = false;
|
||||
}
|
||||
|
||||
// merge overlapped results
|
||||
for (size_t i = 0; i < input_boxes.size(); ++i)
|
||||
{
|
||||
int labeli = input_boxes[i].label;
|
||||
if (skip[i])
|
||||
continue;
|
||||
|
||||
@ -189,7 +163,6 @@ namespace cv {
|
||||
|
||||
for (size_t j = i + 1; j < input_boxes.size(); ++j)
|
||||
{
|
||||
int labelj = input_boxes[j].label;
|
||||
if (skip[j])
|
||||
continue;
|
||||
{
|
||||
@ -206,12 +179,14 @@ namespace cv {
|
||||
float cover = inter / (std::min)(area_i, area_j);
|
||||
|
||||
if (ovr > thr || cover > 0.96){
|
||||
box.x1 = (std::min)(box.x1, input_boxes[j].x1);
|
||||
box.y1 = (std::min)(box.y1, input_boxes[j].y1);
|
||||
box.x2 = (std::max)(box.x2, input_boxes[j].x2);
|
||||
box.y2 = (std::max)(box.y2, input_boxes[j].y2);
|
||||
box.score = (std::max)(box.score, input_boxes[j].score);
|
||||
if (input_boxes[j].score > box.score) {
|
||||
box.x1 = input_boxes[j].x1;
|
||||
box.y1 = input_boxes[j].y1;
|
||||
box.x2 = input_boxes[j].x2;
|
||||
box.y2 = input_boxes[j].y2;
|
||||
box.score = input_boxes[j].score;
|
||||
box.label = 5;
|
||||
}
|
||||
skip[j] = true;
|
||||
}
|
||||
}
|
||||
@ -222,10 +197,10 @@ namespace cv {
|
||||
box.y2 = CLIP(box.y2 / inputHeight, 0, 1);
|
||||
output_boxes.push_back(box);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Detector::nms(std::vector<BoxInfo>& input_boxes, float NMS_THRESH)
|
||||
{
|
||||
void Detector::nms(std::vector<BoxInfo>& input_boxes, float NMS_THRESH)
|
||||
{
|
||||
if (input_boxes.size() <= 1) return;
|
||||
std::sort(input_boxes.begin(), input_boxes.end(), [](BoxInfo a, BoxInfo b) { return a.score > b.score; });
|
||||
std::vector<float> vArea(input_boxes.size());
|
||||
@ -253,7 +228,7 @@ namespace cv {
|
||||
input_boxes.erase(input_boxes.begin() + j);
|
||||
vArea.erase(vArea.begin() + j);
|
||||
}
|
||||
else if (cover >= 0.96) // qiantao
|
||||
else if (cover >= 0.96)
|
||||
{
|
||||
if (vArea[i] > vArea[j])
|
||||
{
|
||||
@ -276,21 +251,21 @@ namespace cv {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline float fast_exp(float x)
|
||||
{
|
||||
inline float fast_exp(float x)
|
||||
{
|
||||
union {
|
||||
uint32_t i;
|
||||
float f;
|
||||
} v{};
|
||||
v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f);
|
||||
return v.f;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename _Tp>
|
||||
int activation_function_softmax(const _Tp* src, _Tp* dst, int length)
|
||||
{
|
||||
template<typename _Tp>
|
||||
int activation_function_softmax(const _Tp* src, _Tp* dst, int length)
|
||||
{
|
||||
const _Tp alpha = *(std::max_element)(src, src + length);
|
||||
_Tp denominator{ 0 };
|
||||
|
||||
@ -305,9 +280,10 @@ namespace cv {
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
void Detector::decode_infer(float *clsPred, float *disPred, int stride, std::vector<std::vector<BoxInfo>> &results, const std::vector<int> &outShapeCls, const std::vector<int> &outShapeDis, float scoreThres,float inputWidth,float inputHeight)
|
||||
{
|
||||
}
|
||||
|
||||
void Detector::decode_infer(float *clsPred, float *disPred, int stride, std::vector<std::vector<BoxInfo>> &results, const std::vector<int> &outShapeCls, const std::vector<int> &outShapeDis, float scoreThres,float inputWidth,float inputHeight)
|
||||
{
|
||||
int numClasses = outShapeCls[2];
|
||||
int RegMax = (outShapeDis[2] / 4) - 1;
|
||||
int lenFeat = outShapeCls[1];
|
||||
@ -322,10 +298,9 @@ namespace cv {
|
||||
float score = pScore[idx * numClasses + label];
|
||||
if (score > scoreThres)
|
||||
{
|
||||
const float * bboxPred = disPred + idx*outShapeDis[2]; //(RegMax + 1)*4;
|
||||
const float * bboxPred = disPred + idx*outShapeDis[2];
|
||||
int row = idx / featWidth;
|
||||
int col = idx % featWidth;
|
||||
// disPred2Bbox
|
||||
float centerX = col * stride;
|
||||
float centerY = row * stride;
|
||||
std::vector<float> disPred_;
|
||||
@ -350,5 +325,5 @@ namespace cv {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace cv
|
||||
@ -45,14 +45,14 @@ namespace cv {
|
||||
Detector(){};
|
||||
~Detector(){};
|
||||
int init(const std::string &config_path);
|
||||
int detect(const Mat &image,std::vector<DetectInfo> &bboxes);
|
||||
void setReferenceSize(int reference_size) {this->reference_size = reference_size;}
|
||||
void setScoreThres(float score_thres) {this->score_thres = score_thres;}
|
||||
void setIouThres(float iou_thres) {this->iou_thres = iou_thres;}
|
||||
bool detect(const Mat &image,std::vector<DetectInfo> &bboxes);
|
||||
void setReferenceSize(int reference_size_) {this->reference_size = reference_size_;}
|
||||
void setScoreThres(float score_thres_) {this->score_thres = score_thres_;}
|
||||
void setIouThres(float iou_thres_) {this->iou_thres = iou_thres_;}
|
||||
|
||||
private:
|
||||
int post_process_det(std::vector<Mat> outputs,float inputWidth,float inputHeight,std::vector<BoxInfo>& dets);
|
||||
int pre_process_det(const Mat &image,Mat &out_blob);
|
||||
bool post_process_det(std::vector<Mat> outputs,float inputWidth,float inputHeight,std::vector<BoxInfo>& dets);
|
||||
bool pre_process_det(const Mat &image,Mat &out_blob);
|
||||
void multiclass_nms(std::vector<BoxInfo> &input_boxes, std::vector<BoxInfo> &output_boxes, float thr, int inputWidth, int inputHeight);
|
||||
void decode_infer(float *clsPred, float *disPred, int stride, std::vector<std::vector<BoxInfo>> &results, const std::vector<int> &outShapeCls, const std::vector<int> &outShapeDis, float scoreThres,float inputHeight,float inputWidth);
|
||||
void nms(std::vector<BoxInfo>& input_boxes, float NMS_THRESH);
|
||||
|
||||
@ -11,57 +11,51 @@
|
||||
#define CLIP(x, x1, x2) max(x1, min(x, x2))
|
||||
namespace cv {
|
||||
int SuperScale::init(const std::string &sr_path) {
|
||||
try
|
||||
{
|
||||
dnn::Net network = dnn::readNetFromONNX(sr_path);
|
||||
if(network.empty())
|
||||
{
|
||||
return -101;
|
||||
}
|
||||
this->qbar_sr = std::make_shared<dnn::Net>(network);
|
||||
}
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
printf("%s", e.what());
|
||||
return -3;
|
||||
}
|
||||
net_loaded_ = true;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::vector<float> SuperScale::getScaleList(const int width, const int height) {
|
||||
if (width < 320 || height < 320) return {1.0, 2.0, 0.5};
|
||||
if (width < 640 && height < 640) return {1.0, 0.5};
|
||||
return {0.5, 1.0};
|
||||
float min_side = min(width, height);
|
||||
if (min_side <= 450.f) {
|
||||
return {1.0f, 300.0f / min_side, 2.0f};
|
||||
}
|
||||
else
|
||||
return {1.0f, 450.f / min_side};
|
||||
}
|
||||
|
||||
Mat SuperScale::ProcessImageScale(const Mat &src, float scale, const bool &use_sr,
|
||||
void SuperScale::processImageScale(const Mat &src, Mat &dst, float scale, bool use_sr,
|
||||
int sr_max_size) {
|
||||
Mat dst = src;
|
||||
if (scale == 1.0) { // src
|
||||
return dst;
|
||||
}
|
||||
|
||||
int width = src.cols;
|
||||
int height = src.rows;
|
||||
if (scale == 2.0) { // upsample
|
||||
int SR_TH = sr_max_size;
|
||||
if (use_sr && (int)sqrt(width * height * 1.0) < SR_TH && net_loaded_) {
|
||||
int ret = SuperResoutionScale(src, dst);
|
||||
if (ret == 0) return dst;
|
||||
}
|
||||
|
||||
{ resize(src, dst, Size(), scale, scale, INTER_CUBIC); }
|
||||
} else if (scale < 1.0) { // downsample
|
||||
scale = min(scale, MAX_SCALE);
|
||||
if (scale > .0 && scale < 1.0)
|
||||
{ // down sample
|
||||
resize(src, dst, Size(), scale, scale, INTER_AREA);
|
||||
}
|
||||
|
||||
return dst;
|
||||
else if (scale >= 1.0 && scale < 2.0)
|
||||
{
|
||||
resize(src, dst, Size(), scale, scale, INTER_CUBIC);
|
||||
}
|
||||
else if (scale >= 2.0)
|
||||
{
|
||||
int width = src.cols;
|
||||
int height = src.rows;
|
||||
if (use_sr && (int) sqrt(width * height * 1.0) < sr_max_size && !qbar_sr->empty())
|
||||
{
|
||||
superResolutionScale(src, dst);
|
||||
if (scale > 2.0)
|
||||
{
|
||||
processImageScale(dst, dst, scale / 2.0f, use_sr);
|
||||
}
|
||||
}
|
||||
else
|
||||
{ resize(src, dst, Size(), scale, scale, INTER_CUBIC); }
|
||||
}
|
||||
}
|
||||
|
||||
int SuperScale::SuperResoutionScale(const Mat &src, Mat &dst) {
|
||||
int SuperScale::superResolutionScale(const Mat &src, Mat &dst) {
|
||||
|
||||
// cv::resize(src, dst, Size(), 2, 2, INTER_CUBIC);
|
||||
Mat blob;
|
||||
dnn::blobFromImage(src, blob, 1.0, Size(src.cols, src.rows), {0.0f}, false, false);
|
||||
|
||||
@ -70,6 +64,7 @@ int SuperScale::SuperResoutionScale(const Mat &src, Mat &dst) {
|
||||
|
||||
dst = Mat(prob.size[2], prob.size[3], CV_32F, prob.ptr<float>());
|
||||
dst.convertTo(dst, CV_8UC1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
} // namesapce cv
|
||||
@ -14,18 +14,19 @@
|
||||
using namespace std;
|
||||
|
||||
namespace cv {
|
||||
constexpr static float MAX_SCALE = 4.0f;
|
||||
|
||||
class SuperScale {
|
||||
public:
|
||||
SuperScale(){};
|
||||
~SuperScale(){};
|
||||
int init(const std::string &config_path);
|
||||
std::vector<float> getScaleList(const int width, const int height);
|
||||
Mat ProcessImageScale(const Mat &src, float scale, const bool &use_sr, int sr_max_size = 160);
|
||||
void processImageScale(const Mat &src, Mat &dst, float scale, bool use_sr, int sr_max_size = 160);
|
||||
|
||||
private:
|
||||
std::shared_ptr<dnn::Net> qbar_sr;
|
||||
bool net_loaded_ = false;
|
||||
int SuperResoutionScale(const Mat &src, Mat &dst);
|
||||
int superResolutionScale(const Mat &src, Mat &dst);
|
||||
};
|
||||
} // namesapce cv
|
||||
#endif // __SCALE_SUPER_SCALE_HPP_
|
||||
Loading…
Reference in New Issue
Block a user