From ec8b7c933a71f168b584c549547ecdcbf46b06c3 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 3 Apr 2021 15:11:38 +0300 Subject: [PATCH 01/93] Update Documentation --- modules/core/include/opencv2/core/types.hpp | 26 ++++----- .../features2d/include/opencv2/features2d.hpp | 45 +++++---------- modules/imgproc/include/opencv2/imgproc.hpp | 57 +++++++++---------- 3 files changed, 55 insertions(+), 73 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 4d04bef90a..0b1d948156 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -702,24 +702,24 @@ public: //! the default constructor CV_WRAP KeyPoint(); /** - @param _pt x & y coordinates of the keypoint - @param _size keypoint diameter - @param _angle keypoint orientation - @param _response keypoint detector response on the keypoint (that is, strength of the keypoint) - @param _octave pyramid octave in which the keypoint has been detected - @param _class_id object id + @param pt x & y coordinates of the keypoint + @param size keypoint diameter + @param angle keypoint orientation + @param response keypoint detector response on the keypoint (that is, strength of the keypoint) + @param octave pyramid octave in which the keypoint has been detected + @param class_id object id */ - KeyPoint(Point2f _pt, float _size, float _angle=-1, float _response=0, int _octave=0, int _class_id=-1); + KeyPoint(Point2f pt, float size, float angle=-1, float response=0, int octave=0, int class_id=-1); /** @param x x-coordinate of the keypoint @param y y-coordinate of the keypoint - @param _size keypoint diameter - @param _angle keypoint orientation - @param _response keypoint detector response on the keypoint (that is, strength of the keypoint) - @param _octave pyramid octave in which the keypoint has been detected - @param _class_id object id + @param size keypoint diameter + @param angle keypoint orientation + @param response keypoint detector response on the keypoint (that is, strength of the keypoint) + @param octave pyramid octave in which the keypoint has been detected + @param class_id object id */ - CV_WRAP KeyPoint(float x, float y, float _size, float _angle=-1, float _response=0, int _octave=0, int _class_id=-1); + CV_WRAP KeyPoint(float x, float y, float size, float angle=-1, float response=0, int octave=0, int class_id=-1); size_t hash() const; diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index 18ec360a1d..86b5e935c8 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -61,25 +61,11 @@ easily switch between different algorithms solving the same problem. This sectio matching descriptors that are represented as vectors in a multidimensional space. All objects that implement vector descriptor matchers inherit the DescriptorMatcher interface. -@note - - An example explaining keypoint matching can be found at - opencv_source_code/samples/cpp/descriptor_extractor_matcher.cpp - - An example on descriptor matching evaluation can be found at - opencv_source_code/samples/cpp/detector_descriptor_matcher_evaluation.cpp - - An example on one to many image matching can be found at - opencv_source_code/samples/cpp/matching_to_many_images.cpp - @defgroup features2d_draw Drawing Function of Keypoints and Matches @defgroup features2d_category Object Categorization This section describes approaches based on local 2D features and used to categorize objects. -@note - - A complete Bag-Of-Words sample can be found at - opencv_source_code/samples/cpp/bagofwords_classification.cpp - - (Python) An example using the features2D framework to perform object categorization can be - found at opencv_source_code/samples/python/find_obj.py - @defgroup feature2d_hal Hardware Acceleration Layer @{ @defgroup features2d_hal_interface Interface @@ -90,7 +76,7 @@ This section describes approaches based on local 2D features and used to categor namespace cv { -//! @addtogroup features2d +//! @addtogroup features2d_main //! @{ // //! writes vector of keypoints to the file storage @@ -241,9 +227,6 @@ the vector descriptor extractors inherit the DescriptorExtractor interface. */ typedef Feature2D DescriptorExtractor; -//! @addtogroup features2d_main -//! @{ - /** @brief Class for implementing the wrapper which makes detectors and extractors to be affine invariant, described as ASIFT in @cite YM11 . @@ -477,20 +460,20 @@ class CV_EXPORTS_W MSER : public Feature2D public: /** @brief Full constructor for %MSER detector - @param _delta it compares \f$(size_{i}-size_{i-delta})/size_{i-delta}\f$ - @param _min_area prune the area which smaller than minArea - @param _max_area prune the area which bigger than maxArea - @param _max_variation prune the area have similar size to its children - @param _min_diversity for color image, trace back to cut off mser with diversity less than min_diversity - @param _max_evolution for color image, the evolution steps - @param _area_threshold for color image, the area threshold to cause re-initialize - @param _min_margin for color image, ignore too small margin - @param _edge_blur_size for color image, the aperture size for edge blur + @param delta it compares \f$(size_{i}-size_{i-delta})/size_{i-delta}\f$ + @param min_area prune the area which smaller than minArea + @param max_area prune the area which bigger than maxArea + @param max_variation prune the area have similar size to its children + @param min_diversity for color image, trace back to cut off mser with diversity less than min_diversity + @param max_evolution for color image, the evolution steps + @param area_threshold for color image, the area threshold to cause re-initialize + @param min_margin for color image, ignore too small margin + @param edge_blur_size for color image, the aperture size for edge blur */ - CV_WRAP static Ptr create( int _delta=5, int _min_area=60, int _max_area=14400, - double _max_variation=0.25, double _min_diversity=.2, - int _max_evolution=200, double _area_threshold=1.01, - double _min_margin=0.003, int _edge_blur_size=5 ); + CV_WRAP static Ptr create( int delta=5, int min_area=60, int max_area=14400, + double max_variation=0.25, double min_diversity=.2, + int max_evolution=200, double area_threshold=1.01, + double min_margin=0.003, int edge_blur_size=5 ); /** @brief Detect %MSER regions diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 2ded15d1a9..aba7821e3e 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1181,7 +1181,7 @@ protected: struct CV_EXPORTS Vertex { Vertex(); - Vertex(Point2f pt, bool _isvirtual, int _firstEdge=0); + Vertex(Point2f pt, bool isvirtual, int firstEdge=0); bool isvirtual() const; bool isfree() const; @@ -1237,9 +1237,9 @@ public: ![image](pics/building_lsd.png) - @param _image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: + @param image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: `lsd_ptr-\>detect(image(roi), lines, ...); lines += Scalar(roi.x, roi.y, roi.x, roi.y);` - @param _lines A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. Where + @param lines A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. Where Vec4i/Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly oriented depending on the gradient. @param width Vector of widths of the regions, where the lines are found. E.g. Width of line. @@ -1251,26 +1251,26 @@ public: - 1 corresponds to 0.1 mean false alarms This vector will be calculated only when the objects type is #LSD_REFINE_ADV. */ - CV_WRAP virtual void detect(InputArray _image, OutputArray _lines, + CV_WRAP virtual void detect(InputArray image, OutputArray lines, OutputArray width = noArray(), OutputArray prec = noArray(), OutputArray nfa = noArray()) = 0; /** @brief Draws the line segments on a given image. - @param _image The image, where the lines will be drawn. Should be bigger or equal to the image, + @param image The image, where the lines will be drawn. Should be bigger or equal to the image, where the lines were found. @param lines A vector of the lines that needed to be drawn. */ - CV_WRAP virtual void drawSegments(InputOutputArray _image, InputArray lines) = 0; + CV_WRAP virtual void drawSegments(InputOutputArray image, InputArray lines) = 0; /** @brief Draws two groups of lines in blue and red, counting the non overlapping (mismatching) pixels. @param size The size of the image, where lines1 and lines2 were found. @param lines1 The first group of lines that needs to be drawn. It is visualized in blue color. @param lines2 The second group of lines. They visualized in red color. - @param _image Optional image, where the lines will be drawn. The image should be color(3-channel) + @param image Optional image, where the lines will be drawn. The image should be color(3-channel) in order for lines1 and lines2 to be drawn in the above mentioned colors. */ - CV_WRAP virtual int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray _image = noArray()) = 0; + CV_WRAP virtual int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray image = noArray()) = 0; virtual ~LineSegmentDetector() { } }; @@ -1280,22 +1280,21 @@ public: The LineSegmentDetector algorithm is defined using the standard values. Only advanced users may want to edit those, as to tailor it for their own application. -@param _refine The way found lines will be refined, see #LineSegmentDetectorModes -@param _scale The scale of the image that will be used to find the lines. Range (0..1]. -@param _sigma_scale Sigma for Gaussian filter. It is computed as sigma = _sigma_scale/_scale. -@param _quant Bound to the quantization error on the gradient norm. -@param _ang_th Gradient angle tolerance in degrees. -@param _log_eps Detection threshold: -log10(NFA) \> log_eps. Used only when advance refinement -is chosen. -@param _density_th Minimal density of aligned region points in the enclosing rectangle. -@param _n_bins Number of bins in pseudo-ordering of gradient modulus. +@param refine The way found lines will be refined, see #LineSegmentDetectorModes +@param scale The scale of the image that will be used to find the lines. Range (0..1]. +@param sigma_scale Sigma for Gaussian filter. It is computed as sigma = sigma_scale/scale. +@param quant Bound to the quantization error on the gradient norm. +@param ang_th Gradient angle tolerance in degrees. +@param log_eps Detection threshold: -log10(NFA) \> log_eps. Used only when advance refinement is chosen. +@param density_th Minimal density of aligned region points in the enclosing rectangle. +@param n_bins Number of bins in pseudo-ordering of gradient modulus. @note Implementation has been removed due original code license conflict */ CV_EXPORTS_W Ptr createLineSegmentDetector( - int _refine = LSD_REFINE_STD, double _scale = 0.8, - double _sigma_scale = 0.6, double _quant = 2.0, double _ang_th = 22.5, - double _log_eps = 0, double _density_th = 0.7, int _n_bins = 1024); + int refine = LSD_REFINE_STD, double scale = 0.8, + double sigma_scale = 0.6, double quant = 2.0, double ang_th = 22.5, + double log_eps = 0, double density_th = 0.7, int n_bins = 1024); //! @} imgproc_feature @@ -1494,7 +1493,7 @@ The unnormalized square box filter can be useful in computing local image statis variance and standard deviation around the neighborhood of a pixel. @param src input image -@param dst output image of the same size and type as _src +@param dst output image of the same size and type as src @param ddepth the output image depth (-1 to use src.depth()) @param ksize kernel size @param anchor kernel anchor point. The default value of Point(-1, -1) denotes that the anchor is at the kernel @@ -2036,8 +2035,8 @@ CV_EXPORTS_W void HoughLinesP( InputArray image, OutputArray lines, The function finds lines in a set of points using a modification of the Hough transform. @include snippets/imgproc_HoughLinesPointSet.cpp -@param _point Input vector of points. Each vector must be encoded as a Point vector \f$(x,y)\f$. Type must be CV_32FC2 or CV_32SC2. -@param _lines Output vector of found lines. Each vector is encoded as a vector \f$(votes, rho, theta)\f$. +@param point Input vector of points. Each vector must be encoded as a Point vector \f$(x,y)\f$. Type must be CV_32FC2 or CV_32SC2. +@param lines Output vector of found lines. Each vector is encoded as a vector \f$(votes, rho, theta)\f$. The larger the value of 'votes', the higher the reliability of the Hough line. @param lines_max Max count of hough lines. @param threshold Accumulator threshold parameter. Only those lines are returned that get enough @@ -2049,7 +2048,7 @@ votes ( \f$>\texttt{threshold}\f$ ) @param max_theta Maximum angle value of the accumulator in radians. @param theta_step Angle resolution of the accumulator in radians. */ -CV_EXPORTS_W void HoughLinesPointSet( InputArray _point, OutputArray _lines, int lines_max, int threshold, +CV_EXPORTS_W void HoughLinesPointSet( InputArray point, OutputArray lines, int lines_max, int threshold, double min_rho, double max_rho, double rho_step, double min_theta, double max_theta, double theta_step ); @@ -4163,9 +4162,9 @@ Examples of how intersectConvexConvex works /** @brief Finds intersection of two convex polygons -@param _p1 First polygon -@param _p2 Second polygon -@param _p12 Output polygon describing the intersecting area +@param p1 First polygon +@param p2 Second polygon +@param p12 Output polygon describing the intersecting area @param handleNested When true, an intersection is found if one of the polygons is fully enclosed in the other. When false, no intersection is found. If the polygons share a side or the vertex of one polygon lies on an edge of the other, they are not considered nested and an intersection will be found regardless of the value of handleNested. @@ -4174,8 +4173,8 @@ of the other, they are not considered nested and an intersection will be found r @note intersectConvexConvex doesn't confirm that both polygons are convex and will return invalid results if they aren't. */ -CV_EXPORTS_W float intersectConvexConvex( InputArray _p1, InputArray _p2, - OutputArray _p12, bool handleNested = true ); +CV_EXPORTS_W float intersectConvexConvex( InputArray p1, InputArray p2, + OutputArray p12, bool handleNested = true ); /** @example samples/cpp/fitellipse.cpp An example using the fitEllipse technique From df7185307545c6a89f7a1b0d222ba47245f442fe Mon Sep 17 00:00:00 2001 From: Andrei Costinescu Date: Fri, 9 Apr 2021 07:17:20 +0200 Subject: [PATCH 02/93] Update text in linux_install.markdown --- doc/tutorials/introduction/linux_install/linux_install.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index 5cd496d173..5083fac282 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -108,7 +108,7 @@ CMake package files will be located in the build root: ## Install @warning -Installation process only copies files to predefined locations and do minor patching. Library installed using this method is not integrated into the system package registry and can not be uninstalled automatically. We do not recommend system-wide installation to regular users due to possible conflicts with system packages. +The installation process only copies files to predefined locations and does minor patching. Installing using this method does not integrate opencv into the system package registry and thus, for example, opencv can not be uninstalled automatically. We do not recommend system-wide installation to regular users due to possible conflicts with system packages. By default OpenCV will be installed to the `/usr/local` directory, all files will be copied to following locations: * `/usr/local/bin` - executable files From 222af8e7e48d35d0d4dd02f4b2b0291e4e091d1c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 9 Apr 2021 15:46:11 +0000 Subject: [PATCH 03/93] core: avoid process cleanup deadlock if TlsStorage is not used --- modules/core/src/precomp.hpp | 2 ++ modules/core/src/system.cpp | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/core/src/precomp.hpp b/modules/core/src/precomp.hpp index 0ffde8855a..eebbda0694 100644 --- a/modules/core/src/precomp.hpp +++ b/modules/core/src/precomp.hpp @@ -388,6 +388,8 @@ cv::Mutex& getInitializationMutex(); #define CV_SINGLETON_LAZY_INIT(TYPE, INITIALIZER) CV_SINGLETON_LAZY_INIT_(TYPE, INITIALIZER, instance) #define CV_SINGLETON_LAZY_INIT_REF(TYPE, INITIALIZER) CV_SINGLETON_LAZY_INIT_(TYPE, INITIALIZER, *instance) +CV_EXPORTS void releaseTlsStorageThread(); + int cv_snprintf(char* buf, int len, const char* fmt, ...); int cv_vsnprintf(char* buf, int len, const char* fmt, va_list args); } diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 6e882e1dde..51c7363b30 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -1583,6 +1583,9 @@ struct ThreadData size_t idx; // Thread index in TLS storage. This is not OS thread ID! }; + +static bool g_isTlsStorageInitialized = false; + // Main TLS storage class class TlsStorage { @@ -1592,6 +1595,7 @@ public: { tlsSlots.reserve(32); threads.reserve(32); + g_isTlsStorageInitialized = true; } ~TlsStorage() { @@ -1810,6 +1814,13 @@ static void WINAPI opencv_fls_destructor(void* pData) } // namespace details using namespace details; +void releaseTlsStorageThread() +{ + if (!g_isTlsStorageInitialized) + return; // nothing to release, so prefer to avoid creation of new global structures + getTlsStorage().releaseThread(); +} + TLSDataContainer::TLSDataContainer() { key_ = (int)getTlsStorage().reserveSlot(this); // Reserve key from TLS storage @@ -1893,7 +1904,7 @@ BOOL WINAPI DllMain(HINSTANCE, DWORD fdwReason, LPVOID lpReserved) { // Not allowed to free resources if lpReserved is non-null // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682583.aspx - cv::getTlsStorage().releaseThread(); + releaseTlsStorageThread(); } } return TRUE; From a9a6801c6dcbe2baa989453e2c95c80388888879 Mon Sep 17 00:00:00 2001 From: Danny <33044223+danielenricocahall@users.noreply.github.com> Date: Fri, 9 Apr 2021 12:56:14 -0400 Subject: [PATCH 04/93] Merge pull request #19884 from danielenricocahall:fix-prediction-features-bug Fix bug with predictions in RTrees/Boost * address bug where predict functions with invalid feature count in rtrees/boost models * compact matrix rep in tests * check 1..n-1 and n+1 in feature size validation test --- modules/ml/src/boost.cpp | 1 + modules/ml/src/rtrees.cpp | 1 + modules/ml/test/test_rtrees.cpp | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/modules/ml/src/boost.cpp b/modules/ml/src/boost.cpp index 4b94410eeb..58f572b90d 100644 --- a/modules/ml/src/boost.cpp +++ b/modules/ml/src/boost.cpp @@ -490,6 +490,7 @@ public: float predict( InputArray samples, OutputArray results, int flags ) const CV_OVERRIDE { + CV_Assert( samples.cols() == getVarCount() && samples.type() == CV_32F ); return impl.predict(samples, results, flags); } diff --git a/modules/ml/src/rtrees.cpp b/modules/ml/src/rtrees.cpp index 1deee6f6c8..46af37ce11 100644 --- a/modules/ml/src/rtrees.cpp +++ b/modules/ml/src/rtrees.cpp @@ -479,6 +479,7 @@ public: float predict( InputArray samples, OutputArray results, int flags ) const CV_OVERRIDE { CV_TRACE_FUNCTION(); + CV_Assert( samples.cols() == getVarCount() && samples.type() == CV_32F ); return impl.predict(samples, results, flags); } diff --git a/modules/ml/test/test_rtrees.cpp b/modules/ml/test/test_rtrees.cpp index 1ec9b8d042..5a4fb34e74 100644 --- a/modules/ml/test/test_rtrees.cpp +++ b/modules/ml/test/test_rtrees.cpp @@ -95,6 +95,25 @@ TEST(ML_RTrees, 11142_sample_weights_classification) EXPECT_GE(error_with_weights, error_without_weights); } +TEST(ML_RTrees, bug_12974_throw_exception_when_predict_different_feature_count) +{ + int numFeatures = 5; + // create a 5 feature dataset and train the model + cv::Ptr model = RTrees::create(); + Mat samples(10, numFeatures, CV_32F); + randu(samples, 0, 10); + Mat labels = (Mat_(10,1) << 0,0,0,0,0,1,1,1,1,1); + cv::Ptr trainData = TrainData::create(samples, cv::ml::ROW_SAMPLE, labels); + model->train(trainData); + // try to predict on data which have fewer features - this should throw an exception + for(int i = 1; i < numFeatures - 1; ++i) { + Mat test(1, i, CV_32FC1); + ASSERT_THROW(model->predict(test), Exception); + } + // try to predict on data which have more features - this should also throw an exception + Mat test(1, numFeatures + 1, CV_32FC1); + ASSERT_THROW(model->predict(test), Exception); +} }} // namespace From b9b19185bcf603e27853260dc1d9c763203e4b2c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 12 Apr 2021 19:05:52 +0000 Subject: [PATCH 05/93] ml: fix legacy import in DTreesImpl --- modules/ml/src/boost.cpp | 2 +- modules/ml/src/tree.cpp | 43 +++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/modules/ml/src/boost.cpp b/modules/ml/src/boost.cpp index 58f572b90d..be9c9a7b46 100644 --- a/modules/ml/src/boost.cpp +++ b/modules/ml/src/boost.cpp @@ -490,7 +490,7 @@ public: float predict( InputArray samples, OutputArray results, int flags ) const CV_OVERRIDE { - CV_Assert( samples.cols() == getVarCount() && samples.type() == CV_32F ); + CV_CheckEQ(samples.cols(), getVarCount(), ""); return impl.predict(samples, results, flags); } diff --git a/modules/ml/src/tree.cpp b/modules/ml/src/tree.cpp index 87181b156c..1f82ff5081 100644 --- a/modules/ml/src/tree.cpp +++ b/modules/ml/src/tree.cpp @@ -43,6 +43,8 @@ #include "precomp.hpp" #include +#include + namespace cv { namespace ml { @@ -1694,9 +1696,9 @@ void DTreesImpl::write( FileStorage& fs ) const void DTreesImpl::readParams( const FileNode& fn ) { _isClassifier = (int)fn["is_classifier"] != 0; - /*int var_all = (int)fn["var_all"]; - int var_count = (int)fn["var_count"]; - int cat_var_count = (int)fn["cat_var_count"]; + int varAll = (int)fn["var_all"]; + int varCount = (int)fn["var_count"]; + /*int cat_var_count = (int)fn["cat_var_count"]; int ord_var_count = (int)fn["ord_var_count"];*/ FileNode tparams_node = fn["training_params"]; @@ -1723,11 +1725,38 @@ void DTreesImpl::readParams( const FileNode& fn ) readVectorOrMat(fn["var_idx"], varIdx); fn["var_type"] >> varType; - int format = 0; - fn["format"] >> format; - bool isLegacy = format < 3; + bool isLegacy = false; + if (fn["format"].empty()) // Export bug until OpenCV 3.2: https://github.com/opencv/opencv/pull/6314 + { + if (!fn["cat_ofs"].empty()) + isLegacy = false; // 2.4 doesn't store "cat_ofs" + else if (!fn["missing_subst"].empty()) + isLegacy = false; // 2.4 doesn't store "missing_subst" + else if (!fn["class_labels"].empty()) + isLegacy = false; // 2.4 doesn't store "class_labels" + else if ((int)varType.size() != varAll) + isLegacy = true; // 3.0+: https://github.com/opencv/opencv/blame/3.0.0/modules/ml/src/tree.cpp#L1576 + else if (/*(int)varType.size() == varAll &&*/ varCount == varAll) + isLegacy = true; + else + { + // 3.0+: + // - https://github.com/opencv/opencv/blame/3.0.0/modules/ml/src/tree.cpp#L1552-L1553 + // - https://github.com/opencv/opencv/blame/3.0.0/modules/ml/src/precomp.hpp#L296 + isLegacy = !(varCount + 1 == varAll); + } + CV_LOG_INFO(NULL, "ML/DTrees: possible missing 'format' field due to bug of OpenCV export implementation. " + "Details: https://github.com/opencv/opencv/issues/5412. Consider re-exporting of saved ML model. " + "isLegacy = " << isLegacy); + } + else + { + int format = 0; + fn["format"] >> format; + CV_CheckGT(format, 0, ""); + isLegacy = format < 3; + } - int varAll = (int)fn["var_all"]; if (isLegacy && (int)varType.size() <= varAll) { std::vector extendedTypes(varAll + 1, 0); From 0bdbc745c444e488021c356ff6197bc6d538e517 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 13 Apr 2021 11:09:14 +0000 Subject: [PATCH 06/93] ml: update checks --- modules/ml/src/rtrees.cpp | 2 +- modules/ml/src/tree.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ml/src/rtrees.cpp b/modules/ml/src/rtrees.cpp index 46af37ce11..56be5c0e22 100644 --- a/modules/ml/src/rtrees.cpp +++ b/modules/ml/src/rtrees.cpp @@ -479,7 +479,7 @@ public: float predict( InputArray samples, OutputArray results, int flags ) const CV_OVERRIDE { CV_TRACE_FUNCTION(); - CV_Assert( samples.cols() == getVarCount() && samples.type() == CV_32F ); + CV_CheckEQ(samples.cols(), getVarCount(), ""); return impl.predict(samples, results, flags); } diff --git a/modules/ml/src/tree.cpp b/modules/ml/src/tree.cpp index 1f82ff5081..5dae889013 100644 --- a/modules/ml/src/tree.cpp +++ b/modules/ml/src/tree.cpp @@ -1701,6 +1701,9 @@ void DTreesImpl::readParams( const FileNode& fn ) /*int cat_var_count = (int)fn["cat_var_count"]; int ord_var_count = (int)fn["ord_var_count"];*/ + if (varAll <= 0) + CV_Error(Error::StsParseError, "The field \"var_all\" of DTree classifier is missing or non-positive"); + FileNode tparams_node = fn["training_params"]; TreeParams params0 = TreeParams(); From 2a48730166bdd3e0f39e61895e1bfc3aad67d0ac Mon Sep 17 00:00:00 2001 From: berak Date: Tue, 13 Apr 2021 12:51:34 +0200 Subject: [PATCH 07/93] docs:fix python retvals --- doc/tools/html_functions.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/doc/tools/html_functions.py b/doc/tools/html_functions.py index b76639cea5..204f6d1c1b 100644 --- a/doc/tools/html_functions.py +++ b/doc/tools/html_functions.py @@ -107,17 +107,10 @@ def add_signature_to_table(soup, table, signature, language, type): """ Add a signature to an html table""" row = soup.new_tag('tr') row.append(soup.new_tag('td', style='width: 20px;')) - - if 'ret' in signature: - row.append(append(soup.new_tag('td'), signature['ret'])) - row.append(append(soup.new_tag('td'), '=')) - else: - row.append(soup.new_tag('td')) # return values - row.append(soup.new_tag('td')) # '=' - row.append(append(soup.new_tag('td'), signature['name'] + '(')) row.append(append(soup.new_tag('td', **{'class': 'paramname'}), signature['arg'])) - row.append(append(soup.new_tag('td'), ')')) + row.append(append(soup.new_tag('td'), ') -> ')) + row.append(append(soup.new_tag('td'), signature['ret'])) table.append(row) From ec32061f5fc63880727b2d252294412b2e024c54 Mon Sep 17 00:00:00 2001 From: CSBVision Date: Wed, 14 Apr 2021 12:05:53 +0200 Subject: [PATCH 08/93] Update __init__.py to support symbolic links Currently, the LOADER_DIR is set as os.path.dirname(os.path.abspath(__file__)). This does not point to the true library path if the cv2 folder is symlinked into the Python package directory such that importing cv2 under Python fails. The proposed change only resolves symbolic links correctly by calling os.path.realpath(__file__) first and does not change anything if __file__ contains no symbolic link. --- modules/python/package/cv2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 940ac65732..de70872839 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -34,7 +34,7 @@ def bootstrap(): import platform if DEBUG: print('OpenCV loader: os.name="{}" platform.system()="{}"'.format(os.name, str(platform.system()))) - LOADER_DIR=os.path.dirname(os.path.abspath(__file__)) + LOADER_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) PYTHON_EXTENSIONS_PATHS = [] BINARIES_PATHS = [] From aeb8dfc52df99d866f4a7c50a34d926fe7140613 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 15 Apr 2021 12:13:15 +0300 Subject: [PATCH 09/93] Fix header sorting for modules without headers --- cmake/OpenCVModule.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index bcbca833f2..5501b99113 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -877,7 +877,9 @@ macro(_ocv_create_module) ocv_compiler_optimization_process_sources(OPENCV_MODULE_${the_module}_SOURCES OPENCV_MODULE_${the_module}_DEPS_EXT ${the_module}) set(__module_headers ${OPENCV_MODULE_${the_module}_HEADERS}) - list(SORT __module_headers) # fix headers order, useful for bindings + if(__module_headers) + list(SORT __module_headers) # fix headers order, useful for bindings + endif() set(OPENCV_MODULE_${the_module}_HEADERS ${__module_headers} CACHE INTERNAL "List of header files for ${the_module}") set(OPENCV_MODULE_${the_module}_SOURCES ${OPENCV_MODULE_${the_module}_SOURCES} CACHE INTERNAL "List of source files for ${the_module}") From 0df61591495bc24275aa8a88cf34cfae06d65144 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 19 Apr 2021 02:01:57 +0000 Subject: [PATCH 10/93] cmake: fix handling of VIDEOIO_ENABLE_PLUGINS=OFF --- modules/videoio/CMakeLists.txt | 16 +++++++++++----- modules/videoio/src/backend_plugin.cpp | 2 ++ modules/videoio/src/videoio_registry.cpp | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 3a79631b86..8db5026455 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -1,12 +1,18 @@ set(VIDEOIO_PLUGIN_LIST "" CACHE STRING "List of videoio backends to be compiled as plugins (ffmpeg, gstreamer, mfx, msmf or special value 'all')") -set(VIDEOIO_ENABLE_PLUGINS "ON" CACHE BOOL "Allow building videoio plugin support") +set(VIDEOIO_ENABLE_PLUGINS "ON" CACHE BOOL "Allow building and using of videoio plugins") mark_as_advanced(VIDEOIO_PLUGIN_LIST VIDEOIO_ENABLE_PLUGINS) string(REPLACE "," ";" VIDEOIO_PLUGIN_LIST "${VIDEOIO_PLUGIN_LIST}") # support comma-separated list (,) too - -# Make virtual opencv_videoio_plugins target -if(NOT TARGET opencv_videoio_plugins) - add_custom_target(opencv_videoio_plugins ALL) +if(NOT VIDEOIO_ENABLE_PLUGINS) + if(VIDEOIO_PLUGIN_LIST) + message(WARNING "VideoIO: plugins are disabled through VIDEOIO_ENABLE_PLUGINS, so VIDEOIO_PLUGIN_LIST='${VIDEOIO_PLUGIN_LIST}' is ignored") + set(VIDEOIO_PLUGIN_LIST "") + endif() +else() + # Make virtual opencv_videoio_plugins target + if(NOT TARGET opencv_videoio_plugins) + add_custom_target(opencv_videoio_plugins ALL) + endif() endif() ocv_add_module(videoio opencv_imgproc opencv_imgcodecs WRAP java objc python) diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index ad34602ee2..fa4a1f6ff6 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -684,6 +684,8 @@ Ptr createPluginBackendFactory(VideoCaptureAPIs id, const char* #if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) return makePtr(id, baseName); //.staticCast(); #else + CV_UNUSED(id); + CV_UNUSED(baseName); return Ptr(); #endif } diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 59d96d162c..9fc75bc40e 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -84,7 +84,7 @@ static const struct VideoBackendInfo builtin_backends[] = #ifdef HAVE_MSMF DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF), -#elif defined(ENABLE_PLUGINS) +#elif defined(ENABLE_PLUGINS) && defined(_WIN32) DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER), #endif From 896bffb543d1c80781f5f217caffdedc98a8d8f1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 19 Apr 2021 01:39:59 +0000 Subject: [PATCH 11/93] videoio(plugin): add query API for plugins --- .../include/opencv2/videoio/registry.hpp | 27 ++++- modules/videoio/src/backend.hpp | 12 ++ modules/videoio/src/backend_plugin.cpp | 94 +++++++++++++++- modules/videoio/src/backend_static.cpp | 4 + modules/videoio/src/videoio_registry.cpp | 75 +++++++++++++ modules/videoio/test/test_plugins.cpp | 105 ++++++++++++++++++ 6 files changed, 311 insertions(+), 6 deletions(-) create mode 100644 modules/videoio/test/test_plugins.cpp diff --git a/modules/videoio/include/opencv2/videoio/registry.hpp b/modules/videoio/include/opencv2/videoio/registry.hpp index 89fb5a836c..cf72247b3f 100644 --- a/modules/videoio/include/opencv2/videoio/registry.hpp +++ b/modules/videoio/include/opencv2/videoio/registry.hpp @@ -39,7 +39,32 @@ CV_EXPORTS_W std::vector getStreamBackends(); CV_EXPORTS_W std::vector getWriterBackends(); /** @brief Returns true if backend is available */ -CV_EXPORTS bool hasBackend(VideoCaptureAPIs api); +CV_EXPORTS_W bool hasBackend(VideoCaptureAPIs api); + +/** @brief Returns true if backend is built in (false if backend is used as plugin) */ +CV_EXPORTS_W bool isBackendBuiltIn(VideoCaptureAPIs api); + +/** @brief Returns description and ABI/API version of videoio plugin's camera interface */ +CV_EXPORTS_W std::string getCameraBackendPluginVersion( + VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); + +/** @brief Returns description and ABI/API version of videoio plugin's stream capture interface */ +CV_EXPORTS_W std::string getStreamBackendPluginVersion( + VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); + +/** @brief Returns description and ABI/API version of videoio plugin's writer interface */ +CV_EXPORTS_W std::string getWriterBackendPluginVersion( + VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); + //! @} }} // namespace diff --git a/modules/videoio/src/backend.hpp b/modules/videoio/src/backend.hpp index ecf0e0d1d3..2a95ec05aa 100644 --- a/modules/videoio/src/backend.hpp +++ b/modules/videoio/src/backend.hpp @@ -27,6 +27,7 @@ class IBackendFactory public: virtual ~IBackendFactory() {} virtual Ptr getBackend() const = 0; + virtual bool isBuiltIn() const = 0; }; //============================================================================= @@ -48,6 +49,17 @@ Ptr createPluginBackendFactory(VideoCaptureAPIs id, const char* void applyParametersFallback(const Ptr& cap, const VideoCaptureParameters& params); +std::string getCapturePluginVersion( + const Ptr& backend_factory, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); +std::string getWriterPluginVersion( + const Ptr& backend_factory, + CV_OUT int& version_ABI, + CV_OUT int& version_API +); + } // namespace cv:: #endif // BACKEND_HPP_DEFINED diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index ad34602ee2..9602032556 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -210,6 +210,24 @@ public: Ptr createCapture(const std::string &filename, const VideoCaptureParameters& params) const CV_OVERRIDE; Ptr createWriter(const std::string& filename, int fourcc, double fps, const cv::Size& sz, const VideoWriterParameters& params) const CV_OVERRIDE; + + std::string getCapturePluginVersion(CV_OUT int& version_ABI, CV_OUT int& version_API) + { + CV_Assert(capture_api_ || plugin_api_); + const OpenCV_API_Header& api_header = capture_api_ ? capture_api_->api_header : plugin_api_->api_header; + version_ABI = api_header.min_api_version; + version_API = api_header.api_version; + return api_header.api_description; + } + + std::string getWriterPluginVersion(CV_OUT int& version_ABI, CV_OUT int& version_API) + { + CV_Assert(writer_api_ || plugin_api_); + const OpenCV_API_Header& api_header = writer_api_ ? writer_api_->api_header : plugin_api_->api_header; + version_ABI = api_header.min_api_version; + version_API = api_header.api_version; + return api_header.api_description; + } }; class PluginBackendFactory : public IBackendFactory @@ -229,14 +247,41 @@ public: Ptr getBackend() const CV_OVERRIDE { - if (!initialized) - { - const_cast(this)->initBackend(); - } + initBackend(); return backend.staticCast(); } + + bool isBuiltIn() const CV_OVERRIDE { return false; } + + std::string getCapturePluginVersion( + CV_OUT int& version_ABI, + CV_OUT int& version_API) const + { + initBackend(); + if (!backend) + CV_Error_(Error::StsNotImplemented, ("Backend '%s' is not available", baseName_)); + return backend->getCapturePluginVersion(version_ABI, version_API); + } + + std::string getWriterPluginVersion( + CV_OUT int& version_ABI, + CV_OUT int& version_API) const + { + initBackend(); + if (!backend) + CV_Error_(Error::StsNotImplemented, ("Backend '%s' is not available", baseName_)); + return backend->getWriterPluginVersion(version_ABI, version_API); + } + protected: - void initBackend() + inline void initBackend() const + { + if (!initialized) + { + const_cast(this)->initBackend_(); + } + } + void initBackend_() { AutoLock lock(getInitializationMutex()); try { @@ -688,4 +733,43 @@ Ptr createPluginBackendFactory(VideoCaptureAPIs id, const char* #endif } + +std::string getCapturePluginVersion( + const Ptr& backend_factory, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ +#if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) + using namespace impl; + CV_Assert(backend_factory); + PluginBackendFactory* plugin_backend_factory = dynamic_cast(backend_factory.get()); + CV_Assert(plugin_backend_factory); + return plugin_backend_factory->getCapturePluginVersion(version_ABI, version_API); +#else + CV_UNUSED(version_ABI); + CV_UNUSED(version_API); + CV_Error(Error::StsBadFunc, "Plugins are not available in this build"); +#endif +} + +std::string getWriterPluginVersion( + const Ptr& backend_factory, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ +#if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) + using namespace impl; + CV_Assert(backend_factory); + PluginBackendFactory* plugin_backend_factory = dynamic_cast(backend_factory.get()); + CV_Assert(plugin_backend_factory); + return plugin_backend_factory->getWriterPluginVersion(version_ABI, version_API); +#else + CV_UNUSED(version_ABI); + CV_UNUSED(version_API); + CV_Error(Error::StsBadFunc, "Plugins are not available in this build"); +#endif +} + } // namespace diff --git a/modules/videoio/src/backend_static.cpp b/modules/videoio/src/backend_static.cpp index 2e0088f558..3001906acf 100644 --- a/modules/videoio/src/backend_static.cpp +++ b/modules/videoio/src/backend_static.cpp @@ -99,6 +99,8 @@ public: { return backend.staticCast(); } + + bool isBuiltIn() const CV_OVERRIDE { return true; } }; @@ -165,6 +167,8 @@ public: { return backend.staticCast(); } + + bool isBuiltIn() const CV_OVERRIDE { return true; } }; diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 59d96d162c..d15145ba6c 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -403,6 +403,81 @@ bool hasBackend(VideoCaptureAPIs api) return false; } +bool isBackendBuiltIn(VideoCaptureAPIs api) +{ + std::vector backends = VideoBackendRegistry::getInstance().getEnabledBackends(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (api == info.id) + { + CV_Assert(!info.backendFactory.empty()); + return info.backendFactory->isBuiltIn(); + } + } + return false; +} + +std::string getCameraBackendPluginVersion(VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ + const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByIndex(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (api == info.id) + { + CV_Assert(!info.backendFactory.empty()); + CV_Assert(!info.backendFactory->isBuiltIn()); + return getCapturePluginVersion(info.backendFactory, version_ABI, version_API); + } + } + CV_Error(Error::StsError, "Unknown or wrong backend ID"); +} + +std::string getStreamBackendPluginVersion(VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ + const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_CaptureByFilename(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (api == info.id) + { + CV_Assert(!info.backendFactory.empty()); + CV_Assert(!info.backendFactory->isBuiltIn()); + return getCapturePluginVersion(info.backendFactory, version_ABI, version_API); + } + } + CV_Error(Error::StsError, "Unknown or wrong backend ID"); +} + + +/** @brief Returns description and ABI/API version of videoio plugin's writer interface */ +std::string getWriterBackendPluginVersion(VideoCaptureAPIs api, + CV_OUT int& version_ABI, + CV_OUT int& version_API +) +{ + const std::vector backends = VideoBackendRegistry::getInstance().getAvailableBackends_Writer(); + for (size_t i = 0; i < backends.size(); i++) + { + const VideoBackendInfo& info = backends[i]; + if (api == info.id) + { + CV_Assert(!info.backendFactory.empty()); + CV_Assert(!info.backendFactory->isBuiltIn()); + return getWriterPluginVersion(info.backendFactory, version_ABI, version_API); + } + } + CV_Error(Error::StsError, "Unknown or wrong backend ID"); +} + + } // namespace registry } // namespace diff --git a/modules/videoio/test/test_plugins.cpp b/modules/videoio/test/test_plugins.cpp new file mode 100644 index 0000000000..3bae600be1 --- /dev/null +++ b/modules/videoio/test/test_plugins.cpp @@ -0,0 +1,105 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +enum VideoBackendMode +{ + MODE_CAMERA, + MODE_STREAM, + MODE_WRITER, +}; + +static +void dumpBackendInfo(VideoCaptureAPIs backend, enum VideoBackendMode mode) +{ + std::string name; + try + { + name = videoio_registry::getBackendName(backend); + } + catch (const std::exception& e) + { + ADD_FAILURE() << "Can't query name of backend=" << backend << ": " << e.what(); + } + catch (...) + { + ADD_FAILURE() << "Can't query name of backend=" << backend << ": unknown C++ exception"; + } + bool isBuiltIn = true; + try + { + isBuiltIn = videoio_registry::isBackendBuiltIn(backend); + } + catch (const std::exception& e) + { + ADD_FAILURE() << "Failed isBackendBuiltIn(backend=" << backend << "): " << e.what(); + cout << name << " - UNKNOWN TYPE" << endl; + return; + } + if (isBuiltIn) + { + cout << name << " - BUILTIN" << endl; + return; + } + + std::string description = "NO_DESCRIPTION"; + int version_ABI = 0; + int version_API = 0; + try + { + if (mode == MODE_CAMERA) + description = videoio_registry::getCameraBackendPluginVersion(backend, version_ABI, version_API); + else if (mode == MODE_STREAM) + description = videoio_registry::getStreamBackendPluginVersion(backend, version_ABI, version_API); + else if (mode == MODE_WRITER) + description = videoio_registry::getWriterBackendPluginVersion(backend, version_ABI, version_API); + else + CV_Error(Error::StsInternal, ""); + cout << name << " - PLUGIN (" << description << ") ABI=" << version_ABI << " API=" << version_API << endl; + return; + } + catch (const cv::Exception& e) + { + if (e.code == Error::StsNotImplemented) + { + cout << name << " - PLUGIN - NOT LOADED" << endl; + return; + } + ADD_FAILURE() << "Failed getBackendPluginDescription(backend=" << backend << "): " << e.what(); + } + catch (const std::exception& e) + { + ADD_FAILURE() << "Failed getBackendPluginDescription(backend=" << backend << "): " << e.what(); + } + cout << name << " - PLUGIN (ERROR on quering information)" << endl; +} + +TEST(VideoIO_Plugins, query) +{ + const std::vector camera_backends = cv::videoio_registry::getCameraBackends(); + cout << "== Camera APIs (" << camera_backends.size() << "):" << endl; + for (auto backend : camera_backends) + { + dumpBackendInfo(backend, MODE_CAMERA); + } + + const std::vector stream_backends = cv::videoio_registry::getStreamBackends(); + cout << "== Stream capture APIs (" << stream_backends.size() << "):" << endl; + for (auto backend : stream_backends) + { + dumpBackendInfo(backend, MODE_STREAM); + } + + const std::vector writer_backends = cv::videoio_registry::getWriterBackends(); + cout << "== Writer APIs (" << writer_backends.size() << "):" << endl; + for (auto backend : writer_backends) + { + dumpBackendInfo(backend, MODE_WRITER); + } +} + +}} From c41650db20909e55f5f28f25fbca89dd2078a65d Mon Sep 17 00:00:00 2001 From: Mary Strodl Date: Mon, 19 Apr 2021 21:01:37 -0400 Subject: [PATCH 12/93] CMakeLists: remove extraneous checks, minimum required version is 3.5.1 --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a0c62eedf..9b5bc9f8bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,7 +238,7 @@ OCV_OPTION(WITH_CAP_IOS "Enable iOS video capture" ON VISIBLE_IF IOS VERIFY HAVE_CAP_IOS) OCV_OPTION(WITH_CAROTENE "Use NVidia carotene acceleration library for ARM platform" ON - VISIBLE_IF (ARM OR AARCH64) AND NOT IOS AND NOT (CMAKE_VERSION VERSION_LESS "2.8.11")) + VISIBLE_IF (ARM OR AARCH64) AND NOT IOS) OCV_OPTION(WITH_CPUFEATURES "Use cpufeatures Android library" ON VISIBLE_IF ANDROID VERIFY HAVE_CPUFEATURES) @@ -498,7 +498,7 @@ OCV_OPTION(OPENCV_WARNINGS_ARE_ERRORS "Treat warnings as errors" OCV_OPTION(ANDROID_EXAMPLES_WITH_LIBS "Build binaries of Android examples with native libraries" OFF IF ANDROID ) OCV_OPTION(ENABLE_IMPL_COLLECTION "Collect implementation data on function call" OFF ) OCV_OPTION(ENABLE_INSTRUMENTATION "Instrument functions to collect calls trace and performance" OFF ) -OCV_OPTION(ENABLE_GNU_STL_DEBUG "Enable GNU STL Debug mode (defines _GLIBCXX_DEBUG)" OFF IF ((NOT CMAKE_VERSION VERSION_LESS "2.8.11") AND CV_GCC) ) +OCV_OPTION(ENABLE_GNU_STL_DEBUG "Enable GNU STL Debug mode (defines _GLIBCXX_DEBUG)" OFF IF CV_GCC ) OCV_OPTION(ENABLE_BUILD_HARDENING "Enable hardening of the resulting binaries (against security attacks, detects memory corruption, etc)" OFF) OCV_OPTION(ENABLE_LTO "Enable Link Time Optimization" OFF IF CV_GCC OR MSVC) OCV_OPTION(ENABLE_THIN_LTO "Enable Thin LTO" OFF IF CV_CLANG) @@ -1041,7 +1041,6 @@ endif() status("") status(" Platform:") if(NOT DEFINED OPENCV_TIMESTAMP - AND NOT CMAKE_VERSION VERSION_LESS 2.8.11 AND NOT BUILD_INFO_SKIP_TIMESTAMP ) string(TIMESTAMP OPENCV_TIMESTAMP "" UTC) From 3386efddbaaaaa96c8b0e4c752f215456c96f240 Mon Sep 17 00:00:00 2001 From: berak Date: Tue, 20 Apr 2021 11:57:42 +0200 Subject: [PATCH 13/93] calib3d: fix masks for usac --- modules/calib3d/src/usac/ransac_solvers.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index 0c7637d582..eb7e177678 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -408,10 +408,11 @@ int mergePoints (InputArray pts1_, InputArray pts2_, Mat &pts, bool ispnp) { void saveMask (OutputArray mask, const std::vector &inliers_mask) { if (mask.needed()) { const int points_size = (int) inliers_mask.size(); - mask.create(points_size, 1, CV_8U); - auto * maskptr = mask.getMat().ptr(); + Mat tmp_mask(points_size, 1, CV_8U); + auto * maskptr = tmp_mask.ptr(); for (int i = 0; i < points_size; i++) maskptr[i] = (uchar) inliers_mask[i]; + tmp_mask.copyTo(mask); } } void setParameters (Ptr ¶ms, EstimationMethod estimator, const UsacParams &usac_params, From 405e820fe13cb50c042996e7cfc8903a6d64a900 Mon Sep 17 00:00:00 2001 From: berak Date: Tue, 20 Apr 2021 12:59:36 +0200 Subject: [PATCH 14/93] Update contrast_preserve.hpp fix a build warning: ``` C:\Slave\workspace\precommit\windows10\opencv\modules\photo\src\contrast_preserve.hpp(289): warning C4244: '=': conversion from 'double' to '_Tp', possible loss of data with [ _Tp=float ] C:\Slave\workspace\precommit\windows10\opencv\modules\photo\src\contrast_preserve.hpp(361): warning C4244: '=': conversion from 'double' to '_Tp', possible loss of data with [ _Tp=float ] ``` (from https://build.opencv.org.cn/job/precommit/job/windows10/1633/console) --- modules/photo/src/contrast_preserve.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/photo/src/contrast_preserve.hpp b/modules/photo/src/contrast_preserve.hpp index 1afd4bc3e3..5681779fc9 100644 --- a/modules/photo/src/contrast_preserve.hpp +++ b/modules/photo/src/contrast_preserve.hpp @@ -285,9 +285,9 @@ void Decolor::grad_system(const Mat &im, vector < vector < double > > &polyGrad, add_vector(comb,idx,r,g,b); for(int i = 0;i(i,j)= + curIm.at(i,j)=static_cast( pow(rgb_channel[2].at(i,j),r)*pow(rgb_channel[1].at(i,j),g)* - pow(rgb_channel[0].at(i,j),b); + pow(rgb_channel[0].at(i,j),b)); vector curGrad; gradvector(curIm,curGrad); add_to_vector_poly(polyGrad,curGrad,idx1); From 3930c9a49201677e9e6b3e17b0a4cc932f9c4fc7 Mon Sep 17 00:00:00 2001 From: danielenricocahall Date: Tue, 20 Apr 2021 22:08:01 -0400 Subject: [PATCH 15/93] fix loop boundary condition --- modules/stitching/src/matchers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/stitching/src/matchers.cpp b/modules/stitching/src/matchers.cpp index 4c6cce8038..b3f1c7a1bf 100644 --- a/modules/stitching/src/matchers.cpp +++ b/modules/stitching/src/matchers.cpp @@ -826,7 +826,7 @@ void BestOf2NearestRangeMatcher::operator ()(const std::vector &f std::vector > near_pairs; for (int i = 0; i < num_images - 1; ++i) - for (int j = i + 1; j < std::min(num_images, i + range_width_); ++j) + for (int j = i + 1; j < std::min(num_images, i + 1 + range_width_); ++j) if (features[i].keypoints.size() > 0 && features[j].keypoints.size() > 0 && mask_(i, j)) near_pairs.push_back(std::make_pair(i, j)); From 5105a937d153d2281be6ba665e1a78cc118c0657 Mon Sep 17 00:00:00 2001 From: "Quella Zhang (Beyondsoft Corporation)" Date: Wed, 21 Apr 2021 14:08:52 +0800 Subject: [PATCH 16/93] Add namespace specifier for format() --- modules/imgproc/test/test_houghcircles.cpp | 2 +- samples/cpp/calibration.cpp | 4 ++-- samples/cpp/digits_svm.cpp | 4 ++-- samples/cpp/select3dobj.cpp | 2 +- .../calib3d/camera_calibration/camera_calibration.cpp | 4 ++-- .../calib3d/real_time_pose_estimation/src/main_detection.cpp | 2 +- samples/tapi/ufacedetect.cpp | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/imgproc/test/test_houghcircles.cpp b/modules/imgproc/test/test_houghcircles.cpp index de92800976..094963211c 100644 --- a/modules/imgproc/test/test_houghcircles.cpp +++ b/modules/imgproc/test/test_houghcircles.cpp @@ -56,7 +56,7 @@ using namespace std; static string getTestCaseName(const string& picture_name, double minDist, double edgeThreshold, double accumThreshold, int minRadius, int maxRadius) { - string results_name = format("circles_%s_%.0f_%.0f_%.0f_%d_%d", + string results_name = cv::format("circles_%s_%.0f_%.0f_%.0f_%d_%d", picture_name.c_str(), minDist, edgeThreshold, accumThreshold, minRadius, maxRadius); string temp(results_name); size_t pos = temp.find_first_of("\\/."); diff --git a/samples/cpp/calibration.cpp b/samples/cpp/calibration.cpp index 3e978736e7..f1ea78b304 100644 --- a/samples/cpp/calibration.cpp +++ b/samples/cpp/calibration.cpp @@ -538,9 +538,9 @@ int main( int argc, char** argv ) if( mode == CAPTURING ) { if(undistortImage) - msg = format( "%d/%d Undist", (int)imagePoints.size(), nframes ); + msg = cv::format( "%d/%d Undist", (int)imagePoints.size(), nframes ); else - msg = format( "%d/%d", (int)imagePoints.size(), nframes ); + msg = cv::format( "%d/%d", (int)imagePoints.size(), nframes ); } putText( view, msg, textOrigin, 1, 1, diff --git a/samples/cpp/digits_svm.cpp b/samples/cpp/digits_svm.cpp index e401ab2b89..c55b320da5 100644 --- a/samples/cpp/digits_svm.cpp +++ b/samples/cpp/digits_svm.cpp @@ -137,7 +137,7 @@ static void evaluate_model(const vector& predictions, const vector& err /= predictions.size(); - cout << format("error: %.2f %%", err * 100) << endl; + cout << cv::format("error: %.2f %%", err * 100) << endl; int confusion[10][10] = {}; @@ -151,7 +151,7 @@ static void evaluate_model(const vector& predictions, const vector& { for (int j = 0; j < 10; j++) { - cout << format("%2d ", confusion[i][j]); + cout << cv::format("%2d ", confusion[i][j]); } cout << endl; } diff --git a/samples/cpp/select3dobj.cpp b/samples/cpp/select3dobj.cpp index 252bc266cc..f946565cf6 100644 --- a/samples/cpp/select3dobj.cpp +++ b/samples/cpp/select3dobj.cpp @@ -490,7 +490,7 @@ int main(int argc, char** argv) setMouseCallback("View", onMouse, 0); bool boardFound = false; - string indexFilename = format("%s_index.yml", outprefix.c_str()); + string indexFilename = cv::format("%s_index.yml", outprefix.c_str()); vector capturedImgList; vector roiList; diff --git a/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp b/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp index fffc8e8357..95eeec07e9 100644 --- a/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp +++ b/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp @@ -402,9 +402,9 @@ int main(int argc, char* argv[]) if( mode == CAPTURING ) { if(s.showUndistorsed) - msg = format( "%d/%d Undist", (int)imagePoints.size(), s.nrFrames ); + msg = cv::format( "%d/%d Undist", (int)imagePoints.size(), s.nrFrames ); else - msg = format( "%d/%d", (int)imagePoints.size(), s.nrFrames ); + msg = cv::format( "%d/%d", (int)imagePoints.size(), s.nrFrames ); } putText( view, msg, textOrigin, 1, 1, mode == CALIBRATED ? GREEN : RED); diff --git a/samples/cpp/tutorial_code/calib3d/real_time_pose_estimation/src/main_detection.cpp b/samples/cpp/tutorial_code/calib3d/real_time_pose_estimation/src/main_detection.cpp index 8313d56c76..6da977a59d 100644 --- a/samples/cpp/tutorial_code/calib3d/real_time_pose_estimation/src/main_detection.cpp +++ b/samples/cpp/tutorial_code/calib3d/real_time_pose_estimation/src/main_detection.cpp @@ -361,7 +361,7 @@ int main(int argc, char *argv[]) frame_vis.copyTo(frameSave); } - string saveFilename = format(string(saveDirectory + "/image_%04d.png").c_str(), frameCount); + string saveFilename = cv::format(string(saveDirectory + "/image_%04d.png").c_str(), frameCount); imwrite(saveFilename, frameSave); frameCount++; } diff --git a/samples/tapi/ufacedetect.cpp b/samples/tapi/ufacedetect.cpp index 0a6d91c3d6..23445cc28d 100644 --- a/samples/tapi/ufacedetect.cpp +++ b/samples/tapi/ufacedetect.cpp @@ -206,7 +206,7 @@ void detectAndDraw( UMat& img, Mat& canvas, CascadeClassifier& cascade, double alpha = nframes > 50 ? 0.01 : 1./nframes; avgfps = avgfps*(1-alpha) + fps*alpha; - putText(canvas, format("OpenCL: %s, fps: %.1f", ocl::useOpenCL() ? "ON" : "OFF", avgfps), Point(50, 30), + putText(canvas, cv::format("OpenCL: %s, fps: %.1f", ocl::useOpenCL() ? "ON" : "OFF", avgfps), Point(50, 30), FONT_HERSHEY_SIMPLEX, 0.8, Scalar(0,255,0), 2); for ( size_t i = 0; i < faces.size(); i++ ) From 2e143b87995e370bf79abd4a4cbbdfc84f294e77 Mon Sep 17 00:00:00 2001 From: Aleksandr Voron Date: Wed, 21 Apr 2021 21:29:19 +0300 Subject: [PATCH 17/93] Merge pull request #19961 from alvoron:dnn_ngraph_int64_fix Explicit usage of int64_t in CropAndResizeLayer (IE backend) * Update crop_and_resize_layer.cpp --- modules/dnn/src/layers/crop_and_resize_layer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/layers/crop_and_resize_layer.cpp b/modules/dnn/src/layers/crop_and_resize_layer.cpp index ba11c33508..261706a78e 100644 --- a/modules/dnn/src/layers/crop_and_resize_layer.cpp +++ b/modules/dnn/src/layers/crop_and_resize_layer.cpp @@ -125,7 +125,8 @@ public: auto input = nodes[0].dynamicCast()->node; auto rois = nodes[1].dynamicCast()->node; - std::vector dims = rois->get_shape(), offsets(4, 0); + auto rois_shape = rois->get_shape(); + std::vector dims(rois_shape.begin(), rois_shape.end()), offsets(4, 0); offsets[3] = 2; dims[3] = 7; @@ -139,7 +140,7 @@ public: lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); // Reshape rois from 4D to 2D - std::vector shapeData = {dims[2], 5}; + std::vector shapeData = {dims[2], 5}; auto shape = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, shapeData.data()); auto reshape = std::make_shared(slice, shape, true); From 1b64851fa88700281f58812e22eff56607eb1dd1 Mon Sep 17 00:00:00 2001 From: Stefano Allegretti Date: Thu, 22 Apr 2021 20:20:12 +0200 Subject: [PATCH 18/93] Merge pull request #19951 from stal12:3.4 * Fix #4363 - wrong hierarchy (CV_RETR_TREE) in findContours * Add regression test for findContours * use C++11 => C++98 on 3.4 branch --- modules/imgproc/src/contours.cpp | 32 +++++++++++++++-- modules/imgproc/test/test_contours.cpp | 49 ++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/modules/imgproc/src/contours.cpp b/modules/imgproc/src/contours.cpp index e433cdb514..241f1443f5 100644 --- a/modules/imgproc/src/contours.cpp +++ b/modules/imgproc/src/contours.cpp @@ -625,7 +625,8 @@ icvFetchContour( schar *ptr, /* trace contour until certain point is met. - returns 1 if met, 0 else. + returns 1 if met and this is the last contour + encountered by a raster scan reaching the point, 0 else. */ static int icvTraceContour( schar *ptr, int step, schar *stop_ptr, int is_hole ) @@ -668,14 +669,39 @@ icvTraceContour( schar *ptr, int step, schar *stop_ptr, int is_hole ) break; } - if( i3 == stop_ptr || (i4 == i0 && i3 == i1) ) + if (i3 == stop_ptr) { + if (!(*i3 & 0x80)) { + /* it's the only contour */ + return 1; + } + + /* check if this is the last contour */ + /* encountered during a raster scan */ + schar *i5; + int t = s; + while (true) + { + t = (t - 1) & 7; + i5 = i3 + deltas[t]; + if (*i5 != 0) + break; + if (t == 0) + return 1; + } + } + + if( (i4 == i0 && i3 == i1) ) break; i3 = i4; s = (s + 4) & 7; } /* end of border following loop */ } - return i3 == stop_ptr; + else { + return i3 == stop_ptr; + } + + return 0; } diff --git a/modules/imgproc/test/test_contours.cpp b/modules/imgproc/test/test_contours.cpp index a5c924829d..c07d19098b 100644 --- a/modules/imgproc/test/test_contours.cpp +++ b/modules/imgproc/test/test_contours.cpp @@ -485,6 +485,55 @@ TEST(Imgproc_FindContours, border) ASSERT_EQ(0, cvtest::norm(img, img_draw_contours, NORM_INF)); } +TEST(Imgproc_FindContours, regression_4363_shared_nbd) +{ + // Create specific test image + Mat1b img(12, 69, (const uchar&)0); + + img(1, 1) = 1; + + // Vertical rectangle with hole sharing the same NBD + for (int r = 1; r <= 10; ++r) { + for (int c = 3; c <= 5; ++c) { + img(r, c) = 1; + } + } + img(9, 4) = 0; + + // 124 small CCs + for (int r = 1; r <= 7; r += 2) { + for (int c = 7; c <= 67; c += 2) { + img(r, c) = 1; + } + } + + // Last CC + img(9, 7) = 1; + + vector< vector > contours; + vector hierarchy; + findContours(img, contours, hierarchy, RETR_TREE, CHAIN_APPROX_NONE); + + bool found = false; + size_t index = 0; + for (vector< vector >::const_iterator i = contours.begin(); i != contours.end(); ++i) + { + const vector& c = *i; + if (!c.empty() && c[0] == Point(7, 9)) + { + found = true; + index = (size_t)(i - contours.begin()); + break; + } + } + EXPECT_TRUE(found) << "Desired result: point (7,9) is a contour - Actual result: point (7,9) is not a contour"; + + if (found) + { + EXPECT_LT(hierarchy[index][3], 0) << "Desired result: (7,9) has no parent - Actual result: parent of (7,9) is another contour. index = " << index; + } +} + TEST(Imgproc_PointPolygonTest, regression_10222) { vector contour; From a53582d7068501c2a2a914af1838de685268072e Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Fri, 23 Apr 2021 13:26:53 +0300 Subject: [PATCH 19/93] Merge pull request #19923 from dbudniko:dbudniko/G-API_mtcnn_demo_PR G-API MTCNN sample * add face detection demo * clean up * enable back accumulate * additional input * meta args workaround * additional arg * add init * roll back * fix shadowing * roll back * clean up and PNet copy from debug branch which now works * try nets operator * more clean up * more clean up * add 6 layers pyramid experimental code * final clean up and ready for PR * original image resize * Remove Pnet declarations. Generic infer is used now. * scales and sizes calculation added * fix assert, and add ceil to size calculation * try doubles for scales * Address comments from Dmitry. * use half scale option * fix half scale * clean up debug outputs * try to get input image width and height * clean up * trailing spaces and review from Maxim * more comments from Maxim are addressed * try to fix warnings * try to fix warnings and address more comments from Dmitry * crop fix and clean up * more warnings fixes * more warnings fixes * more comments from Maxim are addressed * even more consts * copy_n for regressions * address more comments from Dmitry * more comments from Maxim --- .../gapi/include/opencv2/gapi/infer/ie.hpp | 30 + modules/gapi/samples/face_detection.cpp | 748 ++++++++++++++++++ 2 files changed, 778 insertions(+) create mode 100644 modules/gapi/samples/face_detection.cpp diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 60137c960c..2bed13abc3 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -235,6 +235,36 @@ public: return *this; } + Params& cfgInputReshape(std::map> && reshape_table) { + desc.reshape_table = std::move(reshape_table); + return *this; + } + + Params& cfgInputReshape(const std::map>&reshape_table) { + desc.reshape_table = reshape_table; + return *this; + } + + Params& cfgInputReshape(std::string && layer_name, std::vector && layer_dims) { + desc.reshape_table.emplace(layer_name, layer_dims); + return *this; + } + + Params& cfgInputReshape(const std::string & layer_name, const std::vector&layer_dims) { + desc.reshape_table.emplace(layer_name, layer_dims); + return *this; + } + + Params& cfgInputReshape(std::unordered_set && layer_names) { + desc.layer_names_to_reshape = std::move(layer_names); + return *this; + } + + Params& cfgInputReshape(const std::unordered_set&layer_names) { + desc.layer_names_to_reshape = layer_names; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/samples/face_detection.cpp b/modules/gapi/samples/face_detection.cpp new file mode 100644 index 0000000000..73115284ca --- /dev/null +++ b/modules/gapi/samples/face_detection.cpp @@ -0,0 +1,748 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const std::string about = +"This is an OpenCV-based version of OMZ MTCNN Face Detection example"; +const std::string keys = +"{ h help | | Print this help message }" +"{ input | | Path to the input video file }" +"{ mtcnnpm | mtcnn-p.xml | Path to OpenVINO MTCNN P (Proposal) detection model (.xml)}" +"{ mtcnnpd | CPU | Target device for the MTCNN P (e.g. CPU, GPU, VPU, ...) }" +"{ mtcnnrm | mtcnn-r.xml | Path to OpenVINO MTCNN R (Refinement) detection model (.xml)}" +"{ mtcnnrd | CPU | Target device for the MTCNN R (e.g. CPU, GPU, VPU, ...) }" +"{ mtcnnom | mtcnn-o.xml | Path to OpenVINO MTCNN O (Output) detection model (.xml)}" +"{ mtcnnod | CPU | Target device for the MTCNN O (e.g. CPU, GPU, VPU, ...) }" +"{ thrp | 0.6 | MTCNN P confidence threshold}" +"{ thrr | 0.7 | MTCNN R confidence threshold}" +"{ thro | 0.7 | MTCNN O confidence threshold}" +"{ half_scale | false | MTCNN P use half scale pyramid}" +; + +namespace { +std::string weights_path(const std::string& model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + const auto ext = model_path.substr(sz - EXT_LEN); + CV_Assert(cv::toLowerCase(ext) == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +////////////////////////////////////////////////////////////////////// +} // anonymous namespace + +namespace custom { +namespace { + +// Define custom structures and operations +#define NUM_REGRESSIONS 4 +#define NUM_PTS 5 + +struct BBox { + double x1; + double y1; + double x2; + double y2; + + cv::Rect getRect() const { return cv::Rect(static_cast(x1), + static_cast(y1), + static_cast(x2 - x1), + static_cast(y2 - y1)); } + + BBox getSquare() const { + BBox bbox; + double bboxWidth = x2 - x1; + double bboxHeight = y2 - y1; + double side = std::max(bboxWidth, bboxHeight); + bbox.x1 = static_cast(x1) + (bboxWidth - side) * 0.5; + bbox.y1 = static_cast(y1) + (bboxHeight - side) * 0.5; + bbox.x2 = bbox.x1 + side; + bbox.y2 = bbox.y1 + side; + return bbox; + } +}; + +struct Face { + BBox bbox; + double score; + std::array regression; + double ptsCoords[2 * NUM_PTS]; + + static void applyRegression(std::vector& faces, bool addOne = false) { + for (auto& face : faces) { + double bboxWidth = + face.bbox.x2 - face.bbox.x1 + static_cast(addOne); + double bboxHeight = + face.bbox.y2 - face.bbox.y1 + static_cast(addOne); + face.bbox.x1 = face.bbox.x1 + static_cast(face.regression[1]) * bboxWidth; + face.bbox.y1 = face.bbox.y1 + static_cast(face.regression[0]) * bboxHeight; + face.bbox.x2 = face.bbox.x2 + static_cast(face.regression[3]) * bboxWidth; + face.bbox.y2 = face.bbox.y2 + static_cast(face.regression[2]) * bboxHeight; + } + } + + static void bboxes2Squares(std::vector& faces) { + for (auto& face : faces) { + face.bbox = face.bbox.getSquare(); + } + } + + static std::vector runNMS(std::vector& faces, const double threshold, + const bool useMin = false) { + std::vector facesNMS; + if (faces.empty()) { + return facesNMS; + } + + std::sort(faces.begin(), faces.end(), [](const Face& f1, const Face& f2) { + return f1.score > f2.score; + }); + + std::vector indices(faces.size()); + std::iota(indices.begin(), indices.end(), 0); + + while (indices.size() > 0) { + const int idx = indices[0]; + facesNMS.push_back(faces[idx]); + std::vector tmpIndices = indices; + indices.clear(); + const double area1 = (faces[idx].bbox.x2 - faces[idx].bbox.x1 + 1) * + (faces[idx].bbox.y2 - faces[idx].bbox.y1 + 1); + for (size_t i = 1; i < tmpIndices.size(); ++i) { + int tmpIdx = tmpIndices[i]; + const double interX1 = std::max(faces[idx].bbox.x1, faces[tmpIdx].bbox.x1); + const double interY1 = std::max(faces[idx].bbox.y1, faces[tmpIdx].bbox.y1); + const double interX2 = std::min(faces[idx].bbox.x2, faces[tmpIdx].bbox.x2); + const double interY2 = std::min(faces[idx].bbox.y2, faces[tmpIdx].bbox.y2); + + const double bboxWidth = std::max(0.0, (interX2 - interX1 + 1)); + const double bboxHeight = std::max(0.0, (interY2 - interY1 + 1)); + + const double interArea = bboxWidth * bboxHeight; + const double area2 = (faces[tmpIdx].bbox.x2 - faces[tmpIdx].bbox.x1 + 1) * + (faces[tmpIdx].bbox.y2 - faces[tmpIdx].bbox.y1 + 1); + double overlap = 0.0; + if (useMin) { + overlap = interArea / std::min(area1, area2); + } else { + overlap = interArea / (area1 + area2 - interArea); + } + if (overlap <= threshold) { + indices.push_back(tmpIdx); + } + } + } + return facesNMS; + } +}; + +const double P_NET_WINDOW_SIZE = 12.0; +const double P_NET_STRIDE = 2.0; + +std::vector buildFaces(const cv::Mat& scores, + const cv::Mat& regressions, + const double scaleFactor, + const double threshold) { + + auto w = scores.size[3]; + auto h = scores.size[2]; + auto size = w * h; + + const float* scores_data = scores.ptr(); + scores_data += size; + + const float* reg_data = regressions.ptr(); + + std::vector boxes; + + for (int i = 0; i < size; i++) { + if (scores_data[i] >= (threshold)) { + int y = i / w; + int x = i - w * y; + + Face faceInfo; + BBox& faceBox = faceInfo.bbox; + + faceBox.x1 = (static_cast(x) * P_NET_STRIDE) / scaleFactor; + faceBox.y1 = (static_cast(y) * P_NET_STRIDE) / scaleFactor; + faceBox.x2 = (static_cast(x) * P_NET_STRIDE + P_NET_WINDOW_SIZE - 1.f) / scaleFactor; + faceBox.y2 = (static_cast(y) * P_NET_STRIDE + P_NET_WINDOW_SIZE - 1.f) / scaleFactor; + faceInfo.regression[0] = reg_data[i]; + faceInfo.regression[1] = reg_data[i + size]; + faceInfo.regression[2] = reg_data[i + 2 * size]; + faceInfo.regression[3] = reg_data[i + 3 * size]; + faceInfo.score = scores_data[i]; + boxes.push_back(faceInfo); + } + } + + return boxes; +} + +// Define networks for this sample +using GMat2 = std::tuple; +using GMat3 = std::tuple; +using GMats = cv::GArray; +using GRects = cv::GArray; +using GSize = cv::GOpaque; + +G_API_NET(MTCNNRefinement, + , + "sample.custom.mtcnn_refinement"); + +G_API_NET(MTCNNOutput, + , + "sample.custom.mtcnn_output"); + +using GFaces = cv::GArray; +G_API_OP(BuildFaces, + , + "sample.custom.mtcnn.build_faces") { + static cv::GArrayDesc outMeta(const cv::GMatDesc&, + const cv::GMatDesc&, + const double, + const double) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(RunNMS, + , + "sample.custom.mtcnn.run_nms") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, + const double, const bool) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(AccumulatePyramidOutputs, + , + "sample.custom.mtcnn.accumulate_pyramid_outputs") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, + const cv::GArrayDesc&) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(ApplyRegression, + , + "sample.custom.mtcnn.apply_regression") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const bool) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxesToSquares, + , + "sample.custom.mtcnn.bboxes_to_squares") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(R_O_NetPreProcGetROIs, + , + "sample.custom.mtcnn.bboxes_r_o_net_preproc_get_rois") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, const cv::GOpaqueDesc&) { + return cv::empty_array_desc(); + } +}; + + +G_API_OP(RNetPostProc, + , + "sample.custom.mtcnn.rnet_postproc") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const double) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(ONetPostProc, + , + "sample.custom.mtcnn.onet_postproc") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const cv::GArrayDesc&, + const double) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(SwapFaces, + , + "sample.custom.mtcnn.swap_faces") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc&) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(Transpose, + , + "sample.custom.mtcnn.transpose") { + static cv::GMatDesc outMeta(const cv::GMatDesc in) { + return in.withSize(cv::Size(in.size.height, in.size.width)); + } +}; + +//Custom kernels implementation +GAPI_OCV_KERNEL(OCVBuildFaces, BuildFaces) { + static void run(const cv::Mat & in_scores, + const cv::Mat & in_regresssions, + const double scaleFactor, + const double threshold, + std::vector &out_faces) { + out_faces = buildFaces(in_scores, in_regresssions, scaleFactor, threshold); + } +};// GAPI_OCV_KERNEL(BuildFaces) + +GAPI_OCV_KERNEL(OCVRunNMS, RunNMS) { + static void run(const std::vector &in_faces, + const double threshold, + const bool useMin, + std::vector &out_faces) { + std::vector in_faces_copy = in_faces; + out_faces = Face::runNMS(in_faces_copy, threshold, useMin); + } +};// GAPI_OCV_KERNEL(RunNMS) + +GAPI_OCV_KERNEL(OCVAccumulatePyramidOutputs, AccumulatePyramidOutputs) { + static void run(const std::vector &total_faces, + const std::vector &in_faces, + std::vector &out_faces) { + out_faces = total_faces; + out_faces.insert(out_faces.end(), in_faces.begin(), in_faces.end()); + } +};// GAPI_OCV_KERNEL(AccumulatePyramidOutputs) + +GAPI_OCV_KERNEL(OCVApplyRegression, ApplyRegression) { + static void run(const std::vector &in_faces, + const bool addOne, + std::vector &out_faces) { + std::vector in_faces_copy = in_faces; + Face::applyRegression(in_faces_copy, addOne); + out_faces.clear(); + out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end()); + } +};// GAPI_OCV_KERNEL(ApplyRegression) + +GAPI_OCV_KERNEL(OCVBBoxesToSquares, BBoxesToSquares) { + static void run(const std::vector &in_faces, + std::vector &out_faces) { + std::vector in_faces_copy = in_faces; + Face::bboxes2Squares(in_faces_copy); + out_faces.clear(); + out_faces.insert(out_faces.end(), in_faces_copy.begin(), in_faces_copy.end()); + } +};// GAPI_OCV_KERNEL(BBoxesToSquares) + +GAPI_OCV_KERNEL(OCVR_O_NetPreProcGetROIs, R_O_NetPreProcGetROIs) { + static void run(const std::vector &in_faces, + const cv::Size & in_image_size, + std::vector &outs) { + outs.clear(); + for (const auto& face : in_faces) { + cv::Rect tmp_rect = face.bbox.getRect(); + //Compare to transposed sizes width<->height + tmp_rect &= cv::Rect(tmp_rect.x, tmp_rect.y, in_image_size.height - tmp_rect.x - 4, in_image_size.width - tmp_rect.y - 4); + outs.push_back(tmp_rect); + } + } +};// GAPI_OCV_KERNEL(R_O_NetPreProcGetROIs) + + +GAPI_OCV_KERNEL(OCVRNetPostProc, RNetPostProc) { + static void run(const std::vector &in_faces, + const std::vector &in_scores, + const std::vector &in_regresssions, + const double threshold, + std::vector &out_faces) { + out_faces.clear(); + for (unsigned int k = 0; k < in_faces.size(); ++k) { + const float* scores_data = in_scores[k].ptr(); + const float* reg_data = in_regresssions[k].ptr(); + if (scores_data[1] >= threshold) { + Face info = in_faces[k]; + info.score = scores_data[1]; + std::copy_n(reg_data, NUM_REGRESSIONS, info.regression.begin()); + out_faces.push_back(info); + } + } + } +};// GAPI_OCV_KERNEL(RNetPostProc) + +GAPI_OCV_KERNEL(OCVONetPostProc, ONetPostProc) { + static void run(const std::vector &in_faces, + const std::vector &in_scores, + const std::vector &in_regresssions, + const std::vector &in_landmarks, + const double threshold, + std::vector &out_faces) { + out_faces.clear(); + for (unsigned int k = 0; k < in_faces.size(); ++k) { + const float* scores_data = in_scores[k].ptr(); + const float* reg_data = in_regresssions[k].ptr(); + const float* landmark_data = in_landmarks[k].ptr(); + if (scores_data[1] >= threshold) { + Face info = in_faces[k]; + info.score = scores_data[1]; + for (int i = 0; i < 4; ++i) { + info.regression[i] = reg_data[i]; + } + double w = info.bbox.x2 - info.bbox.x1 + 1.0; + double h = info.bbox.y2 - info.bbox.y1 + 1.0; + + for (int p = 0; p < NUM_PTS; ++p) { + info.ptsCoords[2 * p] = + info.bbox.x1 + static_cast(landmark_data[NUM_PTS + p]) * w - 1; + info.ptsCoords[2 * p + 1] = info.bbox.y1 + static_cast(landmark_data[p]) * h - 1; + } + + out_faces.push_back(info); + } + } + } +};// GAPI_OCV_KERNEL(ONetPostProc) + +GAPI_OCV_KERNEL(OCVSwapFaces, SwapFaces) { + static void run(const std::vector &in_faces, + std::vector &out_faces) { + std::vector in_faces_copy = in_faces; + out_faces.clear(); + if (!in_faces_copy.empty()) { + for (size_t i = 0; i < in_faces_copy.size(); ++i) { + std::swap(in_faces_copy[i].bbox.x1, in_faces_copy[i].bbox.y1); + std::swap(in_faces_copy[i].bbox.x2, in_faces_copy[i].bbox.y2); + for (int p = 0; p < NUM_PTS; ++p) { + std::swap(in_faces_copy[i].ptsCoords[2 * p], in_faces_copy[i].ptsCoords[2 * p + 1]); + } + } + out_faces = in_faces_copy; + } + } +};// GAPI_OCV_KERNEL(SwapFaces) + +GAPI_OCV_KERNEL(OCVTranspose, Transpose) { + static void run(const cv::Mat &in_mat, + cv::Mat &out_mat) { + cv::transpose(in_mat, out_mat); + } +};// GAPI_OCV_KERNEL(Transpose) +} // anonymous namespace +} // namespace custom + +namespace vis { +namespace { +void bbox(const cv::Mat& m, const cv::Rect& rc) { + cv::rectangle(m, rc, cv::Scalar{ 0,255,0 }, 2, cv::LINE_8, 0); +}; + +using rectPoints = std::pair>; + +static cv::Mat drawRectsAndPoints(const cv::Mat& img, + const std::vector data) { + cv::Mat outImg; + img.copyTo(outImg); + + for (const auto& el : data) { + vis::bbox(outImg, el.first); + auto pts = el.second; + for (size_t i = 0; i < pts.size(); ++i) { + cv::circle(outImg, pts[i], 3, cv::Scalar(0, 255, 255), 1); + } + } + return outImg; +} +} // anonymous namespace +} // namespace vis + + +//Infer helper function +namespace { +static inline std::tuple run_mtcnn_p(cv::GMat &in, const std::string &id) { + cv::GInferInputs inputs; + inputs["data"] = in; + auto outputs = cv::gapi::infer(id, inputs); + auto regressions = outputs.at("conv4-2"); + auto scores = outputs.at("prob1"); + return std::make_tuple(regressions, scores); +} + +//Operator fot PNet network package creation in the loop +inline cv::gapi::GNetPackage& operator += (cv::gapi::GNetPackage& lhs, const cv::gapi::GNetPackage& rhs) { + lhs.networks.reserve(lhs.networks.size() + rhs.networks.size()); + lhs.networks.insert(lhs.networks.end(), rhs.networks.begin(), rhs.networks.end()); + return lhs; +} + +static inline std::string get_pnet_level_name(const cv::Size &in_size) { + return "MTCNNProposal_" + std::to_string(in_size.width) + "x" + std::to_string(in_size.height); +} + +int calculate_scales(const cv::Size &input_size, std::vector &out_scales, std::vector &out_sizes ) { + //calculate multi - scale and limit the maxinum side to 1000 + //pr_scale: limit the maxinum side to 1000, < 1.0 + double pr_scale = 1.0; + double h = static_cast(input_size.height); + double w = static_cast(input_size.width); + if (std::min(w, h) > 1000) + { + pr_scale = 1000.0 / std::min(h, w); + w = w * pr_scale; + h = h * pr_scale; + } + else if (std::max(w, h) < 1000) + { + w = w * pr_scale; + h = h * pr_scale; + } + //multi - scale + out_scales.clear(); + out_sizes.clear(); + const double factor = 0.709; + int factor_count = 0; + double minl = std::min(h, w); + while (minl >= 12) + { + const double current_scale = pr_scale * std::pow(factor, factor_count); + cv::Size current_size(static_cast(static_cast(input_size.width) * current_scale), + static_cast(static_cast(input_size.height) * current_scale)); + out_scales.push_back(current_scale); + out_sizes.push_back(current_size); + minl *= factor; + factor_count += 1; + } + return factor_count; +} + +int calculate_half_scales(const cv::Size &input_size, std::vector& out_scales, std::vector& out_sizes) { + double pr_scale = 0.5; + const double h = static_cast(input_size.height); + const double w = static_cast(input_size.width); + //multi - scale + out_scales.clear(); + out_sizes.clear(); + const double factor = 0.5; + int factor_count = 0; + double minl = std::min(h, w); + while (minl >= 12.0*2.0) + { + const double current_scale = pr_scale; + cv::Size current_size(static_cast(static_cast(input_size.width) * current_scale), + static_cast(static_cast(input_size.height) * current_scale)); + out_scales.push_back(current_scale); + out_sizes.push_back(current_size); + minl *= factor; + factor_count += 1; + pr_scale *= 0.5; + } + return factor_count; +} + +const int MAX_PYRAMID_LEVELS = 13; +////////////////////////////////////////////////////////////////////// +} // anonymous namespace + +int main(int argc, char* argv[]) { + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + const auto input_file_name = cmd.get("input"); + const auto model_path_p = cmd.get("mtcnnpm"); + const auto target_dev_p = cmd.get("mtcnnpd"); + const auto conf_thresh_p = cmd.get("thrp"); + const auto model_path_r = cmd.get("mtcnnrm"); + const auto target_dev_r = cmd.get("mtcnnrd"); + const auto conf_thresh_r = cmd.get("thrr"); + const auto model_path_o = cmd.get("mtcnnom"); + const auto target_dev_o = cmd.get("mtcnnod"); + const auto conf_thresh_o = cmd.get("thro"); + const auto use_half_scale = cmd.get("half_scale"); + + std::vector level_size; + std::vector scales; + //MTCNN input size + cv::VideoCapture cap; + cap.open(input_file_name); + if (!cap.isOpened()) + CV_Assert(false); + auto in_rsz = cv::Size{ static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH)), + static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT)) }; + //Calculate scales, number of pyramid levels and sizes for PNet pyramid + auto pyramid_levels = use_half_scale ? calculate_half_scales(in_rsz, scales, level_size) : + calculate_scales(in_rsz, scales, level_size); + CV_Assert(pyramid_levels <= MAX_PYRAMID_LEVELS); + + //Proposal part of MTCNN graph + //Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW) + cv::GMat in_original; + cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); + cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); + cv::GMat in_resized[MAX_PYRAMID_LEVELS]; + cv::GMat in_transposed[MAX_PYRAMID_LEVELS]; + cv::GMat regressions[MAX_PYRAMID_LEVELS]; + cv::GMat scores[MAX_PYRAMID_LEVELS]; + cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; + cv::GArray total_faces[MAX_PYRAMID_LEVELS]; + cv::GArray faces_init(std::vector{}); + + //The very first PNet pyramid layer to init total_faces[0] + in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); + in_transposed[0] = custom::Transpose::on(in_resized[0]); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposed[0], get_pnet_level_name(level_size[0])); + cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], scales[0], conf_thresh_p); + nms_p_faces[0] = custom::RunNMS::on(faces0, 0.5, false); + total_faces[0] = custom::AccumulatePyramidOutputs::on(faces_init, nms_p_faces[0]); + //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] + for (int i = 1; i < pyramid_levels; ++i) + { + in_resized[i] = cv::gapi::resize(in_originalRGB, level_size[i]); + in_transposed[i] = custom::Transpose::on(in_resized[i]); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposed[i], get_pnet_level_name(level_size[i])); + cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], scales[i], conf_thresh_p); + nms_p_faces[i] = custom::RunNMS::on(faces, 0.5, false); + total_faces[i] = custom::AccumulatePyramidOutputs::on(total_faces[i - 1], nms_p_faces[i]); + } + + //Proposal post-processing + cv::GArray nms07_p_faces_total = custom::RunNMS::on(total_faces[pyramid_levels - 1], 0.7, false); + cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(nms07_p_faces_total, false); + cv::GArray final_faces_pnet = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); + + //Refinement part of MTCNN graph + cv::GArray faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz); + cv::GArray regressionsRNet, scoresRNet; + cv::GMat in_originalRGB_transposed = custom::Transpose::on(in_originalRGB); + std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_originalRGB_transposed); + + //Refinement post-processing + cv::GArray rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r); + cv::GArray nms07_r_faces_total = custom::RunNMS::on(rnet_post_proc_faces, 0.7, false); + cv::GArray final_r_faces_for_bb2squares = custom::ApplyRegression::on(nms07_r_faces_total, true); + cv::GArray final_faces_rnet = custom::BBoxesToSquares::on(final_r_faces_for_bb2squares); + + //Output part of MTCNN graph + cv::GArray faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz); + cv::GArray regressionsONet, scoresONet, landmarksONet; + std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_originalRGB_transposed); + + //Output post-processing + cv::GArray onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o); + cv::GArray final_o_faces_for_nms07 = custom::ApplyRegression::on(onet_post_proc_faces, true); + cv::GArray nms07_o_faces_total = custom::RunNMS::on(final_o_faces_for_nms07, 0.7, true); + cv::GArray final_faces_onet = custom::SwapFaces::on(nms07_o_faces_total); + + cv::GComputation graph_mtcnn(cv::GIn(in_original), cv::GOut(cv::gapi::copy(in_original), final_faces_onet)); + + // MTCNN Refinement detection network + auto mtcnnr_net = cv::gapi::ie::Params{ + model_path_r, // path to topology IR + weights_path(model_path_r), // path to weights + target_dev_r, // device specifier + }.cfgOutputLayers({ "conv5-2", "prob1" }).cfgInputLayers({ "data" }); + + // MTCNN Output detection network + auto mtcnno_net = cv::gapi::ie::Params{ + model_path_o, // path to topology IR + weights_path(model_path_o), // path to weights + target_dev_o, // device specifier + }.cfgOutputLayers({ "conv6-2", "conv6-3", "prob1" }).cfgInputLayers({ "data" }); + + auto networks_mtcnn = cv::gapi::networks(mtcnnr_net, mtcnno_net); + + // MTCNN Proposal detection network + for (int i = 0; i < pyramid_levels; ++i) + { + std::string net_id = get_pnet_level_name(level_size[i]); + std::vector reshape_dims = { 1, 3, (size_t)level_size[i].width, (size_t)level_size[i].height }; + cv::gapi::ie::Params mtcnnp_net{ + net_id, // tag + model_path_p, // path to topology IR + weights_path(model_path_p), // path to weights + target_dev_p, // device specifier + }; + mtcnnp_net.cfgInputReshape({ {"data", reshape_dims} }); + networks_mtcnn += cv::gapi::networks(mtcnnp_net); + } + + auto kernels_mtcnn = cv::gapi::kernels< custom::OCVBuildFaces + , custom::OCVRunNMS + , custom::OCVAccumulatePyramidOutputs + , custom::OCVApplyRegression + , custom::OCVBBoxesToSquares + , custom::OCVR_O_NetPreProcGetROIs + , custom::OCVRNetPostProc + , custom::OCVONetPostProc + , custom::OCVSwapFaces + , custom::OCVTranspose + >(); + auto pipeline_mtcnn = graph_mtcnn.compileStreaming(cv::compile_args(networks_mtcnn, kernels_mtcnn)); + + std::cout << "Reading " << input_file_name << std::endl; + // Input stream + auto in_src = cv::gapi::wip::make_src(input_file_name); + + // Set the pipeline source & start the pipeline + pipeline_mtcnn.setSource(cv::gin(in_src)); + pipeline_mtcnn.start(); + + // Declare the output data & run the processing loop + cv::TickMeter tm; + cv::Mat image; + std::vector out_faces; + + tm.start(); + int frames = 0; + while (pipeline_mtcnn.pull(cv::gout(image, out_faces))) { + frames++; + std::cout << "Final Faces Size " << out_faces.size() << std::endl; + std::vector data; + // show the image with faces in it + for (const auto& out_face : out_faces) { + std::vector pts; + for (int p = 0; p < NUM_PTS; ++p) { + pts.push_back( + cv::Point(static_cast(out_face.ptsCoords[2 * p]), static_cast(out_face.ptsCoords[2 * p + 1]))); + } + auto rect = out_face.bbox.getRect(); + auto d = std::make_pair(rect, pts); + data.push_back(d); + } + // Visualize results on the frame + auto resultImg = vis::drawRectsAndPoints(image, data); + tm.stop(); + const auto fps_str = std::to_string(frames / tm.getTimeSec()) + " FPS"; + cv::putText(resultImg, fps_str, { 0,32 }, cv::FONT_HERSHEY_SIMPLEX, 1.0, { 0,255,0 }, 2); + cv::imshow("Out", resultImg); + cv::waitKey(1); + out_faces.clear(); + tm.start(); + } + tm.stop(); + std::cout << "Processed " << frames << " frames" + << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; + return 0; +} From fcaeeac931922cd2cab4382419b4af77f0c4b0dd Mon Sep 17 00:00:00 2001 From: Harald Scheirich Date: Fri, 23 Apr 2021 16:48:32 -0400 Subject: [PATCH 20/93] Merge pull request #19780 from HarryDC:feature/index-multiimage-tiff Add reading of specific images from multipage tiff * Add reading of specific images from multipage tiff * Fix build issues * Add missing flag for gdal * Fix unused param warning * Remove duplicated code * change public parameter type to int * Fix warnings * Fix parameter check --- .../imgcodecs/include/opencv2/imgcodecs.hpp | 20 +++ modules/imgcodecs/src/loadsave.cpp | 116 +++++++++++++++--- modules/imgcodecs/test/test_tiff.cpp | 88 +++++++++++++ 3 files changed, 207 insertions(+), 17 deletions(-) diff --git a/modules/imgcodecs/include/opencv2/imgcodecs.hpp b/modules/imgcodecs/include/opencv2/imgcodecs.hpp index 6a389fd471..42227f3788 100644 --- a/modules/imgcodecs/include/opencv2/imgcodecs.hpp +++ b/modules/imgcodecs/include/opencv2/imgcodecs.hpp @@ -215,6 +215,26 @@ The function imreadmulti loads a multi-page image from the specified file into a */ CV_EXPORTS_W bool imreadmulti(const String& filename, CV_OUT std::vector& mats, int flags = IMREAD_ANYCOLOR); +/** @brief Loads a of images of a multi-page image from a file. + +The function imreadmulti loads a specified range from a multi-page image from the specified file into a vector of Mat objects. +@param filename Name of file to be loaded. +@param start Start index of the image to load +@param count Count number of images to load +@param flags Flag that can take values of cv::ImreadModes, default with cv::IMREAD_ANYCOLOR. +@param mats A vector of Mat objects holding each page, if more than one. +@sa cv::imread +*/ +CV_EXPORTS_W bool imreadmulti(const String& filename, CV_OUT std::vector& mats, int start, int count, int flags = IMREAD_ANYCOLOR); + +/** @brief Returns the number of images inside the give file + +The function imcount will return the number of pages in a multi-page image, or 1 for single-page images +@param filename Name of file to be loaded. +@param flags Flag that can take values of cv::ImreadModes, default with cv::IMREAD_ANYCOLOR. +*/ +CV_EXPORTS_W size_t imcount(const String& filename, int flags = IMREAD_ANYCOLOR); + /** @brief Saves an image to a specified file. The function imwrite saves the image to the specified file. The image format is chosen based on the diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 350042cd7d..28d8ff285b 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -495,25 +495,19 @@ imread_( const String& filename, int flags, Mat& mat ) } -/** -* Read an image into memory and return the information -* -* @param[in] filename File to load -* @param[in] flags Flags -* @param[in] mats Reference to C++ vector object to hold the images -* -*/ static bool -imreadmulti_(const String& filename, int flags, std::vector& mats) +imreadmulti_(const String& filename, int flags, std::vector& mats, int start, int count) { /// Search for the relevant decoder to handle the imagery ImageDecoder decoder; + CV_CheckGE(start, 0, "Start index cannont be < 0"); + #ifdef HAVE_GDAL - if (flags != IMREAD_UNCHANGED && (flags & IMREAD_LOAD_GDAL) == IMREAD_LOAD_GDAL){ + if (flags != IMREAD_UNCHANGED && (flags & IMREAD_LOAD_GDAL) == IMREAD_LOAD_GDAL) { decoder = GdalDecoder().newDecoder(); } - else{ + else { #endif decoder = findDecoder(filename); #ifdef HAVE_GDAL @@ -521,10 +515,14 @@ imreadmulti_(const String& filename, int flags, std::vector& mats) #endif /// if no decoder was found, return nothing. - if (!decoder){ + if (!decoder) { return 0; } + if (count < 0) { + count = std::numeric_limits::max(); + } + /// set the filename in the driver decoder->setSource(filename); @@ -532,7 +530,7 @@ imreadmulti_(const String& filename, int flags, std::vector& mats) try { // read the header to make sure it succeeds - if( !decoder->readHeader() ) + if (!decoder->readHeader()) return 0; } catch (const cv::Exception& e) @@ -546,11 +544,22 @@ imreadmulti_(const String& filename, int flags, std::vector& mats) return 0; } - for (;;) + int current = start; + + while (current > 0) + { + if (!decoder->nextPage()) + { + return false; + } + --current; + } + + while (current < count) { // grab the decoded type int type = decoder->type(); - if( (flags & IMREAD_LOAD_GDAL) != IMREAD_LOAD_GDAL && flags != IMREAD_UNCHANGED ) + if ((flags & IMREAD_LOAD_GDAL) != IMREAD_LOAD_GDAL && flags != IMREAD_UNCHANGED) { if ((flags & IMREAD_ANYDEPTH) == 0) type = CV_MAKETYPE(CV_8U, CV_MAT_CN(type)); @@ -585,7 +594,7 @@ imreadmulti_(const String& filename, int flags, std::vector& mats) break; // optionally rotate the data if EXIF' orientation flag says so - if( (flags & IMREAD_IGNORE_ORIENTATION) == 0 && flags != IMREAD_UNCHANGED ) + if ((flags & IMREAD_IGNORE_ORIENTATION) == 0 && flags != IMREAD_UNCHANGED) { ApplyExifOrientation(decoder->getExifTag(ORIENTATION), mat); } @@ -595,6 +604,7 @@ imreadmulti_(const String& filename, int flags, std::vector& mats) { break; } + ++current; } return !mats.empty(); @@ -636,9 +646,81 @@ bool imreadmulti(const String& filename, std::vector& mats, int flags) { CV_TRACE_FUNCTION(); - return imreadmulti_(filename, flags, mats); + return imreadmulti_(filename, flags, mats, 0, -1); } + +bool imreadmulti(const String& filename, std::vector& mats, int start, int count, int flags) +{ + CV_TRACE_FUNCTION(); + + return imreadmulti_(filename, flags, mats, start, count); +} + +static +size_t imcount_(const String& filename, int flags) +{ + /// Search for the relevant decoder to handle the imagery + ImageDecoder decoder; + +#ifdef HAVE_GDAL + if (flags != IMREAD_UNCHANGED && (flags & IMREAD_LOAD_GDAL) == IMREAD_LOAD_GDAL) { + decoder = GdalDecoder().newDecoder(); + } + else { +#else + CV_UNUSED(flags); +#endif + decoder = findDecoder(filename); +#ifdef HAVE_GDAL + } +#endif + + /// if no decoder was found, return nothing. + if (!decoder) { + return 0; + } + + /// set the filename in the driver + decoder->setSource(filename); + + // read the header to make sure it succeeds + try + { + // read the header to make sure it succeeds + if (!decoder->readHeader()) + return 0; + } + catch (const cv::Exception& e) + { + std::cerr << "imcount_('" << filename << "'): can't read header: " << e.what() << std::endl << std::flush; + return 0; + } + catch (...) + { + std::cerr << "imcount_('" << filename << "'): can't read header: unknown exception" << std::endl << std::flush; + return 0; + } + + size_t result = 1; + + + while (decoder->nextPage()) + { + ++result; + } + + return result; +} + +size_t imcount(const String& filename, int flags) +{ + CV_TRACE_FUNCTION(); + + return imcount_(filename, flags); +} + + static bool imwrite_( const String& filename, const std::vector& img_vec, const std::vector& params, bool flipv ) { diff --git a/modules/imgcodecs/test/test_tiff.cpp b/modules/imgcodecs/test/test_tiff.cpp index 2c6fb6249b..dec38014aa 100644 --- a/modules/imgcodecs/test/test_tiff.cpp +++ b/modules/imgcodecs/test/test_tiff.cpp @@ -358,6 +358,94 @@ TEST(Imgcodecs_Tiff, decode_black_and_write_image_pr17275_default) EXPECT_EQ(CV_8UC3, img.type()) << cv::typeToString(img.type()); } +TEST(Imgcodecs_Tiff, count_multipage) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + { + const string filename = root + "readwrite/multipage.tif"; + ASSERT_EQ((size_t)6, imcount(filename)); + } + { + const string filename = root + "readwrite/test32FC3_raw.tiff"; + ASSERT_EQ((size_t)1, imcount(filename)); + } +} + +TEST(Imgcodecs_Tiff, read_multipage_indexed) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filename = root + "readwrite/multipage.tif"; + const string page_files[] = { + "readwrite/multipage_p1.tif", + "readwrite/multipage_p2.tif", + "readwrite/multipage_p3.tif", + "readwrite/multipage_p4.tif", + "readwrite/multipage_p5.tif", + "readwrite/multipage_p6.tif" + }; + const int page_count = sizeof(page_files) / sizeof(page_files[0]); + vector single_pages; + for (int i = 0; i < page_count; i++) + { + // imread and imreadmulti have different default values for the flag + const Mat page = imread(root + page_files[i], IMREAD_ANYCOLOR); + single_pages.push_back(page); + } + ASSERT_EQ((size_t)page_count, single_pages.size()); + + { + SCOPED_TRACE("Edge Cases"); + vector multi_pages; + bool res = imreadmulti(filename, multi_pages, 0, 0); + // If we asked for 0 images and we successfully read 0 images should this be false ? + ASSERT_TRUE(res == false); + ASSERT_EQ((size_t)0, multi_pages.size()); + res = imreadmulti(filename, multi_pages, 0, 123123); + ASSERT_TRUE(res == true); + ASSERT_EQ((size_t)6, multi_pages.size()); + } + + { + SCOPED_TRACE("Read all with indices"); + vector multi_pages; + bool res = imreadmulti(filename, multi_pages, 0, 6); + ASSERT_TRUE(res == true); + ASSERT_EQ((size_t)page_count, multi_pages.size()); + for (int i = 0; i < page_count; i++) + { + EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), multi_pages[i], single_pages[i]); + } + } + + { + SCOPED_TRACE("Read one by one"); + vector multi_pages; + for (int i = 0; i < page_count; i++) + { + bool res = imreadmulti(filename, multi_pages, i, 1); + ASSERT_TRUE(res == true); + ASSERT_EQ((size_t)1, multi_pages.size()); + EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), multi_pages[0], single_pages[i]); + multi_pages.clear(); + } + } + + { + SCOPED_TRACE("Read multiple at a time"); + vector multi_pages; + for (int i = 0; i < page_count/2; i++) + { + bool res = imreadmulti(filename, multi_pages, i*2, 2); + ASSERT_TRUE(res == true); + ASSERT_EQ((size_t)2, multi_pages.size()); + EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), multi_pages[0], single_pages[i * 2]) << i; + EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), multi_pages[1], single_pages[i * 2 + 1]); + multi_pages.clear(); + } + } +} + + #endif }} // namespace From 886835088836d90460f33ebd002a0af23d87824c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 23 Apr 2021 22:30:06 +0000 Subject: [PATCH 21/93] doxygen: fix quotes in add_toggle macro --- doc/Doxyfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 02dd372660..808315efcf 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -31,7 +31,7 @@ MULTILINE_CPP_IS_BRIEF = NO INHERIT_DOCS = YES SEPARATE_MEMBER_PAGES = NO TAB_SIZE = 4 -ALIASES += add_toggle{1}="@htmlonly[block]
\1
@endhtmlonly" +ALIASES += add_toggle{1}="@htmlonly[block]
\1
@endhtmlonly" ALIASES += add_toggle_cpp="@htmlonly[block]