diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a35cebb1f..56b4fa16e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,10 @@ if(POLICY CMP0056) cmake_policy(SET CMP0056 NEW) # try_compile(): link flags endif() +if(POLICY CMP0066) + cmake_policy(SET CMP0066 NEW) # CMake 3.7: try_compile(): use per-config flags, like CMAKE_CXX_FLAGS_RELEASE +endif() + if(POLICY CMP0067) cmake_policy(SET CMP0067 NEW) # CMake 3.8: try_compile(): honor language standard variables (like C++11) endif() diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 40a36b656e..1504fa61c4 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -1,6 +1,8 @@ add_definitions(-D__OPENCV_BUILD=1) add_definitions(-D__OPENCV_APPS=1) +string(REPLACE "," ";" OPENCV_INSTALL_APPS_LIST "${OPENCV_INSTALL_APPS_LIST}") # support comma-separated list (,) too + # Unified function for creating OpenCV applications: # ocv_add_application(tgt [MODULES [ ...]] SRCS [ ...]) function(ocv_add_application the_target) @@ -25,12 +27,14 @@ function(ocv_add_application the_target) set_target_properties(${the_target} PROPERTIES FOLDER "applications") endif() - if(INSTALL_CREATE_DISTRIB) + if(NOT INSTALL_CREATE_DISTRIB + OR (OPENCV_INSTALL_APPS_LIST STREQUAL "all" OR ";${OPENCV_INSTALL_APPS_LIST};" MATCHES ";${the_target};") + ) + install(TARGETS ${the_target} RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT dev) + elseif(INSTALL_CREATE_DISTRIB) if(BUILD_SHARED_LIBS) install(TARGETS ${the_target} RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} CONFIGURATIONS Release COMPONENT dev) endif() - else() - install(TARGETS ${the_target} RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT dev) endif() endfunction() diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake index 5d0ee4a96b..4ff02a77d3 100644 --- a/cmake/OpenCVDetectPython.cmake +++ b/cmake/OpenCVDetectPython.cmake @@ -78,10 +78,10 @@ if(NOT ${found}) AND NOT DEFINED ${executable} ) if(NOT OPENCV_SKIP_PYTHON_WARNING) - message(WARNING "CMake's 'find_host_package(PythonInterp ${__python_package_version})' founds wrong Python version:\n" + message(WARNING "CMake's 'find_host_package(PythonInterp ${__python_package_version})' found wrong Python version:\n" "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}\n" "PYTHON_VERSION_STRING=${PYTHON_VERSION_STRING}\n" - "Consider specify '${executable}' variable via CMake command line or environment variables\n") + "Consider providing the '${executable}' variable via CMake command line or environment variables\n") endif() ocv_clear_vars(PYTHONINTERP_FOUND PYTHON_EXECUTABLE PYTHON_VERSION_STRING PYTHON_VERSION_MAJOR PYTHON_VERSION_MINOR PYTHON_VERSION_PATCH) if(NOT CMAKE_VERSION VERSION_LESS "3.12") diff --git a/doc/js_tutorials/js_gui/js_image_display/js_image_display.markdown b/doc/js_tutorials/js_gui/js_image_display/js_image_display.markdown index efe65e320e..9ad4ce2e53 100644 --- a/doc/js_tutorials/js_gui/js_image_display/js_image_display.markdown +++ b/doc/js_tutorials/js_gui/js_image_display/js_image_display.markdown @@ -13,7 +13,7 @@ OpenCV.js saves images as cv.Mat type. We use HTML canvas element to transfer cv or in reverse. The ImageData interface can represent or set the underlying pixel data of an area of a canvas element. -@sa Please refer to canvas docs for more details. +@note Please refer to canvas docs for more details. First, create an ImageData obj from canvas: @code{.js} diff --git a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown index f56e639005..e337999efd 100644 --- a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown +++ b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown @@ -83,7 +83,7 @@ use 7x6 grid. (Normally a chess board has 8x8 squares and 7x7 internal corners). corner points and retval which will be True if pattern is obtained. These corners will be placed in an order (from left-to-right, top-to-bottom) -@sa This function may not be able to find the required pattern in all the images. So, one good option +@note This function may not be able to find the required pattern in all the images. So, one good option is to write the code such that, it starts the camera and check each frame for required pattern. Once the pattern is obtained, find the corners and store it in a list. Also, provide some interval before reading next frame so that we can adjust our chess board in different direction. Continue this @@ -91,7 +91,7 @@ process until the required number of good patterns are obtained. Even in the exa are not sure how many images out of the 14 given are good. Thus, we must read all the images and take only the good ones. -@sa Instead of chess board, we can alternatively use a circular grid. In this case, we must use the function +@note Instead of chess board, we can alternatively use a circular grid. In this case, we must use the function **cv.findCirclesGrid()** to find the pattern. Fewer images are sufficient to perform camera calibration using a circular grid. Once we find the corners, we can increase their accuracy using **cv.cornerSubPix()**. We can also diff --git a/doc/py_tutorials/py_gui/py_image_display/py_image_display.markdown b/doc/py_tutorials/py_gui/py_image_display/py_image_display.markdown index 8b8cae0e73..edb957bd95 100644 --- a/doc/py_tutorials/py_gui/py_image_display/py_image_display.markdown +++ b/doc/py_tutorials/py_gui/py_image_display/py_image_display.markdown @@ -132,7 +132,7 @@ A screen-shot of the window will look like this : ![image](images/matplotlib_screenshot.jpg) -@sa Plenty of plotting options are available in Matplotlib. Please refer to Matplotlib docs for more +@note Plenty of plotting options are available in Matplotlib. Please refer to Matplotlib docs for more details. Some, we will see on the way. __warning__ diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown index f11f11aed8..e50f9dd6c9 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown @@ -113,7 +113,7 @@ I got following results: See, even image rotation doesn't affect much on this comparison. -@sa [Hu-Moments](http://en.wikipedia.org/wiki/Image_moment#Rotation_invariant_moments) are seven +@note [Hu-Moments](http://en.wikipedia.org/wiki/Image_moment#Rotation_invariant_moments) are seven moments invariant to translation, rotation and scale. Seventh one is skew-invariant. Those values can be found using **cv.HuMoments()** function. diff --git a/doc/py_tutorials/py_imgproc/py_histograms/py_histogram_begins/py_histogram_begins.markdown b/doc/py_tutorials/py_imgproc/py_histograms/py_histogram_begins/py_histogram_begins.markdown index c26449cad4..8cb24139e8 100644 --- a/doc/py_tutorials/py_imgproc/py_histograms/py_histogram_begins/py_histogram_begins.markdown +++ b/doc/py_tutorials/py_imgproc/py_histograms/py_histogram_begins/py_histogram_begins.markdown @@ -94,7 +94,7 @@ hist is same as we calculated before. But bins will have 257 elements, because N as 0-0.99, 1-1.99, 2-2.99 etc. So final range would be 255-255.99. To represent that, they also add 256 at end of bins. But we don't need that 256. Upto 255 is sufficient. -@sa Numpy has another function, **np.bincount()** which is much faster than (around 10X) +@note Numpy has another function, **np.bincount()** which is much faster than (around 10X) np.histogram(). So for one-dimensional histograms, you can better try that. Don't forget to set minlength = 256 in np.bincount. For example, hist = np.bincount(img.ravel(),minlength=256) diff --git a/doc/tutorials/ml/introduction_to_svm/introduction_to_svm.markdown b/doc/tutorials/ml/introduction_to_svm/introduction_to_svm.markdown index 84fae2bad8..f7fca1520a 100644 --- a/doc/tutorials/ml/introduction_to_svm/introduction_to_svm.markdown +++ b/doc/tutorials/ml/introduction_to_svm/introduction_to_svm.markdown @@ -51,7 +51,7 @@ Let's introduce the notation used to define formally a hyperplane: where \f$\beta\f$ is known as the *weight vector* and \f$\beta_{0}\f$ as the *bias*. -@sa A more in depth description of this and hyperplanes you can find in the section 4.5 (*Separating +@note A more in depth description of this and hyperplanes you can find in the section 4.5 (*Separating Hyperplanes*) of the book: *Elements of Statistical Learning* by T. Hastie, R. Tibshirani and J. H. Friedman (@cite HTF01). diff --git a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown index f43790c297..7e214e0962 100644 --- a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown +++ b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown @@ -164,7 +164,7 @@ Describing the methods goes well beyond the purpose of this tutorial. For that I the article introducing it. Nevertheless, you can get a good image of it by looking at the OpenCV implementation below. -@sa +@note SSIM is described more in-depth in the: "Z. Wang, A. C. Bovik, H. R. Sheikh and E. P. Simoncelli, "Image quality assessment: From error visibility to structural similarity," IEEE Transactions on Image Processing, vol. 13, no. 4, pp. 600-612, Apr. 2004." article. diff --git a/modules/calib3d/src/quadsubpix.cpp b/modules/calib3d/src/quadsubpix.cpp index b4100a22f9..24dfdc14a4 100644 --- a/modules/calib3d/src/quadsubpix.cpp +++ b/modules/calib3d/src/quadsubpix.cpp @@ -61,13 +61,13 @@ static void orderContours(const std::vector >& contours, Poin for(i = 0; i < n; i++) { size_t ni = contours[i].size(); - double min_dist = std::numeric_limits::max(); + float min_dist = std::numeric_limits::max(); for(j = 0; j < ni; j++) { double dist = norm(Point2f((float)contours[i][j].x, (float)contours[i][j].y) - point); - min_dist = MIN(min_dist, dist); + min_dist = (float)MIN((double)min_dist, dist); } - order.push_back(std::pair((int)i, (float)min_dist)); + order.push_back(std::pair((int)i, min_dist)); } std::sort(order.begin(), order.end(), is_smaller); diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index cf005c65f3..bdc46643fc 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -93,6 +93,7 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer float forgetBias, cellClip; bool useCellClip, usePeephole; bool reverse; // If true, go in negative direction along the time axis + bool bidirectional; // If true, produces both forward and reversed directions along time axis public: @@ -101,6 +102,7 @@ public: { setParamsFrom(params); + bidirectional = params.get("bidirectional", false); if (!blobs.empty()) { CV_Assert(blobs.size() >= 3); @@ -110,10 +112,11 @@ public: const Mat& Wh = blobs[0]; const Mat& Wx = blobs[1]; const Mat& bias = blobs[2]; - CV_Assert(Wh.dims == 2 && Wx.dims == 2); - CV_Assert(Wh.rows == Wx.rows); - CV_Assert(Wh.rows == 4*Wh.cols); - CV_Assert(Wh.rows == (int)bias.total()); + CV_CheckEQ(Wh.dims, 2, ""); + CV_CheckEQ(Wx.dims, 2, ""); + CV_CheckEQ(Wh.rows, Wx.rows, ""); + CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional))*4*Wh.cols, ""); + CV_CheckEQ(Wh.rows, (int)bias.total(), ""); CV_Assert(Wh.type() == Wx.type() && Wx.type() == bias.type()); // Peephole weights. @@ -135,6 +138,7 @@ public: useCellClip = params.get("use_cell_clip", false); usePeephole = params.get("use_peephole", false); reverse = params.get("reverse", false); + CV_Assert(!reverse || !bidirectional); allocated = false; outTailShape.clear(); @@ -206,6 +210,7 @@ public: outResShape.push_back(_numSamples); outResShape.insert(outResShape.end(), outTailShape_.begin(), outTailShape_.end()); + outResShape.back() *= (1 + static_cast(bidirectional)); size_t noutputs = produceCellOutput ? 2 : 1; outputs.assign(noutputs, outResShape); @@ -252,6 +257,7 @@ public: outTsShape.clear(); outTsShape.push_back(numSamples); outTsShape.insert(outTsShape.end(), outTailShape.begin(), outTailShape.end()); + outTsShape.back() *= (1 + static_cast(bidirectional)); allocated = true; } @@ -272,91 +278,96 @@ public: outputs_arr.getMatVector(output); internals_arr.getMatVector(internals); - const Mat &Wh = blobs[0]; - const Mat &Wx = blobs[1]; - const Mat &bias = blobs[2]; - - int numOut = Wh.size[1]; - - Mat hInternal = internals[0], cInternal = internals[1], - dummyOnes = internals[2], gates = internals[3]; - hInternal.setTo(0.); - cInternal.setTo(0.); - dummyOnes.setTo(1.); - - int numSamplesTotal = numTimeStamps*numSamples; - Mat xTs = input[0].reshape(1, numSamplesTotal); - - Mat hOutTs = output[0].reshape(1, numSamplesTotal); - Mat cOutTs = produceCellOutput ? output[1].reshape(1, numSamplesTotal) : Mat(); - - int tsStart, tsEnd, tsInc; - if (reverse) { - tsStart = numTimeStamps - 1; - tsEnd = -1; - tsInc = -1; - } - else { - tsStart = 0; - tsEnd = numTimeStamps; - tsInc = 1; - } - for (int ts = tsStart; ts != tsEnd; ts += tsInc) + const int numDirs = 1 + static_cast(bidirectional); + for (int i = 0; i < numDirs; ++i) { - Range curRowRange(ts*numSamples, (ts + 1)*numSamples); - Mat xCurr = xTs.rowRange(curRowRange); + const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); + const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); + const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); - gemm(xCurr, Wx, 1, gates, 0, gates, GEMM_2_T); // Wx * x_t - gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T); //+Wh * h_{t-1} - gemm(dummyOnes, bias, 1, gates, 1, gates); //+b + int numOut = Wh.size[1]; - Mat gateI = gates.colRange(0*numOut, 1*numOut); - Mat gateF = gates.colRange(1*numOut, 2*numOut); - Mat gateO = gates.colRange(2*numOut, 3*numOut); - Mat gateG = gates.colRange(3*numOut, 4*numOut); + Mat hInternal = internals[0], cInternal = internals[1], + dummyOnes = internals[2], gates = internals[3]; + hInternal.setTo(0.); + cInternal.setTo(0.); + dummyOnes.setTo(1.); - if (forgetBias) - add(gateF, forgetBias, gateF); + int numSamplesTotal = numTimeStamps*numSamples; + Mat xTs = input[0].reshape(1, numSamplesTotal); - if (usePeephole) - { - Mat gatesIF = gates.colRange(0, 2*numOut); - gemm(cInternal, blobs[3], 1, gateI, 1, gateI); - gemm(cInternal, blobs[4], 1, gateF, 1, gateF); - sigmoid(gatesIF, gatesIF); + Mat hOutTs = output[0].reshape(1, numSamplesTotal); + hOutTs = hOutTs.colRange(i * hOutTs.cols / numDirs, (i + 1) * hOutTs.cols / numDirs); + Mat cOutTs = produceCellOutput ? output[1].reshape(1, numSamplesTotal) : Mat(); + + int tsStart, tsEnd, tsInc; + if (reverse || i == 1) { + tsStart = numTimeStamps - 1; + tsEnd = -1; + tsInc = -1; } - else - { - Mat gatesIFO = gates.colRange(0, 3*numOut); - sigmoid(gatesIFO, gatesIFO); + else { + tsStart = 0; + tsEnd = numTimeStamps; + tsInc = 1; } - - tanh(gateG, gateG); - - //compute c_t - multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1} - multiply(gateI, gateG, gateI); // i_t (*) g_t - add(gateF, gateI, cInternal); // c_t = f_t (*) c_{t-1} + i_t (*) g_t - - if (useCellClip) + for (int ts = tsStart; ts != tsEnd; ts += tsInc) { - min(cInternal, cellClip, cInternal); - max(cInternal, -cellClip, cInternal); - } - if (usePeephole) - { - gemm(cInternal, blobs[5], 1, gateO, 1, gateO); - sigmoid(gateO, gateO); - } + Range curRowRange(ts*numSamples, (ts + 1)*numSamples); + Mat xCurr = xTs.rowRange(curRowRange); - //compute h_t - tanh(cInternal, hInternal); - multiply(gateO, hInternal, hInternal); + gemm(xCurr, Wx, 1, gates, 0, gates, GEMM_2_T); // Wx * x_t + gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T); //+Wh * h_{t-1} + gemm(dummyOnes, bias, 1, gates, 1, gates); //+b - //save results in output blobs - hInternal.copyTo(hOutTs.rowRange(curRowRange)); - if (produceCellOutput) - cInternal.copyTo(cOutTs.rowRange(curRowRange)); + Mat gateI = gates.colRange(0*numOut, 1*numOut); + Mat gateF = gates.colRange(1*numOut, 2*numOut); + Mat gateO = gates.colRange(2*numOut, 3*numOut); + Mat gateG = gates.colRange(3*numOut, 4*numOut); + + if (forgetBias) + add(gateF, forgetBias, gateF); + + if (usePeephole) + { + Mat gatesIF = gates.colRange(0, 2*numOut); + gemm(cInternal, blobs[3], 1, gateI, 1, gateI); + gemm(cInternal, blobs[4], 1, gateF, 1, gateF); + sigmoid(gatesIF, gatesIF); + } + else + { + Mat gatesIFO = gates.colRange(0, 3*numOut); + sigmoid(gatesIFO, gatesIFO); + } + + tanh(gateG, gateG); + + //compute c_t + multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1} + multiply(gateI, gateG, gateI); // i_t (*) g_t + add(gateF, gateI, cInternal); // c_t = f_t (*) c_{t-1} + i_t (*) g_t + + if (useCellClip) + { + min(cInternal, cellClip, cInternal); + max(cInternal, -cellClip, cInternal); + } + if (usePeephole) + { + gemm(cInternal, blobs[5], 1, gateO, 1, gateO); + sigmoid(gateO, gateO); + } + + //compute h_t + tanh(cInternal, hInternal); + multiply(gateO, hInternal, hInternal); + + //save results in output blobs + hInternal.copyTo(hOutTs.rowRange(curRowRange)); + if (produceCellOutput) + cInternal.copyTo(cOutTs.rowRange(curRowRange)); + } } } }; diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 2516cbf145..7672c28ad5 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -49,6 +49,11 @@ class ONNXImporter LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto); bool isCeilMode(const LayerParams& layerParams); + void addLayer(Net& dstNet, LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto, + std::map& layer_id, + std::map& outShapes); + public: ONNXImporter(const char *onnxFile) @@ -259,6 +264,42 @@ Mat ONNXImporter::getBlob(const opencv_onnx::NodeProto& node_proto, return constBlob->second; } +void ONNXImporter::addLayer(Net& dstNet, LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto, + std::map& layer_id, + std::map& outShapes) +{ + std::map::iterator layerId; + std::map::iterator shapeIt; + + int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + for (int i = 0; i < node_proto.output_size(); ++i) + { + layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); + } + + std::vector layerInpShapes, layerOutShapes, layerInternalShapes; + int inpNum = 0; + for (int j = 0; j < node_proto.input_size(); j++) { + layerId = layer_id.find(node_proto.input(j)); + if (layerId != layer_id.end()) { + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); + ++inpNum; + // Collect input shapes. + shapeIt = outShapes.find(node_proto.input(j)); + CV_Assert(shapeIt != outShapes.end()); + layerInpShapes.push_back(shapeIt->second); + } + } + // Compute shape of output blob for this layer. + Ptr layer = dstNet.getLayer(id); + layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); + for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) + { + outShapes[node_proto.output(i)] = layerOutShapes[i]; + } +} + void ONNXImporter::populateNet(Net dstNet) { CV_Assert(model_proto.has_graph()); @@ -455,6 +496,7 @@ void ONNXImporter::populateNet(Net dstNet) runLayer(layerParams, inputs, sliced); CV_Assert(sliced.size() == 1); constBlobs.insert(std::make_pair(layerParams.name, sliced[0])); + outShapes[layerParams.name] = shape(sliced[0]); continue; } } @@ -579,6 +621,70 @@ void ONNXImporter::populateNet(Net dstNet) constBlobs.insert(std::make_pair(layerParams.name, layerParams.blobs[0])); continue; } + else if (layer_type == "LSTM") + { + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() == 7); + Mat Wx = getBlob(node_proto, constBlobs, 1); + Mat Wh = getBlob(node_proto, constBlobs, 2); + Mat b = getBlob(node_proto, constBlobs, 3); + CV_CheckEQ(countNonZero(getBlob(node_proto, constBlobs, 5)), 0, "Unsupported non zero initial_h"); + CV_CheckEQ(countNonZero(getBlob(node_proto, constBlobs, 6)), 0, "Unsupported non zero initial_c"); + b = b.reshape(1, b.size[0]); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + + lstmParams.blobs.resize(3); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(dstNet, lstmParams, node_proto, layer_id, outShapes); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + } else if (layer_type == "ImageScaler") { const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; @@ -882,13 +988,38 @@ void ONNXImporter::populateNet(Net dstNet) { CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); DictValue axes_dict = layerParams.get("axes"); - if (axes_dict.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional squeeze"); + MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes_dict.getIntValue(0); - layerParams.set("axis", axis - 1); - layerParams.set("end_axis", axis); - layerParams.type = "Flatten"; + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + } + else + layerParams.type = "Identity"; + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, constBlobs, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + constBlobs.insert(std::make_pair(layerParams.name, out)); + outShapes[layerParams.name] = shape(out); + continue; + } } else if (layer_type == "Flatten") { @@ -1018,9 +1149,17 @@ void ONNXImporter::populateNet(Net dstNet) else layerParams.type = "Identity"; } - else if (layer_type == "ConstantOfShape") + else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") { - float fill_value = layerParams.blobs.empty() ? 0 : layerParams.blobs[0].at(0, 0); + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + fill_value = layerParams.blobs[0].at(0, 0); + } + else + fill_value = layerParams.get("value", 0); + MatShape inpShape = getBlob(node_proto, constBlobs, 0); for (int i = 0; i < inpShape.size(); i++) CV_CheckGT(inpShape[i], 0, ""); @@ -1032,17 +1171,30 @@ void ONNXImporter::populateNet(Net dstNet) else if (layer_type == "Gather") { CV_Assert(node_proto.input_size() == 2); - CV_Assert(layerParams.has("axis")); Mat input = getBlob(node_proto, constBlobs, 0); Mat indexMat = getBlob(node_proto, constBlobs, 1); CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); int index = indexMat.at(0); - int axis = layerParams.get("axis"); - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); + Mat out; + if (layerParams.has("axis")) + { + int axis = layerParams.get("axis"); - Mat out = input(ranges); + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + } + else + { + CV_Assert(index < input.total()); + const int dims = input.dims; + input = input.reshape(1, 1); + input.dims = 2; + out = input.reshape(1, 1).colRange(index, index + 1); + out.dims = dims; + } constBlobs.insert(std::make_pair(layerParams.name, out)); continue; } @@ -1145,34 +1297,7 @@ void ONNXImporter::populateNet(Net dstNet) layerParams.blobs.push_back(getBlob(node_proto, constBlobs, j)); } } - - int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); - for (int i = 0; i < node_proto.output_size(); ++i) - { - layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); - } - - std::vector layerInpShapes, layerOutShapes, layerInternalShapes; - int inpNum = 0; - for (int j = 0; j < node_proto.input_size(); j++) { - layerId = layer_id.find(node_proto.input(j)); - if (layerId != layer_id.end()) { - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); - ++inpNum; - // Collect input shapes. - shapeIt = outShapes.find(node_proto.input(j)); - CV_Assert(shapeIt != outShapes.end()); - layerInpShapes.push_back(shapeIt->second); - } - } - - // Compute shape of output blob for this layer. - Ptr layer = dstNet.getLayer(id); - layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); - for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) - { - outShapes[node_proto.output(i)] = layerOutShapes[i]; - } + addLayer(dstNet, layerParams, node_proto, layer_id, outShapes); } } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 19b6bfed27..914ad405e0 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -430,6 +430,8 @@ TEST_P(Test_ONNX_layers, Reshape) TEST_P(Test_ONNX_layers, Squeeze) { + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("squeeze"); } @@ -476,6 +478,16 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) testONNXModels("split_max"); } +TEST_P(Test_ONNX_layers, LSTM) +{ + testONNXModels("lstm", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, LSTM_bidirectional) +{ + testONNXModels("lstm_bidirectional", npy, 0, 0, false, false); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers diff --git a/modules/imgproc/src/bilateral_filter.dispatch.cpp b/modules/imgproc/src/bilateral_filter.dispatch.cpp index a27ebb18f5..ed0e71bbab 100644 --- a/modules/imgproc/src/bilateral_filter.dispatch.cpp +++ b/modules/imgproc/src/bilateral_filter.dispatch.cpp @@ -406,6 +406,8 @@ void bilateralFilter( InputArray _src, OutputArray _dst, int d, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + _dst.create( _src.size(), _src.type() ); CV_OCL_RUN(_src.dims() <= 2 && _dst.isUMat(), diff --git a/modules/imgproc/src/box_filter.dispatch.cpp b/modules/imgproc/src/box_filter.dispatch.cpp index 054e7474c6..c9ec693385 100644 --- a/modules/imgproc/src/box_filter.dispatch.cpp +++ b/modules/imgproc/src/box_filter.dispatch.cpp @@ -443,6 +443,8 @@ void boxFilter(InputArray _src, OutputArray _dst, int ddepth, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + CV_OCL_RUN(_dst.isUMat() && (borderType == BORDER_REPLICATE || borderType == BORDER_CONSTANT || borderType == BORDER_REFLECT || borderType == BORDER_REFLECT_101), @@ -514,6 +516,8 @@ void sqrBoxFilter(InputArray _src, OutputArray _dst, int ddepth, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + int srcType = _src.type(), sdepth = CV_MAT_DEPTH(srcType), cn = CV_MAT_CN(srcType); Size size = _src.size(); diff --git a/modules/imgproc/src/deriv.cpp b/modules/imgproc/src/deriv.cpp index b898b648ac..2d8b59926d 100644 --- a/modules/imgproc/src/deriv.cpp +++ b/modules/imgproc/src/deriv.cpp @@ -416,6 +416,8 @@ void cv::Sobel( InputArray _src, OutputArray _dst, int ddepth, int dx, int dy, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + int stype = _src.type(), sdepth = CV_MAT_DEPTH(stype), cn = CV_MAT_CN(stype); if (ddepth < 0) ddepth = sdepth; @@ -468,6 +470,8 @@ void cv::Scharr( InputArray _src, OutputArray _dst, int ddepth, int dx, int dy, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + int stype = _src.type(), sdepth = CV_MAT_DEPTH(stype), cn = CV_MAT_CN(stype); if (ddepth < 0) ddepth = sdepth; @@ -785,6 +789,8 @@ void cv::Laplacian( InputArray _src, OutputArray _dst, int ddepth, int ksize, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + int stype = _src.type(), sdepth = CV_MAT_DEPTH(stype), cn = CV_MAT_CN(stype); if (ddepth < 0) ddepth = sdepth; diff --git a/modules/imgproc/src/filter.dispatch.cpp b/modules/imgproc/src/filter.dispatch.cpp index 65a066b57b..d39c749121 100644 --- a/modules/imgproc/src/filter.dispatch.cpp +++ b/modules/imgproc/src/filter.dispatch.cpp @@ -169,6 +169,9 @@ int FilterEngine::start(const Size& _wholeSize, const Size& sz, const Point& ofs { CV_INSTRUMENT_REGION(); + CV_Assert(!sz.empty()); + CV_Assert(!_wholeSize.empty()); + CV_CPU_DISPATCH(FilterEngine__start, (*this, _wholeSize, sz, ofs), CV_CPU_DISPATCH_MODES_ALL); } @@ -176,6 +179,11 @@ int FilterEngine::start(const Size& _wholeSize, const Size& sz, const Point& ofs int FilterEngine::start(const Mat& src, const Size &wsz, const Point &ofs) { + CV_INSTRUMENT_REGION(); + + CV_Assert(!src.empty()); + CV_Assert(!wsz.empty()); + start( wsz, src.size(), ofs); return startY - ofs.y; } @@ -1398,6 +1406,9 @@ void filter2D(InputArray _src, OutputArray _dst, int ddepth, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + CV_Assert(!_kernel.empty()); + CV_OCL_RUN(_dst.isUMat() && _src.dims() <= 2, ocl_filter2D(_src, _dst, ddepth, _kernel, anchor0, delta, borderType)) @@ -1429,6 +1440,10 @@ void sepFilter2D(InputArray _src, OutputArray _dst, int ddepth, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + CV_Assert(!_kernelX.empty()); + CV_Assert(!_kernelY.empty()); + CV_OCL_RUN(_dst.isUMat() && _src.dims() <= 2 && (size_t)_src.rows() > _kernelY.total() && (size_t)_src.cols() > _kernelX.total(), ocl_sepFilter2D(_src, _dst, ddepth, _kernelX, _kernelY, anchor, delta, borderType)) diff --git a/modules/imgproc/src/median_blur.dispatch.cpp b/modules/imgproc/src/median_blur.dispatch.cpp index 450a9d0167..949a0a9ece 100644 --- a/modules/imgproc/src/median_blur.dispatch.cpp +++ b/modules/imgproc/src/median_blur.dispatch.cpp @@ -280,6 +280,8 @@ void medianBlur( InputArray _src0, OutputArray _dst, int ksize ) { CV_INSTRUMENT_REGION(); + CV_Assert(!_src0.empty()); + CV_Assert( (ksize % 2 == 1) && (_src0.dims() <= 2 )); if( ksize <= 1 || _src0.empty() ) diff --git a/modules/imgproc/src/morph.dispatch.cpp b/modules/imgproc/src/morph.dispatch.cpp index cbb5315f31..45ae3994d9 100644 --- a/modules/imgproc/src/morph.dispatch.cpp +++ b/modules/imgproc/src/morph.dispatch.cpp @@ -939,6 +939,8 @@ static void morphOp( int op, InputArray _src, OutputArray _dst, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + Mat kernel = _kernel.getMat(); Size ksize = !kernel.empty() ? kernel.size() : Size(3,3); anchor = normalizeAnchor(anchor, ksize); @@ -1005,6 +1007,8 @@ void erode( InputArray src, OutputArray dst, InputArray kernel, { CV_INSTRUMENT_REGION(); + CV_Assert(!src.empty()); + morphOp( MORPH_ERODE, src, dst, kernel, anchor, iterations, borderType, borderValue ); } @@ -1015,6 +1019,8 @@ void dilate( InputArray src, OutputArray dst, InputArray kernel, { CV_INSTRUMENT_REGION(); + CV_Assert(!src.empty()); + morphOp( MORPH_DILATE, src, dst, kernel, anchor, iterations, borderType, borderValue ); } @@ -1154,6 +1160,8 @@ void morphologyEx( InputArray _src, OutputArray _dst, int op, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + Mat kernel = _kernel.getMat(); if (kernel.empty()) { diff --git a/modules/imgproc/src/smooth.dispatch.cpp b/modules/imgproc/src/smooth.dispatch.cpp index c90d0828b8..4ac7df8b4d 100644 --- a/modules/imgproc/src/smooth.dispatch.cpp +++ b/modules/imgproc/src/smooth.dispatch.cpp @@ -603,6 +603,8 @@ void GaussianBlur(InputArray _src, OutputArray _dst, Size ksize, { CV_INSTRUMENT_REGION(); + CV_Assert(!_src.empty()); + int type = _src.type(); Size size = _src.size(); _dst.create( size, type ); diff --git a/modules/imgproc/test/test_filter.cpp b/modules/imgproc/test/test_filter.cpp index accf30db9f..68dea24437 100644 --- a/modules/imgproc/test/test_filter.cpp +++ b/modules/imgproc/test/test_filter.cpp @@ -2323,4 +2323,37 @@ TEST(Imgproc_Pyrdown, issue_12961) ASSERT_EQ(0.0, cv::norm(dst)); } + +// https://github.com/opencv/opencv/issues/16857 +TEST(Imgproc, filter_empty_src_16857) +{ +#define CV_TEST_EXPECT_EMPTY_THROW(statement) CV_TEST_EXPECT_EXCEPTION_MESSAGE(statement, ".empty()") + + Mat src, dst, dst2; + + CV_TEST_EXPECT_EMPTY_THROW(bilateralFilter(src, dst, 5, 50, 20)); + CV_TEST_EXPECT_EMPTY_THROW(blur(src, dst, Size(3, 3))); + CV_TEST_EXPECT_EMPTY_THROW(boxFilter(src, dst, CV_8U, Size(3, 3))); + CV_TEST_EXPECT_EMPTY_THROW(sqrBoxFilter(src, dst, CV_8U, Size(3, 3))); + CV_TEST_EXPECT_EMPTY_THROW(medianBlur(src, dst, 3)); + CV_TEST_EXPECT_EMPTY_THROW(GaussianBlur(src, dst, Size(3, 3), 0)); + CV_TEST_EXPECT_EMPTY_THROW(cv::filter2D(src, dst, CV_8U, Mat_::zeros(Size(3, 3)))); + CV_TEST_EXPECT_EMPTY_THROW(sepFilter2D(src, dst, CV_8U, Mat_::zeros(Size(3, 1)), Mat_::zeros(Size(1, 3)))); + CV_TEST_EXPECT_EMPTY_THROW(Sobel(src, dst, CV_8U, 1, 1)); + CV_TEST_EXPECT_EMPTY_THROW(spatialGradient(src, dst, dst2)); + CV_TEST_EXPECT_EMPTY_THROW(Scharr(src, dst, CV_8U, 1, 1)); + CV_TEST_EXPECT_EMPTY_THROW(Laplacian(src, dst, CV_8U)); + + CV_TEST_EXPECT_EMPTY_THROW(cv::dilate(src, dst, Mat())); // cvtest:: by default + CV_TEST_EXPECT_EMPTY_THROW(cv::erode(src, dst, Mat())); // cvtest:: by default + CV_TEST_EXPECT_EMPTY_THROW(morphologyEx(src, dst, MORPH_OPEN, Mat())); + + //debug: CV_TEST_EXPECT_EMPTY_THROW(blur(Mat_(Size(3,3)), dst, Size(3, 3))); + + EXPECT_TRUE(src.empty()); + EXPECT_TRUE(dst.empty()); + EXPECT_TRUE(dst2.empty()); +} + + }} // namespace diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 42518da6b0..116766a619 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -122,9 +122,16 @@ void QRDetect::init(const Mat& src, double eps_vertical_, double eps_horizontal_ eps_vertical = eps_vertical_; eps_horizontal = eps_horizontal_; - adaptiveThreshold(barcode, bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); - adaptiveThreshold(resized_barcode, resized_bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); + if (!barcode.empty()) + adaptiveThreshold(barcode, bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); + else + bin_barcode.release(); + + if (!resized_barcode.empty()) + adaptiveThreshold(resized_barcode, resized_bin_barcode, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 83, 2); + else + resized_bin_barcode.release(); } vector QRDetect::searchHorizontalLines() diff --git a/modules/ts/include/opencv2/ts/ts_ext.hpp b/modules/ts/include/opencv2/ts/ts_ext.hpp index f22a9d42eb..b2a4cac241 100644 --- a/modules/ts/include/opencv2/ts/ts_ext.hpp +++ b/modules/ts/include/opencv2/ts/ts_ext.hpp @@ -161,4 +161,35 @@ bool checkBigDataTests(); #undef TEST_P #define TEST_P(test_case_name, test_name) CV__TEST_P(test_case_name, test_name, Body, CV__TEST_BODY_IMPL) + +#define CV_TEST_EXPECT_EXCEPTION_MESSAGE(statement, msg) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + const char* msg_ = msg; \ + bool hasException = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (const cv::Exception& e) { \ + if (NULL == strstr(e.what(), msg_)) \ + ADD_FAILURE() << "Unexpected cv::Exception is raised: " << #statement << "\n Expected message substring: '" << msg_ << "'. Actual message:\n" << e.what(); \ + hasException = true; \ + } \ + catch (const std::exception& e) { \ + ADD_FAILURE() << "Unexpected std::exception is raised: " << #statement << "\n" << e.what(); \ + hasException = true; \ + } \ + catch (...) { \ + ADD_FAILURE() << "Unexpected C++ exception is raised: " << #statement; \ + hasException = true; \ + } \ + if (!hasException) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_test_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_test_, __LINE__): \ + ADD_FAILURE() << "Failed: Expected: " #statement " throws an '" << msg << "' exception.\n" \ + " Actual: it doesn't." + + #endif // OPENCV_TS_EXT_HPP diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 25f9115ad3..e5b74eed38 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -639,7 +639,8 @@ public: documentation of source stream to know the right URL. @param apiPreference preferred Capture API backends to use. Can be used to enforce a specific reader implementation if multiple are available: e.g. cv::CAP_FFMPEG or cv::CAP_IMAGES or cv::CAP_DSHOW. - @sa The list of supported API backends cv::VideoCaptureAPIs + + @sa cv::VideoCaptureAPIs */ CV_WRAP explicit VideoCapture(const String& filename, int apiPreference = CAP_ANY); @@ -651,7 +652,7 @@ public: @param apiPreference preferred Capture API backends to use. Can be used to enforce a specific reader implementation if multiple are available: e.g. cv::CAP_DSHOW or cv::CAP_MSMF or cv::CAP_V4L. - @sa The list of supported API backends cv::VideoCaptureAPIs + @sa cv::VideoCaptureAPIs */ CV_WRAP explicit VideoCapture(int index, int apiPreference = CAP_ANY);