diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index 9d30d55d17..c081989645 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -285,6 +285,16 @@ public: const DetectorParameters &detectorParams = DetectorParameters(), const RefineParameters& refineParams = RefineParameters()); + /** @brief ArucoDetector constructor for multiple dictionaries + * + * @param dictionaries indicates the type of markers that will be searched. Empty dictionaries will throw an error. + * @param detectorParams marker detection parameters + * @param refineParams marker refine detection parameters + */ + CV_WRAP ArucoDetector(const std::vector &dictionaries, + const DetectorParameters &detectorParams = DetectorParameters(), + const RefineParameters& refineParams = RefineParameters()); + /** @brief Basic marker detection * * @param image input image @@ -297,7 +307,7 @@ public: * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a * correct codification. Useful for debugging purposes. * - * Performs marker detection in the input image. Only markers included in the specific dictionary + * Performs marker detection in the input image. Only markers included in the first specified dictionary * are searched. For each detected marker, it returns the 2D position of its corner in the image * and its corresponding identifier. * Note that this function does not perform pose estimation. @@ -329,6 +339,8 @@ public: * If camera parameters and distortion coefficients are provided, missing markers are reprojected * using projectPoint function. If not, missing marker projections are interpolated using global * homography, and all the marker corners in the board must have the same Z coordinate. + * @note This function assumes that the board only contains markers from one dictionary, so only the + * first configured dictionary is used. It has to match the dictionary of the board to work properly. */ CV_WRAP void refineDetectedMarkers(InputArray image, const Board &board, InputOutputArrayOfArrays detectedCorners, @@ -336,9 +348,59 @@ public: InputArray cameraMatrix = noArray(), InputArray distCoeffs = noArray(), OutputArray recoveredIdxs = noArray()) const; + /** @brief Basic marker detection + * + * @param image input image + * @param corners vector of detected marker corners. For each marker, its four corners + * are provided, (e.g std::vector > ). For N detected markers, + * the dimensions of this array is Nx4. The order of the corners is clockwise. + * @param ids vector of identifiers of the detected markers. The identifier is of type int + * (e.g. std::vector). For N detected markers, the size of ids is also N. + * The identifiers have the same order than the markers in the imgPoints array. + * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a + * correct codification. Useful for debugging purposes. + * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the + * list of corresponding dictionaries. + * + * Performs marker detection in the input image. Only markers included in the specific dictionaries + * are searched. For each detected marker, it returns the 2D position of its corner in the image + * and its corresponding identifier. + * Note that this function does not perform pose estimation. + * @note The function does not correct lens distortion or takes it into account. It's recommended to undistort + * input image with corresponding camera model, if camera parameters are known + * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard + */ + CV_WRAP void detectMarkersMultiDict(InputArray image, OutputArrayOfArrays corners, OutputArray ids, + OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + + /** @brief Returns first dictionary from internal list used for marker detection. + * + * @return The first dictionary from the configured ArucoDetector. + */ CV_WRAP const Dictionary& getDictionary() const; + + /** @brief Sets and replaces the first dictionary in internal list to be used for marker detection. + * + * @param dictionary The new dictionary that will replace the first dictionary in the internal list. + */ CV_WRAP void setDictionary(const Dictionary& dictionary); + /** @brief Returns all dictionaries currently used for marker detection as a vector. + * + * @return A std::vector containing all dictionaries used by the ArucoDetector. + */ + CV_WRAP std::vector getDictionaries() const; + + /** @brief Sets the entire collection of dictionaries to be used for marker detection, replacing any existing dictionaries. + * + * @param dictionaries A std::vector containing the new set of dictionaries to be used. + * + * Configures the ArucoDetector to use the provided vector of dictionaries for marker detection. + * This method replaces any dictionaries that were previously set. + * @note Setting an empty vector of dictionaries will throw an error. + */ + CV_WRAP void setDictionaries(const std::vector& dictionaries); + CV_WRAP const DetectorParameters& getDetectorParameters() const; CV_WRAP void setDetectorParameters(const DetectorParameters& detectorParameters); diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index 8dd407d32f..305d2bdca9 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -458,5 +458,30 @@ class aruco_objdetect_test(NewOpenCVTests): with self.assertRaises(Exception): img = cv.aruco.drawDetectedDiamonds(img, points2, borderColor=255) + def test_multi_dict_arucodetector(self): + aruco_params = cv.aruco.DetectorParameters() + aruco_dicts = [ + cv.aruco.getPredefinedDictionary(cv.aruco.DICT_4X4_250), + cv.aruco.getPredefinedDictionary(cv.aruco.DICT_5X5_250) + ] + aruco_detector = cv.aruco.ArucoDetector(aruco_dicts, aruco_params) + id = 2 + marker_size = 100 + offset = 10 + img_marker1 = cv.aruco.generateImageMarker(aruco_dicts[0], id, marker_size, aruco_params.markerBorderBits) + img_marker1 = np.pad(img_marker1, pad_width=offset, mode='constant', constant_values=255) + img_marker2 = cv.aruco.generateImageMarker(aruco_dicts[1], id, marker_size, aruco_params.markerBorderBits) + img_marker2 = np.pad(img_marker2, pad_width=offset, mode='constant', constant_values=255) + img_markers = np.concatenate((img_marker1, img_marker2), axis=1) + + corners, ids, rejected, dictIndices = aruco_detector.detectMarkersMultiDict(img_markers) + + self.assertEqual(2, len(ids)) + self.assertEqual(id, ids[0]) + self.assertEqual(id, ids[1]) + self.assertEqual(2, len(dictIndices)) + self.assertEqual(0, dictIndices[0]) + self.assertEqual(1, dictIndices[1]) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index a1b04c8181..0075d3e5a8 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -10,6 +10,7 @@ #include "apriltag/apriltag_quad_thresh.hpp" #include "aruco_utils.hpp" #include +#include namespace cv { namespace aruco { @@ -640,9 +641,14 @@ static inline void findCornerInPyrImage(const float scale_init, const int closes } } +enum class DictionaryMode { + Single, + Multi +}; + struct ArucoDetector::ArucoDetectorImpl { - /// dictionary indicates the type of markers that will be searched - Dictionary dictionary; + /// dictionaries indicates the types of markers that will be searched + vector dictionaries; /// marker detection parameters, check DetectorParameters docs to see available settings DetectorParameters detectorParams; @@ -651,9 +657,210 @@ struct ArucoDetector::ArucoDetectorImpl { RefineParameters refineParams; ArucoDetectorImpl() {} - ArucoDetectorImpl(const Dictionary &_dictionary, const DetectorParameters &_detectorParams, - const RefineParameters& _refineParams): dictionary(_dictionary), - detectorParams(_detectorParams), refineParams(_refineParams) {} + ArucoDetectorImpl(const vector&_dictionaries, const DetectorParameters &_detectorParams, + const RefineParameters& _refineParams): dictionaries(_dictionaries), + detectorParams(_detectorParams), refineParams(_refineParams) { + CV_Assert(!dictionaries.empty()); + } + + /* + * @brief Detect markers either using multiple or just first dictionary + */ + void detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices, DictionaryMode dictMode) { + CV_Assert(!_image.empty()); + + CV_Assert(detectorParams.markerBorderBits > 0); + // check that the parameters are set correctly if Aruco3 is used + CV_Assert(!(detectorParams.useAruco3Detection == true && + detectorParams.minSideLengthCanonicalImg == 0 && + detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); + + Mat grey; + _convertToGrey(_image, grey); + + // Aruco3 functionality is the extension of Aruco. + // The description can be found in: + // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. + // if Aruco3 functionality if not wanted + // change some parameters to be sure to turn it off + if (!detectorParams.useAruco3Detection) { + detectorParams.minMarkerLengthRatioOriginalImg = 0.0; + detectorParams.minSideLengthCanonicalImg = 0; + } + else { + // always turn on corner refinement in case of Aruco3, due to upsampling + detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + } + + /// Step 0: equation (2) from paper [1] + const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / + (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* + detectorParams.minMarkerLengthRatioOriginalImg)); + + /// Step 1: create image pyramid. Section 3.4. in [1] + vector grey_pyramid; + int closest_pyr_image_idx = 0, num_levels = 0; + //// Step 1.1: resize image with equation (1) from paper [1] + if (detectorParams.useAruco3Detection) { + const float scale_pyr = 2.f; + const float img_area = static_cast(grey.rows*grey.cols); + const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* + detectorParams.minSideLengthCanonicalImg); + // find max level + num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); + // the closest pyramid image to the downsampled segmentation image + // will later be used as start index for corner upsampling + const float scale_img_area = img_area * fxfy * fxfy; + closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); + } + buildPyramid(grey, grey_pyramid, num_levels); + + // resize to segmentation image + // in this reduces size the contours will be detected + if (fxfy != 1.f) + resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); + + /// STEP 2: Detect marker candidates + vector > candidates; + vector > contours; + vector ids; + + /// STEP 2.a Detect marker candidates :: using AprilTag + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ + _apriltag(grey, detectorParams, candidates, contours); + } + /// STEP 2.b Detect marker candidates :: traditional way + else { + detectCandidates(grey, candidates, contours); + } + + /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS + vector dictIndices; + vector> rejectedImgPoints; + if (DictionaryMode::Single == dictMode) { + Dictionary& dictionary = dictionaries.at(0); + auto selectedCandidates = filterTooCloseCandidates(candidates, contours, dictionary.markerSize); + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, + ids, dictionary, rejectedImgPoints); + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + performCornerSubpixRefinement(grey, grey_pyramid, closest_pyr_image_idx, candidates, dictionary); + } + } else if (DictionaryMode::Multi == dictMode) { + map> candidatesPerDictionarySize; + for (const Dictionary& dictionary : dictionaries) { + candidatesPerDictionarySize.emplace(dictionary.markerSize, vector()); + } + + // create candidate trees for each dictionary size + for (auto& candidatesTreeEntry : candidatesPerDictionarySize) { + // copy candidates + vector> candidatesCopy = candidates; + vector > contoursCopy = contours; + candidatesTreeEntry.second = filterTooCloseCandidates(candidatesCopy, contoursCopy, candidatesTreeEntry.first); + } + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + int dictIndex = 0; + for (const Dictionary& currentDictionary : dictionaries) { + // temporary variable to store the current candidates + vector> currentCandidates; + identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize.at(currentDictionary.markerSize), currentCandidates, contours, + ids, currentDictionary, rejectedImgPoints); + if (_dictIndices.needed()) { + dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); + } + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + performCornerSubpixRefinement(grey, grey_pyramid, closest_pyr_image_idx, currentCandidates, currentDictionary); + } + candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); + dictIndex++; + } + + // Clean up rejectedImgPoints by comparing to itself and all candidates + const float epsilon = 0.000001f; + auto compareCandidates = [epsilon](vector a, vector b) { + for (int i = 0; i < 4; i++) { + if (std::abs(a[i].x - b[i].x) > epsilon || std::abs(a[i].y - b[i].y) > epsilon) { + return false; + } + } + return true; + }; + std::sort(rejectedImgPoints.begin(), rejectedImgPoints.end(), [](const vector& a, const vector&b){ + float avgX = (a[0].x + a[1].x + a[2].x + a[3].x)*.25f; + float avgY = (a[0].y + a[1].y + a[2].y + a[3].y)*.25f; + float aDist = avgX*avgX + avgY*avgY; + avgX = (b[0].x + b[1].x + b[2].x + b[3].x)*.25f; + avgY = (b[0].y + b[1].y + b[2].y + b[3].y)*.25f; + float bDist = avgX*avgX + avgY*avgY; + return aDist < bDist; + }); + auto last = std::unique(rejectedImgPoints.begin(), rejectedImgPoints.end(), compareCandidates); + rejectedImgPoints.erase(last, rejectedImgPoints.end()); + + for (auto it = rejectedImgPoints.begin(); it != rejectedImgPoints.end();) { + bool erased = false; + for (const auto& candidate : candidates) { + if (compareCandidates(candidate, *it)) { + it = rejectedImgPoints.erase(it); + erased = true; + break; + } + } + if (!erased) { + it++; + } + } + } + + /// STEP 3, Optional : Corner refinement :: use contour container + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ + + if (!ids.empty()) { + + // do corner refinement using the contours for each detected markers + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + for (int i = range.start; i < range.end; i++) { + _refineCandidateLines(contours[i], candidates[i]); + } + }); + } + } + + if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + + // scale to orignal size, this however will lead to inaccurate detections! + for (auto &vecPoints : candidates) + for (auto &point : vecPoints) + point *= 1.f/fxfy; + } + + // copy to output arrays + _copyVector2Output(candidates, _corners); + Mat(ids).copyTo(_ids); + if(_rejectedImgPoints.needed()) { + _copyVector2Output(rejectedImgPoints, _rejectedImgPoints); + } + if (_dictIndices.needed()) { + Mat(dictIndices).copyTo(_dictIndices); + } + } + /** * @brief Detect square candidates in the input image */ @@ -671,14 +878,12 @@ struct ArucoDetector::ArucoDetectorImpl { * clear candidates and contours */ vector - filterTooCloseCandidates(vector > &candidates, vector > &contours) { + filterTooCloseCandidates(vector > &candidates, vector > &contours, int markerSize) { CV_Assert(detectorParams.minMarkerDistanceRate >= 0.); vector candidateTree(candidates.size()); for(size_t i = 0ull; i < candidates.size(); i++) { candidateTree[i] = MarkerCandidateTree(std::move(candidates[i]), std::move(contours[i])); } - candidates.clear(); - contours.clear(); // sort candidates from big to small std::stable_sort(candidateTree.begin(), candidateTree.end()); @@ -735,7 +940,7 @@ struct ArucoDetector::ArucoDetectorImpl { for (size_t i = 1ull; i < grouped.size(); i++) { size_t id = grouped[i]; float dist = getAverageDistance(candidateTree[id].corners, candidateTree[currId].corners); - float moduleSize = getAverageModuleSize(candidateTree[id].corners, dictionary.markerSize, detectorParams.markerBorderBits); + float moduleSize = getAverageModuleSize(candidateTree[id].corners, markerSize, detectorParams.markerBorderBits); if (dist > detectorParams.minGroupDistance*moduleSize) { currId = id; candidateTree[grouped[0]].closeContours.push_back(candidateTree[id]); @@ -770,9 +975,8 @@ struct ArucoDetector::ArucoDetectorImpl { */ void identifyCandidates(const Mat& grey, const vector& image_pyr, vector& selectedContours, vector >& accepted, vector >& contours, - vector& ids, OutputArrayOfArrays _rejected = noArray()) { + vector& ids, const Dictionary& currentDictionary, vector>& rejected) const { size_t ncandidates = selectedContours.size(); - vector > rejected; vector idsTmp(ncandidates, -1); vector rotated(ncandidates, 0); @@ -807,11 +1011,11 @@ struct ArucoDetector::ArucoDetectorImpl { } const float scale = detectorParams.useAruco3Detection ? img.cols / static_cast(grey.cols) : 1.f; - validCandidates[v] = _identifyOneCandidate(dictionary, img, selectedContours[v].corners, idsTmp[v], detectorParams, rotated[v], scale); + validCandidates[v] = _identifyOneCandidate(currentDictionary, img, selectedContours[v].corners, idsTmp[v], detectorParams, rotated[v], scale); if (validCandidates[v] == 0 && checkCloseContours) { for (const MarkerCandidate& closeMarkerCandidate: selectedContours[v].closeContours) { - validCandidates[v] = _identifyOneCandidate(dictionary, img, closeMarkerCandidate.corners, idsTmp[v], detectorParams, rotated[v], scale); + validCandidates[v] = _identifyOneCandidate(currentDictionary, img, closeMarkerCandidate.corners, idsTmp[v], detectorParams, rotated[v], scale); if (validCandidates[v] > 0) { selectedContours[v].corners = closeMarkerCandidate.corners; selectedContours[v].contour = closeMarkerCandidate.contour; @@ -852,105 +1056,11 @@ struct ArucoDetector::ArucoDetectorImpl { rejected.push_back(selectedContours[i].corners); } } - - // parse output - if(_rejected.needed()) { - _copyVector2Output(rejected, _rejected); - } } -}; - -ArucoDetector::ArucoDetector(const Dictionary &_dictionary, - const DetectorParameters &_detectorParams, - const RefineParameters& _refineParams) { - arucoDetectorImpl = makePtr(_dictionary, _detectorParams, _refineParams); -} - -void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, - OutputArrayOfArrays _rejectedImgPoints) const { - CV_Assert(!_image.empty()); - DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionary; - - CV_Assert(detectorParams.markerBorderBits > 0); - // check that the parameters are set correctly if Aruco3 is used - CV_Assert(!(detectorParams.useAruco3Detection == true && - detectorParams.minSideLengthCanonicalImg == 0 && - detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); - - Mat grey; - _convertToGrey(_image, grey); - - // Aruco3 functionality is the extension of Aruco. - // The description can be found in: - // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. - // if Aruco3 functionality if not wanted - // change some parameters to be sure to turn it off - if (!detectorParams.useAruco3Detection) { - detectorParams.minMarkerLengthRatioOriginalImg = 0.0; - detectorParams.minSideLengthCanonicalImg = 0; - } - else { - // always turn on corner refinement in case of Aruco3, due to upsampling - detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - } - - /// Step 0: equation (2) from paper [1] - const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / - (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* - detectorParams.minMarkerLengthRatioOriginalImg)); - - /// Step 1: create image pyramid. Section 3.4. in [1] - vector grey_pyramid; - int closest_pyr_image_idx = 0, num_levels = 0; - //// Step 1.1: resize image with equation (1) from paper [1] - if (detectorParams.useAruco3Detection) { - const float scale_pyr = 2.f; - const float img_area = static_cast(grey.rows*grey.cols); - const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* - detectorParams.minSideLengthCanonicalImg); - // find max level - num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); - // the closest pyramid image to the downsampled segmentation image - // will later be used as start index for corner upsampling - const float scale_img_area = img_area * fxfy * fxfy; - closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); - } - buildPyramid(grey, grey_pyramid, num_levels); - - // resize to segmentation image - // in this reduces size the contours will be detected - if (fxfy != 1.f) - resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); - - /// STEP 2: Detect marker candidates - vector > candidates; - vector > contours; - vector ids; - - /// STEP 2.a Detect marker candidates :: using AprilTag - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ - _apriltag(grey, detectorParams, candidates, contours); - } - /// STEP 2.b Detect marker candidates :: traditional way - else { - arucoDetectorImpl->detectCandidates(grey, candidates, contours); - } - - /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS - auto selectedCandidates = arucoDetectorImpl->filterTooCloseCandidates(candidates, contours); - - /// STEP 2: Check candidate codification (identify markers) - arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, - ids, _rejectedImgPoints); - - /// STEP 3: Corner refinement :: use corner subpix - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + void performCornerSubpixRefinement(const Mat& grey, const vector& grey_pyramid, int closest_pyr_image_idx, const vector>& candidates, const Dictionary& dictionary) const { CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); + detectorParams.cornerRefinementMinAccuracy > 0); // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { const int begin = range.start; @@ -960,47 +1070,40 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner if (detectorParams.useAruco3Detection) { const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); - } - else { + } else { int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); + getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); } } }); } +}; - /// STEP 3, Optional : Corner refinement :: use contour container - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ +ArucoDetector::ArucoDetector(const Dictionary &_dictionary, + const DetectorParameters &_detectorParams, + const RefineParameters& _refineParams) { + arucoDetectorImpl = makePtr(vector{_dictionary}, _detectorParams, _refineParams); +} - if (!ids.empty()) { +ArucoDetector::ArucoDetector(const vector &_dictionaries, + const DetectorParameters &_detectorParams, + const RefineParameters& _refineParams) { + arucoDetectorImpl = makePtr(_dictionaries, _detectorParams, _refineParams); +} - // do corner refinement using the contours for each detected markers - parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { - for (int i = range.start; i < range.end; i++) { - _refineCandidateLines(contours[i], candidates[i]); - } - }); - } - } +void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints) const { + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, noArray(), DictionaryMode::Single); +} - if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - - // scale to orignal size, this however will lead to inaccurate detections! - for (auto &vecPoints : candidates) - for (auto &point : vecPoints) - point *= 1.f/fxfy; - } - - // copy to output arrays - _copyVector2Output(candidates, _corners); - Mat(ids).copyTo(_ids); +void ArucoDetector::detectMarkersMultiDict(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices) const { + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, _dictIndices, DictionaryMode::Multi); } /** @@ -1114,7 +1217,7 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board InputOutputArrayOfArrays _rejectedCorners, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _recoveredIdxs) const { DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionary; + const Dictionary& dictionary = arucoDetectorImpl->dictionaries.at(0); RefineParameters& refineParams = arucoDetectorImpl->refineParams; CV_Assert(refineParams.minRepDistance > 0); @@ -1280,25 +1383,58 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } } -void ArucoDetector::write(FileStorage &fs) const -{ - arucoDetectorImpl->dictionary.writeDictionary(fs); +void ArucoDetector::write(FileStorage &fs) const { + // preserve old format for single dictionary case + if (1 == arucoDetectorImpl->dictionaries.size()) { + arucoDetectorImpl->dictionaries[0].writeDictionary(fs); + } else { + fs << "dictionaries" << "["; + for (auto& dictionary : arucoDetectorImpl->dictionaries) { + fs << "{"; + dictionary.writeDictionary(fs); + fs << "}"; + } + fs << "]"; + } arucoDetectorImpl->detectorParams.writeDetectorParameters(fs); arucoDetectorImpl->refineParams.writeRefineParameters(fs); } void ArucoDetector::read(const FileNode &fn) { - arucoDetectorImpl->dictionary.readDictionary(fn); + arucoDetectorImpl->dictionaries.clear(); + if (!fn.empty() && !fn["dictionaries"].empty() && fn["dictionaries"].isSeq()) { + for (const auto& dictionaryNode : fn["dictionaries"]) { + arucoDetectorImpl->dictionaries.emplace_back(); + arucoDetectorImpl->dictionaries.back().readDictionary(dictionaryNode); + } + } else { + // backward compatibility + arucoDetectorImpl->dictionaries.emplace_back(); + arucoDetectorImpl->dictionaries.back().readDictionary(fn); + } arucoDetectorImpl->detectorParams.readDetectorParameters(fn); arucoDetectorImpl->refineParams.readRefineParameters(fn); } const Dictionary& ArucoDetector::getDictionary() const { - return arucoDetectorImpl->dictionary; + return arucoDetectorImpl->dictionaries[0]; } void ArucoDetector::setDictionary(const Dictionary& dictionary) { - arucoDetectorImpl->dictionary = dictionary; + if (arucoDetectorImpl->dictionaries.empty()) { + arucoDetectorImpl->dictionaries.push_back(dictionary); + } else { + arucoDetectorImpl->dictionaries[0] = dictionary; + } +} + +vector ArucoDetector::getDictionaries() const { + return arucoDetectorImpl->dictionaries; +} + +void ArucoDetector::setDictionaries(const vector& dictionaries) { + CV_Assert(!dictionaries.empty()); + arucoDetectorImpl->dictionaries = dictionaries; } const DetectorParameters& ArucoDetector::getDetectorParameters() const { diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 7145b5d663..fd957698b2 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -6,6 +6,18 @@ #include "opencv2/objdetect/aruco_detector.hpp" #include "opencv2/calib3d.hpp" +namespace cv { + namespace aruco { + bool operator==(const Dictionary& d1, const Dictionary& d2); + bool operator==(const Dictionary& d1, const Dictionary& d2) { + return d1.markerSize == d2.markerSize + && std::equal(d1.bytesList.begin>(), d1.bytesList.end>(), d2.bytesList.begin>()) + && std::equal(d2.bytesList.begin>(), d2.bytesList.end>(), d1.bytesList.begin>()) + && d1.maxCorrectionBits == d2.maxCorrectionBits; + }; + } +} + namespace opencv_test { namespace { /** @@ -638,6 +650,155 @@ TEST(CV_ArucoDetectMarkers, regression_contour_24220) } } +TEST(CV_ArucoMultiDict, setGetDictionaries) +{ + vector dictionaries = {aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}; + aruco::ArucoDetector detector(dictionaries); + vector dicts = detector.getDictionaries(); + ASSERT_EQ(dicts.size(), 2ul); + EXPECT_EQ(dicts[0].markerSize, 4); + EXPECT_EQ(dicts[1].markerSize, 5); + dictionaries.clear(); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_6X6_100)); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_25h9)); + detector.setDictionaries(dictionaries); + dicts = detector.getDictionaries(); + ASSERT_EQ(dicts.size(), 3ul); + EXPECT_EQ(dicts[0].markerSize, 6); + EXPECT_EQ(dicts[1].markerSize, 7); + EXPECT_EQ(dicts[2].markerSize, 5); + auto dict = detector.getDictionary(); + EXPECT_EQ(dict.markerSize, 6); + detector.setDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_16h5)); + dicts = detector.getDictionaries(); + ASSERT_EQ(dicts.size(), 3ul); + EXPECT_EQ(dicts[0].markerSize, 4); + EXPECT_EQ(dicts[1].markerSize, 7); + EXPECT_EQ(dicts[2].markerSize, 5); +} + + +TEST(CV_ArucoMultiDict, noDict) +{ + aruco::ArucoDetector detector; + EXPECT_THROW({ + detector.setDictionaries({}); + }, Exception); +} + + +TEST(CV_ArucoMultiDict, multiMarkerDetection) +{ + const int markerSidePixels = 100; + const int imageSize = markerSidePixels * 2 + 3 * (markerSidePixels / 2); + vector usedDictionaries; + + // draw synthetic image + Mat img = Mat(imageSize, imageSize, CV_8UC1, Scalar::all(255)); + for(int y = 0; y < 2; y++) { + for(int x = 0; x < 2; x++) { + Mat marker; + int id = y * 2 + x; + int dictId = x * 4 + y * 8; + auto dict = aruco::getPredefinedDictionary(dictId); + usedDictionaries.push_back(dict); + aruco::generateImageMarker(dict, id, markerSidePixels, marker); + Point2f firstCorner(markerSidePixels / 2.f + x * (1.5f * markerSidePixels), + markerSidePixels / 2.f + y * (1.5f * markerSidePixels)); + Mat aux = img(Rect((int)firstCorner.x, (int)firstCorner.y, markerSidePixels, markerSidePixels)); + marker.copyTo(aux); + } + } + img.convertTo(img, CV_8UC3); + + aruco::ArucoDetector detector(usedDictionaries); + + vector > markerCorners; + vector markerIds; + vector > rejectedImgPts; + vector dictIds; + detector.detectMarkersMultiDict(img, markerCorners, markerIds, rejectedImgPts, dictIds); + ASSERT_EQ(markerIds.size(), 4u); + ASSERT_EQ(dictIds.size(), 4u); + for (size_t i = 0; i < dictIds.size(); ++i) { + EXPECT_EQ(dictIds[i], (int)i); + } +} + + +TEST(CV_ArucoMultiDict, multiMarkerDoubleDetection) +{ + const int markerSidePixels = 100; + const int imageWidth = 2 * markerSidePixels + 3 * (markerSidePixels / 2); + const int imageHeight = markerSidePixels + 2 * (markerSidePixels / 2); + vector usedDictionaries = { + aruco::getPredefinedDictionary(aruco::DICT_5X5_50), + aruco::getPredefinedDictionary(aruco::DICT_5X5_100) + }; + + // draw synthetic image + Mat img = Mat(imageHeight, imageWidth, CV_8UC1, Scalar::all(255)); + for(int y = 0; y < 2; y++) { + Mat marker; + int id = 49 + y; + auto dict = aruco::getPredefinedDictionary(aruco::DICT_5X5_100); + aruco::generateImageMarker(dict, id, markerSidePixels, marker); + Point2f firstCorner(markerSidePixels / 2.f + y * (1.5f * markerSidePixels), + markerSidePixels / 2.f); + Mat aux = img(Rect((int)firstCorner.x, (int)firstCorner.y, markerSidePixels, markerSidePixels)); + marker.copyTo(aux); + } + img.convertTo(img, CV_8UC3); + + aruco::ArucoDetector detector(usedDictionaries); + + vector > markerCorners; + vector markerIds; + vector > rejectedImgPts; + vector dictIds; + detector.detectMarkersMultiDict(img, markerCorners, markerIds, rejectedImgPts, dictIds); + ASSERT_EQ(markerIds.size(), 3u); + ASSERT_EQ(dictIds.size(), 3u); + EXPECT_EQ(dictIds[0], 0); // 5X5_50 + EXPECT_EQ(dictIds[1], 1); // 5X5_100 + EXPECT_EQ(dictIds[2], 1); // 5X5_100 +} + + +TEST(CV_ArucoMultiDict, serialization) +{ + aruco::ArucoDetector detector; + { + FileStorage fs_out(".json", FileStorage::WRITE + FileStorage::MEMORY); + ASSERT_TRUE(fs_out.isOpened()); + detector.write(fs_out); + std::string serialized_string = fs_out.releaseAndGetString(); + FileStorage test_fs(serialized_string, FileStorage::Mode::READ + FileStorage::MEMORY); + ASSERT_TRUE(test_fs.isOpened()); + aruco::ArucoDetector test_detector; + test_detector.read(test_fs.root()); + // compare default constructor result + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), test_detector.getDictionary()); + } + detector.setDictionaries({aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}); + { + FileStorage fs_out(".json", FileStorage::WRITE + FileStorage::MEMORY); + ASSERT_TRUE(fs_out.isOpened()); + detector.write(fs_out); + std::string serialized_string = fs_out.releaseAndGetString(); + FileStorage test_fs(serialized_string, FileStorage::Mode::READ + FileStorage::MEMORY); + ASSERT_TRUE(test_fs.isOpened()); + aruco::ArucoDetector test_detector; + test_detector.read(test_fs.root()); + // check for one additional dictionary + auto dicts = test_detector.getDictionaries(); + ASSERT_EQ(2ul, dicts.size()); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), dicts[0]); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_5X5_100), dicts[1]); + } +} + struct ArucoThreading: public testing::TestWithParam { diff --git a/modules/objdetect/test/test_boarddetection.cpp b/modules/objdetect/test/test_boarddetection.cpp index 1fa6e11994..312bb5b56b 100644 --- a/modules/objdetect/test/test_boarddetection.cpp +++ b/modules/objdetect/test/test_boarddetection.cpp @@ -134,14 +134,17 @@ class CV_ArucoRefine : public cvtest::BaseTest { public: CV_ArucoRefine(ArucoAlgParams arucoAlgParams) { - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(aruco::DICT_6X6_250); + vector dictionaries = {aruco::getPredefinedDictionary(aruco::DICT_6X6_250), + aruco::getPredefinedDictionary(aruco::DICT_5X5_250), + aruco::getPredefinedDictionary(aruco::DICT_4X4_250), + aruco::getPredefinedDictionary(aruco::DICT_7X7_250)}; aruco::DetectorParameters params; params.minDistanceToBorder = 3; params.cornerRefinementMethod = (int)aruco::CORNER_REFINE_SUBPIX; if (arucoAlgParams == ArucoAlgParams::USE_ARUCO3) params.useAruco3Detection = true; aruco::RefineParameters refineParams(10.f, 3.f, true); - detector = aruco::ArucoDetector(dictionary, params, refineParams); + detector = aruco::ArucoDetector(dictionaries, params, refineParams); } protected: diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index a7837a6ff8..4a3181bdde 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -45,6 +45,7 @@ typedef std::vector vector_DMatch; typedef std::vector vector_String; typedef std::vector vector_string; typedef std::vector vector_Scalar; +typedef std::vector vector_Dictionary; typedef std::vector > vector_vector_char; typedef std::vector > vector_vector_Point;