From c759a7cddea6b2f316440e607fc538062376dd2f Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Mon, 17 Feb 2025 16:49:39 +0100 Subject: [PATCH 01/14] Extend ArUcoDetector to run multiple dictionaries in an efficient manner. * Add constructor for multiple dictionaries * Add get/set/remove/add functions for multiple dictionaries * Add unit tests TESTED=unit tests --- .../opencv2/objdetect/aruco_detector.hpp | 26 ++- .../objdetect/src/aruco/aruco_detector.cpp | 177 +++++++++++++----- .../objdetect/test/test_arucodetection.cpp | 88 +++++++++ 3 files changed, 237 insertions(+), 54 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index 9d30d55d17..0d3eb69647 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -285,6 +285,16 @@ public: const DetectorParameters &detectorParams = DetectorParameters(), const RefineParameters& refineParams = RefineParameters()); + /** @brief ArucoDetector constructor for multiple dictionaries + * + * @param dictionaries indicates the type of markers that will be searched + * @param detectorParams marker detection parameters + * @param refineParams marker refine detection parameters + */ + CV_WRAP ArucoDetector(const std::vector &dictionaries, + const DetectorParameters &detectorParams = DetectorParameters(), + const RefineParameters& refineParams = RefineParameters()); + /** @brief Basic marker detection * * @param image input image @@ -296,8 +306,10 @@ public: * The identifiers have the same order than the markers in the imgPoints array. * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a * correct codification. Useful for debugging purposes. + * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the + * list of corresponding dictionaries. * - * Performs marker detection in the input image. Only markers included in the specific dictionary + * Performs marker detection in the input image. Only markers included in the specific dictionaries * are searched. For each detected marker, it returns the 2D position of its corner in the image * and its corresponding identifier. * Note that this function does not perform pose estimation. @@ -306,7 +318,7 @@ public: * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard */ CV_WRAP void detectMarkers(InputArray image, OutputArrayOfArrays corners, OutputArray ids, - OutputArrayOfArrays rejectedImgPoints = noArray()) const; + OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; /** @brief Refine not detected markers based on the already detected and the board layout * @@ -329,6 +341,8 @@ public: * If camera parameters and distortion coefficients are provided, missing markers are reprojected * using projectPoint function. If not, missing marker projections are interpolated using global * homography, and all the marker corners in the board must have the same Z coordinate. + * @note This function assumes that the board only contains markers from one dictionary, so only the + * first configured dictionary is used. */ CV_WRAP void refineDetectedMarkers(InputArray image, const Board &board, InputOutputArrayOfArrays detectedCorners, @@ -336,8 +350,12 @@ public: InputArray cameraMatrix = noArray(), InputArray distCoeffs = noArray(), OutputArray recoveredIdxs = noArray()) const; - CV_WRAP const Dictionary& getDictionary() const; - CV_WRAP void setDictionary(const Dictionary& dictionary); + CV_WRAP const Dictionary& getDictionary(size_t index = 0) const; + CV_WRAP void setDictionary(const Dictionary& dictionary, size_t index = 0); + CV_WRAP const std::vector& getDictionaries() const; + CV_WRAP void setDictionaries(const std::vector& dictionaries); + CV_WRAP void addDictionary(const Dictionary& dictionary); + CV_WRAP void removeDictionary(size_t index); CV_WRAP const DetectorParameters& getDetectorParameters() const; CV_WRAP void setDetectorParameters(const DetectorParameters& detectorParameters); diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index a1b04c8181..2c9ddb9160 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -10,6 +10,7 @@ #include "apriltag/apriltag_quad_thresh.hpp" #include "aruco_utils.hpp" #include +#include namespace cv { namespace aruco { @@ -641,8 +642,8 @@ static inline void findCornerInPyrImage(const float scale_init, const int closes } struct ArucoDetector::ArucoDetectorImpl { - /// dictionary indicates the type of markers that will be searched - Dictionary dictionary; + /// dictionaries indicates the types of markers that will be searched + std::vector dictionaries; /// marker detection parameters, check DetectorParameters docs to see available settings DetectorParameters detectorParams; @@ -651,8 +652,8 @@ struct ArucoDetector::ArucoDetectorImpl { RefineParameters refineParams; ArucoDetectorImpl() {} - ArucoDetectorImpl(const Dictionary &_dictionary, const DetectorParameters &_detectorParams, - const RefineParameters& _refineParams): dictionary(_dictionary), + ArucoDetectorImpl(const std::vector&_dictionaries, const DetectorParameters &_detectorParams, + const RefineParameters& _refineParams): dictionaries(_dictionaries), detectorParams(_detectorParams), refineParams(_refineParams) {} /** * @brief Detect square candidates in the input image @@ -671,14 +672,12 @@ struct ArucoDetector::ArucoDetectorImpl { * clear candidates and contours */ vector - filterTooCloseCandidates(vector > &candidates, vector > &contours) { + filterTooCloseCandidates(vector > &candidates, vector > &contours, int markerSize) { CV_Assert(detectorParams.minMarkerDistanceRate >= 0.); vector candidateTree(candidates.size()); for(size_t i = 0ull; i < candidates.size(); i++) { candidateTree[i] = MarkerCandidateTree(std::move(candidates[i]), std::move(contours[i])); } - candidates.clear(); - contours.clear(); // sort candidates from big to small std::stable_sort(candidateTree.begin(), candidateTree.end()); @@ -735,7 +734,7 @@ struct ArucoDetector::ArucoDetectorImpl { for (size_t i = 1ull; i < grouped.size(); i++) { size_t id = grouped[i]; float dist = getAverageDistance(candidateTree[id].corners, candidateTree[currId].corners); - float moduleSize = getAverageModuleSize(candidateTree[id].corners, dictionary.markerSize, detectorParams.markerBorderBits); + float moduleSize = getAverageModuleSize(candidateTree[id].corners, markerSize, detectorParams.markerBorderBits); if (dist > detectorParams.minGroupDistance*moduleSize) { currId = id; candidateTree[grouped[0]].closeContours.push_back(candidateTree[id]); @@ -770,7 +769,7 @@ struct ArucoDetector::ArucoDetectorImpl { */ void identifyCandidates(const Mat& grey, const vector& image_pyr, vector& selectedContours, vector >& accepted, vector >& contours, - vector& ids, OutputArrayOfArrays _rejected = noArray()) { + vector& ids, const Dictionary& currentDictionary, OutputArrayOfArrays _rejected = noArray()) { size_t ncandidates = selectedContours.size(); vector > rejected; @@ -807,11 +806,11 @@ struct ArucoDetector::ArucoDetectorImpl { } const float scale = detectorParams.useAruco3Detection ? img.cols / static_cast(grey.cols) : 1.f; - validCandidates[v] = _identifyOneCandidate(dictionary, img, selectedContours[v].corners, idsTmp[v], detectorParams, rotated[v], scale); + validCandidates[v] = _identifyOneCandidate(currentDictionary, img, selectedContours[v].corners, idsTmp[v], detectorParams, rotated[v], scale); if (validCandidates[v] == 0 && checkCloseContours) { for (const MarkerCandidate& closeMarkerCandidate: selectedContours[v].closeContours) { - validCandidates[v] = _identifyOneCandidate(dictionary, img, closeMarkerCandidate.corners, idsTmp[v], detectorParams, rotated[v], scale); + validCandidates[v] = _identifyOneCandidate(currentDictionary, img, closeMarkerCandidate.corners, idsTmp[v], detectorParams, rotated[v], scale); if (validCandidates[v] > 0) { selectedContours[v].corners = closeMarkerCandidate.corners; selectedContours[v].contour = closeMarkerCandidate.contour; @@ -864,14 +863,19 @@ struct ArucoDetector::ArucoDetectorImpl { ArucoDetector::ArucoDetector(const Dictionary &_dictionary, const DetectorParameters &_detectorParams, const RefineParameters& _refineParams) { - arucoDetectorImpl = makePtr(_dictionary, _detectorParams, _refineParams); + arucoDetectorImpl = makePtr(vector{_dictionary}, _detectorParams, _refineParams); +} + +ArucoDetector::ArucoDetector(const std::vector &_dictionaries, + const DetectorParameters &_detectorParams, + const RefineParameters& _refineParams) { + arucoDetectorImpl = makePtr(_dictionaries, _detectorParams, _refineParams); } void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, - OutputArrayOfArrays _rejectedImgPoints) const { + OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices) const { CV_Assert(!_image.empty()); DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionary; CV_Assert(detectorParams.markerBorderBits > 0); // check that the parameters are set correctly if Aruco3 is used @@ -940,38 +944,66 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner arucoDetectorImpl->detectCandidates(grey, candidates, contours); } - /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS - auto selectedCandidates = arucoDetectorImpl->filterTooCloseCandidates(candidates, contours); + /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS + unordered_set uniqueMarkerSizes; + for (const Dictionary& dictionary : arucoDetectorImpl->dictionaries) { + uniqueMarkerSizes.insert(dictionary.markerSize); + } + + // create at max 4 marker candidate trees for each dictionary size + vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; + for (int markerSize : uniqueMarkerSizes) { + // min marker size is 4, so subtract 4 to get index + const auto dictionarySizeIndex = markerSize - 4; + // copy candidates + vector> candidatesCopy = candidates; + vector > contoursCopy = contours; + candidatesPerDictionarySize[dictionarySizeIndex] = arucoDetectorImpl->filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); + } + candidates.clear(); + contours.clear(); /// STEP 2: Check candidate codification (identify markers) - arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, - ids, _rejectedImgPoints); + size_t dictIndex = 0; + vector dictIndices; + for (const Dictionary& currentDictionary : arucoDetectorImpl->dictionaries) { + const auto dictionarySizeIndex = currentDictionary.markerSize - 4; + // temporary variable to store the current candidates + vector> currentCandidates; + arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, + ids, currentDictionary, _rejectedImgPoints); + if (_dictIndices.needed()) { + dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); + } - /// STEP 3: Corner refinement :: use corner subpix - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners - parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { - const int begin = range.start; - const int end = range.end; + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; - for (int i = begin; i < end; i++) { - if (detectorParams.useAruco3Detection) { - const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; - findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); + } + else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } } - else { - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - } - }); + }); + } + candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); + dictIndex++; } /// STEP 3, Optional : Corner refinement :: use contour container @@ -1001,6 +1033,12 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner // copy to output arrays _copyVector2Output(candidates, _corners); Mat(ids).copyTo(_ids); + if (_dictIndices.needed()) { + _dictIndices.create(dictIndices.size(), 1, CV_32SC1); + Mat dictIndicesMat = _dictIndices.getMat(); + Mat m = cv::Mat1i(dictIndices).t(); + m.copyTo(dictIndicesMat); + } } /** @@ -1114,7 +1152,7 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board InputOutputArrayOfArrays _rejectedCorners, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _recoveredIdxs) const { DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionary; + const Dictionary& dictionary = arucoDetectorImpl->dictionaries[0]; RefineParameters& refineParams = arucoDetectorImpl->refineParams; CV_Assert(refineParams.minRepDistance > 0); @@ -1280,25 +1318,64 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } } -void ArucoDetector::write(FileStorage &fs) const -{ - arucoDetectorImpl->dictionary.writeDictionary(fs); +void ArucoDetector::write(FileStorage &fs) const { + fs << "dictionaries" << "["; + for (auto& dictionary : arucoDetectorImpl->dictionaries) { + fs << "{"; + dictionary.writeDictionary(fs); + fs << "}"; + } + fs << "]"; arucoDetectorImpl->detectorParams.writeDetectorParameters(fs); arucoDetectorImpl->refineParams.writeRefineParameters(fs); } void ArucoDetector::read(const FileNode &fn) { - arucoDetectorImpl->dictionary.readDictionary(fn); + arucoDetectorImpl->dictionaries.clear(); + if (!fn.empty() && !fn["dictionaries"].empty() && fn["dictionaries"].isSeq()) { + for (const auto& dictionaryNode : fn["dictionaries"]) { + arucoDetectorImpl->dictionaries.emplace_back(); + arucoDetectorImpl->dictionaries.back().readDictionary(dictionaryNode); + } + } else { + // backward compatibility + arucoDetectorImpl->dictionaries.emplace_back(); + arucoDetectorImpl->dictionaries.back().readDictionary(fn); + } arucoDetectorImpl->detectorParams.readDetectorParameters(fn); arucoDetectorImpl->refineParams.readRefineParameters(fn); } -const Dictionary& ArucoDetector::getDictionary() const { - return arucoDetectorImpl->dictionary; +const Dictionary& ArucoDetector::getDictionary(size_t index) const { + CV_Assert(index < arucoDetectorImpl->dictionaries.size()); + return arucoDetectorImpl->dictionaries[index]; } -void ArucoDetector::setDictionary(const Dictionary& dictionary) { - arucoDetectorImpl->dictionary = dictionary; +void ArucoDetector::setDictionary(const Dictionary& dictionary, size_t index) { + // special case: if index is 0, we add the dictionary to the list to preserve the old behavior + CV_Assert(index == 0 || index < arucoDetectorImpl->dictionaries.size()); + if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { + arucoDetectorImpl->dictionaries.push_back(dictionary); + } else { + arucoDetectorImpl->dictionaries.at(index) = dictionary; + } +} + +const vector& ArucoDetector::getDictionaries() const { + return arucoDetectorImpl->dictionaries; +} + +void ArucoDetector::setDictionaries(const vector& dictionaries) { + arucoDetectorImpl->dictionaries = dictionaries; +} + +void ArucoDetector::addDictionary(const Dictionary& dictionary) { + arucoDetectorImpl->dictionaries.push_back(dictionary); +} + +void ArucoDetector::removeDictionary(size_t index) { + CV_Assert(index < arucoDetectorImpl->dictionaries.size()); + arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); } const DetectorParameters& ArucoDetector::getDetectorParameters() const { diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 7145b5d663..94e062eb56 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -638,6 +638,94 @@ TEST(CV_ArucoDetectMarkers, regression_contour_24220) } } +TEST(CV_ArucoMultiDict, addRemoveDictionary) +{ + aruco::ArucoDetector detector; + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_100)); + const auto& dicts = detector.getDictionaries(); + ASSERT_EQ(dicts.size(), 2ul); + EXPECT_EQ(dicts[0].markerSize, 4); + EXPECT_EQ(dicts[1].markerSize, 5); + detector.removeDictionary(0); + ASSERT_EQ(dicts.size(), 1ul); + EXPECT_EQ(dicts[0].markerSize, 5); + detector.removeDictionary(0); + EXPECT_EQ(dicts.size(), 0ul); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_6X6_100)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_25h9)); + ASSERT_EQ(dicts.size(), 3ul); + EXPECT_EQ(dicts[0].markerSize, 6); + EXPECT_EQ(dicts[1].markerSize, 7); + EXPECT_EQ(dicts[2].markerSize, 5); + detector.setDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_36h10), 1); + auto dict = detector.getDictionary(); + EXPECT_EQ(dict.markerSize, 6); + detector.setDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_16h5)); + ASSERT_EQ(dicts.size(), 3ul); + EXPECT_EQ(dicts[0].markerSize, 4); + EXPECT_EQ(dicts[1].markerSize, 6); + EXPECT_EQ(dicts[2].markerSize, 5); +} + + +TEST(CV_ArucoMultiDict, noDict) +{ + aruco::ArucoDetector detector; + detector.removeDictionary(0); + + vector > markerCorners; + vector markerIds; + + string img_path = cvtest::findDataFile("aruco/singlemarkersoriginal.jpg"); + Mat image = imread(img_path); + + detector.detectMarkers(image, markerCorners, markerIds); + + EXPECT_EQ(markerIds.size(), 0u); +} + + +TEST(CV_ArucoMultiDict, multiMarkerDetection) +{ + aruco::ArucoDetector detector; + detector.removeDictionary(0); + + const int markerSidePixels = 100; + const int imageSize = markerSidePixels * 2 + 3 * (markerSidePixels / 2); + + // draw synthetic image + Mat img = Mat(imageSize, imageSize, CV_8UC1, Scalar::all(255)); + for(int y = 0; y < 2; y++) { + for(int x = 0; x < 2; x++) { + Mat marker; + int id = y * 2 + x; + int dictId = x * 4 + y * 8; + auto dict = aruco::getPredefinedDictionary(dictId); + detector.addDictionary(dict); + aruco::generateImageMarker(dict, id, markerSidePixels, marker); + Point2f firstCorner = + Point2f(markerSidePixels / 2.f + x * (1.5f * markerSidePixels), + markerSidePixels / 2.f + y * (1.5f * markerSidePixels)); + Mat aux = img.colRange((int)firstCorner.x, (int)firstCorner.x + markerSidePixels) + .rowRange((int)firstCorner.y, (int)firstCorner.y + markerSidePixels); + marker.copyTo(aux); + } + } + img.convertTo(img, CV_8UC3); + + vector > markerCorners; + vector markerIds; + vector > rejectedImgPts; + vector dictIds; + detector.detectMarkers(img, markerCorners, markerIds, rejectedImgPts, dictIds); + ASSERT_EQ(markerIds.size(), 4u); + ASSERT_EQ(dictIds.size(), 4u); + for (size_t i = 0; i < dictIds.size(); ++i) { + EXPECT_EQ(dictIds[i], (int)i); + } +} + struct ArucoThreading: public testing::TestWithParam { From 379b5a2fdb829a6a62617601e41157de0b187c99 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Tue, 18 Feb 2025 14:08:09 +0100 Subject: [PATCH 02/14] Fix python bindings --- modules/python/src2/cv2.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index a7837a6ff8..4a3181bdde 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -45,6 +45,7 @@ typedef std::vector vector_DMatch; typedef std::vector vector_String; typedef std::vector vector_string; typedef std::vector vector_Scalar; +typedef std::vector vector_Dictionary; typedef std::vector > vector_vector_char; typedef std::vector > vector_vector_Point; From bb07ce74541a1ff2507884f2b41c276da8bd9a31 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Tue, 18 Feb 2025 17:03:37 +0100 Subject: [PATCH 03/14] Address comments, add Python test --- .../opencv2/objdetect/aruco_detector.hpp | 6 ++-- .../misc/python/test/test_objdetect_aruco.py | 31 ++++++++++++++++-- .../objdetect/src/aruco/aruco_detector.cpp | 17 +++++----- .../objdetect/test/test_arucodetection.cpp | 32 +++++++------------ 4 files changed, 51 insertions(+), 35 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index 0d3eb69647..d38fb1753e 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -350,12 +350,12 @@ public: InputArray cameraMatrix = noArray(), InputArray distCoeffs = noArray(), OutputArray recoveredIdxs = noArray()) const; - CV_WRAP const Dictionary& getDictionary(size_t index = 0) const; - CV_WRAP void setDictionary(const Dictionary& dictionary, size_t index = 0); + CV_WRAP const Dictionary& getDictionary(int index = 0) const; + CV_WRAP void setDictionary(const Dictionary& dictionary, int index = 0); CV_WRAP const std::vector& getDictionaries() const; CV_WRAP void setDictionaries(const std::vector& dictionaries); CV_WRAP void addDictionary(const Dictionary& dictionary); - CV_WRAP void removeDictionary(size_t index); + CV_WRAP void removeDictionary(int index); CV_WRAP const DetectorParameters& getDetectorParameters() const; CV_WRAP void setDetectorParameters(const DetectorParameters& detectorParameters); diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index 8dd407d32f..a6991f364d 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -156,7 +156,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array([[offset, offset],[marker_size+offset-1.0,offset], [marker_size+offset-1.0,marker_size+offset-1.0], [offset, marker_size+offset-1.0]], dtype=np.float32) - corners, ids, rejected = aruco_detector.detectMarkers(img_marker) + corners, ids, rejected, _ = aruco_detector.detectMarkers(img_marker) self.assertEqual(1, len(ids)) self.assertEqual(id, ids[0]) @@ -171,7 +171,7 @@ class aruco_objdetect_test(NewOpenCVTests): board = cv.aruco.GridBoard(board_size, 5.0, 1.0, aruco_dict) board_image = board.generateImage((board_size[0]*50, board_size[1]*50), marginSize=10) - corners, ids, rejected = aruco_detector.detectMarkers(board_image) + corners, ids, rejected, _ = aruco_detector.detectMarkers(board_image) self.assertEqual(board_size[0]*board_size[1], len(ids)) part_corners, part_ids, part_rejected = corners[:-1], ids[:-1], list(rejected) @@ -203,7 +203,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array(board.getObjPoints())[:, :, 0:2]*cell_size # detect corners - markerCorners, markerIds, _ = aruco_detector.detectMarkers(image) + markerCorners, markerIds, _, _ = aruco_detector.detectMarkers(image) # test refine rejected = [markerCorners[-1]] @@ -458,5 +458,30 @@ class aruco_objdetect_test(NewOpenCVTests): with self.assertRaises(Exception): img = cv.aruco.drawDetectedDiamonds(img, points2, borderColor=255) + def test_multi_dict_arucodetector(self): + aruco_params = cv.aruco.DetectorParameters() + aruco_dicts = [ + cv.aruco.getPredefinedDictionary(cv.aruco.DICT_4X4_250), + cv.aruco.getPredefinedDictionary(cv.aruco.DICT_5X5_250) + ] + aruco_detector = cv.aruco.ArucoDetector(aruco_dicts, aruco_params) + id = 2 + marker_size = 100 + offset = 10 + img_marker1 = cv.aruco.generateImageMarker(aruco_dicts[0], id, marker_size, aruco_params.markerBorderBits) + img_marker1 = np.pad(img_marker1, pad_width=offset, mode='constant', constant_values=255) + img_marker2 = cv.aruco.generateImageMarker(aruco_dicts[1], id, marker_size, aruco_params.markerBorderBits) + img_marker2 = np.pad(img_marker2, pad_width=offset, mode='constant', constant_values=255) + img_markers = np.concatenate((img_marker1, img_marker2), axis=1) + + corners, ids, rejected, dictIndices = aruco_detector.detectMarkers(img_markers) + + self.assertEqual(2, len(ids)) + self.assertEqual(id, ids[0]) + self.assertEqual(id, ids[1]) + self.assertEqual(2, len(dictIndices)) + self.assertEqual(0, dictIndices[0]) + self.assertEqual(1, dictIndices[1]) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 2c9ddb9160..ab3e325618 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -654,7 +654,9 @@ struct ArucoDetector::ArucoDetectorImpl { ArucoDetectorImpl(const std::vector&_dictionaries, const DetectorParameters &_detectorParams, const RefineParameters& _refineParams): dictionaries(_dictionaries), - detectorParams(_detectorParams), refineParams(_refineParams) {} + detectorParams(_detectorParams), refineParams(_refineParams) { + CV_Assert(!dictionaries.empty()); + } /** * @brief Detect square candidates in the input image */ @@ -1034,10 +1036,7 @@ void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corner _copyVector2Output(candidates, _corners); Mat(ids).copyTo(_ids); if (_dictIndices.needed()) { - _dictIndices.create(dictIndices.size(), 1, CV_32SC1); - Mat dictIndicesMat = _dictIndices.getMat(); - Mat m = cv::Mat1i(dictIndices).t(); - m.copyTo(dictIndicesMat); + Mat(dictIndices).copyTo(_dictIndices); } } @@ -1346,12 +1345,12 @@ void ArucoDetector::read(const FileNode &fn) { arucoDetectorImpl->refineParams.readRefineParameters(fn); } -const Dictionary& ArucoDetector::getDictionary(size_t index) const { +const Dictionary& ArucoDetector::getDictionary(int index) const { CV_Assert(index < arucoDetectorImpl->dictionaries.size()); return arucoDetectorImpl->dictionaries[index]; } -void ArucoDetector::setDictionary(const Dictionary& dictionary, size_t index) { +void ArucoDetector::setDictionary(const Dictionary& dictionary, int index) { // special case: if index is 0, we add the dictionary to the list to preserve the old behavior CV_Assert(index == 0 || index < arucoDetectorImpl->dictionaries.size()); if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { @@ -1373,8 +1372,10 @@ void ArucoDetector::addDictionary(const Dictionary& dictionary) { arucoDetectorImpl->dictionaries.push_back(dictionary); } -void ArucoDetector::removeDictionary(size_t index) { +void ArucoDetector::removeDictionary(int index) { CV_Assert(index < arucoDetectorImpl->dictionaries.size()); + // disallow no dictionaries + CV_Assert(arucoDetectorImpl->dictionaries.size() > 1); arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); } diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 94e062eb56..511f71e74b 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -640,6 +640,7 @@ TEST(CV_ArucoDetectMarkers, regression_contour_24220) TEST(CV_ArucoMultiDict, addRemoveDictionary) { + // using default constructor that pre-configures DICT_4X4_50 aruco::ArucoDetector detector; detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_100)); const auto& dicts = detector.getDictionaries(); @@ -649,11 +650,10 @@ TEST(CV_ArucoMultiDict, addRemoveDictionary) detector.removeDictionary(0); ASSERT_EQ(dicts.size(), 1ul); EXPECT_EQ(dicts[0].markerSize, 5); - detector.removeDictionary(0); - EXPECT_EQ(dicts.size(), 0ul); detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_6X6_100)); detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_25h9)); + detector.removeDictionary(0); ASSERT_EQ(dicts.size(), 3ul); EXPECT_EQ(dicts[0].markerSize, 6); EXPECT_EQ(dicts[1].markerSize, 7); @@ -672,27 +672,17 @@ TEST(CV_ArucoMultiDict, addRemoveDictionary) TEST(CV_ArucoMultiDict, noDict) { aruco::ArucoDetector detector; - detector.removeDictionary(0); - - vector > markerCorners; - vector markerIds; - - string img_path = cvtest::findDataFile("aruco/singlemarkersoriginal.jpg"); - Mat image = imread(img_path); - - detector.detectMarkers(image, markerCorners, markerIds); - - EXPECT_EQ(markerIds.size(), 0u); + EXPECT_THROW({ + detector.removeDictionary(0); + }, Exception); } TEST(CV_ArucoMultiDict, multiMarkerDetection) { - aruco::ArucoDetector detector; - detector.removeDictionary(0); - const int markerSidePixels = 100; const int imageSize = markerSidePixels * 2 + 3 * (markerSidePixels / 2); + vector usedDictionaries; // draw synthetic image Mat img = Mat(imageSize, imageSize, CV_8UC1, Scalar::all(255)); @@ -702,18 +692,18 @@ TEST(CV_ArucoMultiDict, multiMarkerDetection) int id = y * 2 + x; int dictId = x * 4 + y * 8; auto dict = aruco::getPredefinedDictionary(dictId); - detector.addDictionary(dict); + usedDictionaries.push_back(dict); aruco::generateImageMarker(dict, id, markerSidePixels, marker); - Point2f firstCorner = - Point2f(markerSidePixels / 2.f + x * (1.5f * markerSidePixels), + Point2f firstCorner(markerSidePixels / 2.f + x * (1.5f * markerSidePixels), markerSidePixels / 2.f + y * (1.5f * markerSidePixels)); - Mat aux = img.colRange((int)firstCorner.x, (int)firstCorner.x + markerSidePixels) - .rowRange((int)firstCorner.y, (int)firstCorner.y + markerSidePixels); + Mat aux = img(Rect((int)firstCorner.x, (int)firstCorner.y, markerSidePixels, markerSidePixels)); marker.copyTo(aux); } } img.convertTo(img, CV_8UC3); + aruco::ArucoDetector detector(usedDictionaries); + vector > markerCorners; vector markerIds; vector > rejectedImgPts; From 9ae23a7f51a5d98e443cd221518bad5d62726ca5 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Wed, 19 Feb 2025 10:53:03 +0100 Subject: [PATCH 04/14] Fix index comparison warnings --- modules/objdetect/src/aruco/aruco_detector.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index ab3e325618..1fb2592b21 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1346,13 +1346,13 @@ void ArucoDetector::read(const FileNode &fn) { } const Dictionary& ArucoDetector::getDictionary(int index) const { - CV_Assert(index < arucoDetectorImpl->dictionaries.size()); + CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); return arucoDetectorImpl->dictionaries[index]; } void ArucoDetector::setDictionary(const Dictionary& dictionary, int index) { // special case: if index is 0, we add the dictionary to the list to preserve the old behavior - CV_Assert(index == 0 || index < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index == 0 || static_cast(index) < arucoDetectorImpl->dictionaries.size()); if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { arucoDetectorImpl->dictionaries.push_back(dictionary); } else { @@ -1373,9 +1373,9 @@ void ArucoDetector::addDictionary(const Dictionary& dictionary) { } void ArucoDetector::removeDictionary(int index) { - CV_Assert(index < arucoDetectorImpl->dictionaries.size()); + CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); // disallow no dictionaries - CV_Assert(arucoDetectorImpl->dictionaries.size() > 1); + CV_Assert(arucoDetectorImpl->dictionaries.size() > 1ul); arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); } From f212c163e311bfc516ccca6cc00a64002a7a30ec Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Wed, 19 Feb 2025 18:37:49 +0100 Subject: [PATCH 05/14] have two detectMarkers functions for python backwards compatibility using multiple dictionaries for refinement (function split not necessary as it's backwards compatible) --- .../opencv2/objdetect/aruco_detector.hpp | 33 +- .../misc/python/test/test_objdetect_aruco.py | 8 +- .../objdetect/src/aruco/aruco_detector.cpp | 626 ++++++++++-------- .../objdetect/test/test_arucodetection.cpp | 2 +- .../objdetect/test/test_boarddetection.cpp | 3 + 5 files changed, 394 insertions(+), 278 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index d38fb1753e..281d772db8 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -287,7 +287,7 @@ public: /** @brief ArucoDetector constructor for multiple dictionaries * - * @param dictionaries indicates the type of markers that will be searched + * @param dictionaries indicates the type of markers that will be searched. Empty dictionaries will throw an error. * @param detectorParams marker detection parameters * @param refineParams marker refine detection parameters */ @@ -306,10 +306,8 @@ public: * The identifiers have the same order than the markers in the imgPoints array. * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a * correct codification. Useful for debugging purposes. - * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the - * list of corresponding dictionaries. * - * Performs marker detection in the input image. Only markers included in the specific dictionaries + * Performs marker detection in the input image. Only markers included in the first specified dictionary * are searched. For each detected marker, it returns the 2D position of its corner in the image * and its corresponding identifier. * Note that this function does not perform pose estimation. @@ -318,7 +316,7 @@ public: * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard */ CV_WRAP void detectMarkers(InputArray image, OutputArrayOfArrays corners, OutputArray ids, - OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + OutputArrayOfArrays rejectedImgPoints = noArray()) const; /** @brief Refine not detected markers based on the already detected and the board layout * @@ -350,6 +348,31 @@ public: InputArray cameraMatrix = noArray(), InputArray distCoeffs = noArray(), OutputArray recoveredIdxs = noArray()) const; + /** @brief Basic marker detection + * + * @param image input image + * @param corners vector of detected marker corners. For each marker, its four corners + * are provided, (e.g std::vector > ). For N detected markers, + * the dimensions of this array is Nx4. The order of the corners is clockwise. + * @param ids vector of identifiers of the detected markers. The identifier is of type int + * (e.g. std::vector). For N detected markers, the size of ids is also N. + * The identifiers have the same order than the markers in the imgPoints array. + * @param rejectedImgPoints contains the imgPoints of those squares whose inner code has not a + * correct codification. Useful for debugging purposes. + * @param dictIndices vector of dictionary indices for each detected marker. Use getDictionaries() to get the + * list of corresponding dictionaries. + * + * Performs marker detection in the input image. Only markers included in the specific dictionaries + * are searched. For each detected marker, it returns the 2D position of its corner in the image + * and its corresponding identifier. + * Note that this function does not perform pose estimation. + * @note The function does not correct lens distortion or takes it into account. It's recommended to undistort + * input image with corresponding camera model, if camera parameters are known + * @sa undistort, estimatePoseSingleMarkers, estimatePoseBoard + */ + CV_WRAP void detectMarkersMultiDict(InputArray image, OutputArrayOfArrays corners, OutputArray ids, + OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + CV_WRAP const Dictionary& getDictionary(int index = 0) const; CV_WRAP void setDictionary(const Dictionary& dictionary, int index = 0); CV_WRAP const std::vector& getDictionaries() const; diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index a6991f364d..305d2bdca9 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -156,7 +156,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array([[offset, offset],[marker_size+offset-1.0,offset], [marker_size+offset-1.0,marker_size+offset-1.0], [offset, marker_size+offset-1.0]], dtype=np.float32) - corners, ids, rejected, _ = aruco_detector.detectMarkers(img_marker) + corners, ids, rejected = aruco_detector.detectMarkers(img_marker) self.assertEqual(1, len(ids)) self.assertEqual(id, ids[0]) @@ -171,7 +171,7 @@ class aruco_objdetect_test(NewOpenCVTests): board = cv.aruco.GridBoard(board_size, 5.0, 1.0, aruco_dict) board_image = board.generateImage((board_size[0]*50, board_size[1]*50), marginSize=10) - corners, ids, rejected, _ = aruco_detector.detectMarkers(board_image) + corners, ids, rejected = aruco_detector.detectMarkers(board_image) self.assertEqual(board_size[0]*board_size[1], len(ids)) part_corners, part_ids, part_rejected = corners[:-1], ids[:-1], list(rejected) @@ -203,7 +203,7 @@ class aruco_objdetect_test(NewOpenCVTests): gold_corners = np.array(board.getObjPoints())[:, :, 0:2]*cell_size # detect corners - markerCorners, markerIds, _, _ = aruco_detector.detectMarkers(image) + markerCorners, markerIds, _ = aruco_detector.detectMarkers(image) # test refine rejected = [markerCorners[-1]] @@ -474,7 +474,7 @@ class aruco_objdetect_test(NewOpenCVTests): img_marker2 = np.pad(img_marker2, pad_width=offset, mode='constant', constant_values=255) img_markers = np.concatenate((img_marker1, img_marker2), axis=1) - corners, ids, rejected, dictIndices = aruco_detector.detectMarkers(img_markers) + corners, ids, rejected, dictIndices = aruco_detector.detectMarkersMultiDict(img_markers) self.assertEqual(2, len(ids)) self.assertEqual(id, ids[0]) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 1fb2592b21..881b627d00 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -641,6 +641,11 @@ static inline void findCornerInPyrImage(const float scale_init, const int closes } } +enum class DictionaryMode { + Single, + Multi +}; + struct ArucoDetector::ArucoDetectorImpl { /// dictionaries indicates the types of markers that will be searched std::vector dictionaries; @@ -657,6 +662,252 @@ struct ArucoDetector::ArucoDetectorImpl { detectorParams(_detectorParams), refineParams(_refineParams) { CV_Assert(!dictionaries.empty()); } + + /* + * @brief Detect markers either using multiple or just first dictionary + */ + void detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices, DictionaryMode dictMode) { + CV_Assert(!_image.empty()); + + CV_Assert(detectorParams.markerBorderBits > 0); + // check that the parameters are set correctly if Aruco3 is used + CV_Assert(!(detectorParams.useAruco3Detection == true && + detectorParams.minSideLengthCanonicalImg == 0 && + detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); + + Mat grey; + _convertToGrey(_image, grey); + + // Aruco3 functionality is the extension of Aruco. + // The description can be found in: + // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. + // if Aruco3 functionality if not wanted + // change some parameters to be sure to turn it off + if (!detectorParams.useAruco3Detection) { + detectorParams.minMarkerLengthRatioOriginalImg = 0.0; + detectorParams.minSideLengthCanonicalImg = 0; + } + else { + // always turn on corner refinement in case of Aruco3, due to upsampling + detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + } + + /// Step 0: equation (2) from paper [1] + const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / + (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* + detectorParams.minMarkerLengthRatioOriginalImg)); + + /// Step 1: create image pyramid. Section 3.4. in [1] + vector grey_pyramid; + int closest_pyr_image_idx = 0, num_levels = 0; + //// Step 1.1: resize image with equation (1) from paper [1] + if (detectorParams.useAruco3Detection) { + const float scale_pyr = 2.f; + const float img_area = static_cast(grey.rows*grey.cols); + const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* + detectorParams.minSideLengthCanonicalImg); + // find max level + num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); + // the closest pyramid image to the downsampled segmentation image + // will later be used as start index for corner upsampling + const float scale_img_area = img_area * fxfy * fxfy; + closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); + } + buildPyramid(grey, grey_pyramid, num_levels); + + // resize to segmentation image + // in this reduces size the contours will be detected + if (fxfy != 1.f) + resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); + + /// STEP 2: Detect marker candidates + vector > candidates; + vector > contours; + vector ids; + + /// STEP 2.a Detect marker candidates :: using AprilTag + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ + _apriltag(grey, detectorParams, candidates, contours); + } + /// STEP 2.b Detect marker candidates :: traditional way + else { + detectCandidates(grey, candidates, contours); + } + + /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS + vector dictIndices; + vector> rejectedImgPoints; + if (DictionaryMode::Single == dictMode) { + Dictionary& dictionary = dictionaries.at(0); + auto selectedCandidates = filterTooCloseCandidates(candidates, contours, dictionary.markerSize); + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + identifyCandidates(grey, grey_pyramid, selectedCandidates, candidates, contours, + ids, dictionary, rejectedImgPoints); + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); + } else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + } + }); + } + } else if (DictionaryMode::Multi == dictMode) { + unordered_set uniqueMarkerSizes; + for (const Dictionary& dictionary : dictionaries) { + uniqueMarkerSizes.insert(dictionary.markerSize); + } + + // create at max 4 marker candidate trees for each dictionary size + vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; + for (int markerSize : uniqueMarkerSizes) { + // min marker size is 4, so subtract 4 to get index + const auto dictionarySizeIndex = markerSize - 4; + // copy candidates + vector> candidatesCopy = candidates; + vector > contoursCopy = contours; + candidatesPerDictionarySize[dictionarySizeIndex] = filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); + } + candidates.clear(); + contours.clear(); + + /// STEP 2: Check candidate codification (identify markers) + int dictIndex = 0; + for (const Dictionary& currentDictionary : dictionaries) { + const auto dictionarySizeIndex = currentDictionary.markerSize - 4; + // temporary variable to store the current candidates + vector> currentCandidates; + identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, + ids, currentDictionary, rejectedImgPoints); + if (_dictIndices.needed()) { + dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); + } + + /// STEP 3: Corner refinement :: use corner subpix + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); + } + else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + } + }); + } + candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); + dictIndex++; + } + + // Clean up rejectedImgPoints by comparing to itself and all candidates + const float epsilon = 0.000001; + auto compareCandidates = [epsilon](std::vector a, std::vector b) { + for (int i = 0; i < 4; i++) { + if (std::abs(a[i].x - b[i].x) > epsilon || std::abs(a[i].y - b[i].y) > epsilon) { + return false; + } + } + return true; + }; + std::sort(rejectedImgPoints.begin(), rejectedImgPoints.end(), [](const vector& a, const vector&b){ + float avgX = (a[0].x + a[1].x + a[2].x + a[3].x)*.25f; + float avgY = (a[0].y + a[1].y + a[2].y + a[3].y)*.25f; + float aDist = avgX*avgX + avgY*avgY; + avgX = (b[0].x + b[1].x + b[2].x + b[3].x)*.25f; + avgY = (b[0].y + b[1].y + b[2].y + b[3].y)*.25f; + float bDist = avgX*avgX + avgY*avgY; + return aDist < bDist; + }); + auto last = std::unique(rejectedImgPoints.begin(), rejectedImgPoints.end(), compareCandidates); + rejectedImgPoints.erase(last, rejectedImgPoints.end()); + + for (auto it = rejectedImgPoints.begin(); it != rejectedImgPoints.end();) { + bool erased = false; + for (const auto& candidate : candidates) { + if (compareCandidates(candidate, *it)) { + it = rejectedImgPoints.erase(it); + erased = true; + break; + } + } + if (!erased) { + it++; + } + } + } + + /// STEP 3, Optional : Corner refinement :: use contour container + if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ + + if (!ids.empty()) { + + // do corner refinement using the contours for each detected markers + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + for (int i = range.start; i < range.end; i++) { + _refineCandidateLines(contours[i], candidates[i]); + } + }); + } + } + + if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { + // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection + // Todo: update other CORNER_REFINE methods + + // scale to orignal size, this however will lead to inaccurate detections! + for (auto &vecPoints : candidates) + for (auto &point : vecPoints) + point *= 1.f/fxfy; + } + + // copy to output arrays + _copyVector2Output(candidates, _corners); + Mat(ids).copyTo(_ids); + if(_rejectedImgPoints.needed()) { + _copyVector2Output(rejectedImgPoints, _rejectedImgPoints); + } + if (_dictIndices.needed()) { + Mat(dictIndices).copyTo(_dictIndices); + } + } + /** * @brief Detect square candidates in the input image */ @@ -771,9 +1022,8 @@ struct ArucoDetector::ArucoDetectorImpl { */ void identifyCandidates(const Mat& grey, const vector& image_pyr, vector& selectedContours, vector >& accepted, vector >& contours, - vector& ids, const Dictionary& currentDictionary, OutputArrayOfArrays _rejected = noArray()) { + vector& ids, const Dictionary& currentDictionary, vector>& rejected) const { size_t ncandidates = selectedContours.size(); - vector > rejected; vector idsTmp(ncandidates, -1); vector rotated(ncandidates, 0); @@ -853,11 +1103,6 @@ struct ArucoDetector::ArucoDetectorImpl { rejected.push_back(selectedContours[i].corners); } } - - // parse output - if(_rejected.needed()) { - _copyVector2Output(rejected, _rejected); - } } }; @@ -875,169 +1120,13 @@ ArucoDetector::ArucoDetector(const std::vector &_dictionaries, } void ArucoDetector::detectMarkers(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, + OutputArrayOfArrays _rejectedImgPoints) const { + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, noArray(), DictionaryMode::Single); +} + +void ArucoDetector::detectMarkersMultiDict(InputArray _image, OutputArrayOfArrays _corners, OutputArray _ids, OutputArrayOfArrays _rejectedImgPoints, OutputArray _dictIndices) const { - CV_Assert(!_image.empty()); - DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - - CV_Assert(detectorParams.markerBorderBits > 0); - // check that the parameters are set correctly if Aruco3 is used - CV_Assert(!(detectorParams.useAruco3Detection == true && - detectorParams.minSideLengthCanonicalImg == 0 && - detectorParams.minMarkerLengthRatioOriginalImg == 0.0)); - - Mat grey; - _convertToGrey(_image, grey); - - // Aruco3 functionality is the extension of Aruco. - // The description can be found in: - // [1] Speeded up detection of squared fiducial markers, 2018, FJ Romera-Ramirez et al. - // if Aruco3 functionality if not wanted - // change some parameters to be sure to turn it off - if (!detectorParams.useAruco3Detection) { - detectorParams.minMarkerLengthRatioOriginalImg = 0.0; - detectorParams.minSideLengthCanonicalImg = 0; - } - else { - // always turn on corner refinement in case of Aruco3, due to upsampling - detectorParams.cornerRefinementMethod = (int)CORNER_REFINE_SUBPIX; - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - } - - /// Step 0: equation (2) from paper [1] - const float fxfy = (!detectorParams.useAruco3Detection ? 1.f : detectorParams.minSideLengthCanonicalImg / - (detectorParams.minSideLengthCanonicalImg + std::max(grey.cols, grey.rows)* - detectorParams.minMarkerLengthRatioOriginalImg)); - - /// Step 1: create image pyramid. Section 3.4. in [1] - vector grey_pyramid; - int closest_pyr_image_idx = 0, num_levels = 0; - //// Step 1.1: resize image with equation (1) from paper [1] - if (detectorParams.useAruco3Detection) { - const float scale_pyr = 2.f; - const float img_area = static_cast(grey.rows*grey.cols); - const float min_area_marker = static_cast(detectorParams.minSideLengthCanonicalImg* - detectorParams.minSideLengthCanonicalImg); - // find max level - num_levels = static_cast(log2(img_area / min_area_marker)/scale_pyr); - // the closest pyramid image to the downsampled segmentation image - // will later be used as start index for corner upsampling - const float scale_img_area = img_area * fxfy * fxfy; - closest_pyr_image_idx = cvRound(log2(img_area / scale_img_area)/scale_pyr); - } - buildPyramid(grey, grey_pyramid, num_levels); - - // resize to segmentation image - // in this reduces size the contours will be detected - if (fxfy != 1.f) - resize(grey, grey, Size(cvRound(fxfy * grey.cols), cvRound(fxfy * grey.rows))); - - /// STEP 2: Detect marker candidates - vector > candidates; - vector > contours; - vector ids; - - /// STEP 2.a Detect marker candidates :: using AprilTag - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_APRILTAG){ - _apriltag(grey, detectorParams, candidates, contours); - } - /// STEP 2.b Detect marker candidates :: traditional way - else { - arucoDetectorImpl->detectCandidates(grey, candidates, contours); - } - - /// STEP 2.c FILTER OUT NEAR CANDIDATE PAIRS - unordered_set uniqueMarkerSizes; - for (const Dictionary& dictionary : arucoDetectorImpl->dictionaries) { - uniqueMarkerSizes.insert(dictionary.markerSize); - } - - // create at max 4 marker candidate trees for each dictionary size - vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; - for (int markerSize : uniqueMarkerSizes) { - // min marker size is 4, so subtract 4 to get index - const auto dictionarySizeIndex = markerSize - 4; - // copy candidates - vector> candidatesCopy = candidates; - vector > contoursCopy = contours; - candidatesPerDictionarySize[dictionarySizeIndex] = arucoDetectorImpl->filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); - } - candidates.clear(); - contours.clear(); - - /// STEP 2: Check candidate codification (identify markers) - size_t dictIndex = 0; - vector dictIndices; - for (const Dictionary& currentDictionary : arucoDetectorImpl->dictionaries) { - const auto dictionarySizeIndex = currentDictionary.markerSize - 4; - // temporary variable to store the current candidates - vector> currentCandidates; - arucoDetectorImpl->identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, - ids, currentDictionary, _rejectedImgPoints); - if (_dictIndices.needed()) { - dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); - } - - /// STEP 3: Corner refinement :: use corner subpix - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners - parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { - const int begin = range.start; - const int end = range.end; - - for (int i = begin; i < end; i++) { - if (detectorParams.useAruco3Detection) { - const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; - findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); - } - else { - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - } - }); - } - candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); - dictIndex++; - } - - /// STEP 3, Optional : Corner refinement :: use contour container - if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_CONTOUR){ - - if (!ids.empty()) { - - // do corner refinement using the contours for each detected markers - parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { - for (int i = range.start; i < range.end; i++) { - _refineCandidateLines(contours[i], candidates[i]); - } - }); - } - } - - if (detectorParams.cornerRefinementMethod != (int)CORNER_REFINE_SUBPIX && fxfy != 1.f) { - // only CORNER_REFINE_SUBPIX implement correctly for useAruco3Detection - // Todo: update other CORNER_REFINE methods - - // scale to orignal size, this however will lead to inaccurate detections! - for (auto &vecPoints : candidates) - for (auto &point : vecPoints) - point *= 1.f/fxfy; - } - - // copy to output arrays - _copyVector2Output(candidates, _corners); - Mat(ids).copyTo(_ids); - if (_dictIndices.needed()) { - Mat(dictIndices).copyTo(_dictIndices); - } + arucoDetectorImpl->detectMarkers(_image, _corners, _ids, _rejectedImgPoints, _dictIndices, DictionaryMode::Multi); } /** @@ -1151,7 +1240,6 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board InputOutputArrayOfArrays _rejectedCorners, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _recoveredIdxs) const { DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; - const Dictionary& dictionary = arucoDetectorImpl->dictionaries[0]; RefineParameters& refineParams = arucoDetectorImpl->refineParams; CV_Assert(refineParams.minRepDistance > 0); @@ -1174,10 +1262,6 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board // list of missing markers indicating if they have been assigned to a candidate vector alreadyIdentified(_rejectedCorners.total(), false); - // maximum bits that can be corrected - int maxCorrectionRecalculated = - int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); - Mat grey; _convertToGrey(_image, grey); @@ -1193,106 +1277,112 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } vector recoveredIdxs; // original indexes of accepted markers in _rejectedCorners - // for each missing marker, try to find a correspondence - for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { + for (const auto& dictionary : arucoDetectorImpl->dictionaries) { + // maximum bits that can be corrected + int maxCorrectionRecalculated = + int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); - // best match at the moment - int closestCandidateIdx = -1; - double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; - Mat closestRotatedMarker; + // for each missing marker, try to find a correspondence + for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { - for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { - if(alreadyIdentified[j]) continue; + // best match at the moment + int closestCandidateIdx = -1; + double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; + Mat closestRotatedMarker; - // check distance - double minDistance = closestCandidateDistance + 1; - bool valid = false; - int validRot = 0; - for(int c = 0; c < 4; c++) { // first corner in rejected candidate - double currentMaxDistance = 0; - for(int k = 0; k < 4; k++) { - Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; - Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; - double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; - currentMaxDistance = max(currentMaxDistance, cornerDist); + for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { + if(alreadyIdentified[j]) continue; + + // check distance + double minDistance = closestCandidateDistance + 1; + bool valid = false; + int validRot = 0; + for(int c = 0; c < 4; c++) { // first corner in rejected candidate + double currentMaxDistance = 0; + for(int k = 0; k < 4; k++) { + Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; + Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; + double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; + currentMaxDistance = max(currentMaxDistance, cornerDist); + } + // if distance is better than current best distance + if(currentMaxDistance < closestCandidateDistance) { + valid = true; + validRot = c; + minDistance = currentMaxDistance; + } + if(!refineParams.checkAllOrders) break; } - // if distance is better than current best distance - if(currentMaxDistance < closestCandidateDistance) { - valid = true; - validRot = c; - minDistance = currentMaxDistance; + + if(!valid) continue; + + // apply rotation + Mat rotatedMarker; + if(refineParams.checkAllOrders) { + rotatedMarker = Mat(4, 1, CV_32FC2); + for(int c = 0; c < 4; c++) + rotatedMarker.ptr()[c] = + _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; + } + else rotatedMarker = _rejectedCorners.getMat(j); + + // last filter, check if inner code is close enough to the assigned marker code + int codeDistance = 0; + // if errorCorrectionRate, dont check code + if(refineParams.errorCorrectionRate >= 0) { + + // extract bits + Mat bits = _extractBits( + grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, + detectorParams.perspectiveRemovePixelPerCell, + detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); + + Mat onlyBits = + bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) + .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); + + codeDistance = + dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); + } + + // if everythin is ok, assign values to current best match + if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { + closestCandidateIdx = j; + closestCandidateDistance = minDistance; + closestRotatedMarker = rotatedMarker; } - if(!refineParams.checkAllOrders) break; } - if(!valid) continue; + // if at least one good match, we have rescue the missing marker + if(closestCandidateIdx >= 0) { - // apply rotation - Mat rotatedMarker; - if(refineParams.checkAllOrders) { - rotatedMarker = Mat(4, 1, CV_32FC2); - for(int c = 0; c < 4; c++) - rotatedMarker.ptr()[c] = - _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; + // subpixel refinement + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && + detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + + std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, closestRotatedMarker, + Size(cornerRefinementWinSize, cornerRefinementWinSize), + Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + + // remove from rejected + alreadyIdentified[closestCandidateIdx] = true; + + // add to detected + finalAcceptedCorners.push_back(closestRotatedMarker); + finalAcceptedIds.push_back(undetectedMarkersIds[i]); + + // add the original index of the candidate + recoveredIdxs.push_back(closestCandidateIdx); } - else rotatedMarker = _rejectedCorners.getMat(j); - - // last filter, check if inner code is close enough to the assigned marker code - int codeDistance = 0; - // if errorCorrectionRate, dont check code - if(refineParams.errorCorrectionRate >= 0) { - - // extract bits - Mat bits = _extractBits( - grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, - detectorParams.perspectiveRemovePixelPerCell, - detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); - - Mat onlyBits = - bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) - .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); - - codeDistance = - dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); - } - - // if everythin is ok, assign values to current best match - if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { - closestCandidateIdx = j; - closestCandidateDistance = minDistance; - closestRotatedMarker = rotatedMarker; - } - } - - // if at least one good match, we have rescue the missing marker - if(closestCandidateIdx >= 0) { - - // subpixel refinement - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && - detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - - std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, closestRotatedMarker, - Size(cornerRefinementWinSize, cornerRefinementWinSize), - Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - - // remove from rejected - alreadyIdentified[closestCandidateIdx] = true; - - // add to detected - finalAcceptedCorners.push_back(closestRotatedMarker); - finalAcceptedIds.push_back(undetectedMarkersIds[i]); - - // add the original index of the candidate - recoveredIdxs.push_back(closestCandidateIdx); } } @@ -1346,13 +1436,13 @@ void ArucoDetector::read(const FileNode &fn) { } const Dictionary& ArucoDetector::getDictionary(int index) const { - CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); return arucoDetectorImpl->dictionaries[index]; } void ArucoDetector::setDictionary(const Dictionary& dictionary, int index) { // special case: if index is 0, we add the dictionary to the list to preserve the old behavior - CV_Assert(index == 0 || static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index == 0 || (index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size())); if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { arucoDetectorImpl->dictionaries.push_back(dictionary); } else { @@ -1373,7 +1463,7 @@ void ArucoDetector::addDictionary(const Dictionary& dictionary) { } void ArucoDetector::removeDictionary(int index) { - CV_Assert(static_cast(index) < arucoDetectorImpl->dictionaries.size()); + CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); // disallow no dictionaries CV_Assert(arucoDetectorImpl->dictionaries.size() > 1ul); arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 511f71e74b..c3138312a4 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -708,7 +708,7 @@ TEST(CV_ArucoMultiDict, multiMarkerDetection) vector markerIds; vector > rejectedImgPts; vector dictIds; - detector.detectMarkers(img, markerCorners, markerIds, rejectedImgPts, dictIds); + detector.detectMarkersMultiDict(img, markerCorners, markerIds, rejectedImgPts, dictIds); ASSERT_EQ(markerIds.size(), 4u); ASSERT_EQ(dictIds.size(), 4u); for (size_t i = 0; i < dictIds.size(); ++i) { diff --git a/modules/objdetect/test/test_boarddetection.cpp b/modules/objdetect/test/test_boarddetection.cpp index 1fa6e11994..ac767f24a7 100644 --- a/modules/objdetect/test/test_boarddetection.cpp +++ b/modules/objdetect/test/test_boarddetection.cpp @@ -142,6 +142,9 @@ class CV_ArucoRefine : public cvtest::BaseTest { params.useAruco3Detection = true; aruco::RefineParameters refineParams(10.f, 3.f, true); detector = aruco::ArucoDetector(dictionary, params, refineParams); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_250)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_4X4_250)); + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); } protected: From 1f9d6aa6cf08538b390e3ccc2c15cd47b6fb2bee Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Thu, 20 Feb 2025 15:12:56 +0100 Subject: [PATCH 06/14] Fixed warning on Windows, clarified refineDetectedMarkers method --- modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp | 2 +- modules/objdetect/src/aruco/aruco_detector.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index 281d772db8..2770a23e36 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -340,7 +340,7 @@ public: * using projectPoint function. If not, missing marker projections are interpolated using global * homography, and all the marker corners in the board must have the same Z coordinate. * @note This function assumes that the board only contains markers from one dictionary, so only the - * first configured dictionary is used. + * first configured dictionary is used. It has to match the dictionary of the board to work properly. */ CV_WRAP void refineDetectedMarkers(InputArray image, const Board &board, InputOutputArrayOfArrays detectedCorners, diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 881b627d00..baa8cfc33b 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -837,7 +837,7 @@ struct ArucoDetector::ArucoDetectorImpl { } // Clean up rejectedImgPoints by comparing to itself and all candidates - const float epsilon = 0.000001; + const float epsilon = 0.000001f; auto compareCandidates = [epsilon](std::vector a, std::vector b) { for (int i = 0; i < 4; i++) { if (std::abs(a[i].x - b[i].x) > epsilon || std::abs(a[i].y - b[i].y) > epsilon) { From 364eedb87ee1118d4e90cb57a82a245c0f76d699 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Thu, 20 Feb 2025 15:27:19 +0100 Subject: [PATCH 07/14] Undo multi dict functionality of refineDetectedMarkers method --- .../objdetect/src/aruco/aruco_detector.cpp | 193 +++++++++--------- 1 file changed, 96 insertions(+), 97 deletions(-) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index baa8cfc33b..62f34e486b 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1240,6 +1240,7 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board InputOutputArrayOfArrays _rejectedCorners, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _recoveredIdxs) const { DetectorParameters& detectorParams = arucoDetectorImpl->detectorParams; + const Dictionary& dictionary = arucoDetectorImpl->dictionaries.at(0); RefineParameters& refineParams = arucoDetectorImpl->refineParams; CV_Assert(refineParams.minRepDistance > 0); @@ -1262,6 +1263,10 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board // list of missing markers indicating if they have been assigned to a candidate vector alreadyIdentified(_rejectedCorners.total(), false); + // maximum bits that can be corrected + int maxCorrectionRecalculated = + int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); + Mat grey; _convertToGrey(_image, grey); @@ -1277,112 +1282,106 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } vector recoveredIdxs; // original indexes of accepted markers in _rejectedCorners - for (const auto& dictionary : arucoDetectorImpl->dictionaries) { - // maximum bits that can be corrected - int maxCorrectionRecalculated = - int(double(dictionary.maxCorrectionBits) * refineParams.errorCorrectionRate); + // for each missing marker, try to find a correspondence + for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { - // for each missing marker, try to find a correspondence - for(unsigned int i = 0; i < undetectedMarkersIds.size(); i++) { + // best match at the moment + int closestCandidateIdx = -1; + double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; + Mat closestRotatedMarker; - // best match at the moment - int closestCandidateIdx = -1; - double closestCandidateDistance = refineParams.minRepDistance * refineParams.minRepDistance + 1; - Mat closestRotatedMarker; + for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { + if(alreadyIdentified[j]) continue; - for(unsigned int j = 0; j < _rejectedCorners.total(); j++) { - if(alreadyIdentified[j]) continue; - - // check distance - double minDistance = closestCandidateDistance + 1; - bool valid = false; - int validRot = 0; - for(int c = 0; c < 4; c++) { // first corner in rejected candidate - double currentMaxDistance = 0; - for(int k = 0; k < 4; k++) { - Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; - Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; - double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; - currentMaxDistance = max(currentMaxDistance, cornerDist); - } - // if distance is better than current best distance - if(currentMaxDistance < closestCandidateDistance) { - valid = true; - validRot = c; - minDistance = currentMaxDistance; - } - if(!refineParams.checkAllOrders) break; + // check distance + double minDistance = closestCandidateDistance + 1; + bool valid = false; + int validRot = 0; + for(int c = 0; c < 4; c++) { // first corner in rejected candidate + double currentMaxDistance = 0; + for(int k = 0; k < 4; k++) { + Point2f rejCorner = _rejectedCorners.getMat(j).ptr()[(c + k) % 4]; + Point2f distVector = undetectedMarkersCorners[i][k] - rejCorner; + double cornerDist = distVector.x * distVector.x + distVector.y * distVector.y; + currentMaxDistance = max(currentMaxDistance, cornerDist); } - - if(!valid) continue; - - // apply rotation - Mat rotatedMarker; - if(refineParams.checkAllOrders) { - rotatedMarker = Mat(4, 1, CV_32FC2); - for(int c = 0; c < 4; c++) - rotatedMarker.ptr()[c] = - _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; - } - else rotatedMarker = _rejectedCorners.getMat(j); - - // last filter, check if inner code is close enough to the assigned marker code - int codeDistance = 0; - // if errorCorrectionRate, dont check code - if(refineParams.errorCorrectionRate >= 0) { - - // extract bits - Mat bits = _extractBits( - grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, - detectorParams.perspectiveRemovePixelPerCell, - detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); - - Mat onlyBits = - bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) - .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); - - codeDistance = - dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); - } - - // if everythin is ok, assign values to current best match - if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { - closestCandidateIdx = j; - closestCandidateDistance = minDistance; - closestRotatedMarker = rotatedMarker; + // if distance is better than current best distance + if(currentMaxDistance < closestCandidateDistance) { + valid = true; + validRot = c; + minDistance = currentMaxDistance; } + if(!refineParams.checkAllOrders) break; } - // if at least one good match, we have rescue the missing marker - if(closestCandidateIdx >= 0) { + if(!valid) continue; - // subpixel refinement - if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && - detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - - std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, closestRotatedMarker, - Size(cornerRefinementWinSize, cornerRefinementWinSize), - Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - - // remove from rejected - alreadyIdentified[closestCandidateIdx] = true; - - // add to detected - finalAcceptedCorners.push_back(closestRotatedMarker); - finalAcceptedIds.push_back(undetectedMarkersIds[i]); - - // add the original index of the candidate - recoveredIdxs.push_back(closestCandidateIdx); + // apply rotation + Mat rotatedMarker; + if(refineParams.checkAllOrders) { + rotatedMarker = Mat(4, 1, CV_32FC2); + for(int c = 0; c < 4; c++) + rotatedMarker.ptr()[c] = + _rejectedCorners.getMat(j).ptr()[(c + 4 + validRot) % 4]; } + else rotatedMarker = _rejectedCorners.getMat(j); + + // last filter, check if inner code is close enough to the assigned marker code + int codeDistance = 0; + // if errorCorrectionRate, dont check code + if(refineParams.errorCorrectionRate >= 0) { + + // extract bits + Mat bits = _extractBits( + grey, rotatedMarker, dictionary.markerSize, detectorParams.markerBorderBits, + detectorParams.perspectiveRemovePixelPerCell, + detectorParams.perspectiveRemoveIgnoredMarginPerCell, detectorParams.minOtsuStdDev); + + Mat onlyBits = + bits.rowRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits) + .colRange(detectorParams.markerBorderBits, bits.rows - detectorParams.markerBorderBits); + + codeDistance = + dictionary.getDistanceToId(onlyBits, undetectedMarkersIds[i], false); + } + + // if everythin is ok, assign values to current best match + if(refineParams.errorCorrectionRate < 0 || codeDistance < maxCorrectionRecalculated) { + closestCandidateIdx = j; + closestCandidateDistance = minDistance; + closestRotatedMarker = rotatedMarker; + } + } + + // if at least one good match, we have rescue the missing marker + if(closestCandidateIdx >= 0) { + + // subpixel refinement + if(detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && + detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + + std::vector marker(closestRotatedMarker.begin(), closestRotatedMarker.end()); + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(marker, dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, closestRotatedMarker, + Size(cornerRefinementWinSize, cornerRefinementWinSize), + Size(-1, -1), TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + + // remove from rejected + alreadyIdentified[closestCandidateIdx] = true; + + // add to detected + finalAcceptedCorners.push_back(closestRotatedMarker); + finalAcceptedIds.push_back(undetectedMarkersIds[i]); + + // add the original index of the candidate + recoveredIdxs.push_back(closestCandidateIdx); } } From 3c88a001a24c54a77a499fbfedb7437ec0fc69e3 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Mon, 24 Feb 2025 14:30:48 +0100 Subject: [PATCH 08/14] Add docs to Dictionary get/set/add/remove functions --- .../opencv2/objdetect/aruco_detector.hpp | 54 +++++++++++++++++++ .../objdetect/src/aruco/aruco_detector.cpp | 9 ++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index 2770a23e36..e944c09a54 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -373,11 +373,65 @@ public: CV_WRAP void detectMarkersMultiDict(InputArray image, OutputArrayOfArrays corners, OutputArray ids, OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; + /** @brief Returns a specific dictionary from the set of dictionaries used for marker detection. + * + * @param index Index of the dictionary to retrieve. Default is 0, returning the first dictionary if multiple are set. + * + * Returns the dictionary at the specified index from the internal collection of dictionaries used by the ArucoDetector. + * If only one dictionary is set, or if the index is 0, this method will return that dictionary. + * If multiple dictionaries are in use (e.g., for dictionary cascade), this allows access to each dictionary individually. + * @note If the index is out of bounds, the function throws an error. + */ CV_WRAP const Dictionary& getDictionary(int index = 0) const; + + /** @brief Sets a specific dictionary in the list of dictionaries used for marker detection, replacing the dictionary at the given index. + * + * @param dictionary The dictionary to set at the specified index. + * @param index Index of the dictionary to set. Default is 0, replacing the first dictionary if multiple are set. + * + * Sets the dictionary at the specified index within the internal collection of dictionaries used by the ArucoDetector. + * If implementing a dictionary cascade or similar, this method allows for replacing specific dictionaries within the collection. + * @note If the index is out of bounds, the method is going to throw an error. + */ CV_WRAP void setDictionary(const Dictionary& dictionary, int index = 0); + + /** @brief Returns all dictionaries currently used for marker detection as a vector. + * + * @return A constant reference to a std::vector containing all dictionaries used by the ArucoDetector. + * + * Provides access to the entire set of Dictionary objects currently configured within the ArucoDetector. + * This is useful for inspecting the dictionaries being used, iterating through them, or determining the number of dictionaries in use. + */ CV_WRAP const std::vector& getDictionaries() const; + + /** @brief Sets the entire collection of dictionaries to be used for marker detection, replacing any existing dictionaries. + * + * @param dictionaries A std::vector containing the new set of dictionaries to be used. + * + * Configures the ArucoDetector to use the provided vector of Dictionary objects for marker detection. + * This method replaces any dictionaries that were previously set. + * @note Setting an empty vector of dictionaries will throw an error. + */ CV_WRAP void setDictionaries(const std::vector& dictionaries); + + /** @brief Adds a new dictionary to the collection of dictionaries used for marker detection. + * + * @param dictionary The dictionary to add to the collection. + * + * Appends the provided Dictionary object to the internal collection of dictionaries used by the ArucoDetector. + * This method is useful when you want to extend the set of dictionaries already in use without replacing them entirely. + */ CV_WRAP void addDictionary(const Dictionary& dictionary); + + /** @brief Removes a dictionary from the collection of dictionaries at the specified index. + * + * @param index Index of the dictionary to remove from the collection. + * + * Removes the Dictionary object at the specified index from the internal collection of dictionaries. + * After removing a dictionary, the indices of subsequent dictionaries in the collection will be shifted. + * @note If the index is out of bounds, the function will throw. It will also not allow removing the last + * dictionary of the internal collection. + */ CV_WRAP void removeDictionary(int index); CV_WRAP const DetectorParameters& getDetectorParameters() const; diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 62f34e486b..3fabaf94c7 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -648,7 +648,7 @@ enum class DictionaryMode { struct ArucoDetector::ArucoDetectorImpl { /// dictionaries indicates the types of markers that will be searched - std::vector dictionaries; + vector dictionaries; /// marker detection parameters, check DetectorParameters docs to see available settings DetectorParameters detectorParams; @@ -657,7 +657,7 @@ struct ArucoDetector::ArucoDetectorImpl { RefineParameters refineParams; ArucoDetectorImpl() {} - ArucoDetectorImpl(const std::vector&_dictionaries, const DetectorParameters &_detectorParams, + ArucoDetectorImpl(const vector&_dictionaries, const DetectorParameters &_detectorParams, const RefineParameters& _refineParams): dictionaries(_dictionaries), detectorParams(_detectorParams), refineParams(_refineParams) { CV_Assert(!dictionaries.empty()); @@ -838,7 +838,7 @@ struct ArucoDetector::ArucoDetectorImpl { // Clean up rejectedImgPoints by comparing to itself and all candidates const float epsilon = 0.000001f; - auto compareCandidates = [epsilon](std::vector a, std::vector b) { + auto compareCandidates = [epsilon](vector a, vector b) { for (int i = 0; i < 4; i++) { if (std::abs(a[i].x - b[i].x) > epsilon || std::abs(a[i].y - b[i].y) > epsilon) { return false; @@ -1113,7 +1113,7 @@ ArucoDetector::ArucoDetector(const Dictionary &_dictionary, arucoDetectorImpl = makePtr(vector{_dictionary}, _detectorParams, _refineParams); } -ArucoDetector::ArucoDetector(const std::vector &_dictionaries, +ArucoDetector::ArucoDetector(const vector &_dictionaries, const DetectorParameters &_detectorParams, const RefineParameters& _refineParams) { arucoDetectorImpl = makePtr(_dictionaries, _detectorParams, _refineParams); @@ -1454,6 +1454,7 @@ const vector& ArucoDetector::getDictionaries() const { } void ArucoDetector::setDictionaries(const vector& dictionaries) { + CV_Assert(!dictionaries.empty()); arucoDetectorImpl->dictionaries = dictionaries; } From 6c3b195a57ae6605486d8d2b7cf767a536a960e4 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Mon, 24 Feb 2025 17:33:09 +0100 Subject: [PATCH 09/14] Make sure serialization with single dict preserves old behavior --- .../objdetect/src/aruco/aruco_detector.cpp | 17 +++++--- .../objdetect/test/test_arucodetection.cpp | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 3fabaf94c7..7f062b01cd 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1407,13 +1407,18 @@ void ArucoDetector::refineDetectedMarkers(InputArray _image, const Board& _board } void ArucoDetector::write(FileStorage &fs) const { - fs << "dictionaries" << "["; - for (auto& dictionary : arucoDetectorImpl->dictionaries) { - fs << "{"; - dictionary.writeDictionary(fs); - fs << "}"; + // preserve old format for single dictionary case + if (1 == arucoDetectorImpl->dictionaries.size()) { + arucoDetectorImpl->dictionaries[0].writeDictionary(fs); + } else { + fs << "dictionaries" << "["; + for (auto& dictionary : arucoDetectorImpl->dictionaries) { + fs << "{"; + dictionary.writeDictionary(fs); + fs << "}"; + } + fs << "]"; } - fs << "]"; arucoDetectorImpl->detectorParams.writeDetectorParameters(fs); arucoDetectorImpl->refineParams.writeRefineParameters(fs); } diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index c3138312a4..00cce307a0 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -6,6 +6,15 @@ #include "opencv2/objdetect/aruco_detector.hpp" #include "opencv2/calib3d.hpp" +namespace cv::aruco { + bool operator==(const Dictionary& d1, const Dictionary& d2); + bool operator==(const Dictionary& d1, const Dictionary& d2) { + return d1.markerSize == d2.markerSize + && std::equal(d1.bytesList.begin(), d1.bytesList.end(), d2.bytesList.begin()) + && d1.maxCorrectionBits == d2.maxCorrectionBits; + }; +} + namespace opencv_test { namespace { /** @@ -675,6 +684,9 @@ TEST(CV_ArucoMultiDict, noDict) EXPECT_THROW({ detector.removeDictionary(0); }, Exception); + EXPECT_THROW({ + detector.setDictionaries({}); + }, Exception); } @@ -717,6 +729,36 @@ TEST(CV_ArucoMultiDict, multiMarkerDetection) } +TEST(CV_ArucoMultiDict, serialization) +{ + const std::string fileName("test_aruco_serialization.json"); + aruco::ArucoDetector detector; + { + FileStorage fs(fileName, FileStorage::Mode::WRITE); + detector.write(fs); + fs.release(); + FileStorage test_fs(fileName, FileStorage::Mode::READ); + aruco::ArucoDetector test_detector; + test_detector.read(test_fs.root()); + // compare default constructor result + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), test_detector.getDictionary()); + } + detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_100)); + { + FileStorage fs(fileName, FileStorage::Mode::WRITE); + detector.write(fs); + fs.release(); + FileStorage test_fs(fileName, FileStorage::Mode::READ); + aruco::ArucoDetector test_detector; + test_detector.read(test_fs.root()); + // check for one additional dictionary + ASSERT_EQ(2ul, test_detector.getDictionaries().size()); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), test_detector.getDictionary()); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_5X5_100), test_detector.getDictionary(1)); + } +} + + struct ArucoThreading: public testing::TestWithParam { struct NumThreadsSetter { From 314f99f7a03e1fb543ad771db87c80dd2c68d5aa Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Mon, 24 Feb 2025 18:01:10 +0100 Subject: [PATCH 10/14] Remove add/removeDictionary and retain ABI of set/getDictionary functions --- .../opencv2/objdetect/aruco_detector.hpp | 51 ++++--------------- .../objdetect/src/aruco/aruco_detector.cpp | 26 +++------- .../objdetect/test/test_arucodetection.cpp | 38 +++++++------- .../objdetect/test/test_boarddetection.cpp | 10 ++-- 4 files changed, 37 insertions(+), 88 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp index e944c09a54..c081989645 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_detector.hpp @@ -373,67 +373,34 @@ public: CV_WRAP void detectMarkersMultiDict(InputArray image, OutputArrayOfArrays corners, OutputArray ids, OutputArrayOfArrays rejectedImgPoints = noArray(), OutputArray dictIndices = noArray()) const; - /** @brief Returns a specific dictionary from the set of dictionaries used for marker detection. + /** @brief Returns first dictionary from internal list used for marker detection. * - * @param index Index of the dictionary to retrieve. Default is 0, returning the first dictionary if multiple are set. - * - * Returns the dictionary at the specified index from the internal collection of dictionaries used by the ArucoDetector. - * If only one dictionary is set, or if the index is 0, this method will return that dictionary. - * If multiple dictionaries are in use (e.g., for dictionary cascade), this allows access to each dictionary individually. - * @note If the index is out of bounds, the function throws an error. + * @return The first dictionary from the configured ArucoDetector. */ - CV_WRAP const Dictionary& getDictionary(int index = 0) const; + CV_WRAP const Dictionary& getDictionary() const; - /** @brief Sets a specific dictionary in the list of dictionaries used for marker detection, replacing the dictionary at the given index. + /** @brief Sets and replaces the first dictionary in internal list to be used for marker detection. * - * @param dictionary The dictionary to set at the specified index. - * @param index Index of the dictionary to set. Default is 0, replacing the first dictionary if multiple are set. - * - * Sets the dictionary at the specified index within the internal collection of dictionaries used by the ArucoDetector. - * If implementing a dictionary cascade or similar, this method allows for replacing specific dictionaries within the collection. - * @note If the index is out of bounds, the method is going to throw an error. + * @param dictionary The new dictionary that will replace the first dictionary in the internal list. */ - CV_WRAP void setDictionary(const Dictionary& dictionary, int index = 0); + CV_WRAP void setDictionary(const Dictionary& dictionary); /** @brief Returns all dictionaries currently used for marker detection as a vector. * - * @return A constant reference to a std::vector containing all dictionaries used by the ArucoDetector. - * - * Provides access to the entire set of Dictionary objects currently configured within the ArucoDetector. - * This is useful for inspecting the dictionaries being used, iterating through them, or determining the number of dictionaries in use. + * @return A std::vector containing all dictionaries used by the ArucoDetector. */ - CV_WRAP const std::vector& getDictionaries() const; + CV_WRAP std::vector getDictionaries() const; /** @brief Sets the entire collection of dictionaries to be used for marker detection, replacing any existing dictionaries. * * @param dictionaries A std::vector containing the new set of dictionaries to be used. * - * Configures the ArucoDetector to use the provided vector of Dictionary objects for marker detection. + * Configures the ArucoDetector to use the provided vector of dictionaries for marker detection. * This method replaces any dictionaries that were previously set. * @note Setting an empty vector of dictionaries will throw an error. */ CV_WRAP void setDictionaries(const std::vector& dictionaries); - /** @brief Adds a new dictionary to the collection of dictionaries used for marker detection. - * - * @param dictionary The dictionary to add to the collection. - * - * Appends the provided Dictionary object to the internal collection of dictionaries used by the ArucoDetector. - * This method is useful when you want to extend the set of dictionaries already in use without replacing them entirely. - */ - CV_WRAP void addDictionary(const Dictionary& dictionary); - - /** @brief Removes a dictionary from the collection of dictionaries at the specified index. - * - * @param index Index of the dictionary to remove from the collection. - * - * Removes the Dictionary object at the specified index from the internal collection of dictionaries. - * After removing a dictionary, the indices of subsequent dictionaries in the collection will be shifted. - * @note If the index is out of bounds, the function will throw. It will also not allow removing the last - * dictionary of the internal collection. - */ - CV_WRAP void removeDictionary(int index); - CV_WRAP const DetectorParameters& getDetectorParameters() const; CV_WRAP void setDetectorParameters(const DetectorParameters& detectorParameters); diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 7f062b01cd..4ea80e5faa 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1439,22 +1439,19 @@ void ArucoDetector::read(const FileNode &fn) { arucoDetectorImpl->refineParams.readRefineParameters(fn); } -const Dictionary& ArucoDetector::getDictionary(int index) const { - CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); - return arucoDetectorImpl->dictionaries[index]; +const Dictionary& ArucoDetector::getDictionary() const { + return arucoDetectorImpl->dictionaries[0]; } -void ArucoDetector::setDictionary(const Dictionary& dictionary, int index) { - // special case: if index is 0, we add the dictionary to the list to preserve the old behavior - CV_Assert(index == 0 || (index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size())); - if (index == 0 && arucoDetectorImpl->dictionaries.empty()) { +void ArucoDetector::setDictionary(const Dictionary& dictionary) { + if (arucoDetectorImpl->dictionaries.empty()) { arucoDetectorImpl->dictionaries.push_back(dictionary); } else { - arucoDetectorImpl->dictionaries.at(index) = dictionary; + arucoDetectorImpl->dictionaries[0] = dictionary; } } -const vector& ArucoDetector::getDictionaries() const { +vector ArucoDetector::getDictionaries() const { return arucoDetectorImpl->dictionaries; } @@ -1463,17 +1460,6 @@ void ArucoDetector::setDictionaries(const vector& dictionaries) { arucoDetectorImpl->dictionaries = dictionaries; } -void ArucoDetector::addDictionary(const Dictionary& dictionary) { - arucoDetectorImpl->dictionaries.push_back(dictionary); -} - -void ArucoDetector::removeDictionary(int index) { - CV_Assert(index >= 0 && static_cast(index) < arucoDetectorImpl->dictionaries.size()); - // disallow no dictionaries - CV_Assert(arucoDetectorImpl->dictionaries.size() > 1ul); - arucoDetectorImpl->dictionaries.erase(arucoDetectorImpl->dictionaries.begin() + index); -} - const DetectorParameters& ArucoDetector::getDetectorParameters() const { return arucoDetectorImpl->detectorParams; } diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 00cce307a0..6dfb3c821b 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -647,33 +647,31 @@ TEST(CV_ArucoDetectMarkers, regression_contour_24220) } } -TEST(CV_ArucoMultiDict, addRemoveDictionary) +TEST(CV_ArucoMultiDict, setGetDictionaries) { - // using default constructor that pre-configures DICT_4X4_50 - aruco::ArucoDetector detector; - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_100)); - const auto& dicts = detector.getDictionaries(); + vector dictionaries = {aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}; + aruco::ArucoDetector detector(dictionaries); + vector dicts = detector.getDictionaries(); ASSERT_EQ(dicts.size(), 2ul); EXPECT_EQ(dicts[0].markerSize, 4); EXPECT_EQ(dicts[1].markerSize, 5); - detector.removeDictionary(0); - ASSERT_EQ(dicts.size(), 1ul); - EXPECT_EQ(dicts[0].markerSize, 5); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_6X6_100)); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_25h9)); - detector.removeDictionary(0); + dictionaries.clear(); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_6X6_100)); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); + dictionaries.push_back(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_25h9)); + detector.setDictionaries(dictionaries); + dicts = detector.getDictionaries(); ASSERT_EQ(dicts.size(), 3ul); EXPECT_EQ(dicts[0].markerSize, 6); EXPECT_EQ(dicts[1].markerSize, 7); EXPECT_EQ(dicts[2].markerSize, 5); - detector.setDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_36h10), 1); auto dict = detector.getDictionary(); EXPECT_EQ(dict.markerSize, 6); detector.setDictionary(aruco::getPredefinedDictionary(aruco::DICT_APRILTAG_16h5)); + dicts = detector.getDictionaries(); ASSERT_EQ(dicts.size(), 3ul); EXPECT_EQ(dicts[0].markerSize, 4); - EXPECT_EQ(dicts[1].markerSize, 6); + EXPECT_EQ(dicts[1].markerSize, 7); EXPECT_EQ(dicts[2].markerSize, 5); } @@ -681,9 +679,6 @@ TEST(CV_ArucoMultiDict, addRemoveDictionary) TEST(CV_ArucoMultiDict, noDict) { aruco::ArucoDetector detector; - EXPECT_THROW({ - detector.removeDictionary(0); - }, Exception); EXPECT_THROW({ detector.setDictionaries({}); }, Exception); @@ -743,7 +738,7 @@ TEST(CV_ArucoMultiDict, serialization) // compare default constructor result EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), test_detector.getDictionary()); } - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_100)); + detector.setDictionaries({aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}); { FileStorage fs(fileName, FileStorage::Mode::WRITE); detector.write(fs); @@ -752,9 +747,10 @@ TEST(CV_ArucoMultiDict, serialization) aruco::ArucoDetector test_detector; test_detector.read(test_fs.root()); // check for one additional dictionary - ASSERT_EQ(2ul, test_detector.getDictionaries().size()); - EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), test_detector.getDictionary()); - EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_5X5_100), test_detector.getDictionary(1)); + auto dicts = test_detector.getDictionaries(); + ASSERT_EQ(2ul, dicts.size()); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_4X4_50), dicts[0]); + EXPECT_EQ(aruco::getPredefinedDictionary(aruco::DICT_5X5_100), dicts[1]); } } diff --git a/modules/objdetect/test/test_boarddetection.cpp b/modules/objdetect/test/test_boarddetection.cpp index ac767f24a7..312bb5b56b 100644 --- a/modules/objdetect/test/test_boarddetection.cpp +++ b/modules/objdetect/test/test_boarddetection.cpp @@ -134,17 +134,17 @@ class CV_ArucoRefine : public cvtest::BaseTest { public: CV_ArucoRefine(ArucoAlgParams arucoAlgParams) { - aruco::Dictionary dictionary = aruco::getPredefinedDictionary(aruco::DICT_6X6_250); + vector dictionaries = {aruco::getPredefinedDictionary(aruco::DICT_6X6_250), + aruco::getPredefinedDictionary(aruco::DICT_5X5_250), + aruco::getPredefinedDictionary(aruco::DICT_4X4_250), + aruco::getPredefinedDictionary(aruco::DICT_7X7_250)}; aruco::DetectorParameters params; params.minDistanceToBorder = 3; params.cornerRefinementMethod = (int)aruco::CORNER_REFINE_SUBPIX; if (arucoAlgParams == ArucoAlgParams::USE_ARUCO3) params.useAruco3Detection = true; aruco::RefineParameters refineParams(10.f, 3.f, true); - detector = aruco::ArucoDetector(dictionary, params, refineParams); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_5X5_250)); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_4X4_250)); - detector.addDictionary(aruco::getPredefinedDictionary(aruco::DICT_7X7_250)); + detector = aruco::ArucoDetector(dictionaries, params, refineParams); } protected: From d869b12e89a9e9d8bed70849d8a7ca59fc013c16 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Tue, 25 Feb 2025 11:50:13 +0100 Subject: [PATCH 11/14] Fixing warnings in tests --- .../objdetect/test/test_arucodetection.cpp | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 6dfb3c821b..141f732114 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -6,13 +6,15 @@ #include "opencv2/objdetect/aruco_detector.hpp" #include "opencv2/calib3d.hpp" -namespace cv::aruco { - bool operator==(const Dictionary& d1, const Dictionary& d2); - bool operator==(const Dictionary& d1, const Dictionary& d2) { - return d1.markerSize == d2.markerSize - && std::equal(d1.bytesList.begin(), d1.bytesList.end(), d2.bytesList.begin()) - && d1.maxCorrectionBits == d2.maxCorrectionBits; - }; +namespace cv { + namespace aruco { + bool operator==(const Dictionary& d1, const Dictionary& d2); + bool operator==(const Dictionary& d1, const Dictionary& d2) { + return d1.markerSize == d2.markerSize + && std::equal(d1.bytesList.begin(), d1.bytesList.end(), d2.bytesList.begin()) + && d1.maxCorrectionBits == d2.maxCorrectionBits; + }; + } } namespace opencv_test { namespace { @@ -730,9 +732,19 @@ TEST(CV_ArucoMultiDict, serialization) aruco::ArucoDetector detector; { FileStorage fs(fileName, FileStorage::Mode::WRITE); + if (!fs.isOpened()) { + cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" + << " - Skipping due to 'File system write error.'" << endl; + return; + } detector.write(fs); fs.release(); FileStorage test_fs(fileName, FileStorage::Mode::READ); + if (!test_fs.isOpened()) { + cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" + << " - Skipping due to 'File system read error.'" << endl; + return; + } aruco::ArucoDetector test_detector; test_detector.read(test_fs.root()); // compare default constructor result @@ -741,9 +753,19 @@ TEST(CV_ArucoMultiDict, serialization) detector.setDictionaries({aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}); { FileStorage fs(fileName, FileStorage::Mode::WRITE); + if (!fs.isOpened()) { + cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" + << " - Skipping due to 'File system write error.'" << endl; + return; + } detector.write(fs); fs.release(); FileStorage test_fs(fileName, FileStorage::Mode::READ); + if (!test_fs.isOpened()) { + cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" + << " - Skipping due to 'File system read error.'" << endl; + return; + } aruco::ArucoDetector test_detector; test_detector.read(test_fs.root()); // check for one additional dictionary From 3084f950cf5d7d592936dda61d3d25694933ae9d Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Thu, 27 Feb 2025 10:52:27 +0100 Subject: [PATCH 12/14] Fix dictionary comparison in test --- modules/objdetect/test/test_arucodetection.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index 141f732114..e580575ee5 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -11,7 +11,8 @@ namespace cv { bool operator==(const Dictionary& d1, const Dictionary& d2); bool operator==(const Dictionary& d1, const Dictionary& d2) { return d1.markerSize == d2.markerSize - && std::equal(d1.bytesList.begin(), d1.bytesList.end(), d2.bytesList.begin()) + && std::equal(d1.bytesList.begin>(), d1.bytesList.end>(), d2.bytesList.begin>()) + && std::equal(d2.bytesList.begin>(), d2.bytesList.end>(), d1.bytesList.begin>()) && d1.maxCorrectionBits == d2.maxCorrectionBits; }; } From 1aa658fa75d9d8c0d8f85ea03f95e80eb8e7a05f Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Tue, 4 Mar 2025 15:24:03 +0100 Subject: [PATCH 13/14] Address more comments Use map to manage unique marker size candidate trees. Avoid code duplication. Add a test to show double detection with overlapping dictionaries. Generalize to marker sizes of not only predefined dictionaries. --- .../objdetect/src/aruco/aruco_detector.cpp | 89 +++++++------------ .../objdetect/test/test_arucodetection.cpp | 80 +++++++++++------ 2 files changed, 84 insertions(+), 85 deletions(-) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index 4ea80e5faa..a128710f4d 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -10,7 +10,7 @@ #include "apriltag/apriltag_quad_thresh.hpp" #include "aruco_utils.hpp" #include -#include +#include namespace cv { namespace aruco { @@ -752,44 +752,20 @@ struct ArucoDetector::ArucoDetectorImpl { /// STEP 3: Corner refinement :: use corner subpix if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners - parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { - const int begin = range.start; - const int end = range.end; - - for (int i = begin; i < end; i++) { - if (detectorParams.useAruco3Detection) { - const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; - findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); - } else { - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - } - }); + performCornerSubpixRefinement(grey, grey_pyramid, closest_pyr_image_idx, candidates, dictionary); } } else if (DictionaryMode::Multi == dictMode) { - unordered_set uniqueMarkerSizes; + map> candidatesPerDictionarySize; for (const Dictionary& dictionary : dictionaries) { - uniqueMarkerSizes.insert(dictionary.markerSize); + candidatesPerDictionarySize.emplace(dictionary.markerSize, vector()); } - // create at max 4 marker candidate trees for each dictionary size - vector> candidatesPerDictionarySize = {{}, {}, {}, {}}; - for (int markerSize : uniqueMarkerSizes) { - // min marker size is 4, so subtract 4 to get index - const auto dictionarySizeIndex = markerSize - 4; + // create candidate trees for each dictionary size + for (auto& candidatesTreeEntry : candidatesPerDictionarySize) { // copy candidates vector> candidatesCopy = candidates; vector > contoursCopy = contours; - candidatesPerDictionarySize[dictionarySizeIndex] = filterTooCloseCandidates(candidatesCopy, contoursCopy, markerSize); + candidatesTreeEntry.second = filterTooCloseCandidates(candidatesCopy, contoursCopy, candidatesTreeEntry.first); } candidates.clear(); contours.clear(); @@ -797,10 +773,9 @@ struct ArucoDetector::ArucoDetectorImpl { /// STEP 2: Check candidate codification (identify markers) int dictIndex = 0; for (const Dictionary& currentDictionary : dictionaries) { - const auto dictionarySizeIndex = currentDictionary.markerSize - 4; // temporary variable to store the current candidates vector> currentCandidates; - identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize[dictionarySizeIndex], currentCandidates, contours, + identifyCandidates(grey, grey_pyramid, candidatesPerDictionarySize.at(currentDictionary.markerSize), currentCandidates, contours, ids, currentDictionary, rejectedImgPoints); if (_dictIndices.needed()) { dictIndices.insert(dictIndices.end(), currentCandidates.size(), dictIndex); @@ -808,29 +783,7 @@ struct ArucoDetector::ArucoDetectorImpl { /// STEP 3: Corner refinement :: use corner subpix if (detectorParams.cornerRefinementMethod == (int)CORNER_REFINE_SUBPIX) { - CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && - detectorParams.cornerRefinementMinAccuracy > 0); - // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners - parallel_for_(Range(0, (int)currentCandidates.size()), [&](const Range& range) { - const int begin = range.start; - const int end = range.end; - - for (int i = begin; i < end; i++) { - if (detectorParams.useAruco3Detection) { - const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; - findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(currentCandidates[i]), detectorParams); - } - else { - int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* - getAverageModuleSize(currentCandidates[i], currentDictionary.markerSize, detectorParams.markerBorderBits))); - cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); - cornerSubPix(grey, Mat(currentCandidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), - TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, - detectorParams.cornerRefinementMaxIterations, - detectorParams.cornerRefinementMinAccuracy)); - } - } - }); + performCornerSubpixRefinement(grey, grey_pyramid, closest_pyr_image_idx, currentCandidates, currentDictionary); } candidates.insert(candidates.end(), currentCandidates.begin(), currentCandidates.end()); dictIndex++; @@ -1105,6 +1058,30 @@ struct ArucoDetector::ArucoDetectorImpl { } } + void performCornerSubpixRefinement(const Mat& grey, const vector& grey_pyramid, int closest_pyr_image_idx, const vector>& candidates, const Dictionary& dictionary) const { + CV_Assert(detectorParams.cornerRefinementWinSize > 0 && detectorParams.cornerRefinementMaxIterations > 0 && + detectorParams.cornerRefinementMinAccuracy > 0); + // Do subpixel estimation. In Aruco3 start on the lowest pyramid level and upscale the corners + parallel_for_(Range(0, (int)candidates.size()), [&](const Range& range) { + const int begin = range.start; + const int end = range.end; + + for (int i = begin; i < end; i++) { + if (detectorParams.useAruco3Detection) { + const float scale_init = (float) grey_pyramid[closest_pyr_image_idx].cols / grey.cols; + findCornerInPyrImage(scale_init, closest_pyr_image_idx, grey_pyramid, Mat(candidates[i]), detectorParams); + } else { + int cornerRefinementWinSize = std::max(1, cvRound(detectorParams.relativeCornerRefinmentWinSize* + getAverageModuleSize(candidates[i], dictionary.markerSize, detectorParams.markerBorderBits))); + cornerRefinementWinSize = min(cornerRefinementWinSize, detectorParams.cornerRefinementWinSize); + cornerSubPix(grey, Mat(candidates[i]), Size(cornerRefinementWinSize, cornerRefinementWinSize), Size(-1, -1), + TermCriteria(TermCriteria::MAX_ITER | TermCriteria::EPS, + detectorParams.cornerRefinementMaxIterations, + detectorParams.cornerRefinementMinAccuracy)); + } + } + }); + } }; ArucoDetector::ArucoDetector(const Dictionary &_dictionary, diff --git a/modules/objdetect/test/test_arucodetection.cpp b/modules/objdetect/test/test_arucodetection.cpp index e580575ee5..fd957698b2 100644 --- a/modules/objdetect/test/test_arucodetection.cpp +++ b/modules/objdetect/test/test_arucodetection.cpp @@ -727,25 +727,55 @@ TEST(CV_ArucoMultiDict, multiMarkerDetection) } +TEST(CV_ArucoMultiDict, multiMarkerDoubleDetection) +{ + const int markerSidePixels = 100; + const int imageWidth = 2 * markerSidePixels + 3 * (markerSidePixels / 2); + const int imageHeight = markerSidePixels + 2 * (markerSidePixels / 2); + vector usedDictionaries = { + aruco::getPredefinedDictionary(aruco::DICT_5X5_50), + aruco::getPredefinedDictionary(aruco::DICT_5X5_100) + }; + + // draw synthetic image + Mat img = Mat(imageHeight, imageWidth, CV_8UC1, Scalar::all(255)); + for(int y = 0; y < 2; y++) { + Mat marker; + int id = 49 + y; + auto dict = aruco::getPredefinedDictionary(aruco::DICT_5X5_100); + aruco::generateImageMarker(dict, id, markerSidePixels, marker); + Point2f firstCorner(markerSidePixels / 2.f + y * (1.5f * markerSidePixels), + markerSidePixels / 2.f); + Mat aux = img(Rect((int)firstCorner.x, (int)firstCorner.y, markerSidePixels, markerSidePixels)); + marker.copyTo(aux); + } + img.convertTo(img, CV_8UC3); + + aruco::ArucoDetector detector(usedDictionaries); + + vector > markerCorners; + vector markerIds; + vector > rejectedImgPts; + vector dictIds; + detector.detectMarkersMultiDict(img, markerCorners, markerIds, rejectedImgPts, dictIds); + ASSERT_EQ(markerIds.size(), 3u); + ASSERT_EQ(dictIds.size(), 3u); + EXPECT_EQ(dictIds[0], 0); // 5X5_50 + EXPECT_EQ(dictIds[1], 1); // 5X5_100 + EXPECT_EQ(dictIds[2], 1); // 5X5_100 +} + + TEST(CV_ArucoMultiDict, serialization) { - const std::string fileName("test_aruco_serialization.json"); aruco::ArucoDetector detector; { - FileStorage fs(fileName, FileStorage::Mode::WRITE); - if (!fs.isOpened()) { - cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" - << " - Skipping due to 'File system write error.'" << endl; - return; - } - detector.write(fs); - fs.release(); - FileStorage test_fs(fileName, FileStorage::Mode::READ); - if (!test_fs.isOpened()) { - cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" - << " - Skipping due to 'File system read error.'" << endl; - return; - } + FileStorage fs_out(".json", FileStorage::WRITE + FileStorage::MEMORY); + ASSERT_TRUE(fs_out.isOpened()); + detector.write(fs_out); + std::string serialized_string = fs_out.releaseAndGetString(); + FileStorage test_fs(serialized_string, FileStorage::Mode::READ + FileStorage::MEMORY); + ASSERT_TRUE(test_fs.isOpened()); aruco::ArucoDetector test_detector; test_detector.read(test_fs.root()); // compare default constructor result @@ -753,20 +783,12 @@ TEST(CV_ArucoMultiDict, serialization) } detector.setDictionaries({aruco::getPredefinedDictionary(aruco::DICT_4X4_50), aruco::getPredefinedDictionary(aruco::DICT_5X5_100)}); { - FileStorage fs(fileName, FileStorage::Mode::WRITE); - if (!fs.isOpened()) { - cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" - << " - Skipping due to 'File system write error.'" << endl; - return; - } - detector.write(fs); - fs.release(); - FileStorage test_fs(fileName, FileStorage::Mode::READ); - if (!test_fs.isOpened()) { - cout << "[ SKIPPED ] " << "CV_ArucoMultiDict.serialization" - << " - Skipping due to 'File system read error.'" << endl; - return; - } + FileStorage fs_out(".json", FileStorage::WRITE + FileStorage::MEMORY); + ASSERT_TRUE(fs_out.isOpened()); + detector.write(fs_out); + std::string serialized_string = fs_out.releaseAndGetString(); + FileStorage test_fs(serialized_string, FileStorage::Mode::READ + FileStorage::MEMORY); + ASSERT_TRUE(test_fs.isOpened()); aruco::ArucoDetector test_detector; test_detector.read(test_fs.root()); // check for one additional dictionary From d80fd565b4222b1a425b11e847a23f5482aca8a2 Mon Sep 17 00:00:00 2001 From: Benjamin Knecht Date: Tue, 4 Mar 2025 16:24:50 +0100 Subject: [PATCH 14/14] Attempt to fix Windows int type warning --- modules/objdetect/src/aruco/aruco_detector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index a128710f4d..0075d3e5a8 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -755,7 +755,7 @@ struct ArucoDetector::ArucoDetectorImpl { performCornerSubpixRefinement(grey, grey_pyramid, closest_pyr_image_idx, candidates, dictionary); } } else if (DictionaryMode::Multi == dictMode) { - map> candidatesPerDictionarySize; + map> candidatesPerDictionarySize; for (const Dictionary& dictionary : dictionaries) { candidatesPerDictionarySize.emplace(dictionary.markerSize, vector()); }