Merge pull request #23647 from AleksandrPanov:fix_charuco_board_detect

Add charuco board check #23647

Added charuco board checking to avoid detection of incorrect board.
Fixes #23517

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

modules/objdetect/src/aruco/charuco_detector.cpp

@@ -23,6 +23,54 @@ struct CharucoDetector::CharucoDetectorImpl {
                                                               arucoDetector(_arucoDetector)
     {}
 
+    bool checkBoard(InputArrayOfArrays markerCorners, InputArray markerIds, InputArray charucoCorners, InputArray charucoIds) {
+        vector<Mat> mCorners;
+        markerCorners.getMatVector(mCorners);
+        Mat mIds = markerIds.getMat();
+
+        Mat chCorners = charucoCorners.getMat();
+        Mat chIds = charucoIds.getMat();
+
+        vector<vector<int> > nearestMarkerIdx = board.getNearestMarkerIdx();
+        vector<Point2f> distance(board.getNearestMarkerIdx().size(), Point2f(0.f, std::numeric_limits<float>::max()));
+        // distance[i].x: max distance from the i-th charuco corner to charuco corner-forming markers.
+        // The two charuco corner-forming markers of i-th charuco corner are defined in getNearestMarkerIdx()[i]
+        // distance[i].y: min distance from the charuco corner to other markers.
+        for (size_t i = 0ull; i < chIds.total(); i++) {
+            int chId = chIds.ptr<int>(0)[i];
+            Point2f charucoCorner(chCorners.ptr<Point2f>(0)[i]);
+            for (size_t j = 0ull; j < mIds.total(); j++) {
+                int idMaker = mIds.ptr<int>(0)[j];
+                Point2f centerMarker((mCorners[j].ptr<Point2f>(0)[0] + mCorners[j].ptr<Point2f>(0)[1] +
+                                      mCorners[j].ptr<Point2f>(0)[2] + mCorners[j].ptr<Point2f>(0)[3]) / 4.f);
+                float dist = sqrt(normL2Sqr<float>(centerMarker - charucoCorner));
+                // check distance from the charuco corner to charuco corner-forming markers
+                if (nearestMarkerIdx[chId][0] == idMaker || nearestMarkerIdx[chId][1] == idMaker) {
+                    int nearestCornerId =  nearestMarkerIdx[chId][0] == idMaker ? board.getNearestMarkerCorners()[chId][0] : board.getNearestMarkerCorners()[chId][1];
+                    Point2f nearestCorner = mCorners[j].ptr<Point2f>(0)[nearestCornerId];
+                    float distToNearest = sqrt(normL2Sqr<float>(nearestCorner - charucoCorner));
+                    distance[chId].x = max(distance[chId].x, distToNearest);
+                    // check that nearestCorner is nearest point
+                    {
+                        Point2f mid1 = (mCorners[j].ptr<Point2f>(0)[(nearestCornerId + 1) % 4]+nearestCorner)*0.5f;
+                        Point2f mid2 = (mCorners[j].ptr<Point2f>(0)[(nearestCornerId + 3) % 4]+nearestCorner)*0.5f;
+                        float tmpDist = min(sqrt(normL2Sqr<float>(mid1 - charucoCorner)), sqrt(normL2Sqr<float>(mid2 - charucoCorner)));
+                        if (tmpDist < distToNearest)
+                            return false;
+                    }
+                }
+                // check distance from the charuco corner to other markers
+                else
+                    distance[chId].y = min(distance[chId].y, dist);
+            }
+            // if distance from the charuco corner to charuco corner-forming markers more then distance from the charuco corner to other markers,
+            // then a false board is found.
+            if (distance[chId].x > 0.f && distance[chId].y < std::numeric_limits<float>::max() && distance[chId].x > distance[chId].y)
+                return false;
+        }
+        return true;
+    }
+
     /** Calculate the maximum window sizes for corner refinement for each charuco corner based on the distance
      * to their closest markers */
     vector<Size> getMaximumSubPixWindowSizes(InputArrayOfArrays markerCorners, InputArray markerIds,
@@ -246,6 +294,31 @@ struct CharucoDetector::CharucoDetectorImpl {
         Mat(filteredCharucoIds).copyTo(_filteredCharucoIds);
         return (int)_filteredCharucoIds.total();
     }
+
+    void detectBoard(InputArray image, OutputArray charucoCorners, OutputArray charucoIds,
+                     InputOutputArrayOfArrays markerCorners, InputOutputArray markerIds) {
+        CV_Assert((markerCorners.empty() && markerIds.empty() && !image.empty()) || (markerCorners.total() == markerIds.total()));
+        vector<vector<Point2f>> tmpMarkerCorners;
+        vector<int> tmpMarkerIds;
+        InputOutputArrayOfArrays _markerCorners = markerCorners.needed() ? markerCorners : tmpMarkerCorners;
+        InputOutputArray _markerIds = markerIds.needed() ? markerIds : tmpMarkerIds;
+
+        if (markerCorners.empty() && markerIds.empty()) {
+            vector<vector<Point2f> > rejectedMarkers;
+            arucoDetector.detectMarkers(image, _markerCorners, _markerIds, rejectedMarkers);
+            if (charucoParameters.tryRefineMarkers)
+                arucoDetector.refineDetectedMarkers(image, board, _markerCorners,  _markerIds, rejectedMarkers);
+        }
+        // if camera parameters are avaible, use approximated calibration
+        if(!charucoParameters.cameraMatrix.empty())
+            interpolateCornersCharucoApproxCalib(_markerCorners, _markerIds, image, charucoCorners, charucoIds);
+        // else use local homography
+        else
+            interpolateCornersCharucoLocalHom(_markerCorners, _markerIds, image, charucoCorners, charucoIds);
+        // to return a charuco corner, its closest aruco markers should have been detected
+        filterCornersWithoutMinMarkers(charucoCorners, charucoIds, _markerIds, charucoCorners, charucoIds);
+}
+
 };
 
 CharucoDetector::CharucoDetector(const CharucoBoard &board, const CharucoParameters &charucoParams,
@@ -288,30 +361,11 @@ void CharucoDetector::setRefineParameters(const RefineParameters& refineParamete
 
 void CharucoDetector::detectBoard(InputArray image, OutputArray charucoCorners, OutputArray charucoIds,
                                   InputOutputArrayOfArrays markerCorners, InputOutputArray markerIds) const {
-    CV_Assert((markerCorners.empty() && markerIds.empty() && !image.empty()) || (markerCorners.total() == markerIds.total()));
+    charucoDetectorImpl->detectBoard(image, charucoCorners, charucoIds, markerCorners, markerIds);
-    vector<vector<Point2f>> tmpMarkerCorners;
+    if (charucoDetectorImpl->checkBoard(markerCorners, markerIds, charucoCorners, charucoIds) == false) {
-    vector<int> tmpMarkerIds;
+        charucoCorners.release();
-    InputOutputArrayOfArrays _markerCorners = markerCorners.needed() ? markerCorners : tmpMarkerCorners;
+        charucoIds.release();
-    InputOutputArray _markerIds = markerIds.needed() ? markerIds : tmpMarkerIds;
-
-    if (markerCorners.empty() && markerIds.empty()) {
-        vector<vector<Point2f> > rejectedMarkers;
-        charucoDetectorImpl->arucoDetector.detectMarkers(image, _markerCorners, _markerIds, rejectedMarkers);
-        if (charucoDetectorImpl->charucoParameters.tryRefineMarkers)
-            charucoDetectorImpl->arucoDetector.refineDetectedMarkers(image, charucoDetectorImpl->board, _markerCorners,
-                                                                     _markerIds, rejectedMarkers);
     }
-    // if camera parameters are avaible, use approximated calibration
-    if(!charucoDetectorImpl->charucoParameters.cameraMatrix.empty())
-        charucoDetectorImpl->interpolateCornersCharucoApproxCalib(_markerCorners, _markerIds, image, charucoCorners,
-                                                                  charucoIds);
-    // else use local homography
-    else
-        charucoDetectorImpl->interpolateCornersCharucoLocalHom(_markerCorners, _markerIds, image, charucoCorners,
-                                                               charucoIds);
-    // to return a charuco corner, its closest aruco markers should have been detected
-    charucoDetectorImpl->filterCornersWithoutMinMarkers(charucoCorners, charucoIds, _markerIds, charucoCorners,
-                                                        charucoIds);
 }
 
 void CharucoDetector::detectDiamonds(InputArray image, OutputArrayOfArrays _diamondCorners, OutputArray _diamondIds,
@@ -416,7 +470,7 @@ void CharucoDetector::detectDiamonds(InputArray image, OutputArrayOfArrays _diam
             // interpolate the charuco corners of the diamond
             vector<Point2f> currentMarkerCorners;
             Mat aux;
-            detectBoard(grey, currentMarkerCorners, aux, currentMarker, currentMarkerId);
+            charucoDetectorImpl->detectBoard(grey, currentMarkerCorners, aux, currentMarker, currentMarkerId);
 
             // if everything is ok, save the diamond
             if(currentMarkerCorners.size() > 0ull) {

modules/objdetect/test/test_charucodetection.cpp

@@ -689,4 +689,32 @@ TEST(Charuco, testmatchImagePoints)
     }
 }
 
+typedef testing::TestWithParam<cv::Size> CharucoBoard;
+INSTANTIATE_TEST_CASE_P(/**/, CharucoBoard, testing::Values(Size(3, 2), Size(3, 2), Size(6, 2), Size(2, 6),
+                                                            Size(3, 4), Size(4, 3), Size(7, 3), Size(3, 7)));
+TEST_P(CharucoBoard, testWrongSizeDetection)
+{
+    cv::Size boardSize = GetParam();
+    ASSERT_FALSE(boardSize.width == boardSize.height);
+    aruco::CharucoBoard board(boardSize, 1.f, 0.5f, aruco::getPredefinedDictionary(aruco::DICT_4X4_50));
+
+    vector<int> detectedCharucoIds, detectedArucoIds;
+    vector<Point2f> detectedCharucoCorners;
+    vector<vector<Point2f>> detectedArucoCorners;
+    Mat boardImage;
+    board.generateImage(boardSize*40, boardImage);
+
+    swap(boardSize.width, boardSize.height);
+    aruco::CharucoDetector detector(aruco::CharucoBoard(boardSize, 1.f, 0.5f, aruco::getPredefinedDictionary(aruco::DICT_4X4_50)));
+    // try detect board with wrong size
+    detector.detectBoard(boardImage, detectedCharucoCorners, detectedCharucoIds, detectedArucoCorners, detectedArucoIds);
+
+    // aruco markers must be found
+    ASSERT_EQ(detectedArucoIds.size(), board.getIds().size());
+    ASSERT_EQ(detectedArucoCorners.size(), board.getIds().size());
+    // charuco corners should not be found in board with wrong size
+    ASSERT_TRUE(detectedCharucoCorners.empty());
+    ASSERT_TRUE(detectedCharucoIds.empty());
+}
+
 }} // namespace