Merge pull request #26742 from ivashmak:fix_homography_inliers

Bug fix for #25546 - Updating inliers for homography estimation #26742

Fixes #25546

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/calib3d/src/fundam.cpp

@@ -414,6 +414,11 @@ cv::Mat cv::findHomography( InputArray _points1, InputArray _points2,
 
     if( result && npoints > 4 && method != RHO)
     {
+        // save the original points before compressing
+        const int npoints_input = npoints;
+        const Mat src_input = src.clone();
+        const Mat dst_input = dst.clone();
+
         compressElems( src.ptr<Point2f>(), tempMask.ptr<uchar>(), 1, npoints );
         npoints = compressElems( dst.ptr<Point2f>(), tempMask.ptr<uchar>(), 1, npoints );
         if( npoints > 0 )
@@ -427,6 +432,16 @@ cv::Mat cv::findHomography( InputArray _points1, InputArray _points2,
             Mat H8(9, 1, CV_64F, H.ptr<double>());
             LMSolver::create(makePtr<HomographyRefineCallback>(src, dst), 10)->run(H8);
             H.convertTo(H, H.type(), scaleFor(H.at<double>(2,2)));
+
+            // find new inliers
+            const float thr_sqr = static_cast<float>(ransacReprojThreshold * ransacReprojThreshold);
+            cv::Mat errors;
+            cb->computeError(src_input, dst_input, H, errors);
+            uchar* maskptr = tempMask.ptr<uchar>();
+            const float * const errors_ptr = errors.ptr<float>();
+            for (int i = 0; i < npoints_input; i++) {
+                maskptr[i] = static_cast<uchar>(errors_ptr[i] <= thr_sqr);
+            }
         }
     }