diff --git a/modules/calib/include/opencv2/calib.hpp b/modules/calib/include/opencv2/calib.hpp index 9b2d009498..f777a2f963 100644 --- a/modules/calib/include/opencv2/calib.hpp +++ b/modules/calib/include/opencv2/calib.hpp @@ -1288,14 +1288,22 @@ points in all the available views from all cameras. @sa findChessboardCorners, findCirclesGrid, calibrateCamera, fisheye::calibrate, registerCameras */ +CV_EXPORTS_AS(calibrateMultiviewExtended) double calibrateMultiview ( + InputArrayOfArrays objPoints, const std::vector> &imagePoints, + const std::vector& imageSize, InputArray detectionMask, InputArray models, + InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + OutputArray initializationPairs, OutputArrayOfArrays rvecs0, + OutputArrayOfArrays tvecs0, OutputArray perFrameErrors, + InputArray flagsForIntrinsics=noArray(), int flags = 0); + +/// @overload CV_EXPORTS_W double calibrateMultiview ( InputArrayOfArrays objPoints, const std::vector> &imagePoints, const std::vector& imageSize, InputArray detectionMask, InputArray models, - InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, - int flags = 0, InputArray flagsForIntrinsics=noArray(), - OutputArrayOfArrays rvecs0=noArray(), OutputArrayOfArrays tvecs0=noArray(), - OutputArray perFrameErrors=noArray(), OutputArray initializationPairs=noArray()); + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + InputArray flagsForIntrinsics=noArray(), int flags = 0); /** @brief Computes Hand-Eye calibration: \f$_{}^{g}\textrm{T}_c\f$ diff --git a/modules/calib/src/multiview_calibration.cpp b/modules/calib/src/multiview_calibration.cpp index ba01051a57..12c8b3caf8 100644 --- a/modules/calib/src/multiview_calibration.cpp +++ b/modules/calib/src/multiview_calibration.cpp @@ -569,16 +569,14 @@ static void checkConnected (const std::vector> &detection_mask } } -//TODO: use Input/OutputArrays for imagePoints(?) double calibrateMultiview( InputArrayOfArrays objPoints, const std::vector> &imagePoints, const std::vector& imageSize, InputArray detectionMask, InputArray models, - InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, - int flags, InputArray flagsForIntrinsics, - OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0, - OutputArray perFrameErrors, OutputArray initializationPairs) { - + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + OutputArray initializationPairs, OutputArrayOfArrays rvecs0, + OutputArrayOfArrays tvecs0, OutputArray perFrameErrors, + InputArray flagsForIntrinsics, int flags) { CV_CheckFalse(objPoints.empty(), "Objects points must not be empty!"); CV_CheckFalse(imagePoints.empty(), "Image points must not be empty!"); CV_CheckFalse(imageSize.empty(), "Image size per camera must not be empty!"); @@ -588,7 +586,7 @@ double calibrateMultiview( Mat detection_mask_ = detectionMask.getMat(); Mat models_mat = models.getMat(); - CV_CheckEQ(detection_mask_.type(), CV_8U, "detectionMask must be of type CV_8U"); + CV_Assert(detection_mask_.type() == CV_8UC1 || detection_mask_.type() == CV_BoolC1); CV_CheckEQ(models_mat.type(), CV_8U, "models must be of type CV_8U"); if(flags & cv::CALIB_STEREO_REGISTRATION) @@ -694,57 +692,59 @@ double calibrateMultiview( distortions.create(NUM_CAMERAS, 1, CV_64F); // calibrate each camera independently to find intrinsic parameters - K and distortion coefficients - for (int camera = 0; camera < NUM_CAMERAS; camera++) { - Mat K, dist; - Mat rvecs, tvecs; - std::vector obj_points_, img_points_; - std::vector errors_per_view; - obj_points_.reserve(num_visible_frames_per_camera[camera]); - img_points_.reserve(num_visible_frames_per_camera[camera]); - for (int f = 0; f < NUM_FRAMES; f++) { - if (detection_mask_mat[camera][f]) { - obj_points_.emplace_back((models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ? - objPoints_norm[f].reshape(3): objPoints_norm[f]); - img_points_.emplace_back((models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ? - imagePoints[camera][f].reshape(2) : imagePoints[camera][f]); - } - } - double repr_err; - if (models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE) { - repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera], - K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at(camera)); - // calibrate does not compute error per view, so compute it manually - errors_per_view = std::vector(obj_points_.size()); - for (int f = 0; f < (int) obj_points_.size(); f++) { - double err2 = multiview::computeReprojectionMSE(obj_points_[f], - img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE); - errors_per_view[f] = sqrt(err2); - } - } else { - repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist, - rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at(camera)); - } - CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!"); - int cnt_visible_frame = 0; - for (int f = 0; f < NUM_FRAMES; f++) { - if (detection_mask_mat[camera][f]) { - rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data)); - tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data)); - double err = errors_per_view[cnt_visible_frame]; - double err2 = err * err; - if (camera_rt_errors[f] > err2) { - camera_rt_errors[f] = err2; - camera_rt_best[f] = camera; + parallel_for_(Range(0, NUM_CAMERAS), [&](const Range& range) { + for (int camera = range.start; camera < range.end; camera++) { + Mat K, dist; + Mat rvecs, tvecs; + std::vector obj_points_, img_points_; + std::vector errors_per_view; + obj_points_.reserve(num_visible_frames_per_camera[camera]); + img_points_.reserve(num_visible_frames_per_camera[camera]); + for (int f = 0; f < NUM_FRAMES; f++) { + if (detection_mask_mat[camera][f]) { + obj_points_.emplace_back((models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ? + objPoints_norm[f].reshape(3): objPoints_norm[f]); + img_points_.emplace_back((models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ? + imagePoints[camera][f].reshape(2) : imagePoints[camera][f]); + } + } + double repr_err; + if (models_mat.at(camera) == cv::CALIB_MODEL_FISHEYE) { + repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera], + K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at(camera)); + // calibrate does not compute error per view, so compute it manually + errors_per_view = std::vector(obj_points_.size()); + for (int f = 0; f < (int) obj_points_.size(); f++) { + double err2 = multiview::computeReprojectionMSE(obj_points_[f], + img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE); + errors_per_view[f] = sqrt(err2); + } + } else { + repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist, + rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at(camera)); + } + CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!"); + int cnt_visible_frame = 0; + for (int f = 0; f < NUM_FRAMES; f++) { + if (detection_mask_mat[camera][f]) { + rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data)); + tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data)); + double err = errors_per_view[cnt_visible_frame]; + double err2 = err * err; + if (camera_rt_errors[f] > err2) { + camera_rt_errors[f] = err2; + camera_rt_best[f] = camera; + } + cnt_visible_frame++; } - cnt_visible_frame++; } - } - Ks.create(K.rows, K.cols, CV_64F, camera); - distortions.create(dist.rows, dist.cols, CV_64F, camera, true); - K.copyTo(Ks.getMat(camera)); - dist.copyTo(distortions.getMat(camera)); - } + Ks.create(K.rows, K.cols, CV_64F, camera); + distortions.create(dist.rows, dist.cols, CV_64F, camera, true); + K.copyTo(Ks.getMat(camera)); + dist.copyTo(distortions.getMat(camera)); + } + }); } else { // use PnP to compute rvecs and tvecs for (int i = 0; i < NUM_FRAMES; i++) { @@ -961,4 +961,17 @@ double calibrateMultiview( return sqrt(sum_errors / cnt_errors); } + +double calibrateMultiview( + InputArrayOfArrays objPoints, const std::vector> &imagePoints, + const std::vector& imageSize, InputArray detectionMask, InputArray models, + InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, + InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts, + InputArray flagsForIntrinsics, int flags) { + + return calibrateMultiview(objPoints, imagePoints, imageSize, detectionMask, models, Ks, distortions, + Rs, Ts, noArray(), noArray(), noArray(), noArray(), flagsForIntrinsics, flags); +} + + } diff --git a/modules/calib/test/test_cameracalibration.cpp b/modules/calib/test/test_cameracalibration.cpp index 1b10159ac7..52e92e1afc 100644 --- a/modules/calib/test/test_cameracalibration.cpp +++ b/modules/calib/test/test_cameracalibration.cpp @@ -2485,7 +2485,7 @@ double CV_MultiviewCalibrationTest_CPP::calibrateStereoCamera( const vector models(2, cv::CALIB_MODEL_PINHOLE); std::vector all_flags(2, flags); double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models, - Rs, Ts, Ks, distortions, 0, all_flags, rvecs, tvecs, errors_mat); + Ks, distortions, Rs, Ts, /*pairs*/ noArray(), rvecs, tvecs, errors_mat, all_flags); if (perViewErrors1.size() != (size_t)numImgs) { diff --git a/modules/calib/test/test_fisheye.cpp b/modules/calib/test/test_fisheye.cpp index e5d528cd22..bf091fedd9 100644 --- a/modules/calib/test/test_fisheye.cpp +++ b/modules/calib/test/test_fisheye.cpp @@ -616,17 +616,12 @@ TEST_F(fisheyeTest, multiview_calibration) } std::vector image_sizes(2, imageSize); cv::Mat visibility_mat = cv::Mat_::ones(2, (int)leftPoints.size()); - std::vector Rs, Ts, Ks, distortions, rvecs0, tvecs0; + std::vector Rs, Ts, Ks, distortions; std::vector models(2, cv::CALIB_MODEL_FISHEYE); - int flag = 0; - flag |= cv::CALIB_RECOMPUTE_EXTRINSIC; - flag |= cv::CALIB_CHECK_COND; - flag |= cv::CALIB_FIX_SKEW; + std::vector all_flags(2, cv::CALIB_RECOMPUTE_EXTRINSIC | cv::CALIB_CHECK_COND | cv::CALIB_FIX_SKEW); - std::vector all_flags(2, flag); - - calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models, - Rs, Ts, Ks, distortions, 0, all_flags, rvecs0, tvecs0); + calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, + models, Ks, distortions, Rs, Ts, all_flags); cv::Matx33d R_correct( 0.9975587205950972, 0.06953016383322372, 0.006492709911733523, -0.06956823121068059, 0.9975601387249519, 0.005833595226966235, -0.006071257768382089, -0.006271040135405457, 0.9999619062167968); diff --git a/modules/calib/test/test_multiview_calib.cpp b/modules/calib/test/test_multiview_calib.cpp index 160bc5f8ca..cf262c7341 100644 --- a/modules/calib/test/test_multiview_calib.cpp +++ b/modules/calib/test/test_multiview_calib.cpp @@ -133,7 +133,7 @@ TEST(multiview_calibration, accuracy) { std::vector Ks, distortions, Rs, Ts; calibrateMultiview(objPoints, image_points_all, image_sizes, visibility_mat, - models, Rs, Ts, Ks, distortions); + models, Ks, distortions, Rs, Ts); const double K_err_tol = 1e1, dist_tol = 5e-2, R_tol = 1e-2, T_tol = 1e-2; for (int c = 0; c < num_cameras; c++) { @@ -169,7 +169,7 @@ struct MultiViewTest : public ::testing::Test std::vector>& image_points_all, cv::Mat& visibility) { image_points_all.clear(); - visibility.create(static_cast(cameras.size()), frameCount, CV_8UC1); + visibility.create(static_cast(cameras.size()), frameCount, CV_BoolC1); for (int c = 0; c < static_cast(cameras.size()); c++) { std::vector camera_image_points; @@ -183,7 +183,7 @@ struct MultiViewTest : public ::testing::Test if (!node.empty()) { camera_image_points.push_back(node.mat().reshape(2, 1)); - visibility.at(c, i) = 255; + visibility.at(c, i) = 1; } else { @@ -315,7 +315,7 @@ TEST_F(MultiViewTest, OneLine) std::vector Ks, distortions, Rs, Rs_rvec, Ts; double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, - Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); + Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, .3); @@ -420,7 +420,7 @@ TEST_F(MultiViewTest, OneLineInitialGuess) int flags = cv::CALIB_USE_EXTRINSIC_GUESS | cv::CALIB_USE_INTRINSIC_GUESS | cv::CALIB_STEREO_REGISTRATION; double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, - Rs_rvec, Ts, Ks, distortions, flags, flagsForIntrinsics); + Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics, flags); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, .3); @@ -486,7 +486,7 @@ TEST_F(MultiViewTest, CamsToFloor) std::vector Ks, distortions, Rs, Rs_rvec, Ts; double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, - Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); + Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, 1.); @@ -555,7 +555,7 @@ TEST_F(MultiViewTest, Hetero) std::vector Ks, distortions, Rs, Rs_rvec, Ts; double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, - Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); + Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics); CV_LOG_INFO(NULL, "RMS: " << rms); EXPECT_LE(rms, 2.5); diff --git a/samples/cpp/multiview_calibration_sample.cpp b/samples/cpp/multiview_calibration_sample.cpp index 24d47bb0b7..2a1fa5beb4 100644 --- a/samples/cpp/multiview_calibration_sample.cpp +++ b/samples/cpp/multiview_calibration_sample.cpp @@ -163,7 +163,7 @@ static void detectPointsAndCalibrate (cv::Size pattern_size, float pattern_dista // ! [multiview_calib] const double rmse = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, - models, Rs, Ts, Ks, distortions); + models, Ks, distortions, Rs, Ts); // ! [multiview_calib] std::cout << "average RMSE over detection mask " << rmse << "\n"; for (int c = 0; c < (int)Rs.size(); c++) { diff --git a/samples/python/multiview_calibration.py b/samples/python/multiview_calibration.py index d7453a1dca..b0a7a69240 100644 --- a/samples/python/multiview_calibration.py +++ b/samples/python/multiview_calibration.py @@ -423,7 +423,7 @@ def calibrateFromPoints( # try: # [multiview_calib] rmse, Rs, Ts, Ks, distortions, rvecs0, tvecs0, errors_per_frame, output_pairs = \ - cv.calibrateMultiview( + cv.calibrateMultiviewExtended( objPoints=pattern_points_all, imagePoints=image_points, imageSize=image_sizes,