mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-06-07 17:44:04 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Multiple calibrateMultiview improvements.

Browse Source 
- Added function overload for the simple case
- Added CV_Bool type support for masks
- `parallel_for_` for intrinsics calibration for faster inference
- Homogenize parameters order with other calibrateXXX functions
This commit is contained in:
Alexander Smorkalov 2024-10-04 12:47:57 +03:00
parent 97681bdfce
commit 110b701bba
7 changed files with 94 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/calib/include/opencv2/calib.hpp
View File

@ -1288,14 +1288,22 @@ points in all the available views from all cameras.
@sa findChessboardCorners, findCirclesGrid, calibrateCamera, fisheye::calibrate, registerCameras @sa findChessboardCorners, findCirclesGrid, calibrateCamera, fisheye::calibrate, registerCameras
*/ */
CV_EXPORTS_AS(calibrateMultiviewExtended) double calibrateMultiview (
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
OutputArray initializationPairs, OutputArrayOfArrays rvecs0,
OutputArrayOfArrays tvecs0, OutputArray perFrameErrors,
InputArray flagsForIntrinsics=noArray(), int flags = 0);
/// @overload
CV_EXPORTS_W double calibrateMultiview ( CV_EXPORTS_W double calibrateMultiview (
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints, InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models, const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
int flags = 0, InputArray flagsForIntrinsics=noArray(), InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
OutputArrayOfArrays rvecs0=noArray(), OutputArrayOfArrays tvecs0=noArray(), InputArray flagsForIntrinsics=noArray(), int flags = 0);
OutputArray perFrameErrors=noArray(), OutputArray initializationPairs=noArray());
/** @brief Computes Hand-Eye calibration: \f$_{}^{g}\textrm{T}_c\f$ /** @brief Computes Hand-Eye calibration: \f$_{}^{g}\textrm{T}_c\f$

123
modules/calib/src/multiview_calibration.cpp
View File

@ -569,16 +569,14 @@ static void checkConnected (const std::vector<std::vector<bool>> &detection_mask
} }
} }
//TODO: use Input/OutputArrays for imagePoints(?)
double calibrateMultiview( double calibrateMultiview(
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints, InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models, const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions, InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
int flags, InputArray flagsForIntrinsics, InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0, OutputArray initializationPairs, OutputArrayOfArrays rvecs0,
OutputArray perFrameErrors, OutputArray initializationPairs) { OutputArrayOfArrays tvecs0, OutputArray perFrameErrors,
InputArray flagsForIntrinsics, int flags) {
CV_CheckFalse(objPoints.empty(), "Objects points must not be empty!"); CV_CheckFalse(objPoints.empty(), "Objects points must not be empty!");
CV_CheckFalse(imagePoints.empty(), "Image points must not be empty!"); CV_CheckFalse(imagePoints.empty(), "Image points must not be empty!");
CV_CheckFalse(imageSize.empty(), "Image size per camera must not be empty!"); CV_CheckFalse(imageSize.empty(), "Image size per camera must not be empty!");
@ -588,7 +586,7 @@ double calibrateMultiview(
Mat detection_mask_ = detectionMask.getMat(); Mat detection_mask_ = detectionMask.getMat();
Mat models_mat = models.getMat(); Mat models_mat = models.getMat();
CV_CheckEQ(detection_mask_.type(), CV_8U, "detectionMask must be of type CV_8U"); CV_Assert(detection_mask_.type() == CV_8UC1 || detection_mask_.type() == CV_BoolC1);
CV_CheckEQ(models_mat.type(), CV_8U, "models must be of type CV_8U"); CV_CheckEQ(models_mat.type(), CV_8U, "models must be of type CV_8U");
if(flags & cv::CALIB_STEREO_REGISTRATION) if(flags & cv::CALIB_STEREO_REGISTRATION)
@ -694,57 +692,59 @@ double calibrateMultiview(
distortions.create(NUM_CAMERAS, 1, CV_64F); distortions.create(NUM_CAMERAS, 1, CV_64F);
// calibrate each camera independently to find intrinsic parameters - K and distortion coefficients // calibrate each camera independently to find intrinsic parameters - K and distortion coefficients
for (int camera = 0; camera < NUM_CAMERAS; camera++) { parallel_for_(Range(0, NUM_CAMERAS), [&](const Range& range) {
Mat K, dist; for (int camera = range.start; camera < range.end; camera++) {
Mat rvecs, tvecs; Mat K, dist;
std::vector<Mat> obj_points_, img_points_; Mat rvecs, tvecs;
std::vector<double> errors_per_view; std::vector<Mat> obj_points_, img_points_;
obj_points_.reserve(num_visible_frames_per_camera[camera]); std::vector<double> errors_per_view;
img_points_.reserve(num_visible_frames_per_camera[camera]); obj_points_.reserve(num_visible_frames_per_camera[camera]);
for (int f = 0; f < NUM_FRAMES; f++) { img_points_.reserve(num_visible_frames_per_camera[camera]);
if (detection_mask_mat[camera][f]) { for (int f = 0; f < NUM_FRAMES; f++) {
obj_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ? if (detection_mask_mat[camera][f]) {
objPoints_norm[f].reshape(3): objPoints_norm[f]); obj_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ?
img_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ? objPoints_norm[f].reshape(3): objPoints_norm[f]);
imagePoints[camera][f].reshape(2) : imagePoints[camera][f]); img_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ?
} imagePoints[camera][f].reshape(2) : imagePoints[camera][f]);
} }
double repr_err; }
if (models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE) { double repr_err;
repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera], if (models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE) {
K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at<int>(camera)); repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera],
// calibrate does not compute error per view, so compute it manually K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at<int>(camera));
errors_per_view = std::vector<double>(obj_points_.size()); // calibrate does not compute error per view, so compute it manually
for (int f = 0; f < (int) obj_points_.size(); f++) { errors_per_view = std::vector<double>(obj_points_.size());
double err2 = multiview::computeReprojectionMSE(obj_points_[f], for (int f = 0; f < (int) obj_points_.size(); f++) {
img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE); double err2 = multiview::computeReprojectionMSE(obj_points_[f],
errors_per_view[f] = sqrt(err2); img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE);
} errors_per_view[f] = sqrt(err2);
} else { }
repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist, } else {
rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at<int>(camera)); repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist,
} rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at<int>(camera));
CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!"); }
int cnt_visible_frame = 0; CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!");
for (int f = 0; f < NUM_FRAMES; f++) { int cnt_visible_frame = 0;
if (detection_mask_mat[camera][f]) { for (int f = 0; f < NUM_FRAMES; f++) {
rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data)); if (detection_mask_mat[camera][f]) {
tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data)); rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data));
double err = errors_per_view[cnt_visible_frame]; tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data));
double err2 = err * err; double err = errors_per_view[cnt_visible_frame];
if (camera_rt_errors[f] > err2) { double err2 = err * err;
camera_rt_errors[f] = err2; if (camera_rt_errors[f] > err2) {
camera_rt_best[f] = camera; camera_rt_errors[f] = err2;
camera_rt_best[f] = camera;
}
cnt_visible_frame++;
} }
cnt_visible_frame++;
} }
}
Ks.create(K.rows, K.cols, CV_64F, camera); Ks.create(K.rows, K.cols, CV_64F, camera);
distortions.create(dist.rows, dist.cols, CV_64F, camera, true); distortions.create(dist.rows, dist.cols, CV_64F, camera, true);
K.copyTo(Ks.getMat(camera)); K.copyTo(Ks.getMat(camera));
dist.copyTo(distortions.getMat(camera)); dist.copyTo(distortions.getMat(camera));
} }
});
} else { } else {
// use PnP to compute rvecs and tvecs // use PnP to compute rvecs and tvecs
for (int i = 0; i < NUM_FRAMES; i++) { for (int i = 0; i < NUM_FRAMES; i++) {
@ -961,4 +961,17 @@ double calibrateMultiview(
return sqrt(sum_errors / cnt_errors); return sqrt(sum_errors / cnt_errors);
} }
double calibrateMultiview(
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputArray flagsForIntrinsics, int flags) {
return calibrateMultiview(objPoints, imagePoints, imageSize, detectionMask, models, Ks, distortions,
Rs, Ts, noArray(), noArray(), noArray(), noArray(), flagsForIntrinsics, flags);
}
} }

2
modules/calib/test/test_cameracalibration.cpp
View File

@ -2485,7 +2485,7 @@ double CV_MultiviewCalibrationTest_CPP::calibrateStereoCamera( const vector<vect
std::vector<uchar> models(2, cv::CALIB_MODEL_PINHOLE); std::vector<uchar> models(2, cv::CALIB_MODEL_PINHOLE);
std::vector<int> all_flags(2, flags); std::vector<int> all_flags(2, flags);
double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models, double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models,
Rs, Ts, Ks, distortions, 0, all_flags, rvecs, tvecs, errors_mat); Ks, distortions, Rs, Ts, /*pairs*/ noArray(), rvecs, tvecs, errors_mat, all_flags);
if (perViewErrors1.size() != (size_t)numImgs) if (perViewErrors1.size() != (size_t)numImgs)
{ {

13
modules/calib/test/test_fisheye.cpp
View File

@ -616,17 +616,12 @@ TEST_F(fisheyeTest, multiview_calibration)
} }
std::vector<cv::Size> image_sizes(2, imageSize); std::vector<cv::Size> image_sizes(2, imageSize);
cv::Mat visibility_mat = cv::Mat_<uchar>::ones(2, (int)leftPoints.size()); cv::Mat visibility_mat = cv::Mat_<uchar>::ones(2, (int)leftPoints.size());
std::vector<cv::Mat> Rs, Ts, Ks, distortions, rvecs0, tvecs0; std::vector<cv::Mat> Rs, Ts, Ks, distortions;
std::vector<uchar> models(2, cv::CALIB_MODEL_FISHEYE); std::vector<uchar> models(2, cv::CALIB_MODEL_FISHEYE);
int flag = 0; std::vector<int> all_flags(2, cv::CALIB_RECOMPUTE_EXTRINSIC | cv::CALIB_CHECK_COND | cv::CALIB_FIX_SKEW);
flag |= cv::CALIB_RECOMPUTE_EXTRINSIC;
flag |= cv::CALIB_CHECK_COND;
flag |= cv::CALIB_FIX_SKEW;
std::vector<int> all_flags(2, flag); calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat,
models, Ks, distortions, Rs, Ts, all_flags);
calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models,
Rs, Ts, Ks, distortions, 0, all_flags, rvecs0, tvecs0);
cv::Matx33d R_correct( 0.9975587205950972, 0.06953016383322372, 0.006492709911733523, cv::Matx33d R_correct( 0.9975587205950972, 0.06953016383322372, 0.006492709911733523,
-0.06956823121068059, 0.9975601387249519, 0.005833595226966235, -0.06956823121068059, 0.9975601387249519, 0.005833595226966235,
-0.006071257768382089, -0.006271040135405457, 0.9999619062167968); -0.006071257768382089, -0.006271040135405457, 0.9999619062167968);

14
modules/calib/test/test_multiview_calib.cpp
View File

@ -133,7 +133,7 @@ TEST(multiview_calibration, accuracy) {
std::vector<cv::Mat> Ks, distortions, Rs, Ts; std::vector<cv::Mat> Ks, distortions, Rs, Ts;
calibrateMultiview(objPoints, image_points_all, image_sizes, visibility_mat, calibrateMultiview(objPoints, image_points_all, image_sizes, visibility_mat,
models, Rs, Ts, Ks, distortions); models, Ks, distortions, Rs, Ts);
const double K_err_tol = 1e1, dist_tol = 5e-2, R_tol = 1e-2, T_tol = 1e-2; const double K_err_tol = 1e1, dist_tol = 5e-2, R_tol = 1e-2, T_tol = 1e-2;
for (int c = 0; c < num_cameras; c++) { for (int c = 0; c < num_cameras; c++) {
@ -169,7 +169,7 @@ struct MultiViewTest : public ::testing::Test
std::vector<std::vector<cv::Mat>>& image_points_all, cv::Mat& visibility) std::vector<std::vector<cv::Mat>>& image_points_all, cv::Mat& visibility)
{ {
image_points_all.clear(); image_points_all.clear();
visibility.create(static_cast<int>(cameras.size()), frameCount, CV_8UC1); visibility.create(static_cast<int>(cameras.size()), frameCount, CV_BoolC1);
for (int c = 0; c < static_cast<int>(cameras.size()); c++) for (int c = 0; c < static_cast<int>(cameras.size()); c++)
{ {
std::vector<cv::Mat> camera_image_points; std::vector<cv::Mat> camera_image_points;
@ -183,7 +183,7 @@ struct MultiViewTest : public ::testing::Test
if (!node.empty()) if (!node.empty())
{ {
camera_image_points.push_back(node.mat().reshape(2, 1)); camera_image_points.push_back(node.mat().reshape(2, 1));
visibility.at<uchar>(c, i) = 255; visibility.at<uchar>(c, i) = 1;
} }
else else
{ {
@ -315,7 +315,7 @@ TEST_F(MultiViewTest, OneLine)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts; std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms); CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, .3); EXPECT_LE(rms, .3);
@ -420,7 +420,7 @@ TEST_F(MultiViewTest, OneLineInitialGuess)
int flags = cv::CALIB_USE_EXTRINSIC_GUESS | cv::CALIB_USE_INTRINSIC_GUESS | cv::CALIB_STEREO_REGISTRATION; int flags = cv::CALIB_USE_EXTRINSIC_GUESS | cv::CALIB_USE_INTRINSIC_GUESS | cv::CALIB_STEREO_REGISTRATION;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, flags, flagsForIntrinsics); Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics, flags);
CV_LOG_INFO(NULL, "RMS: " << rms); CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, .3); EXPECT_LE(rms, .3);
@ -486,7 +486,7 @@ TEST_F(MultiViewTest, CamsToFloor)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts; std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms); CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, 1.); EXPECT_LE(rms, 1.);
@ -555,7 +555,7 @@ TEST_F(MultiViewTest, Hetero)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts; std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models, double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics); Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms); CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, 2.5); EXPECT_LE(rms, 2.5);

2
samples/cpp/multiview_calibration_sample.cpp
View File

@ -163,7 +163,7 @@ static void detectPointsAndCalibrate (cv::Size pattern_size, float pattern_dista
// ! [multiview_calib] // ! [multiview_calib]
const double rmse = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, const double rmse = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility,
models, Rs, Ts, Ks, distortions); models, Ks, distortions, Rs, Ts);
// ! [multiview_calib] // ! [multiview_calib]
std::cout << "average RMSE over detection mask " << rmse << "\n"; std::cout << "average RMSE over detection mask " << rmse << "\n";
for (int c = 0; c < (int)Rs.size(); c++) { for (int c = 0; c < (int)Rs.size(); c++) {

2
samples/python/multiview_calibration.py
View File

@ -423,7 +423,7 @@ def calibrateFromPoints(
# try: # try:
# [multiview_calib] # [multiview_calib]
rmse, Rs, Ts, Ks, distortions, rvecs0, tvecs0, errors_per_frame, output_pairs = \ rmse, Rs, Ts, Ks, distortions, rvecs0, tvecs0, errors_per_frame, output_pairs = \
cv.calibrateMultiview( cv.calibrateMultiviewExtended(
objPoints=pattern_points_all, objPoints=pattern_points_all,
imagePoints=image_points, imagePoints=image_points,
imageSize=image_sizes, imageSize=image_sizes,