mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-01-19 15:04:01 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #26248 from asmorkalov:as/multiview_split

Browse Source 
Multiple calibrateMultiview improvements.
This commit is contained in:
Alexander Smorkalov 2024-10-04 16:37:15 +03:00 committed by GitHub
commit 91775af2dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 94 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/calib/include/opencv2/calib.hpp
View File

@ -1288,14 +1288,22 @@ points in all the available views from all cameras.
@sa findChessboardCorners, findCirclesGrid, calibrateCamera, fisheye::calibrate, registerCameras
*/
CV_EXPORTS_AS(calibrateMultiviewExtended) double calibrateMultiview (
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
OutputArray initializationPairs, OutputArrayOfArrays rvecs0,
OutputArrayOfArrays tvecs0, OutputArray perFrameErrors,
InputArray flagsForIntrinsics=noArray(), int flags = 0);
/// @overload
CV_EXPORTS_W double calibrateMultiview (
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
int flags = 0, InputArray flagsForIntrinsics=noArray(),
OutputArrayOfArrays rvecs0=noArray(), OutputArrayOfArrays tvecs0=noArray(),
OutputArray perFrameErrors=noArray(), OutputArray initializationPairs=noArray());
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputArray flagsForIntrinsics=noArray(), int flags = 0);
/** @brief Computes Hand-Eye calibration: \f$_{}^{g}\textrm{T}_c\f$

123
modules/calib/src/multiview_calibration.cpp
View File

@ -569,16 +569,14 @@ static void checkConnected (const std::vector<std::vector<bool>> &detection_mask
}
}
//TODO: use Input/OutputArrays for imagePoints(?)
double calibrateMultiview(
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
int flags, InputArray flagsForIntrinsics,
OutputArrayOfArrays rvecs0, OutputArrayOfArrays tvecs0,
OutputArray perFrameErrors, OutputArray initializationPairs) {
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
OutputArray initializationPairs, OutputArrayOfArrays rvecs0,
OutputArrayOfArrays tvecs0, OutputArray perFrameErrors,
InputArray flagsForIntrinsics, int flags) {
CV_CheckFalse(objPoints.empty(), "Objects points must not be empty!");
CV_CheckFalse(imagePoints.empty(), "Image points must not be empty!");
CV_CheckFalse(imageSize.empty(), "Image size per camera must not be empty!");
@ -588,7 +586,7 @@ double calibrateMultiview(
Mat detection_mask_ = detectionMask.getMat();
Mat models_mat = models.getMat();
CV_CheckEQ(detection_mask_.type(), CV_8U, "detectionMask must be of type CV_8U");
CV_Assert(detection_mask_.type() == CV_8UC1 || detection_mask_.type() == CV_BoolC1);
CV_CheckEQ(models_mat.type(), CV_8U, "models must be of type CV_8U");
if(flags & cv::CALIB_STEREO_REGISTRATION)
@ -694,57 +692,59 @@ double calibrateMultiview(
distortions.create(NUM_CAMERAS, 1, CV_64F);
// calibrate each camera independently to find intrinsic parameters - K and distortion coefficients
for (int camera = 0; camera < NUM_CAMERAS; camera++) {
Mat K, dist;
Mat rvecs, tvecs;
std::vector<Mat> obj_points_, img_points_;
std::vector<double> errors_per_view;
obj_points_.reserve(num_visible_frames_per_camera[camera]);
img_points_.reserve(num_visible_frames_per_camera[camera]);
for (int f = 0; f < NUM_FRAMES; f++) {
if (detection_mask_mat[camera][f]) {
obj_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ?
objPoints_norm[f].reshape(3): objPoints_norm[f]);
img_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ?
imagePoints[camera][f].reshape(2) : imagePoints[camera][f]);
}
}
double repr_err;
if (models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE) {
repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera],
K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at<int>(camera));
// calibrate does not compute error per view, so compute it manually
errors_per_view = std::vector<double>(obj_points_.size());
for (int f = 0; f < (int) obj_points_.size(); f++) {
double err2 = multiview::computeReprojectionMSE(obj_points_[f],
img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE);
errors_per_view[f] = sqrt(err2);
}
} else {
repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist,
rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at<int>(camera));
}
CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!");
int cnt_visible_frame = 0;
for (int f = 0; f < NUM_FRAMES; f++) {
if (detection_mask_mat[camera][f]) {
rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data));
tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data));
double err = errors_per_view[cnt_visible_frame];
double err2 = err * err;
if (camera_rt_errors[f] > err2) {
camera_rt_errors[f] = err2;
camera_rt_best[f] = camera;
parallel_for_(Range(0, NUM_CAMERAS), [&](const Range& range) {
for (int camera = range.start; camera < range.end; camera++) {
Mat K, dist;
Mat rvecs, tvecs;
std::vector<Mat> obj_points_, img_points_;
std::vector<double> errors_per_view;
obj_points_.reserve(num_visible_frames_per_camera[camera]);
img_points_.reserve(num_visible_frames_per_camera[camera]);
for (int f = 0; f < NUM_FRAMES; f++) {
if (detection_mask_mat[camera][f]) {
obj_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && objPoints_norm[f].channels() != 3) ?
objPoints_norm[f].reshape(3): objPoints_norm[f]);
img_points_.emplace_back((models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE && imagePoints[camera][f].channels() != 2) ?
imagePoints[camera][f].reshape(2) : imagePoints[camera][f]);
}
}
double repr_err;
if (models_mat.at<uchar>(camera) == cv::CALIB_MODEL_FISHEYE) {
repr_err = fisheye::calibrate(obj_points_, img_points_, imageSize[camera],
K, dist, rvecs, tvecs, flagsForIntrinsics_mat.at<int>(camera));
// calibrate does not compute error per view, so compute it manually
errors_per_view = std::vector<double>(obj_points_.size());
for (int f = 0; f < (int) obj_points_.size(); f++) {
double err2 = multiview::computeReprojectionMSE(obj_points_[f],
img_points_[f], K, dist, rvecs.row(f), tvecs.row(f), noArray(), noArray(), cv::CALIB_MODEL_FISHEYE);
errors_per_view[f] = sqrt(err2);
}
} else {
repr_err = calibrateCamera(obj_points_, img_points_, imageSize[camera], K, dist,
rvecs, tvecs, noArray(), noArray(), errors_per_view, flagsForIntrinsics_mat.at<int>(camera));
}
CV_LOG_IF_WARNING(NULL, repr_err > WARNING_RMSE, "Warning! Mean RMSE of intrinsics calibration is higher than "+std::to_string(WARNING_RMSE)+" pixels!");
int cnt_visible_frame = 0;
for (int f = 0; f < NUM_FRAMES; f++) {
if (detection_mask_mat[camera][f]) {
rvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, rvecs.row(cnt_visible_frame).data));
tvecs_all[camera][f] = Vec3d(Mat(3, 1, CV_64F, tvecs.row(cnt_visible_frame).data));
double err = errors_per_view[cnt_visible_frame];
double err2 = err * err;
if (camera_rt_errors[f] > err2) {
camera_rt_errors[f] = err2;
camera_rt_best[f] = camera;
}
cnt_visible_frame++;
}
cnt_visible_frame++;
}
}
Ks.create(K.rows, K.cols, CV_64F, camera);
distortions.create(dist.rows, dist.cols, CV_64F, camera, true);
K.copyTo(Ks.getMat(camera));
dist.copyTo(distortions.getMat(camera));
}
Ks.create(K.rows, K.cols, CV_64F, camera);
distortions.create(dist.rows, dist.cols, CV_64F, camera, true);
K.copyTo(Ks.getMat(camera));
dist.copyTo(distortions.getMat(camera));
}
});
} else {
// use PnP to compute rvecs and tvecs
for (int i = 0; i < NUM_FRAMES; i++) {
@ -961,4 +961,17 @@ double calibrateMultiview(
return sqrt(sum_errors / cnt_errors);
}
double calibrateMultiview(
InputArrayOfArrays objPoints, const std::vector<std::vector<Mat>> &imagePoints,
const std::vector<cv::Size>& imageSize, InputArray detectionMask, InputArray models,
InputOutputArrayOfArrays Ks, InputOutputArrayOfArrays distortions,
InputOutputArrayOfArrays Rs, InputOutputArrayOfArrays Ts,
InputArray flagsForIntrinsics, int flags) {
return calibrateMultiview(objPoints, imagePoints, imageSize, detectionMask, models, Ks, distortions,
Rs, Ts, noArray(), noArray(), noArray(), noArray(), flagsForIntrinsics, flags);
}
}

2
modules/calib/test/test_cameracalibration.cpp
View File

@ -2485,7 +2485,7 @@ double CV_MultiviewCalibrationTest_CPP::calibrateStereoCamera( const vector<vect
std::vector<uchar> models(2, cv::CALIB_MODEL_PINHOLE);
std::vector<int> all_flags(2, flags);
double rms = calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models,
Rs, Ts, Ks, distortions, 0, all_flags, rvecs, tvecs, errors_mat);
Ks, distortions, Rs, Ts, /*pairs*/ noArray(), rvecs, tvecs, errors_mat, all_flags);
if (perViewErrors1.size() != (size_t)numImgs)
{

13
modules/calib/test/test_fisheye.cpp
View File

@ -616,17 +616,12 @@ TEST_F(fisheyeTest, multiview_calibration)
}
std::vector<cv::Size> image_sizes(2, imageSize);
cv::Mat visibility_mat = cv::Mat_<uchar>::ones(2, (int)leftPoints.size());
std::vector<cv::Mat> Rs, Ts, Ks, distortions, rvecs0, tvecs0;
std::vector<cv::Mat> Rs, Ts, Ks, distortions;
std::vector<uchar> models(2, cv::CALIB_MODEL_FISHEYE);
int flag = 0;
flag |= cv::CALIB_RECOMPUTE_EXTRINSIC;
flag |= cv::CALIB_CHECK_COND;
flag |= cv::CALIB_FIX_SKEW;
std::vector<int> all_flags(2, cv::CALIB_RECOMPUTE_EXTRINSIC | cv::CALIB_CHECK_COND | cv::CALIB_FIX_SKEW);
std::vector<int> all_flags(2, flag);
calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat, models,
Rs, Ts, Ks, distortions, 0, all_flags, rvecs0, tvecs0);
calibrateMultiview(objectPoints, image_points_all, image_sizes, visibility_mat,
models, Ks, distortions, Rs, Ts, all_flags);
cv::Matx33d R_correct( 0.9975587205950972, 0.06953016383322372, 0.006492709911733523,
-0.06956823121068059, 0.9975601387249519, 0.005833595226966235,
-0.006071257768382089, -0.006271040135405457, 0.9999619062167968);

14
modules/calib/test/test_multiview_calib.cpp
View File

@ -133,7 +133,7 @@ TEST(multiview_calibration, accuracy) {
std::vector<cv::Mat> Ks, distortions, Rs, Ts;
calibrateMultiview(objPoints, image_points_all, image_sizes, visibility_mat,
models, Rs, Ts, Ks, distortions);
models, Ks, distortions, Rs, Ts);
const double K_err_tol = 1e1, dist_tol = 5e-2, R_tol = 1e-2, T_tol = 1e-2;
for (int c = 0; c < num_cameras; c++) {
@ -169,7 +169,7 @@ struct MultiViewTest : public ::testing::Test
std::vector<std::vector<cv::Mat>>& image_points_all, cv::Mat& visibility)
{
image_points_all.clear();
visibility.create(static_cast<int>(cameras.size()), frameCount, CV_8UC1);
visibility.create(static_cast<int>(cameras.size()), frameCount, CV_BoolC1);
for (int c = 0; c < static_cast<int>(cameras.size()); c++)
{
std::vector<cv::Mat> camera_image_points;
@ -183,7 +183,7 @@ struct MultiViewTest : public ::testing::Test
if (!node.empty())
{
camera_image_points.push_back(node.mat().reshape(2, 1));
visibility.at<uchar>(c, i) = 255;
visibility.at<uchar>(c, i) = 1;
}
else
{
@ -315,7 +315,7 @@ TEST_F(MultiViewTest, OneLine)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics);
Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, .3);
@ -420,7 +420,7 @@ TEST_F(MultiViewTest, OneLineInitialGuess)
int flags = cv::CALIB_USE_EXTRINSIC_GUESS | cv::CALIB_USE_INTRINSIC_GUESS | cv::CALIB_STEREO_REGISTRATION;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, flags, flagsForIntrinsics);
Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics, flags);
CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, .3);
@ -486,7 +486,7 @@ TEST_F(MultiViewTest, CamsToFloor)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics);
Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, 1.);
@ -555,7 +555,7 @@ TEST_F(MultiViewTest, Hetero)
std::vector<cv::Mat> Ks, distortions, Rs, Rs_rvec, Ts;
double rms = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility, models,
Rs_rvec, Ts, Ks, distortions, 0, flagsForIntrinsics);
Ks, distortions, Rs_rvec, Ts, flagsForIntrinsics);
CV_LOG_INFO(NULL, "RMS: " << rms);
EXPECT_LE(rms, 2.5);

2
samples/cpp/multiview_calibration_sample.cpp
View File

@ -163,7 +163,7 @@ static void detectPointsAndCalibrate (cv::Size pattern_size, float pattern_dista
// ! [multiview_calib]
const double rmse = calibrateMultiview(objPoints, image_points_all, image_sizes, visibility,
models, Rs, Ts, Ks, distortions);
models, Ks, distortions, Rs, Ts);
// ! [multiview_calib]
std::cout << "average RMSE over detection mask " << rmse << "\n";
for (int c = 0; c < (int)Rs.size(); c++) {

2
samples/python/multiview_calibration.py
View File

@ -423,7 +423,7 @@ def calibrateFromPoints(
# try:
# [multiview_calib]
rmse, Rs, Ts, Ks, distortions, rvecs0, tvecs0, errors_per_frame, output_pairs = \
cv.calibrateMultiview(
cv.calibrateMultiviewExtended(
objPoints=pattern_points_all,
imagePoints=image_points,
imageSize=image_sizes,