/*M/////////////////////////////////////////////////////////////////////////////////////// // // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. // // By downloading, copying, installing or using the software you agree to this license. // If you do not agree to this license, do not download, install, // copy or use the software. // // // License Agreement // For Open Source Computer Vision Library // // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. // Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, // are permitted provided that the following conditions are met: // // * Redistribution's of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // * Redistribution's in binary form must reproduce the above copyright notice, // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // // * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and // any express or implied warranties, including, but not limited to, the implied // warranties of merchantability and fitness for a particular purpose are disclaimed. // In no event shall the Intel Corporation or contributors be liable for any direct, // indirect, incidental, special, exemplary, or consequential damages // (including, but not limited to, procurement of substitute goods or services; // loss of use, data, or profits; or business interruption) however caused // and on any theory of liability, whether in contract, strict liability, // or tort (including negligence or otherwise) arising in any way out of // the use of this software, even if advised of the possibility of such damage. // //M*/ #include "precomp.hpp" #include #include "opencv2/videostab/inpainting.hpp" #include "opencv2/videostab/global_motion.hpp" #include "opencv2/videostab/fast_marching.hpp" #include "opencv2/videostab/ring_buffer.hpp" #include "opencv2/opencv_modules.hpp" namespace cv { namespace videostab { void InpaintingPipeline::setRadius(int val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setRadius(val); InpainterBase::setRadius(val); } void InpaintingPipeline::setFrames(const std::vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setFrames(val); InpainterBase::setFrames(val); } void InpaintingPipeline::setMotionModel(MotionModel val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setMotionModel(val); InpainterBase::setMotionModel(val); } void InpaintingPipeline::setMotions(const std::vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setMotions(val); InpainterBase::setMotions(val); } void InpaintingPipeline::setStabilizedFrames(const std::vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setStabilizedFrames(val); InpainterBase::setStabilizedFrames(val); } void InpaintingPipeline::setStabilizationMotions(const std::vector &val) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->setStabilizationMotions(val); InpainterBase::setStabilizationMotions(val); } void InpaintingPipeline::inpaint(int idx, Mat &frame, Mat &mask) { for (size_t i = 0; i < inpainters_.size(); ++i) inpainters_[i]->inpaint(idx, frame, mask); } struct Pixel3 { float intens; Point3_ color; bool operator <(const Pixel3 &other) const { return intens < other.intens; } }; ConsistentMosaicInpainter::ConsistentMosaicInpainter() { setStdevThresh(20.f); } void ConsistentMosaicInpainter::inpaint(int idx, Mat &frame, Mat &mask) { CV_Assert(frame.type() == CV_8UC3); CV_Assert(mask.size() == frame.size() && mask.type() == CV_8U); Mat invS = at(idx, *stabilizationMotions_).inv(); std::vector > vmotions(2*radius_ + 1); for (int i = -radius_; i <= radius_; ++i) vmotions[radius_ + i] = getMotion(idx, idx + i, *motions_) * invS; int n; float mean, var; std::vector pixels(2*radius_ + 1); Mat_ > frame_(frame); Mat_ mask_(mask); for (int y = 0; y < mask.rows; ++y) { for (int x = 0; x < mask.cols; ++x) { if (!mask_(y, x)) { n = 0; mean = 0; var = 0; for (int i = -radius_; i <= radius_; ++i) { const Mat_ > &framei = at(idx + i, *frames_); const Mat_ &Mi = vmotions[radius_ + i]; int xi = cvRound(Mi(0,0)*x + Mi(0,1)*y + Mi(0,2)); int yi = cvRound(Mi(1,0)*x + Mi(1,1)*y + Mi(1,2)); if (xi >= 0 && xi < framei.cols && yi >= 0 && yi < framei.rows) { pixels[n].color = framei(yi, xi); mean += pixels[n].intens = intensity(pixels[n].color); n++; } } if (n > 0) { mean /= n; for (int i = 0; i < n; ++i) var += sqr(pixels[i].intens - mean); var /= std::max(n - 1, 1); if (var < stdevThresh_ * stdevThresh_) { std::sort(pixels.begin(), pixels.begin() + n); int nh = (n-1)/2; int c1 = pixels[nh].color.x; int c2 = pixels[nh].color.y; int c3 = pixels[nh].color.z; if (n-2*nh) { c1 = (c1 + pixels[nh].color.x) / 2; c2 = (c2 + pixels[nh].color.y) / 2; c3 = (c3 + pixels[nh].color.z) / 2; } frame_(y, x) = Point3_( static_cast(c1), static_cast(c2), static_cast(c3)); mask_(y, x) = 255; } } } } } } static float alignementError( const Mat &M, const Mat &frame0, const Mat &mask0, const Mat &frame1) { CV_Assert(frame0.type() == CV_8UC3 && frame1.type() == CV_8UC3); CV_Assert(mask0.type() == CV_8U && mask0.size() == frame0.size()); CV_Assert(frame0.size() == frame1.size()); CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); Mat_ mask0_(mask0); Mat_ M_(M); float err = 0; for (int y0 = 0; y0 < frame0.rows; ++y0) { for (int x0 = 0; x0 < frame0.cols; ++x0) { if (mask0_(y0,x0)) { int x1 = cvRound(M_(0,0)*x0 + M_(0,1)*y0 + M_(0,2)); int y1 = cvRound(M_(1,0)*x0 + M_(1,1)*y0 + M_(1,2)); if (y1 >= 0 && y1 < frame1.rows && x1 >= 0 && x1 < frame1.cols) err += std::abs(intensity(frame1.at >(y1,x1)) - intensity(frame0.at >(y0,x0))); } } } return err; } class MotionInpaintBody { public: void operator ()(int x, int y) { float uEst = 0.f, vEst = 0.f, wSum = 0.f; for (int dy = -rad; dy <= rad; ++dy) { for (int dx = -rad; dx <= rad; ++dx) { int qx0 = x + dx; int qy0 = y + dy; if (qy0 >= 0 && qy0 < mask0.rows && qx0 >= 0 && qx0 < mask0.cols && mask0(qy0,qx0)) { int qx1 = cvRound(qx0 + flowX(qy0,qx0)); int qy1 = cvRound(qy0 + flowY(qy0,qx0)); int px1 = qx1 - dx; int py1 = qy1 - dy; if (qx1 >= 0 && qx1 < mask1.cols && qy1 >= 0 && qy1 < mask1.rows && mask1(qy1,qx1) && px1 >= 0 && px1 < mask1.cols && py1 >= 0 && py1 < mask1.rows && mask1(py1,px1)) { float dudx = 0.f, dvdx = 0.f, dudy = 0.f, dvdy = 0.f; if (qx0 > 0 && mask0(qy0,qx0-1)) { if (qx0+1 < mask0.cols && mask0(qy0,qx0+1)) { dudx = (flowX(qy0,qx0+1) - flowX(qy0,qx0-1)) * 0.5f; dvdx = (flowY(qy0,qx0+1) - flowY(qy0,qx0-1)) * 0.5f; } else { dudx = flowX(qy0,qx0) - flowX(qy0,qx0-1); dvdx = flowY(qy0,qx0) - flowY(qy0,qx0-1); } } else if (qx0+1 < mask0.cols && mask0(qy0,qx0+1)) { dudx = flowX(qy0,qx0+1) - flowX(qy0,qx0); dvdx = flowY(qy0,qx0+1) - flowY(qy0,qx0); } if (qy0 > 0 && mask0(qy0-1,qx0)) { if (qy0+1 < mask0.rows && mask0(qy0+1,qx0)) { dudy = (flowX(qy0+1,qx0) - flowX(qy0-1,qx0)) * 0.5f; dvdy = (flowY(qy0+1,qx0) - flowY(qy0-1,qx0)) * 0.5f; } else { dudy = flowX(qy0,qx0) - flowX(qy0-1,qx0); dvdy = flowY(qy0,qx0) - flowY(qy0-1,qx0); } } else if (qy0+1 < mask0.rows && mask0(qy0+1,qx0)) { dudy = flowX(qy0+1,qx0) - flowX(qy0,qx0); dvdy = flowY(qy0+1,qx0) - flowY(qy0,qx0); } Point3_ cp = frame1(py1,px1), cq = frame1(qy1,qx1); float distColor = sqr(static_cast(cp.x-cq.x)) + sqr(static_cast(cp.y-cq.y)) + sqr(static_cast(cp.z-cq.z)); float w = 1.f / (std::sqrt(distColor * (dx*dx + dy*dy)) + eps); uEst += w * (flowX(qy0,qx0) - dudx*dx - dudy*dy); vEst += w * (flowY(qy0,qx0) - dvdx*dx - dvdy*dy); wSum += w; } } } } if (wSum > 0.f) { flowX(y,x) = uEst / wSum; flowY(y,x) = vEst / wSum; mask0(y,x) = 255; } } Mat_ > frame1; Mat_ mask0, mask1; Mat_ flowX, flowY; float eps; int rad; }; MotionInpainter::MotionInpainter() { #ifdef HAVE_OPENCV_CUDAOPTFLOW setOptFlowEstimator(new DensePyrLkOptFlowEstimatorGpu()); #else CV_Error(Error::StsNotImplemented, "Current implementation of MotionInpainter requires GPU"); #endif setFlowErrorThreshold(1e-4f); setDistThreshold(5.f); setBorderMode(BORDER_REPLICATE); } void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask) { std::priority_queue > neighbors; std::vector vmotions(2*radius_ + 1); for (int i = -radius_; i <= radius_; ++i) { Mat motion0to1 = getMotion(idx, idx + i, *motions_) * at(idx, *stabilizationMotions_).inv(); vmotions[radius_ + i] = motion0to1; if (i != 0) { float err = alignementError(motion0to1, frame, mask, at(idx + i, *frames_)); neighbors.push(std::make_pair(-err, idx + i)); } } if (mask1_.size() != mask.size()) { mask1_.create(mask.size()); mask1_.setTo(255); } cvtColor(frame, grayFrame_, COLOR_BGR2GRAY); MotionInpaintBody body; body.rad = 2; body.eps = 1e-4f; while (!neighbors.empty()) { int neighbor = neighbors.top().second; neighbors.pop(); Mat motion1to0 = vmotions[radius_ + neighbor - idx].inv(); // warp frame frame1_ = at(neighbor, *frames_); if (motionModel_ != MM_HOMOGRAPHY) warpAffine( frame1_, transformedFrame1_, motion1to0(Rect(0,0,3,2)), frame1_.size(), INTER_LINEAR, borderMode_); else warpPerspective( frame1_, transformedFrame1_, motion1to0, frame1_.size(), INTER_LINEAR, borderMode_); cvtColor(transformedFrame1_, transformedGrayFrame1_, COLOR_BGR2GRAY); // warp mask if (motionModel_ != MM_HOMOGRAPHY) warpAffine( mask1_, transformedMask1_, motion1to0(Rect(0,0,3,2)), mask1_.size(), INTER_NEAREST); else warpPerspective(mask1_, transformedMask1_, motion1to0, mask1_.size(), INTER_NEAREST); erode(transformedMask1_, transformedMask1_, Mat()); // update flow optFlowEstimator_->run(grayFrame_, transformedGrayFrame1_, flowX_, flowY_, flowErrors_); calcFlowMask( flowX_, flowY_, flowErrors_, flowErrorThreshold_, mask, transformedMask1_, flowMask_); body.flowX = flowX_; body.flowY = flowY_; body.mask0 = flowMask_; body.mask1 = transformedMask1_; body.frame1 = transformedFrame1_; fmm_.run(flowMask_, body); completeFrameAccordingToFlow( flowMask_, flowX_, flowY_, transformedFrame1_, transformedMask1_, distThresh_, frame, mask); } } class ColorAverageInpaintBody { public: void operator ()(int x, int y) { float c1 = 0, c2 = 0, c3 = 0; float wSum = 0; static const int lut[8][2] = {{-1,-1}, {-1,0}, {-1,1}, {0,-1}, {0,1}, {1,-1}, {1,0}, {1,1}}; for (int i = 0; i < 8; ++i) { int qx = x + lut[i][0]; int qy = y + lut[i][1]; if (qy >= 0 && qy < mask.rows && qx >= 0 && qx < mask.cols && mask(qy,qx)) { c1 += frame.at(qy,3*qx); c2 += frame.at(qy,3*qx+1); c3 += frame.at(qy,3*qx+2); wSum += 1; } } float wSumInv = 1.f / wSum; frame(y,x) = Point3_( static_cast(c1*wSumInv), static_cast(c2*wSumInv), static_cast(c3*wSumInv)); mask(y,x) = 255; } cv::Mat_ mask; cv::Mat_ > frame; }; void ColorAverageInpainter::inpaint(int /*idx*/, Mat &frame, Mat &mask) { ColorAverageInpaintBody body; body.mask = mask; body.frame = frame; fmm_.run(mask, body); } void ColorInpainter::inpaint(int /*idx*/, Mat &frame, Mat &mask) { bitwise_not(mask, invMask_); cv::inpaint(frame, invMask_, frame, radius_, method_); } void calcFlowMask( const Mat &flowX, const Mat &flowY, const Mat &errors, float maxError, const Mat &mask0, const Mat &mask1, Mat &flowMask) { CV_Assert(flowX.type() == CV_32F && flowX.size() == mask0.size()); CV_Assert(flowY.type() == CV_32F && flowY.size() == mask0.size()); CV_Assert(errors.type() == CV_32F && errors.size() == mask0.size()); CV_Assert(mask0.type() == CV_8U); CV_Assert(mask1.type() == CV_8U && mask1.size() == mask0.size()); Mat_ flowX_(flowX), flowY_(flowY), errors_(errors); Mat_ mask0_(mask0), mask1_(mask1); flowMask.create(mask0.size(), CV_8U); flowMask.setTo(0); Mat_ flowMask_(flowMask); for (int y0 = 0; y0 < flowMask_.rows; ++y0) { for (int x0 = 0; x0 < flowMask_.cols; ++x0) { if (mask0_(y0,x0) && errors_(y0,x0) < maxError) { int x1 = cvRound(x0 + flowX_(y0,x0)); int y1 = cvRound(y0 + flowY_(y0,x0)); if (x1 >= 0 && x1 < mask1_.cols && y1 >= 0 && y1 < mask1_.rows && mask1_(y1,x1)) flowMask_(y0,x0) = 255; } } } } void completeFrameAccordingToFlow( const Mat &flowMask, const Mat &flowX, const Mat &flowY, const Mat &frame1, const Mat &mask1, float distThresh, Mat &frame0, Mat &mask0) { CV_Assert(flowMask.type() == CV_8U); CV_Assert(flowX.type() == CV_32F && flowX.size() == flowMask.size()); CV_Assert(flowY.type() == CV_32F && flowY.size() == flowMask.size()); CV_Assert(frame1.type() == CV_8UC3 && frame1.size() == flowMask.size()); CV_Assert(mask1.type() == CV_8U && mask1.size() == flowMask.size()); CV_Assert(frame0.type() == CV_8UC3 && frame0.size() == flowMask.size()); CV_Assert(mask0.type() == CV_8U && mask0.size() == flowMask.size()); Mat_ flowMask_(flowMask), mask1_(mask1), mask0_(mask0); Mat_ flowX_(flowX), flowY_(flowY); for (int y0 = 0; y0 < frame0.rows; ++y0) { for (int x0 = 0; x0 < frame0.cols; ++x0) { if (!mask0_(y0,x0) && flowMask_(y0,x0)) { int x1 = cvRound(x0 + flowX_(y0,x0)); int y1 = cvRound(y0 + flowY_(y0,x0)); if (x1 >= 0 && x1 < frame1.cols && y1 >= 0 && y1 < frame1.rows && mask1_(y1,x1) && sqr(flowX_(y0,x0)) + sqr(flowY_(y0,x0)) < sqr(distThresh)) { frame0.at >(y0,x0) = frame1.at >(y1,x1); mask0_(y0,x0) = 255; } } } } } } // namespace videostab } // namespace cv