From 68f26cca78ad278e72a0dc030ea7561b326e9669 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 21 Mar 2012 09:34:27 +0000 Subject: [PATCH] Refactored videostab module --- .../opencv2/videostab/global_motion.hpp | 7 - ...n_filtering.hpp => motion_stabilizing.hpp} | 8 +- .../include/opencv2/videostab/stabilizer.hpp | 2 +- modules/videostab/src/deblurring.cpp | 4 +- modules/videostab/src/global_motion.cpp | 165 -------------- modules/videostab/src/inpainting.cpp | 31 +++ modules/videostab/src/motion_filtering.cpp | 79 ------- modules/videostab/src/motion_stabilizing.cpp | 214 ++++++++++++++++++ 8 files changed, 254 insertions(+), 256 deletions(-) rename modules/videostab/include/opencv2/videostab/{motion_filtering.hpp => motion_stabilizing.hpp} (91%) delete mode 100644 modules/videostab/src/motion_filtering.cpp create mode 100644 modules/videostab/src/motion_stabilizing.cpp diff --git a/modules/videostab/include/opencv2/videostab/global_motion.hpp b/modules/videostab/include/opencv2/videostab/global_motion.hpp index de924bb148..e046bd0f4f 100644 --- a/modules/videostab/include/opencv2/videostab/global_motion.hpp +++ b/modules/videostab/include/opencv2/videostab/global_motion.hpp @@ -131,13 +131,6 @@ private: CV_EXPORTS Mat getMotion(int from, int to, const std::vector &motions); -CV_EXPORTS Mat ensureInclusionConstraint(const Mat &M, Size size, float trimRatio); - -CV_EXPORTS float estimateOptimalTrimRatio(const Mat &M, Size size); - -// frame1 is non-transformed frame -CV_EXPORTS float alignementError(const Mat &M, const Mat &frame0, const Mat &mask0, const Mat &frame1); - } // namespace videostab } // namespace cv diff --git a/modules/videostab/include/opencv2/videostab/motion_filtering.hpp b/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp similarity index 91% rename from modules/videostab/include/opencv2/videostab/motion_filtering.hpp rename to modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp index ca0ecf2697..f163a9a906 100644 --- a/modules/videostab/include/opencv2/videostab/motion_filtering.hpp +++ b/modules/videostab/include/opencv2/videostab/motion_stabilizing.hpp @@ -40,8 +40,8 @@ // //M*/ -#ifndef __OPENCV_VIDEOSTAB_MOTION_FILTERING_HPP__ -#define __OPENCV_VIDEOSTAB_MOTION_FILTERING_HPP__ +#ifndef __OPENCV_VIDEOSTAB_MOTION_STABILIZING_HPP__ +#define __OPENCV_VIDEOSTAB_MOTION_STABILIZING_HPP__ #include #include "opencv2/core/core.hpp" @@ -71,6 +71,10 @@ private: std::vector weight_; }; +CV_EXPORTS Mat ensureInclusionConstraint(const Mat &M, Size size, float trimRatio); + +CV_EXPORTS float estimateOptimalTrimRatio(const Mat &M, Size size); + } // namespace videostab } // namespace diff --git a/modules/videostab/include/opencv2/videostab/stabilizer.hpp b/modules/videostab/include/opencv2/videostab/stabilizer.hpp index 05a0cdf323..99906312d8 100644 --- a/modules/videostab/include/opencv2/videostab/stabilizer.hpp +++ b/modules/videostab/include/opencv2/videostab/stabilizer.hpp @@ -47,7 +47,7 @@ #include "opencv2/core/core.hpp" #include "opencv2/imgproc/imgproc.hpp" #include "opencv2/videostab/global_motion.hpp" -#include "opencv2/videostab/motion_filtering.hpp" +#include "opencv2/videostab/motion_stabilizing.hpp" #include "opencv2/videostab/frame_source.hpp" #include "opencv2/videostab/log.hpp" #include "opencv2/videostab/inpainting.hpp" diff --git a/modules/videostab/src/deblurring.cpp b/modules/videostab/src/deblurring.cpp index 752d1e6360..813e908be5 100644 --- a/modules/videostab/src/deblurring.cpp +++ b/modules/videostab/src/deblurring.cpp @@ -102,8 +102,8 @@ void WeightingDeblurer::deblur(int idx, Mat &frame) { for (int x = 0; x < frame.cols; ++x) { - int x1 = static_cast(M(0,0)*x + M(0,1)*y + M(0,2)); - int y1 = static_cast(M(1,0)*x + M(1,1)*y + M(1,2)); + int x1 = cvRound(M(0,0)*x + M(0,1)*y + M(0,2)); + int y1 = cvRound(M(1,0)*x + M(1,1)*y + M(1,2)); if (x1 >= 0 && x1 < neighbor.cols && y1 >= 0 && y1 < neighbor.rows) { diff --git a/modules/videostab/src/global_motion.cpp b/modules/videostab/src/global_motion.cpp index 081381489c..d015d1b1b3 100644 --- a/modules/videostab/src/global_motion.cpp +++ b/modules/videostab/src/global_motion.cpp @@ -313,170 +313,5 @@ Mat getMotion(int from, int to, const vector &motions) return M; } - -static inline int areaSign(Point2f a, Point2f b, Point2f c) -{ - double area = (b-a).cross(c-a); - if (area < -1e-5) return -1; - if (area > 1e-5) return 1; - return 0; -} - - -static inline bool segmentsIntersect(Point2f a, Point2f b, Point2f c, Point2f d) -{ - return areaSign(a,b,c) * areaSign(a,b,d) < 0 && - areaSign(c,d,a) * areaSign(c,d,b) < 0; -} - - -// Checks if rect a (with sides parallel to axis) is inside rect b (arbitrary). -// Rects must be passed in the [(0,0), (w,0), (w,h), (0,h)] order. -static inline bool isRectInside(const Point2f a[4], const Point2f b[4]) -{ - for (int i = 0; i < 4; ++i) - if (b[i].x > a[0].x && b[i].x < a[2].x && b[i].y > a[0].y && b[i].y < a[2].y) - return false; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) - if (segmentsIntersect(a[i], a[(i+1)%4], b[j], b[(j+1)%4])) - return false; - return true; -} - - -static inline bool isGoodMotion(const float M[], float w, float h, float dx, float dy) -{ - Point2f pt[4] = {Point2f(0,0), Point2f(w,0), Point2f(w,h), Point2f(0,h)}; - Point2f Mpt[4]; - - for (int i = 0; i < 4; ++i) - { - Mpt[i].x = M[0]*pt[i].x + M[1]*pt[i].y + M[2]; - Mpt[i].y = M[3]*pt[i].x + M[4]*pt[i].y + M[5]; - } - - pt[0] = Point2f(dx, dy); - pt[1] = Point2f(w - dx, dy); - pt[2] = Point2f(w - dx, h - dy); - pt[3] = Point2f(dx, h - dy); - - return isRectInside(pt, Mpt); -} - - -static inline void relaxMotion(const float M[], float t, float res[]) -{ - res[0] = M[0]*(1.f-t) + t; - res[1] = M[1]*(1.f-t); - res[2] = M[2]*(1.f-t); - res[3] = M[3]*(1.f-t); - res[4] = M[4]*(1.f-t) + t; - res[5] = M[5]*(1.f-t); -} - - -Mat ensureInclusionConstraint(const Mat &M, Size size, float trimRatio) -{ - CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); - - const float w = static_cast(size.width); - const float h = static_cast(size.height); - const float dx = floor(w * trimRatio); - const float dy = floor(h * trimRatio); - const float srcM[6] = - {M.at(0,0), M.at(0,1), M.at(0,2), - M.at(1,0), M.at(1,1), M.at(1,2)}; - - float curM[6]; - float t = 0; - relaxMotion(srcM, t, curM); - if (isGoodMotion(curM, w, h, dx, dy)) - return M; - - float l = 0, r = 1; - while (r - l > 1e-3f) - { - t = (l + r) * 0.5f; - relaxMotion(srcM, t, curM); - if (isGoodMotion(curM, w, h, dx, dy)) - r = t; - else - l = t; - t = r; - relaxMotion(srcM, r, curM); - } - - return (1 - r) * M + r * Mat::eye(3, 3, CV_32F); -} - - -// TODO can be estimated for O(1) time -float estimateOptimalTrimRatio(const Mat &M, Size size) -{ - CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); - - const float w = static_cast(size.width); - const float h = static_cast(size.height); - Mat_ M_(M); - - Point2f pt[4] = {Point2f(0,0), Point2f(w,0), Point2f(w,h), Point2f(0,h)}; - Point2f Mpt[4]; - - for (int i = 0; i < 4; ++i) - { - Mpt[i].x = M_(0,0)*pt[i].x + M_(0,1)*pt[i].y + M_(0,2); - Mpt[i].y = M_(1,0)*pt[i].x + M_(1,1)*pt[i].y + M_(1,2); - } - - float l = 0, r = 0.5f; - while (r - l > 1e-3f) - { - float t = (l + r) * 0.5f; - float dx = floor(w * t); - float dy = floor(h * t); - pt[0] = Point2f(dx, dy); - pt[1] = Point2f(w - dx, dy); - pt[2] = Point2f(w - dx, h - dy); - pt[3] = Point2f(dx, h - dy); - if (isRectInside(pt, Mpt)) - r = t; - else - l = t; - } - - return r; -} - - -float alignementError(const Mat &M, const Mat &frame0, const Mat &mask0, const Mat &frame1) -{ - CV_Assert(frame0.type() == CV_8UC3 && frame1.type() == CV_8UC3); - CV_Assert(mask0.type() == CV_8U && mask0.size() == frame0.size()); - CV_Assert(frame0.size() == frame1.size()); - CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); - - Mat_ mask0_(mask0); - Mat_ M_(M); - float err = 0; - - for (int y0 = 0; y0 < frame0.rows; ++y0) - { - for (int x0 = 0; x0 < frame0.cols; ++x0) - { - if (mask0_(y0,x0)) - { - int x1 = cvRound(M_(0,0)*x0 + M_(0,1)*y0 + M_(0,2)); - int y1 = cvRound(M_(1,0)*x0 + M_(1,1)*y0 + M_(1,2)); - if (y1 >= 0 && y1 < frame1.rows && x1 >= 0 && x1 < frame1.cols) - err += std::abs(intensity(frame1.at >(y1,x1)) - - intensity(frame0.at >(y0,x0))); - } - } - } - - return err; -} - } // namespace videostab } // namespace cv diff --git a/modules/videostab/src/inpainting.cpp b/modules/videostab/src/inpainting.cpp index 7e4a5f38e2..479e5af2ed 100644 --- a/modules/videostab/src/inpainting.cpp +++ b/modules/videostab/src/inpainting.cpp @@ -188,6 +188,37 @@ void ConsistentMosaicInpainter::inpaint(int idx, Mat &frame, Mat &mask) } +static float alignementError( + const Mat &M, const Mat &frame0, const Mat &mask0, const Mat &frame1) +{ + CV_Assert(frame0.type() == CV_8UC3 && frame1.type() == CV_8UC3); + CV_Assert(mask0.type() == CV_8U && mask0.size() == frame0.size()); + CV_Assert(frame0.size() == frame1.size()); + CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); + + Mat_ mask0_(mask0); + Mat_ M_(M); + float err = 0; + + for (int y0 = 0; y0 < frame0.rows; ++y0) + { + for (int x0 = 0; x0 < frame0.cols; ++x0) + { + if (mask0_(y0,x0)) + { + int x1 = cvRound(M_(0,0)*x0 + M_(0,1)*y0 + M_(0,2)); + int y1 = cvRound(M_(1,0)*x0 + M_(1,1)*y0 + M_(1,2)); + if (y1 >= 0 && y1 < frame1.rows && x1 >= 0 && x1 < frame1.cols) + err += std::abs(intensity(frame1.at >(y1,x1)) - + intensity(frame0.at >(y0,x0))); + } + } + } + + return err; +} + + class MotionInpaintBody { public: diff --git a/modules/videostab/src/motion_filtering.cpp b/modules/videostab/src/motion_filtering.cpp deleted file mode 100644 index d7413e5c75..0000000000 --- a/modules/videostab/src/motion_filtering.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "precomp.hpp" -#include "opencv2/videostab/motion_filtering.hpp" -#include "opencv2/videostab/global_motion.hpp" - -using namespace std; - -namespace cv -{ -namespace videostab -{ - -GaussianMotionFilter::GaussianMotionFilter(int radius, float stdev) : radius_(radius) -{ - float sum = 0; - weight_.resize(2*radius_ + 1); - for (int i = -radius_; i <= radius_; ++i) - sum += weight_[radius_ + i] = std::exp(-i*i/(stdev*stdev)); - for (int i = -radius_; i <= radius_; ++i) - weight_[radius_ + i] /= sum; -} - - -Mat GaussianMotionFilter::apply(int idx, vector &motions) const -{ - const Mat &cur = at(idx, motions); - Mat res = Mat::zeros(cur.size(), cur.type()); - float sum = 0.f; - for (int i = std::max(idx - radius_, 0); i <= idx + radius_; ++i) - { - res += weight_[radius_ + i - idx] * getMotion(idx, i, motions); - sum += weight_[radius_ + i - idx]; - } - return res / sum; -} - -} // namespace videostab -} // namespace cv diff --git a/modules/videostab/src/motion_stabilizing.cpp b/modules/videostab/src/motion_stabilizing.cpp new file mode 100644 index 0000000000..047f627ab8 --- /dev/null +++ b/modules/videostab/src/motion_stabilizing.cpp @@ -0,0 +1,214 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "opencv2/videostab/motion_stabilizing.hpp" +#include "opencv2/videostab/global_motion.hpp" + +using namespace std; + +namespace cv +{ +namespace videostab +{ + +GaussianMotionFilter::GaussianMotionFilter(int radius, float stdev) : radius_(radius) +{ + float sum = 0; + weight_.resize(2*radius_ + 1); + for (int i = -radius_; i <= radius_; ++i) + sum += weight_[radius_ + i] = std::exp(-i*i/(stdev*stdev)); + for (int i = -radius_; i <= radius_; ++i) + weight_[radius_ + i] /= sum; +} + + +Mat GaussianMotionFilter::apply(int idx, vector &motions) const +{ + const Mat &cur = at(idx, motions); + Mat res = Mat::zeros(cur.size(), cur.type()); + float sum = 0.f; + for (int i = std::max(idx - radius_, 0); i <= idx + radius_; ++i) + { + res += weight_[radius_ + i - idx] * getMotion(idx, i, motions); + sum += weight_[radius_ + i - idx]; + } + return res / sum; +} + + +static inline int areaSign(Point2f a, Point2f b, Point2f c) +{ + double area = (b-a).cross(c-a); + if (area < -1e-5) return -1; + if (area > 1e-5) return 1; + return 0; +} + + +static inline bool segmentsIntersect(Point2f a, Point2f b, Point2f c, Point2f d) +{ + return areaSign(a,b,c) * areaSign(a,b,d) < 0 && + areaSign(c,d,a) * areaSign(c,d,b) < 0; +} + + +// Checks if rect a (with sides parallel to axis) is inside rect b (arbitrary). +// Rects must be passed in the [(0,0), (w,0), (w,h), (0,h)] order. +static inline bool isRectInside(const Point2f a[4], const Point2f b[4]) +{ + for (int i = 0; i < 4; ++i) + if (b[i].x > a[0].x && b[i].x < a[2].x && b[i].y > a[0].y && b[i].y < a[2].y) + return false; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) + if (segmentsIntersect(a[i], a[(i+1)%4], b[j], b[(j+1)%4])) + return false; + return true; +} + + +static inline bool isGoodMotion(const float M[], float w, float h, float dx, float dy) +{ + Point2f pt[4] = {Point2f(0,0), Point2f(w,0), Point2f(w,h), Point2f(0,h)}; + Point2f Mpt[4]; + + for (int i = 0; i < 4; ++i) + { + Mpt[i].x = M[0]*pt[i].x + M[1]*pt[i].y + M[2]; + Mpt[i].y = M[3]*pt[i].x + M[4]*pt[i].y + M[5]; + } + + pt[0] = Point2f(dx, dy); + pt[1] = Point2f(w - dx, dy); + pt[2] = Point2f(w - dx, h - dy); + pt[3] = Point2f(dx, h - dy); + + return isRectInside(pt, Mpt); +} + + +static inline void relaxMotion(const float M[], float t, float res[]) +{ + res[0] = M[0]*(1.f-t) + t; + res[1] = M[1]*(1.f-t); + res[2] = M[2]*(1.f-t); + res[3] = M[3]*(1.f-t); + res[4] = M[4]*(1.f-t) + t; + res[5] = M[5]*(1.f-t); +} + + +Mat ensureInclusionConstraint(const Mat &M, Size size, float trimRatio) +{ + CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); + + const float w = static_cast(size.width); + const float h = static_cast(size.height); + const float dx = floor(w * trimRatio); + const float dy = floor(h * trimRatio); + const float srcM[6] = + {M.at(0,0), M.at(0,1), M.at(0,2), + M.at(1,0), M.at(1,1), M.at(1,2)}; + + float curM[6]; + float t = 0; + relaxMotion(srcM, t, curM); + if (isGoodMotion(curM, w, h, dx, dy)) + return M; + + float l = 0, r = 1; + while (r - l > 1e-3f) + { + t = (l + r) * 0.5f; + relaxMotion(srcM, t, curM); + if (isGoodMotion(curM, w, h, dx, dy)) + r = t; + else + l = t; + t = r; + relaxMotion(srcM, r, curM); + } + + return (1 - r) * M + r * Mat::eye(3, 3, CV_32F); +} + + +// TODO can be estimated for O(1) time +float estimateOptimalTrimRatio(const Mat &M, Size size) +{ + CV_Assert(M.size() == Size(3,3) && M.type() == CV_32F); + + const float w = static_cast(size.width); + const float h = static_cast(size.height); + Mat_ M_(M); + + Point2f pt[4] = {Point2f(0,0), Point2f(w,0), Point2f(w,h), Point2f(0,h)}; + Point2f Mpt[4]; + + for (int i = 0; i < 4; ++i) + { + Mpt[i].x = M_(0,0)*pt[i].x + M_(0,1)*pt[i].y + M_(0,2); + Mpt[i].y = M_(1,0)*pt[i].x + M_(1,1)*pt[i].y + M_(1,2); + } + + float l = 0, r = 0.5f; + while (r - l > 1e-3f) + { + float t = (l + r) * 0.5f; + float dx = floor(w * t); + float dy = floor(h * t); + pt[0] = Point2f(dx, dy); + pt[1] = Point2f(w - dx, dy); + pt[2] = Point2f(w - dx, h - dy); + pt[3] = Point2f(dx, h - dy); + if (isRectInside(pt, Mpt)) + r = t; + else + l = t; + } + + return r; +} + +} // namespace videostab +} // namespace cv