mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-01-18 22:44:02 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Perf testing: added ERROR_RELATIVE mode to SANITY_CHECK

Browse Source
This commit is contained in:
Andrey Kamaev 2011-12-28 14:59:09 +00:00
parent fb051f78de
commit 58b980ea88
4 changed files with 146 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
modules/core/perf/perf_core_arithm.cpp
View File

@ -52,14 +52,12 @@ PERF_TEST_P__CORE_ARITHM(add, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM(subtract, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM(min, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM(max, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM(absdiff, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_and, TYPICAL_MATS_BITW_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_or, TYPICAL_MATS_BITW_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(bitwise_xor, TYPICAL_MATS_BITW_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(add, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(subtract, TYPICAL_MATS_CORE_ARITHM)
PERF_TEST_P__CORE_ARITHM_SCALAR(absdiff, TYPICAL_MATS_CORE_ARITHM)
#ifdef ANDROID
PERF_TEST(convert, cvRound)
@ -75,3 +73,41 @@ PERF_TEST(convert, cvRound)
}
}
#endif
PERF_TEST_P(Size_MatType, core_arithm__absdiff, TYPICAL_MATS_CORE_ARITHM)
{
Size sz = std::tr1::get<0>(GetParam());
int type = std::tr1::get<1>(GetParam());
cv::Mat a = Mat(sz, type);
cv::Mat b = Mat(sz, type);
cv::Mat c = Mat(sz, type);
declare.in(a, b, WARMUP_RNG)
.out(c);
TEST_CYCLE(100) absdiff(a,b, c);
#if CV_SSE2 //see ticket 1529: absdiff can be without saturation if SSE is enabled
if (CV_MAT_DEPTH(type) != CV_32S)
#endif
SANITY_CHECK(c, 1e-8);
}
PERF_TEST_P(Size_MatType, core_arithm__absdiff__Scalar, TYPICAL_MATS_CORE_ARITHM)
{
Size sz = std::tr1::get<0>(GetParam());
int type = std::tr1::get<1>(GetParam());
cv::Mat a = Mat(sz, type);
cv::Scalar b;
cv::Mat c = Mat(sz, type);
declare.in(a, b, WARMUP_RNG)
.out(c);
TEST_CYCLE(100) absdiff(a,b, c);
#if CV_SSE2 //see ticket 1529: absdiff can be without saturation if SSE is enabled
if (CV_MAT_DEPTH(type) != CV_32S)
#endif
SANITY_CHECK(c, 1e-8);
}

20
modules/core/perf/perf_stat.cpp
View File

@ -21,7 +21,7 @@ PERF_TEST_P( Size_MatType, sum, TYPICAL_MATS )
TEST_CYCLE(100) { s = sum(arr); }
SANITY_CHECK(s, 1e-6);
SANITY_CHECK(s, 1e-6, ERROR_RELATIVE);
}
@ -89,7 +89,7 @@ PERF_TEST_P( Size_MatType_NormType, norm,
TEST_CYCLE(100) { n = norm(src1, normType); }
SANITY_CHECK(n, 1e-5);
SANITY_CHECK(n, 1e-6, ERROR_RELATIVE);
}
@ -116,7 +116,7 @@ PERF_TEST_P( Size_MatType_NormType, norm_mask,
TEST_CYCLE(100) { n = norm(src1, normType, mask); }
SANITY_CHECK(n, 1e-5);
SANITY_CHECK(n, 1e-6, ERROR_RELATIVE);
}
@ -143,7 +143,7 @@ PERF_TEST_P( Size_MatType_NormType, norm2,
TEST_CYCLE(100) { n = norm(src1, src2, normType); }
SANITY_CHECK(n, 1e-5);
SANITY_CHECK(n, 1e-5, ERROR_RELATIVE);
}
@ -171,7 +171,7 @@ PERF_TEST_P( Size_MatType_NormType, norm2_mask,
TEST_CYCLE(100) { n = norm(src1, src2, normType, mask); }
SANITY_CHECK(n, 1e-5);
SANITY_CHECK(n, 1e-5, ERROR_RELATIVE);
}
@ -258,8 +258,8 @@ PERF_TEST_P( Size_MatType_NormType, normalize_32f,
declare.in(src, WARMUP_RNG).out(dst);
TEST_CYCLE(100) { normalize(src, dst, alpha, 0., normType, CV_32F); }
SANITY_CHECK(dst, 1e-6);
SANITY_CHECK(dst, 1e-6, ERROR_RELATIVE);
}
@ -404,7 +404,7 @@ PERF_TEST_P( Size_MatType_ROp, reduceR,
TEST_CYCLE(100) { reduce(src, vec, 0, reduceOp, ddepth); }
SANITY_CHECK(vec);
SANITY_CHECK(vec, 1);
}
/*
@ -431,6 +431,6 @@ PERF_TEST_P( Size_MatType_ROp, reduceC,
declare.in(src, WARMUP_RNG);
TEST_CYCLE(100) { reduce(src, vec, 1, reduceOp, ddepth); }
SANITY_CHECK(vec);
SANITY_CHECK(vec, 1);
}

14
modules/ts/include/opencv2/ts/ts_perf.hpp
View File

@ -149,13 +149,19 @@ CV_ENUM(MatDepth, CV_8U, CV_8S, CV_16U, CV_16S, CV_32S, CV_32F, CV_64F, CV_USRTY
/*****************************************************************************************\
* Regression control utility for performance testing *
\*****************************************************************************************/
enum ERROR_TYPE
{
ERROR_ABSOLUTE = 0,
ERROR_RELATIVE = 1
};
class CV_EXPORTS Regression
{
public:
static Regression& add(const std::string& name, cv::InputArray array, double eps = DBL_EPSILON);
static Regression& add(const std::string& name, cv::InputArray array, double eps = DBL_EPSILON, ERROR_TYPE err = ERROR_ABSOLUTE);
static void Init(const std::string& testSuitName, const std::string& ext = ".xml");
Regression& operator() (const std::string& name, cv::InputArray array, double eps = DBL_EPSILON);
Regression& operator() (const std::string& name, cv::InputArray array, double eps = DBL_EPSILON, ERROR_TYPE err = ERROR_ABSOLUTE);
private:
static Regression& instance();
@ -181,8 +187,8 @@ private:
void init(const std::string& testSuitName, const std::string& ext);
void write(cv::InputArray array);
void write(cv::Mat m);
void verify(cv::FileNode node, cv::InputArray array, double eps);
void verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname);
void verify(cv::FileNode node, cv::InputArray array, double eps, ERROR_TYPE err);
void verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname, ERROR_TYPE err);
};
#define SANITY_CHECK(array, ...) ::perf::Regression::add(#array, array , ## __VA_ARGS__)

106
modules/ts/src/ts_perf.cpp
View File

@ -43,9 +43,9 @@ Regression& Regression::instance()
return single;
}
Regression& Regression::add(const std::string& name, cv::InputArray array, double eps)
Regression& Regression::add(const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err)
{
return instance()(name, array, eps);
return instance()(name, array, eps, err);
}
void Regression::Init(const std::string& testSuitName, const std::string& ext)
@ -202,13 +202,25 @@ void Regression::write(cv::Mat m)
write() << "val" << getElem(m, y, x, cn) << "}";
}
void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname)
static double evalEps(double expected, double actual, double _eps, ERROR_TYPE err)
{
if (err == ERROR_ABSOLUTE)
return _eps;
else if (err == ERROR_RELATIVE)
return std::max(std::abs(expected), std::abs(actual)) * err;
return 0;
}
void Regression::verify(cv::FileNode node, cv::Mat actual, double _eps, std::string argname, ERROR_TYPE err)
{
double actual_min, actual_max;
cv::minMaxLoc(actual, &actual_min, &actual_max);
double eps = evalEps((double)node["min"], actual_min, _eps, err);
ASSERT_NEAR((double)node["min"], actual_min, eps)
<< " " << argname << " has unexpected minimal value";
eps = evalEps((double)node["max"], actual_max, _eps, err);
ASSERT_NEAR((double)node["max"], actual_max, eps)
<< " " << argname << " has unexpected maximal value";
@ -218,6 +230,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri
<< " " << argname << " has unexpected number of columns";
ASSERT_EQ((int)last["y"], actual.rows - 1)
<< " " << argname << " has unexpected number of rows";
eps = evalEps((double)last["val"], actualLast, _eps, err);
ASSERT_NEAR((double)last["val"], actualLast, eps)
<< " " << argname << " has unexpected value of last element";
@ -225,6 +239,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri
int x1 = rng1["x"];
int y1 = rng1["y"];
int cn1 = rng1["cn"];
eps = evalEps((double)rng1["val"], getElem(actual, y1, x1, cn1), _eps, err);
ASSERT_NEAR((double)rng1["val"], getElem(actual, y1, x1, cn1), eps)
<< " " << argname << " has unexpected value of ["<< x1 << ":" << y1 << ":" << cn1 <<"] element";
@ -232,6 +248,8 @@ void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::stri
int x2 = rng2["x"];
int y2 = rng2["y"];
int cn2 = rng2["cn"];
eps = evalEps((double)rng2["val"], getElem(actual, y2, x2, cn2), _eps, err);
ASSERT_NEAR((double)rng2["val"], getElem(actual, y2, x2, cn2), eps)
<< " " << argname << " has unexpected value of ["<< x2 << ":" << y2 << ":" << cn2 <<"] element";
}
@ -263,7 +281,31 @@ void Regression::write(cv::InputArray array)
}
}
void Regression::verify(cv::FileNode node, cv::InputArray array, double eps)
static int countViolations(const cv::Mat& expected, const cv::Mat& actual, const cv::Mat& diff, double eps, double* max_violation = 0, double* max_allowed = 0)
{
cv::Mat diff64f;
diff.reshape(1).convertTo(diff64f, CV_64F);
cv::Mat expected_abs = cv::abs(expected.reshape(1));
cv::Mat actual_abs = cv::abs(actual.reshape(1));
cv::Mat maximum, mask;
cv::max(expected_abs, actual_abs, maximum);
cv::multiply(maximum, cv::Vec<double, 1>(eps), maximum, CV_64F);
cv::compare(diff64f, maximum, mask, cv::CMP_GT);
int v = cv::countNonZero(mask);
if (v > 0 && max_violation != 0 && max_allowed != 0)
{
int loc[10];
cv::minMaxIdx(maximum, 0, max_allowed, 0, loc, mask);
*max_violation = diff64f.at<double>(loc[1], loc[0]);
}
return v;
}
void Regression::verify(cv::FileNode node, cv::InputArray array, double eps, ERROR_TYPE err)
{
ASSERT_EQ((int)node["kind"], array.kind()) << " Argument \"" << node.name() << "\" has unexpected kind";
ASSERT_EQ((int)node["type"], array.type()) << " Argument \"" << node.name() << "\" has unexpected type";
@ -280,7 +322,7 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps)
{
ASSERT_LE((size_t)26, actual.total() * (size_t)actual.channels())
<< " \"" << node.name() << "[" << idx << "]\" has unexpected number of elements";
verify(node, actual, eps, cv::format("%s[%d]", node.name().c_str(), idx));
verify(node, actual, eps, cv::format("%s[%d]", node.name().c_str(), idx), err);
}
else
{
@ -292,12 +334,26 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps)
cv::Mat diff;
cv::absdiff(expected, actual, diff);
if (!cv::checkRange(diff, true, 0, 0, eps))
if (err == ERROR_ABSOLUTE)
{
double max;
cv::minMaxLoc(diff, 0, &max);
FAIL() << " Difference (=" << max << ") between argument \""
<< node.name() << "[" << idx << "]\" and expected value is bugger than " << eps;
if (!cv::checkRange(diff, true, 0, 0, eps))
{
double max;
cv::minMaxLoc(diff.reshape(1), 0, &max);
FAIL() << " Absolute difference (=" << max << ") between argument \""
<< node.name() << "[" << idx << "]\" and expected value is bugger than " << eps;
}
}
else if (err == ERROR_RELATIVE)
{
double maxv, maxa;
int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa);
if (violations > 0)
{
FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \""
<< node.name() << "[" << idx << "]\" and expected value is bugger than " << eps << " in " << violations << " points";
}
}
}
}
@ -307,7 +363,7 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps)
{
ASSERT_LE((size_t)26, array.total() * (size_t)array.channels())
<< " Argument \"" << node.name() << "\" has unexpected number of elements";
verify(node, array.getMat(), eps, "Argument " + node.name());
verify(node, array.getMat(), eps, "Argument " + node.name(), err);
}
else
{
@ -320,18 +376,32 @@ void Regression::verify(cv::FileNode node, cv::InputArray array, double eps)
cv::Mat diff;
cv::absdiff(expected, actual, diff);
if (!cv::checkRange(diff, true, 0, 0, eps))
if (err == ERROR_ABSOLUTE)
{
double max;
cv::minMaxLoc(diff, 0, &max);
FAIL() << " Difference (=" << max << ") between argument \"" << node.name()
<< "\" and expected value is bugger than " << eps;
if (!cv::checkRange(diff, true, 0, 0, eps))
{
double max;
cv::minMaxLoc(diff.reshape(1), 0, &max);
FAIL() << " Difference (=" << max << ") between argument \"" << node.name()
<< "\" and expected value is bugger than " << eps;
}
}
else if (err == ERROR_RELATIVE)
{
double maxv, maxa;
int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa);
if (violations > 0)
{
FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \"" << node.name()
<< "\" and expected value is bugger than " << eps << " in " << violations << " points";
}
}
}
}
}
Regression& Regression::operator() (const std::string& name, cv::InputArray array, double eps)
Regression& Regression::operator() (const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err)
{
std::string nodename = getCurrentTestNodeName();
@ -356,7 +426,7 @@ Regression& Regression::operator() (const std::string& name, cv::InputArray arra
if (!this_arg.isMap())
ADD_FAILURE() << " No regression data for " << name << " argument";
else
verify(this_arg, array, eps);
verify(this_arg, array, eps, err);
}
return *this;
}