mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-01-18 22:44:02 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #15313 from andrey-golubev:map_subst_to_pattern

Browse Source 
G-API: add transformation logic to GCompiler

* Introduce transformation logic to GCOmpiler

* Remove partialOk() method

* Fix minor issues

* Refactor code according to code review

1. Re-design matchPatternToSubstitute logic
2. Update transformations order
3. Replace check_transformations pass with a
   one time check in GCompiler ctor

* Revert unused nodes handling in pattern matching

* Address minor code review issues

* Address code review comments:

1) Fix some mistakes
2) Add new tests for endless loops
3) Update GCompiler's transformations logic

* Simplify GCompiler check for endless loops

1. Simplify transformations endless loops check:
 - Original idea wasn't a full solution
 - Need to develop a good method (heuristic?) to find loops
   in general case (TODO)
2. Remove irrelevant Endless Loops tests
3. Add new "bad arg" tests and unit tests

* Update comments
This commit is contained in:
Andrey Golubev 2019-09-25 18:19:45 +03:00 committed by Alexander Alekhin
parent 2fef9bc1d8
commit 9f4f9000bc
9 changed files with 986 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/gapi/CMakeLists.txt
View File

@ -62,7 +62,9 @@ set(gapi_srcs
src/compiler/passes/meta.cpp
src/compiler/passes/kernels.cpp
src/compiler/passes/exec.cpp
src/compiler/passes/transformations.cpp
src/compiler/passes/pattern_matching.cpp
src/compiler/passes/perform_substitution.cpp
# Executor
src/executor/gexecutor.cpp

113
modules/gapi/src/compiler/gcompiler.cpp
View File

@ -29,6 +29,7 @@
#include "compiler/gcompiler.hpp"
#include "compiler/gcompiled_priv.hpp"
#include "compiler/passes/passes.hpp"
#include "compiler/passes/pattern_matching.hpp"
#include "executor/gexecutor.hpp"
#include "backends/common/gbackend.hpp"
@ -103,6 +104,84 @@ namespace
return result;
}
// Creates ADE graph from input/output proto args
std::unique_ptr<ade::Graph> makeGraph(const cv::GProtoArgs &ins, const cv::GProtoArgs &outs) {
std::unique_ptr<ade::Graph> pG(new ade::Graph);
ade::Graph& g = *pG;
cv::gimpl::GModel::Graph gm(g);
cv::gimpl::GModel::init(gm);
cv::gimpl::GModelBuilder builder(g);
auto proto_slots = builder.put(ins, outs);
// Store Computation's protocol in metadata
cv::gimpl::Protocol p;
std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots;
gm.metadata().set(p);
return pG;
}
using adeGraphs = std::vector<std::unique_ptr<ade::Graph>>;
// Creates ADE graphs (patterns and substitutes) from pkg's transformations
void makeTransformationGraphs(const cv::gapi::GKernelPackage& pkg,
adeGraphs& patterns,
adeGraphs& substitutes) {
const auto& transforms = pkg.get_transformations();
const auto size = transforms.size();
if (0u == size) return;
// pre-generate all required graphs
patterns.resize(size);
substitutes.resize(size);
for (auto it : ade::util::zip(ade::util::toRange(transforms),
ade::util::toRange(patterns),
ade::util::toRange(substitutes))) {
const auto& t = std::get<0>(it);
auto& p = std::get<1>(it);
auto& s = std::get<2>(it);
auto pattern_comp = t.pattern();
p = makeGraph(pattern_comp.priv().m_ins, pattern_comp.priv().m_outs);
auto substitute_comp = t.substitute();
s = makeGraph(substitute_comp.priv().m_ins, substitute_comp.priv().m_outs);
}
}
void checkTransformations(const cv::gapi::GKernelPackage& pkg,
const adeGraphs& patterns,
const adeGraphs& substitutes) {
const auto& transforms = pkg.get_transformations();
const auto size = transforms.size();
if (0u == size) return;
GAPI_Assert(size == patterns.size());
GAPI_Assert(size == substitutes.size());
const auto empty = [] (const cv::gimpl::SubgraphMatch& m) {
return m.inputDataNodes.empty() && m.startOpNodes.empty()
&& m.finishOpNodes.empty() && m.outputDataNodes.empty()
&& m.inputTestDataNodes.empty() && m.outputTestDataNodes.empty();
};
// **FIXME**: verify other types of endless loops. now, only checking if pattern exists in
// substitute within __the same__ transformation
for (size_t i = 0; i < size; ++i) {
const auto& p = patterns[i];
const auto& s = substitutes[i];
auto matchInSubstitute = cv::gimpl::findMatches(*p, *s);
if (!empty(matchInSubstitute)) {
std::stringstream ss;
ss << "Error: (in transformation with description: '"
<< transforms[i].description
<< "') pattern is detected inside substitute";
throw std::runtime_error(ss.str());
}
}
}
} // anonymous namespace
@ -129,10 +208,26 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c,
// NN backends (present here via network package) always add their
// inference kernels via auxiliary...()
// sanity check transformations
{
adeGraphs patterns, substitutes;
// FIXME: returning vectors of unique_ptrs from makeTransformationGraphs results in
// compile error (at least) on GCC 9 with usage of copy-ctor of std::unique_ptr, so
// using initialization by lvalue reference instead
makeTransformationGraphs(m_all_kernels, patterns, substitutes);
checkTransformations(m_all_kernels, patterns, substitutes);
// NB: saving generated patterns to m_all_patterns to be used later in passes
m_all_patterns = std::move(patterns);
}
auto dump_path = getGraphDumpDirectory(m_args);
m_e.addPassStage("init");
m_e.addPass("init", "check_cycles", ade::passes::CheckCycles());
m_e.addPass("init", "apply_transformations",
std::bind(passes::applyTransformations, _1, std::cref(m_all_kernels),
std::cref(m_all_patterns))); // Note: and re-using patterns here
m_e.addPass("init", "expand_kernels",
std::bind(passes::expandKernels, _1,
m_all_kernels)); // NB: package is copied
@ -255,23 +350,7 @@ cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::generateGraph()
{
validateInputMeta();
validateOutProtoArgs();
// Generate ADE graph from expression-based computation
std::unique_ptr<ade::Graph> pG(new ade::Graph);
ade::Graph& g = *pG;
GModel::Graph gm(g);
cv::gimpl::GModel::init(gm);
cv::gimpl::GModelBuilder builder(g);
auto proto_slots = builder.put(m_c.priv().m_ins, m_c.priv().m_outs);
GAPI_LOG_INFO(NULL, "Generated graph: " << g.nodes().size() << " nodes" << std::endl);
// Store Computation's protocol in metadata
Protocol p;
std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots;
gm.metadata().set(p);
return pG;
return makeGraph(m_c.priv().m_ins, m_c.priv().m_outs);
}
void cv::gimpl::GCompiler::runPasses(ade::Graph &g)

2
modules/gapi/src/compiler/gcompiler.hpp
View File

@ -29,6 +29,8 @@ class GAPI_EXPORTS GCompiler
cv::gapi::GKernelPackage m_all_kernels;
cv::gapi::GNetPackage m_all_networks;
std::vector<std::unique_ptr<ade::Graph>> m_all_patterns; // built patterns from transformations
void validateInputMeta();
void validateOutProtoArgs();

4
modules/gapi/src/compiler/passes/passes.hpp
View File

@ -59,6 +59,10 @@ void resolveKernels(ade::passes::PassContext &ctx,
void fuseIslands(ade::passes::PassContext &ctx);
void syncIslandTags(ade::passes::PassContext &ctx);
void applyTransformations(ade::passes::PassContext &ctx,
const gapi::GKernelPackage &pkg,
const std::vector<std::unique_ptr<ade::Graph>> &preGeneratedPatterns);
}} // namespace gimpl::passes
} // namespace cv

9
modules/gapi/src/compiler/passes/pattern_matching.cpp
View File

@ -54,8 +54,7 @@ bool compareDataNodes(const ade::NodeHandle& first, const std::vector<std::size_
"shall be NodeType::DATA!");
}
if (firstMeta.get<cv::gimpl::Data>().shape !=
secondMeta.get<cv::gimpl::Data>().shape) {
if (firstMeta.get<cv::gimpl::Data>().shape != secondMeta.get<cv::gimpl::Data>().shape) {
return false;
}
@ -180,7 +179,7 @@ inline bool IS_ENDPOINT(const ade::NodeHandle& nh){
// Try to rely on the nh Data::Storage::OUTPUT
return nh->outEdges().empty();
}
}
} // anonymous namespace
// Routine relies on the logic that 1 DATA node may have only 1 input edge.
cv::gimpl::SubgraphMatch
@ -509,7 +508,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph,
// Create vector with the correctly ordered IN data nodes in the test subgraph
std::vector<ade::NodeHandle> inputTestDataNodes;
for (const auto& patternInNode : patternInputDataNodes) {
inputTestDataNodes.push_back(inputApiMatch[patternInNode]);
inputTestDataNodes.push_back(inputApiMatch.at(patternInNode));
}
// Traversing current result for ending OPs
@ -560,7 +559,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph,
// Create vector with the correctly ordered OUT data nodes in the test subgraph
std::vector<ade::NodeHandle> outputTestDataNodes;
for (const auto& patternOutNode : patternOutputDataNodes) {
outputTestDataNodes.push_back(outputApiMatch[patternOutNode]);
outputTestDataNodes.push_back(outputApiMatch.at(patternOutNode));
}
SubgraphMatch subgraph;

6
modules/gapi/src/compiler/passes/pattern_matching.hpp
View File

@ -41,7 +41,6 @@ namespace gimpl {
return !inputDataNodes.empty() && !startOpNodes.empty()
&& !finishOpNodes.empty() && !outputDataNodes.empty()
&& !inputTestDataNodes.empty() && !outputTestDataNodes.empty();
}
S nodes() const {
@ -93,6 +92,11 @@ namespace gimpl {
GAPI_EXPORTS SubgraphMatch findMatches(const cv::gimpl::GModel::Graph& patternGraph,
const cv::gimpl::GModel::Graph& compGraph);
GAPI_EXPORTS void performSubstitution(cv::gimpl::GModel::Graph& graph,
const cv::gimpl::Protocol& patternP,
const cv::gimpl::Protocol& substituteP,
const cv::gimpl::SubgraphMatch& patternToGraphMatch);
} //namespace gimpl
} //namespace cv
#endif // OPENCV_GAPI_PATTERN_MATCHING_HPP

85
modules/gapi/src/compiler/passes/perform_substitution.cpp Normal file
View File

@ -0,0 +1,85 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
#include "pattern_matching.hpp"
#include "ade/util/zip_range.hpp"
namespace cv { namespace gimpl {
namespace {
using Graph = GModel::Graph;
template<typename Iterator>
ade::NodeHandle getNh(Iterator it) { return *it; }
template<>
ade::NodeHandle getNh(SubgraphMatch::M::const_iterator it) { return it->second; }
template<typename Container>
void erase(Graph& g, const Container& c)
{
for (auto first = c.begin(); first != c.end(); ++first) {
ade::NodeHandle node = getNh(first);
if (node == nullptr) continue; // some nodes might already be erased
g.erase(node);
}
}
} // anonymous namespace
void performSubstitution(GModel::Graph& graph,
const Protocol& patternP,
const Protocol& substituteP,
const SubgraphMatch& patternToGraphMatch)
{
// 1. substitute input nodes
const auto& patternIns = patternP.in_nhs;
const auto& substituteIns = substituteP.in_nhs;
for (auto it : ade::util::zip(ade::util::toRange(patternIns),
ade::util::toRange(substituteIns))) {
// Note: we don't replace input DATA nodes here, only redirect their output edges
const auto& patternDataNode = std::get<0>(it);
const auto& substituteDataNode = std::get<1>(it);
const auto& graphDataNode = patternToGraphMatch.inputDataNodes.at(patternDataNode);
GModel::redirectReaders(graph, substituteDataNode, graphDataNode);
}
// 2. substitute output nodes
const auto& patternOuts = patternP.out_nhs;
const auto& substituteOuts = substituteP.out_nhs;
for (auto it : ade::util::zip(ade::util::toRange(patternOuts),
ade::util::toRange(substituteOuts))) {
// Note: we don't replace output DATA nodes here, only redirect their input edges
const auto& patternDataNode = std::get<0>(it);
const auto& substituteDataNode = std::get<1>(it);
const auto& graphDataNode = patternToGraphMatch.outputDataNodes.at(patternDataNode);
// delete existing edges (otherwise we cannot redirect)
for (auto e : graphDataNode->inEdges()) {
graph.erase(e);
}
GModel::redirectWriter(graph, substituteDataNode, graphDataNode);
}
// 3. erase redundant nodes:
// erase input data nodes of __substitute__
erase(graph, substituteIns);
// erase old start OP nodes of __main graph__
erase(graph, patternToGraphMatch.startOpNodes);
// erase old internal nodes of __main graph__
erase(graph, patternToGraphMatch.internalLayers);
// erase old finish OP nodes of __main graph__
erase(graph, patternToGraphMatch.finishOpNodes);
// erase output data nodes of __substitute__
erase(graph, substituteOuts);
}
} // namespace gimpl
} // namespace cv

139
modules/gapi/src/compiler/passes/transformations.cpp Normal file
View File

@ -0,0 +1,139 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
#include "precomp.hpp"
#include <ade/util/zip_range.hpp>
#include <ade/graph.hpp>
#include "api/gcomputation_priv.hpp"
#include "compiler/gmodel.hpp"
#include "compiler/gmodelbuilder.hpp"
#include "compiler/passes/passes.hpp"
#include "compiler/passes/pattern_matching.hpp"
#include <sstream>
namespace cv { namespace gimpl { namespace passes {
namespace
{
using Graph = GModel::Graph;
using Metadata = typename Graph::CMetadataT;
// Checks pairs of {pattern node, substitute node} and asserts if there are any incompatibilities
void checkDataNodes(const Graph& pattern,
const Graph& substitute,
const std::vector<ade::NodeHandle>& patternNodes,
const std::vector<ade::NodeHandle>& substituteNodes)
{
for (auto it : ade::util::zip(patternNodes, substituteNodes)) {
auto pNodeMeta = pattern.metadata(std::get<0>(it));
auto sNodeMeta = substitute.metadata(std::get<1>(it));
GAPI_Assert(pNodeMeta.get<NodeType>().t == NodeType::DATA);
GAPI_Assert(pNodeMeta.get<NodeType>().t == sNodeMeta.get<NodeType>().t);
GAPI_Assert(pNodeMeta.get<Data>().shape == sNodeMeta.get<Data>().shape);
}
}
// Checks compatibility of pattern and substitute graphs based on in/out nodes
void checkCompatibility(const Graph& pattern,
const Graph& substitute,
const Protocol& patternP,
const Protocol& substituteP)
{
const auto& patternDataInputs = patternP.in_nhs;
const auto& patternDataOutputs = patternP.out_nhs;
const auto& substituteDataInputs = substituteP.in_nhs;
const auto& substituteDataOutputs = substituteP.out_nhs;
// number of data nodes must be the same
GAPI_Assert(patternDataInputs.size() == substituteDataInputs.size());
GAPI_Assert(patternDataOutputs.size() == substituteDataOutputs.size());
// for each pattern input node, verify a corresponding substitute input node
checkDataNodes(pattern, substitute, patternDataInputs, substituteDataInputs);
// for each pattern output node, verify a corresponding substitute output node
checkDataNodes(pattern, substitute, patternDataOutputs, substituteDataOutputs);
}
// Tries to substitute __single__ pattern with substitute in the given graph
bool tryToSubstitute(ade::Graph& main,
const std::unique_ptr<ade::Graph>& patternG,
const cv::GComputation& substitute)
{
GModel::Graph gm(main);
// 1. find a pattern in main graph
auto match1 = findMatches(*patternG, gm);
if (!match1.ok()) {
return false;
}
// 2. build substitute graph inside the main graph
cv::gimpl::GModelBuilder builder(main);
const auto& proto_slots = builder.put(substitute.priv().m_ins, substitute.priv().m_outs);
Protocol substituteP;
std::tie(substituteP.inputs, substituteP.outputs, substituteP.in_nhs, substituteP.out_nhs) =
proto_slots;
const Protocol& patternP = GModel::Graph(*patternG).metadata().get<Protocol>();
// 3. check that pattern and substitute are compatible
// FIXME: in theory, we should always have compatible pattern/substitute. if not, we're in
// half-completed state where some transformations are already applied - what can we do
// to handle the situation better? -- use transactional API as in fuse_islands pass?
checkCompatibility(*patternG, gm, patternP, substituteP);
// 4. make substitution
performSubstitution(gm, patternP, substituteP, match1);
return true;
}
} // anonymous namespace
void applyTransformations(ade::passes::PassContext& ctx,
const gapi::GKernelPackage& pkg,
const std::vector<std::unique_ptr<ade::Graph>>& patterns)
{
const auto& transforms = pkg.get_transformations();
const auto size = transforms.size();
if (0u == size) return;
// Note: patterns are already generated at this point
GAPI_Assert(patterns.size() == transforms.size());
// transform as long as it is possible
bool canTransform = true;
while (canTransform)
{
canTransform = false;
// iterate through every transformation and try to transform graph parts
for (auto it : ade::util::zip(ade::util::toRange(transforms), ade::util::toRange(patterns)))
{
const auto& t = std::get<0>(it);
auto& pattern = std::get<1>(it); // Note: using pre-created graphs
GAPI_Assert(nullptr != pattern);
// if transformation is successful (pattern found and substituted), it is possible that
// other transformations will also be successful, so set canTransform to the returned
// value from tryToSubstitute
canTransform = tryToSubstitute(ctx.graph, pattern, t.substitute());
// Note: apply the __same__ substitution as many times as possible and only after go to
// the next one. BUT it can happen that after applying some substitution, some
// _previous_ patterns will also be found and these will be applied first
if (canTransform) {
break;
}
}
}
}
} // namespace passes
} // namespace gimpl
} // namespace cv

649
modules/gapi/test/internal/gapi_int_perform_substitution_test.cpp Normal file
View File

@ -0,0 +1,649 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
#include "../test_precomp.hpp"
#include <stdexcept>
#include <opencv2/gapi/gtransform.hpp>
#include <opencv2/gapi/cpu/core.hpp>
#include <opencv2/gapi/cpu/imgproc.hpp>
#include "compiler/gmodel.hpp"
#include "compiler/gmodel_priv.hpp"
#include "api/gcomputation_priv.hpp"
#include "compiler/gcompiler.hpp"
#include "compiler/gmodelbuilder.hpp"
#include "compiler/passes/passes.hpp"
#include "compiler/passes/pattern_matching.hpp"
#include "../common/gapi_tests_common.hpp"
#include "logger.hpp"
namespace opencv_test
{
// --------------------------------------------------------------------------------------
// Accuracy integration tests (GComputation-level)
namespace {
// FIXME: replace listener with something better (e.g. check graph via GModel?)
// custom "listener" to check what kernels are called within the test
struct KernelListener { std::map<std::string, size_t> counts; };
KernelListener& getListener() {
static KernelListener l;
return l;
}
using CompCreator = std::function<cv::GComputation()>;
using CompileArgsCreator = std::function<cv::GCompileArgs()>;
using Verifier = std::function<void(KernelListener)>;
} // anonymous namespace
// Custom kernels && transformations below:
G_TYPED_KERNEL(MyNV12toBGR, <GMat(GMat, GMat)>, "test.my_nv12_to_bgr") {
static GMatDesc outMeta(GMatDesc in_y, GMatDesc in_uv) {
return cv::gapi::imgproc::GNV12toBGR::outMeta(in_y, in_uv);
}
};
GAPI_OCV_KERNEL(MyNV12toBGRImpl, MyNV12toBGR)
{
static void run(const cv::Mat& in_y, const cv::Mat& in_uv, cv::Mat &out)
{
getListener().counts[MyNV12toBGR::id()]++;
cv::cvtColorTwoPlane(in_y, in_uv, out, cv::COLOR_YUV2BGR_NV12);
}
};
G_TYPED_KERNEL(MyPlanarResize, <GMatP(GMatP, Size, int)>, "test.my_planar_resize") {
static GMatDesc outMeta(GMatDesc in, Size sz, int interp) {
return cv::gapi::core::GResizeP::outMeta(in, sz, interp);
}
};
GAPI_OCV_KERNEL(MyPlanarResizeImpl, MyPlanarResize) {
static void run(const cv::Mat& in, cv::Size out_sz, int interp, cv::Mat &out)
{
getListener().counts[MyPlanarResize::id()]++;
int inH = in.rows / 3;
int inW = in.cols;
int outH = out.rows / 3;
int outW = out.cols;
for (int i = 0; i < 3; i++) {
auto in_plane = in(cv::Rect(0, i*inH, inW, inH));
auto out_plane = out(cv::Rect(0, i*outH, outW, outH));
cv::resize(in_plane, out_plane, out_sz, 0, 0, interp);
}
}
};
G_TYPED_KERNEL(MyInterleavedResize, <GMat(GMat, Size, int)>, "test.my_interleaved_resize") {
static GMatDesc outMeta(GMatDesc in, Size sz, int interp) {
return cv::gapi::core::GResize::outMeta(in, sz, 0.0, 0.0, interp);
}
};
GAPI_OCV_KERNEL(MyInterleavedResizeImpl, MyInterleavedResize) {
static void run(const cv::Mat& in, cv::Size out_sz, int interp, cv::Mat &out)
{
getListener().counts[MyInterleavedResize::id()]++;
cv::resize(in, out, out_sz, 0.0, 0.0, interp);
}
};
G_TYPED_KERNEL(MyToNCHW, <GMatP(GMat)>, "test.my_to_nchw") {
static GMatDesc outMeta(GMatDesc in) {
GAPI_Assert(in.depth == CV_8U);
GAPI_Assert(in.chan == 3);
GAPI_Assert(in.planar == false);
return in.asPlanar();
}
};
GAPI_OCV_KERNEL(MyToNCHWImpl, MyToNCHW) {
static void run(const cv::Mat& in, cv::Mat& out)
{
getListener().counts[MyToNCHW::id()]++;
auto sz = in.size();
auto w = sz.width;
auto h = sz.height;
cv::Mat ins[3] = {};
cv::split(in, ins);
for (int i = 0; i < 3; i++) {
auto in_plane = ins[i];
auto out_plane = out(cv::Rect(0, i*h, w, h));
in_plane.copyTo(out_plane);
}
}
};
using GMat4 = std::tuple<GMat, GMat, GMat, GMat>;
G_TYPED_KERNEL_M(MySplit4, <GMat4(GMat)>, "test.my_split4") {
static std::tuple<GMatDesc, GMatDesc, GMatDesc, GMatDesc> outMeta(GMatDesc in) {
const auto out_depth = in.depth;
const auto out_desc = in.withType(out_depth, 1);
return std::make_tuple(out_desc, out_desc, out_desc, out_desc);
}
};
GAPI_OCV_KERNEL(MySplit4Impl, MySplit4) {
static void run(const cv::Mat& in, cv::Mat& out1, cv::Mat& out2, cv::Mat& out3, cv::Mat& out4)
{
getListener().counts[MySplit4::id()]++;
cv::Mat outs[] = { out1, out2, out3, out4 };
cv::split(in, outs);
}
};
GAPI_TRANSFORM(NV12Transform, <cv::GMat(cv::GMat, cv::GMat)>, "test.nv12_transform")
{
static cv::GMat pattern(const cv::GMat& y, const cv::GMat& uv)
{
GMat out = cv::gapi::NV12toBGR(y, uv);
return out;
}
static cv::GMat substitute(const cv::GMat& y, const cv::GMat& uv)
{
GMat out = MyNV12toBGR::on(y, uv);
return out;
}
};
GAPI_TRANSFORM(ResizeTransform, <cv::GMat(cv::GMat)>, "3 x Resize -> Interleaved Resize")
{
static cv::GMat pattern(const cv::GMat& in)
{
GMat b, g, r;
std::tie(b, g, r) = cv::gapi::split3(in);
const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1,
cv::Size(100, 100), 0, 0, cv::INTER_AREA);
return cv::gapi::merge3(resize(b), resize(g), resize(r));
}
static cv::GMat substitute(const cv::GMat& in)
{
return MyInterleavedResize::on(in, cv::Size(100, 100), cv::INTER_AREA);
}
};
GAPI_TRANSFORM(ResizeTransformToCustom, <cv::GMat(cv::GMat)>, "Resize -> Custom Resize")
{
static cv::GMat pattern(const cv::GMat& in)
{
return cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_AREA);
}
static cv::GMat substitute(const cv::GMat& in)
{
return MyInterleavedResize::on(in, cv::Size(100, 100), cv::INTER_AREA);
}
};
GAPI_TRANSFORM(ChainTransform1, <GMatP(GMat)>, "Resize + toNCHW -> toNCHW + PlanarResize")
{
static GMatP pattern(const cv::GMat& in)
{
return MyToNCHW::on(cv::gapi::resize(in, cv::Size(60, 60)));
}
static GMatP substitute(const cv::GMat& in)
{
return MyPlanarResize::on(MyToNCHW::on(in), cv::Size(60, 60), cv::INTER_LINEAR);
}
};
GAPI_TRANSFORM(ChainTransform2, <GMatP(GMat, GMat)>, "NV12toBGR + toNCHW -> NV12toBGRp")
{
static GMatP pattern(const GMat& y, const GMat& uv)
{
return MyToNCHW::on(MyNV12toBGR::on(y, uv));
}
static GMatP substitute(const GMat& y, const GMat& uv)
{
return cv::gapi::NV12toBGRp(y, uv);
}
};
GAPI_TRANSFORM(Split4Transform, <GMat4(GMat)>, "Split4 -> Custom Split4")
{
static GMat4 pattern(const GMat& in)
{
return cv::gapi::split4(in);
}
static GMat4 substitute(const GMat& in)
{
return MySplit4::on(in);
}
};
GAPI_TRANSFORM(Split4Merge3Transform, <GMat(GMat)>, "Split4 + Merge3 -> Custom Split4 + Merge3")
{
static GMat pattern(const GMat& in)
{
GMat tmp1, tmp2, tmp3, unused;
std::tie(tmp1, tmp2, tmp3, unused) = cv::gapi::split4(in);
return cv::gapi::merge3(tmp1, tmp2, tmp3);
}
static GMat substitute(const GMat& in)
{
GMat tmp1, tmp2, tmp3, unused;
std::tie(tmp1, tmp2, tmp3, unused) = MySplit4::on(in);
return cv::gapi::merge3(tmp1, tmp2, tmp3);
}
};
GAPI_TRANSFORM(Merge4Split4Transform, <GMat4(GMat, GMat, GMat, GMat)>,
"Merge4 + Split4 -> Merge4 + Custom Split4")
{
static GMat4 pattern(const GMat& in1, const GMat& in2, const GMat& in3,
const GMat& in4)
{
return cv::gapi::split4(cv::gapi::merge4(in1, in2, in3, in4));
}
static GMat4 substitute(const GMat& in1, const GMat& in2, const GMat& in3,
const GMat& in4)
{
return MySplit4::on(cv::gapi::merge4(in1, in2, in3, in4));
}
};
// --------------------------------------------------------------------------------------
// Integration tests
TEST(PatternMatchingIntegrationBasic, OneTransformationApplied)
{
cv::Size in_sz(640, 480);
cv::Mat input(in_sz, CV_8UC3);
cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100));
cv::Mat orig_graph_output, transformed_graph_output;
auto orig_args = cv::compile_args();
auto transform_args = cv::compile_args(
cv::gapi::kernels<MyInterleavedResizeImpl, ResizeTransform>());
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
const auto make_computation = [] () {
GMat in;
GMat b, g, r;
std::tie(b, g, r) = cv::gapi::split3(in);
const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1,
cv::Size(100, 100), 0, 0, cv::INTER_AREA);
GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r));
return cv::GComputation(cv::GIn(in), cv::GOut(out));
};
{
// Run original graph
auto mainC = make_computation();
mainC.apply(cv::gin(input), cv::gout(orig_graph_output), std::move(orig_args));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = make_computation(); // get new copy with new Priv
mainC.apply(cv::gin(input), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id()));
ASSERT_EQ(1u, listener.counts.at(MyInterleavedResize::id()));
}
TEST(PatternMatchingIntegrationBasic, SameTransformationAppliedSeveralTimes)
{
cv::Size in_sz(640, 480);
cv::Mat input(in_sz, CV_8UC3);
cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100));
cv::Mat orig_graph_output, transformed_graph_output;
auto orig_args = cv::compile_args();
auto transform_args = cv::compile_args(
cv::gapi::kernels<MyInterleavedResizeImpl, ResizeTransformToCustom>());
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
const auto make_computation = [] () {
GMat in;
GMat b, g, r;
std::tie(b, g, r) = cv::gapi::split3(in);
const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1,
cv::Size(100, 100), 0, 0, cv::INTER_AREA);
GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r));
return cv::GComputation(cv::GIn(in), cv::GOut(out));
};
{
// Run original graph
auto mainC = make_computation();
mainC.apply(cv::gin(input), cv::gout(orig_graph_output), std::move(orig_args));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = make_computation(); // get new copy with new Priv
mainC.apply(cv::gin(input), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id()));
ASSERT_EQ(3u, listener.counts.at(MyInterleavedResize::id()));
}
TEST(PatternMatchingIntegrationBasic, OneNV12toBGRTransformationApplied)
{
cv::Size in_sz(640, 480);
cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2);
cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100));
cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200));
cv::Mat orig_graph_output, transformed_graph_output;
auto orig_args = cv::compile_args();
auto transform_args = cv::compile_args(cv::gapi::kernels<MyNV12toBGRImpl, NV12Transform>());
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
const auto make_computation = [] () {
GMat in1, in2;
GMat bgr = cv::gapi::NV12toBGR(in1, in2);
GMat out = cv::gapi::resize(bgr, cv::Size(100, 100));
return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out));
};
{
// Run original graph
auto mainC = make_computation();
mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), std::move(orig_args));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = make_computation(); // get new copy with new Priv
mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id()));
ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id()));
}
TEST(PatternMatchingIntegrationBasic, TwoTransformationsApplied)
{
cv::Size in_sz(640, 480);
cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2);
cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100));
cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200));
cv::Mat orig_graph_output, transformed_graph_output;
auto orig_args = cv::compile_args();
auto transform_args = cv::compile_args(
cv::gapi::kernels<MyNV12toBGRImpl, MyInterleavedResizeImpl, ResizeTransform,
NV12Transform>()); // compile args with transformations
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
const auto make_computation = [] () {
GMat in1, in2;
GMat bgr = cv::gapi::NV12toBGR(in1, in2);
GMat b, g, r;
std::tie(b, g, r) = cv::gapi::split3(bgr);
const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1,
cv::Size(100, 100), 0, 0, cv::INTER_AREA);
GMat tmp1 = cv::gapi::resize(bgr, cv::Size(90, 90));
GMat tmp2 = cv::gapi::bitwise_not(cv::gapi::merge3(resize(b), resize(g), resize(r)));
GMat out = cv::gapi::resize(tmp1 + GScalar(10.0), cv::Size(100, 100)) + tmp2;
return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out));
};
{
// Run original graph
auto mainC = make_computation();
mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), std::move(orig_args));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = make_computation(); // get new copy with new Priv
mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(2u, listener.counts.size());
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id()));
ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id()));
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id()));
ASSERT_EQ(1u, listener.counts.at(MyInterleavedResize::id()));
}
struct PatternMatchingIntegrationE2E : testing::Test
{
cv::GComputation makeComputation() {
GMat in1, in2;
GMat bgr = MyNV12toBGR::on(in1, in2);
GMat resized = cv::gapi::resize(bgr, cv::Size(60, 60));
GMatP out = MyToNCHW::on(resized);
return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out));
}
void runTest(cv::GCompileArgs&& transform_args) {
cv::Size in_sz(640, 480);
cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2);
cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100));
cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200));
cv::Mat orig_graph_output, transformed_graph_output;
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
{
// Run original graph
auto mainC = makeComputation();
mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output),
cv::compile_args(cv::gapi::kernels<MyNV12toBGRImpl, MyToNCHWImpl>()));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = makeComputation(); // get new copy with new Priv
mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(3u, listener.counts.size());
// called in original graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id()));
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyToNCHW::id()));
ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id()));
ASSERT_EQ(1u, listener.counts.at(MyToNCHW::id()));
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MyPlanarResize::id()));
ASSERT_EQ(1u, listener.counts.at(MyPlanarResize::id()));
}
};
TEST_F(PatternMatchingIntegrationE2E, ChainTransformationsApplied)
{
runTest(cv::compile_args(
cv::gapi::kernels<MyPlanarResizeImpl, ChainTransform1, ChainTransform2>()));
}
TEST_F(PatternMatchingIntegrationE2E, ReversedChainTransformationsApplied)
{
runTest(cv::compile_args(
cv::gapi::kernels<ChainTransform2, MyPlanarResizeImpl, ChainTransform1>()));
}
struct PatternMatchingIntegrationUnusedNodes : testing::Test
{
cv::GComputation makeComputation() {
GMat in1, in2;
GMat bgr = cv::gapi::NV12toBGR(in1, in2);
GMat b1, g1, r1;
std::tie(b1, g1, r1) = cv::gapi::split3(bgr);
// FIXME: easier way to call split4??
GMat merged4 = cv::gapi::merge4(b1, g1, r1, b1);
GMat b2, g2, r2, unused;
std::tie(b2, g2, r2, unused) = cv::gapi::split4(merged4);
GMat out = cv::gapi::merge3(b2, g2, r2);
return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out));
}
void runTest(cv::GCompileArgs&& transform_args) {
cv::Size in_sz(640, 480);
cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2);
cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100));
cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200));
cv::Mat orig_graph_output, transformed_graph_output;
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
{
// Run original graph
auto mainC = makeComputation();
mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output),
cv::compile_args(cv::gapi::kernels<MyNV12toBGRImpl, MyToNCHWImpl>()));
}
// Generate transformed graph (passing transformations via compile args)
auto mainC = makeComputation(); // get new copy with new Priv
mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args));
// Compare
ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
// called in transformed graph:
ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id()));
ASSERT_EQ(1u, listener.counts.at(MySplit4::id()));
}
};
TEST_F(PatternMatchingIntegrationUnusedNodes, SingleOpTransformApplied)
{
runTest(cv::compile_args(cv::gapi::kernels<MySplit4Impl, Split4Transform>()));
}
// FIXME: enable once unused nodes are properly handled by Transformation API
TEST_F(PatternMatchingIntegrationUnusedNodes, DISABLED_TransformWithInternalUnusedNodeApplied)
{
runTest(cv::compile_args(cv::gapi::kernels<MySplit4Impl, Split4Merge3Transform>()));
}
TEST_F(PatternMatchingIntegrationUnusedNodes, TransformWithOutputUnusedNodeApplied)
{
runTest(cv::compile_args(cv::gapi::kernels<MySplit4Impl, Merge4Split4Transform>()));
}
// --------------------------------------------------------------------------------------
// Bad arg integration tests (GCompiler-level) - General
struct PatternMatchingIntegrationBadArgTests : testing::Test
{
cv::GComputation makeComputation() {
GMat in;
GMat a, b, c, d;
std::tie(a, b, c, d) = MySplit4::on(in); // using custom Split4 to check if it's called
GMat out = cv::gapi::merge3(a + b, cv::gapi::bitwise_not(c), d * cv::GScalar(2.0));
return cv::GComputation(cv::GIn(in), cv::GOut(out));
}
void runTest(cv::GCompileArgs&& transform_args) {
cv::Size in_sz(640, 480);
cv::Mat input(in_sz, CV_8UC4);
cv::randu(input, cv::Scalar::all(70), cv::Scalar::all(140));
cv::Mat output;
// Generate transformed graph (passing transformations via compile args)
auto mainC = makeComputation(); // get new copy with new Priv
ASSERT_NO_THROW(mainC.apply(cv::gin(input), cv::gout(output), std::move(transform_args)));
}
};
TEST_F(PatternMatchingIntegrationBadArgTests, NoTransformations)
{
auto transform_args = cv::compile_args(cv::gapi::kernels<MySplit4Impl>());
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
runTest(std::move(transform_args));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id()));
ASSERT_EQ(1u, listener.counts.at(MySplit4::id()));
}
TEST_F(PatternMatchingIntegrationBadArgTests, WrongTransformation)
{
// Here Split4Transform::pattern is "looking for" cv::gapi::split4 but it's not used
auto transform_args = cv::compile_args(cv::gapi::kernels<MySplit4Impl, Split4Transform>());
auto& listener = getListener();
listener.counts.clear(); // clear counters before testing
runTest(std::move(transform_args));
// Custom verification via listener
ASSERT_EQ(1u, listener.counts.size());
ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id()));
ASSERT_EQ(1u, listener.counts.at(MySplit4::id()));
}
// --------------------------------------------------------------------------------------
// Bad arg integration tests (GCompiler-level) - Endless Loops
GAPI_TRANSFORM(EndlessLoopTransform, <cv::GMat(cv::GMat)>, "pattern in substitute")
{
static cv::GMat pattern(const cv::GMat& in)
{
return cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR);
}
static cv::GMat substitute(const cv::GMat& in)
{
cv::GMat b, g, r;
std::tie(b, g, r) = cv::gapi::split3(in);
auto resize = std::bind(&cv::gapi::resize,
std::placeholders::_1, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR);
cv::GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r));
return out;
}
};
TEST(PatternMatchingIntegrationEndlessLoops, PatternInSubstituteInOneTransform)
{
cv::Size in_sz(640, 480);
cv::Mat input(in_sz, CV_8UC3);
cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100));
auto c = [] () {
GMat in;
GMat tmp = cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR);
GMat out = cv::gapi::bitwise_not(tmp);
return cv::GComputation(cv::GIn(in), cv::GOut(out));
}();
EXPECT_THROW(
cv::gimpl::GCompiler(c, cv::descr_of(cv::gin(input)),
cv::compile_args(cv::gapi::kernels<EndlessLoopTransform>())),
std::exception);
}
} // namespace opencv_test