diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 9a3cfa15d7..20a73a5595 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -208,6 +208,10 @@ if(OPENCV_GAPI_INF_ENGINE) ocv_target_link_libraries(${the_module} PRIVATE ${INF_ENGINE_TARGET}) endif() +if (HAVE_NGRAPH) + ocv_target_link_libraries(${the_module} PRIVATE ngraph::ngraph) +endif() + if(HAVE_TBB) ocv_target_link_libraries(${the_module} PRIVATE tbb) endif() @@ -223,6 +227,9 @@ set(__test_extra_deps "") if(OPENCV_GAPI_INF_ENGINE) list(APPEND __test_extra_deps ${INF_ENGINE_TARGET}) endif() +if(HAVE_NGRAPH) + list(APPEND __test_extra_deps ngraph::ngraph) +endif() ocv_add_accuracy_tests(${__test_extra_deps}) # FIXME: test binary is linked with ADE directly since ADE symbols @@ -232,6 +239,9 @@ ocv_add_accuracy_tests(${__test_extra_deps}) if(TARGET opencv_test_gapi) target_include_directories(opencv_test_gapi PRIVATE "${CMAKE_CURRENT_LIST_DIR}/src") target_link_libraries(opencv_test_gapi PRIVATE ade) + if (HAVE_NGRAPH) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_NGRAPH) + endif() endif() if(HAVE_TBB AND TARGET opencv_test_gapi) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 0e9c127fa1..e6b7be58ad 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -80,7 +80,10 @@ struct ParamDesc { // NB: An optional config to setup RemoteContext for IE cv::util::any context_config; - size_t batch_size; + // NB: batch_size can't be equal to 1 by default, because some of models + // have 2D (Layout::NC) input and if the first dimension not equal to 1 + // net.setBatchSize(1) will overwrite it. + cv::optional batch_size; }; } // namespace detail @@ -123,7 +126,7 @@ public: , {} , 1u , {} - , 1u} { + , {}} { }; /** @overload @@ -145,7 +148,7 @@ public: , {} , 1u , {} - , 1u} { + , {}} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -329,7 +332,7 @@ public: @return reference to this parameter structure. */ Params& cfgBatchSize(const size_t size) { - desc.batch_size = size; + desc.batch_size = cv::util::make_optional(size); return *this; } @@ -367,7 +370,7 @@ public: const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}, 1u}, + {}, {}}, m_tag(tag) { }; @@ -385,7 +388,7 @@ public: const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}, 1u}, + {}, {}}, m_tag(tag) { }; @@ -454,7 +457,7 @@ public: /** @see ie::Params::cfgBatchSize */ Params& cfgBatchSize(const size_t size) { - desc.batch_size = size; + desc.batch_size = cv::util::make_optional(size); return *this; } diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index a633431fd8..5e0c63dc3a 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -127,7 +127,6 @@ inline int toCV(IE::Precision prec) { inline IE::TensorDesc toIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { const auto &sz = mat.size; - // NB: For some reason RGB image is 2D image // (since channel component is not counted here). // Note: regular 2D vectors also fall into this category @@ -148,7 +147,6 @@ inline IE::TensorDesc toIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { return IE::TensorDesc(toIE(mat.depth()), IE::SizeVector{1, channels, height, width}, bdesc); } - return IE::TensorDesc(toIE(mat.depth()), toIE(sz), toIELayout(sz.dims())); } @@ -241,7 +239,10 @@ struct IEUnit { if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); - net.setBatchSize(params.batch_size); + // NB: Set batch size only if user asked. (don't set by default) + if (params.batch_size.has_value()) { + net.setBatchSize(params.batch_size.value()); + } } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); @@ -520,7 +521,9 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { blob_params->second); } -inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { +inline IE::Blob::Ptr extractBlob(IECallContext& ctx, + std::size_t i, + cv::gapi::ie::TraitAs hint) { if (ctx.uu.rctx != nullptr) { return extractRemoteBlob(ctx, i); } @@ -532,7 +535,7 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { return wrapIE(*(ctx.views.back()), frame.desc()); } case cv::GShape::GMAT: { - return wrapIE(ctx.inMat(i), cv::gapi::ie::TraitAs::IMAGE); + return wrapIE(ctx.inMat(i), hint); } default: GAPI_Assert("Unsupported input shape for IE backend"); @@ -545,6 +548,9 @@ static void setBlob(InferenceEngine::InferRequest& req, const std::string& layer_name, const IE::Blob::Ptr& blob, const IECallContext& ctx) { + // TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead, + // and redirect our data producers to this memory + // (A memory dialog comes to the picture again) using namespace cv::gapi::ie::detail; if (ctx.uu.params.kind == ParamDesc::Kind::Load) { req.SetBlob(layer_name, blob); @@ -950,7 +956,14 @@ struct Infer: public cv::detail::KernelTag { uu.params.layer_names_to_reshape.end()) { configureInputReshapeByImage(ii, mm, input_reshape_table); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + + // NB: Configure resize only for NCHW/NHWC layout, + // since it isn't supposed to work with others. + auto layout = ii->getTensorDesc().getLayout(); + if (layout == IE::Layout::NCHW || + layout == IE::Layout::NHWC) { + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + } } // FIXME: This isn't the best place to call reshape function. @@ -1000,11 +1013,16 @@ struct Infer: public cv::detail::KernelTag { // non-generic version for now: // - assumes all inputs/outputs are always Mats for (auto i : ade::util::iota(ctx->uu.params.num_in)) { - // TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead, - // and redirect our data producers to this memory - // (A memory dialog comes to the picture again) - IE::Blob::Ptr this_blob = extractBlob(*ctx, i); - setBlob(req, ctx->uu.params.input_names[i], this_blob, *ctx); + const auto& layer_name = ctx->uu.params.input_names[i]; + auto layout = + ctx->uu.this_network.GetInputsInfo(). + at(layer_name)->getTensorDesc().getLayout(); + auto hint = + (layout == IE::Layout::NCHW || layout == IE::Layout::NHWC) + ? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR; + + IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint); + setBlob(req, layer_name, this_blob, *ctx); } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? @@ -1092,7 +1110,10 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(ctx->uu.params.num_in == 1); auto&& this_roi = ctx->inArg(0).rref(); - IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); + // NB: This blob will be used to make roi from its, so + // it should be treated as image + IE::Blob::Ptr this_blob = + extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); setBlob(req, *(ctx->uu.params.input_names.begin()), IE::make_shared_blob(this_blob, toIE(this_roi)), @@ -1187,7 +1208,9 @@ struct InferList: public cv::detail::KernelTag { return; } - IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); + // NB: This blob will be used to make roi from its, so + // it should be treated as image + IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { @@ -1331,7 +1354,9 @@ struct InferList2: public cv::detail::KernelTag { cv::gimpl::ie::RequestPool &reqPool) { GAPI_Assert(ctx->inArgs().size() > 1u && "This operation must have at least two arguments"); - IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0); + // NB: This blob will be used to make roi from its, so + // it should be treated as image + IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE); const auto list_size = ctx->inArg(1u).size(); if (list_size == 0u) { for (auto i : ade::util::iota(ctx->uu.params.num_out)) { diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index 80d27c0298..e6c56515ce 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -22,6 +22,20 @@ #include "backends/ie/util.hpp" #include "backends/ie/giebackend/giewrapper.hpp" +#ifdef HAVE_NGRAPH +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include +#endif + namespace opencv_test { namespace { @@ -2858,6 +2872,66 @@ TEST(TestAgeGender, ThrowBlobAndInputPrecisionMismatch) cv::compile_args(cv::gapi::networks(pp)))); } +#ifdef HAVE_NGRAPH + +TEST(Infer, ModelWith2DInputs) +{ + const std::string model_name = "ModelWith2DInputs"; + const std::string model_path = model_name + ".xml"; + const std::string weights_path = model_name + ".bin"; + const std::string device_id = "CPU"; + const int W = 10; + const int H = 5; + + // NB: Define model with 2D inputs. + auto in1 = std::make_shared( + ngraph::element::Type_t::u8, + ngraph::Shape(std::vector{{H, W}}) + ); + auto in2 = std::make_shared( + ngraph::element::Type_t::u8, + ngraph::Shape(std::vector{{H, W}}) + ); + auto result = std::make_shared(in1, in2); + auto func = std::make_shared( + ngraph::OutputVector{result}, + ngraph::ParameterVector{in1, in2} + ); + + cv::Mat in_mat1(std::vector{H, W}, CV_8U), + in_mat2(std::vector{H, W}, CV_8U), + gapi_mat, ref_mat; + + cv::randu(in_mat1, 0, 100); + cv::randu(in_mat2, 0, 100); + cv::add(in_mat1, in_mat2, ref_mat, cv::noArray(), CV_32F); + + // Compile xml file + IE::CNNNetwork(func).serialize(model_path); + + // Configure & run G-API + cv::GMat g_in1, g_in2; + cv::GInferInputs inputs; + inputs[in1->get_name()] = g_in1; + inputs[in2->get_name()] = g_in2; + auto outputs = cv::gapi::infer(model_name, inputs); + auto out = outputs.at(result->get_name()); + + cv::GComputation comp(cv::GIn(g_in1, g_in2), cv::GOut(out)); + + auto pp = cv::gapi::ie::Params(model_name, + model_path, + weights_path, + device_id); + + comp.apply(cv::gin(in_mat1, in_mat2), cv::gout(gapi_mat), + cv::compile_args(cv::gapi::networks(pp))); + + normAssert(ref_mat, gapi_mat, "Test model output"); +} + +#endif // HAVE_NGRAPH + } // namespace opencv_test #endif // HAVE_INF_ENGINE