Merge pull request #22583 from TolyaTalamanov:at/add-cfg-output-precision-for-ie-backend

G-API: API for configuring model output precision for IE backend

modules/gapi/include/opencv2/gapi/infer/ie.hpp

@@ -88,6 +88,19 @@ struct ParamDesc {
 
     cv::optional<cv::gapi::wip::onevpl::Device> vpl_preproc_device;
     cv::optional<cv::gapi::wip::onevpl::Context> vpl_preproc_ctx;
+
+    using PrecisionT = int;
+    using PrecisionMapT = std::unordered_map<std::string, PrecisionT>;
+    // NB: This parameter can contain:
+    // 1. cv::util::monostate - Don't specify precision, but use default from IR/Blob.
+    // 2. PrecisionT (CV_8U, CV_32F, ...) - Specifies precision for all output layers.
+    // 3. PrecisionMapT ({{"layer0", CV_32F}, {"layer1", CV_16F}} - Specifies precision for certain output layer.
+    // cv::util::monostate is default value that means precision wasn't specified.
+    using PrecisionVariantT = cv::util::variant<cv::util::monostate,
+                                                PrecisionT,
+                                                PrecisionMapT>;
+    PrecisionVariantT output_precision;
+
 };
 } // namespace detail
 
@@ -132,6 +145,7 @@ public:
               , {}
               , {}
               , {}
+              , {}
               , {}} {
     };
 
@@ -156,6 +170,7 @@ public:
               , {}
               , {}
               , {}
+              , {}
               , {}} {
     };
 
@@ -351,6 +366,31 @@ public:
         return *this;
     }
 
+    /** @brief Specifies the output precision for model.
+
+    The function is used to set an output precision for model.
+
+    @param precision Precision in OpenCV format (CV_8U, CV_32F, ...)
+    will be applied to all output layers.
+    @return reference to this parameter structure.
+    */
+    Params<Net>& cfgOutputPrecision(detail::ParamDesc::PrecisionT precision) {
+        desc.output_precision = precision;
+        return *this;
+    }
+
+    /** @overload
+
+    @param precision_map Map of pairs: name of corresponding output layer
+    and its precision in OpenCV format (CV_8U, CV_32F, ...)
+    @return reference to this parameter structure.
+    */
+    Params<Net>&
+    cfgOutputPrecision(detail::ParamDesc::PrecisionMapT precision_map) {
+        desc.output_precision = precision_map;
+        return *this;
+    }
+
     // BEGIN(G-API's network parametrization API)
     GBackend      backend()    const { return cv::gapi::ie::backend();  }
     std::string   tag()        const { return Net::tag(); }
@@ -385,7 +425,7 @@ public:
            const std::string &device)
         : desc{ model, weights, device, {}, {}, {}, 0u, 0u,
                 detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u,
-                {}, {}, {}, {}},
+                {}, {}, {}, {}, {}},
           m_tag(tag) {
     };
 
@@ -403,7 +443,7 @@ public:
            const std::string &device)
         : desc{ model, {}, device, {}, {}, {}, 0u, 0u,
                 detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u,
-                {}, {}, {}, {}},
+                {}, {}, {}, {}, {}},
           m_tag(tag) {
     };
 
@@ -476,6 +516,19 @@ public:
         return *this;
     }
 
+    /** @see ie::Params::cfgOutputPrecision */
+    Params& cfgOutputPrecision(detail::ParamDesc::PrecisionT precision) {
+        desc.output_precision = precision;
+        return *this;
+    }
+
+    /** @overload */
+    Params&
+    cfgOutputPrecision(detail::ParamDesc::PrecisionMapT precision_map) {
+        desc.output_precision = precision_map;
+        return *this;
+    }
+
     // BEGIN(G-API's network parametrization API)
     GBackend      backend()    const { return cv::gapi::ie::backend();  }
     std::string   tag()        const { return m_tag; }

modules/gapi/samples/pipeline_modeling_tool.cpp

@@ -210,6 +210,12 @@ InferParams read<InferParams>(const cv::FileNode& fn) {
     params.input_layers  = readList<std::string>(fn, "input_layers", name);
     params.output_layers = readList<std::string>(fn, "output_layers", name);
     params.config        = readMap<std::string>(fn["config"]);
+
+    auto out_prec_str = readOpt<std::string>(fn["output_precision"]);
+    if (out_prec_str.has_value()) {
+        params.out_precision =
+            cv::optional<int>(strToPrecision(out_prec_str.value()));
+    }
     return params;
 }
 

modules/gapi/samples/pipeline_modeling_tool/pipeline_builder.hpp

@@ -258,6 +258,7 @@ struct InferParams {
     std::vector<std::string> input_layers;
     std::vector<std::string> output_layers;
     std::map<std::string, std::string> config;
+    cv::util::optional<int> out_precision;
 };
 
 class PipelineBuilder {
@@ -362,6 +363,9 @@ void PipelineBuilder::addInfer(const CallParams&  call_params,
     }
 
     pp->pluginConfig(infer_params.config);
+    if (infer_params.out_precision) {
+        pp->cfgOutputPrecision(infer_params.out_precision.value());
+    }
     m_state->networks += cv::gapi::networks(*pp);
 
     addCall(call_params,

modules/gapi/src/backends/ie/giebackend.cpp

@@ -197,6 +197,16 @@ inline IE::Blob::Ptr wrapIE(const cv::MediaFrame::View& view,
 
 template<class MatType>
 inline void copyFromIE(const IE::Blob::Ptr &blob, MatType &mat) {
+    const auto& desc = blob->getTensorDesc();
+    const auto ie_type = toCV(desc.getPrecision());
+    if (ie_type != mat.type()) {
+        std::stringstream ss;
+        ss << "Failed to copy blob from IE to OCV: "
+           << "Blobs have different data types "
+           << "(IE type: " << ie_type
+           << " vs OCV type: " << mat.type() << ")." << std::endl;
+        throw std::logic_error(ss.str());
+    }
     switch (blob->getTensorDesc().getPrecision()) {
 #define HANDLE(E,T)                                                 \
         case IE::Precision::E: std::copy_n(blob->buffer().as<T*>(), \
@@ -365,6 +375,13 @@ struct IEUnit {
             cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind"));
         }
 
+        if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import &&
+            !cv::util::holds_alternative<cv::util::monostate>(params.output_precision)) {
+            cv::util::throw_error(
+                    std::logic_error("Setting output precision isn't supported for imported network"));
+        }
+
+
         using namespace cv::gapi::wip::onevpl;
         if (params.vpl_preproc_device.has_value() && params.vpl_preproc_ctx.has_value()) {
             using namespace cv::gapi::wip;
@@ -1122,6 +1139,28 @@ static IE::PreProcessInfo configurePreProcInfo(const IE::InputInfo::CPtr& ii,
     return info;
 }
 
+using namespace cv::gapi::ie::detail;
+static void configureOutputPrecision(const IE::OutputsDataMap           &outputs_info,
+                                     const ParamDesc::PrecisionVariantT &output_precision) {
+    cv::util::visit(cv::util::overload_lambdas(
+            [&outputs_info](ParamDesc::PrecisionT cvdepth) {
+                auto precision = toIE(cvdepth);
+                for (auto it : outputs_info) {
+                    it.second->setPrecision(precision);
+                }
+            },
+            [&outputs_info](const ParamDesc::PrecisionMapT& precision_map) {
+                for (auto it : precision_map) {
+                    outputs_info.at(it.first)->setPrecision(toIE(it.second));
+                }
+            },
+            [&outputs_info](cv::util::monostate) {
+                // Do nothing.
+            }
+        ), output_precision
+    );
+}
+
 // NB: This is a callback used by async infer
 // to post outputs blobs (cv::GMat's).
 static void PostOutputs(InferenceEngine::InferRequest &request,
@@ -1241,7 +1280,7 @@ struct Infer: public cv::detail::KernelTag {
         GAPI_Assert(uu.params.input_names.size() == in_metas.size()
                     && "Known input layers count doesn't match input meta count");
 
-        // NB: Configuring input precision and network reshape must be done
+        // NB: Configuring input/output precision and network reshape must be done
         // only in the loadNetwork case.
         using namespace cv::gapi::ie::detail;
         if (uu.params.kind == ParamDesc::Kind::Load) {
@@ -1275,6 +1314,7 @@ struct Infer: public cv::detail::KernelTag {
             if (!input_reshape_table.empty()) {
                 const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
             }
+            configureOutputPrecision(uu.net.getOutputsInfo(), uu.params.output_precision);
         } else {
             GAPI_Assert(uu.params.kind == ParamDesc::Kind::Import);
             auto inputs = uu.this_network.GetInputsInfo();
@@ -1393,6 +1433,7 @@ struct InferROI: public cv::detail::KernelTag {
                 const_cast<IEUnit::InputFramesDesc &>(uu.net_input_params)
                             .set_param(input_name, ii->getTensorDesc());
             }
+            configureOutputPrecision(uu.net.getOutputsInfo(), uu.params.output_precision);
         } else {
             GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import);
             auto inputs = uu.this_network.GetInputsInfo();
@@ -1513,6 +1554,7 @@ struct InferList: public cv::detail::KernelTag {
             if (!input_reshape_table.empty()) {
                 const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
             }
+            configureOutputPrecision(uu.net.getOutputsInfo(), uu.params.output_precision);
         } else {
             GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import);
             std::size_t idx = 1u;
@@ -1667,6 +1709,7 @@ struct InferList2: public cv::detail::KernelTag {
                     if (!input_reshape_table.empty()) {
                         const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
                     }
+                    configureOutputPrecision(uu.net.getOutputsInfo(), uu.params.output_precision);
                 } else {
                     GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import);
                     auto inputs = uu.this_network.GetInputsInfo();

modules/gapi/test/infer/gapi_infer_ie_test.cpp

@@ -2956,6 +2956,111 @@ TEST(TestAgeGender, ThrowBlobAndInputPrecisionMismatchStreaming)
     }
 }
 
+TEST(TestAgeGenderIE, ChangeOutputPrecision)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Mat in_mat(cv::Size(320, 240), CV_8UC3);
+    cv::randu(in_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net);
+        for (auto it : net.getOutputsInfo()) {
+            it.second->setPrecision(IE::Precision::U8);
+        }
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat));
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GMat in;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+    cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" })
+     .cfgOutputPrecision(CV_8U);
+    comp.apply(cv::gin(in_mat), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
+TEST(TestAgeGenderIE, ChangeSpecificOutputPrecison)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Mat in_mat(cv::Size(320, 240), CV_8UC3);
+    cv::randu(in_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net    = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net);
+
+        // NB: Specify precision only for "prob" output.
+        net.getOutputsInfo().at("prob")->setPrecision(IE::Precision::U8);
+
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat));
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GMat in;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+    cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" })
+     .cfgOutputPrecision({{"prob", CV_8U}});
+    comp.apply(cv::gin(in_mat), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
 } // namespace opencv_test
 
 #endif //  HAVE_INF_ENGINE