Merge pull request #20546 from sivanov-work:initial_vpl_source

G-API: oneVPL (simplification) source base commit

* oneVPL source initial

* Fix compilation

* Fix compilation path

* Fix NO VPL compile

* Fix unused vars

* Fix unused vars in example

* Simplify oneVPL search: no custom path & download

* Fix standalone GAPI

* Apply comments

modules/gapi/CMakeLists.txt

@@ -163,6 +163,10 @@ set(gapi_srcs
     src/backends/ie/bindings_ie.cpp
     src/backends/python/gpythonbackend.cpp
 
+    # Streaming source
+    src/streaming/onevpl/onevpl_source.cpp
+    src/streaming/onevpl/onevpl_source_priv.cpp
+
     # Utils (ITT tracing)
     src/utils/itt.cpp
     )
@@ -234,6 +238,17 @@ if(HAVE_PLAIDML)
   ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS})
 endif()
 
+if(HAVE_GAPI_ONEVPL)
+  if(TARGET opencv_test_gapi)
+    ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL)
+    ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS})
+  endif()
+  ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL)
+  ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS})
+  if(HAVE_D3D11 AND HAVE_OPENCL)
+    ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS})
+  endif()
+endif()
 
 if(WIN32)
   # Required for htonl/ntohl on Windows

modules/gapi/cmake/init.cmake

@@ -32,3 +32,10 @@ if(WITH_PLAIDML)
       set(HAVE_PLAIDML TRUE)
   endif()
 endif()
+
+if(WITH_GAPI_ONEVPL)
+    find_package(VPL)
+    if(VPL_FOUND)
+        set(HAVE_GAPI_ONEVPL TRUE)
+    endif()
+endif()

modules/gapi/cmake/standalone.cmake

@@ -6,6 +6,13 @@ if (NOT TARGET ade )
   find_package(ade 0.1.0 REQUIRED)
 endif()
 
+if (WITH_GAPI_ONEVPL)
+    find_package(VPL)
+    if(VPL_FOUND)
+        set(HAVE_GAPI_ONEVPL TRUE)
+    endif()
+endif()
+
 set(FLUID_TARGET fluid)
 set(FLUID_ROOT "${CMAKE_CURRENT_LIST_DIR}/../")
 

modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp

@@ -0,0 +1,44 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP
+#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP
+
+#include <opencv2/gapi/garg.hpp>
+#include <opencv2/gapi/streaming/meta.hpp>
+#include <opencv2/gapi/streaming/source.hpp>
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+class GAPI_EXPORTS OneVPLSource : public IStreamSource
+{
+public:
+    struct Priv;
+
+    explicit OneVPLSource(const std::string& filePath);
+    ~OneVPLSource() override;
+
+    bool pull(cv::gapi::wip::Data& data) override;
+    GMetaArg descr_of() const override;
+
+private:
+    explicit OneVPLSource(std::unique_ptr<Priv>&& impl);
+    std::unique_ptr<Priv> m_priv;
+};
+
+template<class... Args>
+GAPI_EXPORTS_W cv::Ptr<IStreamSource> inline make_vpl_src(const std::string& filePath, Args&&... args)
+{
+    return make_src<OneVPLSource>(filePath, std::forward<Args>(args)...);
+}
+
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP

modules/gapi/samples/onevpl_infer_single_roi.cpp

@@ -0,0 +1,254 @@
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <cctype>
+
+#include <opencv2/imgproc.hpp>
+#include <opencv2/gapi.hpp>
+#include <opencv2/gapi/core.hpp>
+#include <opencv2/gapi/cpu/gcpukernel.hpp>
+#include <opencv2/gapi/infer/ie.hpp>
+#include <opencv2/gapi/render.hpp>
+#include <opencv2/gapi/streaming/onevpl/onevpl_source.hpp>
+#include <opencv2/highgui.hpp> // CommandLineParser
+
+const std::string about =
+    "This is an OpenCV-based version of oneVPLSource decoder example";
+const std::string keys =
+    "{ h help       |                                           | Print this help message }"
+    "{ input        |                                           | Path to the input demultiplexed video file }"
+    "{ output       |                                           | Path to the output RAW video file. Use .avi extension }"
+    "{ facem        | face-detection-adas-0001.xml              | Path to OpenVINO IE face detection model (.xml) }"
+    "{ faced        | CPU                                       | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }";
+
+namespace {
+std::string get_weights_path(const std::string &model_path) {
+    const auto EXT_LEN = 4u;
+    const auto sz = model_path.size();
+    CV_Assert(sz > EXT_LEN);
+
+    auto ext = model_path.substr(sz - EXT_LEN);
+    std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
+            return static_cast<unsigned char>(std::tolower(c));
+        });
+    CV_Assert(ext == ".xml");
+    return model_path.substr(0u, sz - EXT_LEN) + ".bin";
+}
+} // anonymous namespace
+
+namespace custom {
+G_API_NET(FaceDetector,   <cv::GMat(cv::GMat)>, "face-detector");
+
+using GDetections = cv::GArray<cv::Rect>;
+using GRect       = cv::GOpaque<cv::Rect>;
+using GSize       = cv::GOpaque<cv::Size>;
+using GPrims      = cv::GArray<cv::gapi::wip::draw::Prim>;
+
+G_API_OP(LocateROI, <GRect(GSize)>, "sample.custom.locate-roi") {
+    static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) {
+        return cv::empty_gopaque_desc();
+    }
+};
+
+G_API_OP(ParseSSD, <GDetections(cv::GMat, GRect, GSize)>, "sample.custom.parse-ssd") {
+    static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) {
+        return cv::empty_array_desc();
+    }
+};
+
+G_API_OP(BBoxes, <GPrims(GDetections, GRect)>, "sample.custom.b-boxes") {
+    static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) {
+        return cv::empty_array_desc();
+    }
+};
+
+GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
+    // This is the place where we can run extra analytics
+    // on the input image frame and select the ROI (region
+    // of interest) where we want to detect our objects (or
+    // run any other inference).
+    //
+    // Currently it doesn't do anything intelligent,
+    // but only crops the input image to square (this is
+    // the most convenient aspect ratio for detectors to use)
+
+    static void run(const cv::Size& in_size, cv::Rect &out_rect) {
+
+        // Identify the central point & square size (- some padding)
+        const auto center = cv::Point{in_size.width/2, in_size.height/2};
+        auto sqside = std::min(in_size.width, in_size.height);
+
+        // Now build the central square ROI
+        out_rect = cv::Rect{ center.x - sqside/2
+                           , center.y - sqside/2
+                           , sqside
+                           , sqside
+                           };
+    }
+};
+
+GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) {
+    static void run(const cv::Mat &in_ssd_result,
+                    const cv::Rect &in_roi,
+                    const cv::Size &in_parent_size,
+                    std::vector<cv::Rect> &out_objects) {
+        const auto &in_ssd_dims = in_ssd_result.size;
+        CV_Assert(in_ssd_dims.dims() == 4u);
+
+        const int MAX_PROPOSALS = in_ssd_dims[2];
+        const int OBJECT_SIZE   = in_ssd_dims[3];
+        CV_Assert(OBJECT_SIZE  == 7); // fixed SSD object size
+
+        const cv::Size up_roi = in_roi.size();
+        const cv::Rect surface({0,0}, in_parent_size);
+
+        out_objects.clear();
+
+        const float *data = in_ssd_result.ptr<float>();
+        for (int i = 0; i < MAX_PROPOSALS; i++) {
+            const float image_id   = data[i * OBJECT_SIZE + 0];
+            const float label      = data[i * OBJECT_SIZE + 1];
+            const float confidence = data[i * OBJECT_SIZE + 2];
+            const float rc_left    = data[i * OBJECT_SIZE + 3];
+            const float rc_top     = data[i * OBJECT_SIZE + 4];
+            const float rc_right   = data[i * OBJECT_SIZE + 5];
+            const float rc_bottom  = data[i * OBJECT_SIZE + 6];
+            (void) label; // unused
+
+            if (image_id < 0.f) {
+                break;    // marks end-of-detections
+            }
+            if (confidence < 0.5f) {
+                continue; // skip objects with low confidence
+            }
+
+            // map relative coordinates to the original image scale
+            // taking the ROI into account
+            cv::Rect rc;
+            rc.x      = static_cast<int>(rc_left   * up_roi.width);
+            rc.y      = static_cast<int>(rc_top    * up_roi.height);
+            rc.width  = static_cast<int>(rc_right  * up_roi.width)  - rc.x;
+            rc.height = static_cast<int>(rc_bottom * up_roi.height) - rc.y;
+            rc.x += in_roi.x;
+            rc.y += in_roi.y;
+            out_objects.emplace_back(rc & surface);
+        }
+    }
+};
+
+GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
+    // This kernel converts the rectangles into G-API's
+    // rendering primitives
+    static void run(const std::vector<cv::Rect> &in_face_rcs,
+                    const             cv::Rect  &in_roi,
+                          std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
+        out_prims.clear();
+        const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
+            return cv::gapi::wip::draw::Rect(rc, clr, 2);
+        };
+        out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan
+        for (auto &&rc : in_face_rcs) {
+            out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0)));   // green
+        }
+    }
+};
+
+} // namespace custom
+
+int main(int argc, char *argv[]) {
+
+    cv::CommandLineParser cmd(argc, argv, keys);
+    cmd.about(about);
+    if (cmd.has("help")) {
+        cmd.printMessage();
+        return 0;
+    }
+
+    // get file name
+    std::string file_path = cmd.get<std::string>("input");
+    const std::string output = cmd.get<std::string>("output");
+    const auto face_model_path = cmd.get<std::string>("facem");
+
+    // check ouput file extension
+    if (!output.empty()) {
+        auto ext = output.find_last_of(".");
+        if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) {
+            std::cerr << "Output file should have *.avi extension for output video" << std::endl;
+            return -1;
+        }
+    }
+
+    auto face_net = cv::gapi::ie::Params<custom::FaceDetector> {
+        face_model_path,                 // path to topology IR
+        get_weights_path(face_model_path),   // path to weights
+        cmd.get<std::string>("faced"),   // device specifier
+    };
+    auto kernels = cv::gapi::kernels
+        < custom::OCVLocateROI
+        , custom::OCVParseSSD
+        , custom::OCVBBoxes>();
+    auto networks = cv::gapi::networks(face_net);
+
+    // Create source
+    cv::Ptr<cv::gapi::wip::IStreamSource> cap;
+    try {
+        cap = cv::gapi::wip::make_vpl_src(file_path);
+        std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl;
+    } catch (const std::exception& ex) {
+        std::cerr << "Cannot create source: " << ex.what() << std::endl;
+        return -1;
+    }
+
+    cv::GMetaArg descr = cap->descr_of();
+    auto frame_descr = cv::util::get<cv::GFrameDesc>(descr);
+
+    // Now build the graph
+    cv::GFrame in;
+    auto size = cv::gapi::streaming::size(in);
+    auto roi = custom::LocateROI::on(size);
+    auto blob = cv::gapi::infer<custom::FaceDetector>(roi, in);
+    auto rcs = custom::ParseSSD::on(blob, roi, size);
+    auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi));
+    auto out = cv::gapi::streaming::BGR(out_frame);
+
+    cv::GStreamingCompiled pipeline;
+    try {
+        pipeline  = cv::GComputation(cv::GIn(in), cv::GOut(out))
+                .compileStreaming(cv::compile_args(kernels, networks));
+    } catch (const std::exception& ex) {
+        std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl;
+        return -1;
+    }
+    // The execution part
+
+    // TODO USE may set pool size from outside and set queue_capacity size,
+    // compile arg: cv::gapi::streaming::queue_capacity
+    pipeline.setSource(std::move(cap));
+    pipeline.start();
+
+    int framesCount = 0;
+    cv::TickMeter t;
+    cv::VideoWriter writer;
+    if (!output.empty() && !writer.isOpened()) {
+        const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height};
+        writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
+        CV_Assert(writer.isOpened());
+    }
+
+    cv::Mat outMat;
+    t.start();
+    while (pipeline.pull(cv::gout(outMat))) {
+        cv::imshow("Out", outMat);
+        cv::waitKey(1);
+        if (!output.empty()) {
+            writer << outMat;
+        }
+        framesCount++;
+    }
+    t.stop();
+    std::cout << "Elapsed time: " << t.getTimeSec() << std::endl;
+    std::cout << "FPS: " << framesCount /  t.getTimeSec() << std::endl;
+    std::cout << "framesCount: " << framesCount << std::endl;
+
+    return 0;
+}

modules/gapi/src/streaming/onevpl/onevpl_source.cpp

@@ -0,0 +1,48 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include <opencv2/gapi/streaming/onevpl/onevpl_source.hpp>
+
+#include "streaming/onevpl/onevpl_source_priv.hpp"
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+#ifdef HAVE_ONEVPL
+OneVPLSource::OneVPLSource(const std::string& filePath) :
+    OneVPLSource(std::unique_ptr<Priv>(new OneVPLSource::Priv(filePath))) {
+
+    if (filePath.empty()) {
+        util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name"));
+    }
+}
+#else
+OneVPLSource::OneVPLSource(const std::string&) {
+    GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
+}
+#endif
+
+OneVPLSource::OneVPLSource(std::unique_ptr<Priv>&& impl) :
+    IStreamSource(),
+    m_priv(std::move(impl)) {
+}
+
+OneVPLSource::~OneVPLSource() {
+}
+
+bool OneVPLSource::pull(cv::gapi::wip::Data& data)
+{
+    return m_priv->pull(data);
+}
+
+GMetaArg OneVPLSource::descr_of() const
+{
+    return m_priv->descr_of();
+}
+} // namespace wip
+} // namespace gapi
+} // namespace cv

modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp

@@ -0,0 +1,63 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include <algorithm>
+#include <sstream>
+
+#include "streaming/onevpl/onevpl_source_priv.hpp"
+#include "logger.hpp"
+
+#ifndef HAVE_ONEVPL
+namespace cv {
+namespace gapi {
+namespace wip {
+bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) {
+    return true;
+}
+GMetaArg OneVPLSource::Priv::descr_of() const {
+    return {};
+}
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#else // HAVE_ONEVPL
+
+namespace cv {
+namespace gapi {
+namespace wip {
+OneVPLSource::Priv::Priv() :
+    mfx_handle(MFXLoad())
+{
+    GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle);
+    description_is_valid = false;
+}
+
+OneVPLSource::Priv::Priv(const std::string&) :
+    OneVPLSource::Priv()
+{
+}
+
+OneVPLSource::Priv::~Priv()
+{
+    GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle);
+    MFXUnload(mfx_handle);
+}
+
+bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&)
+{
+    return false;
+}
+
+GMetaArg OneVPLSource::Priv::descr_of() const
+{
+    return {};
+}
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#endif // HAVE_ONEVPL

modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp

@@ -0,0 +1,62 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP
+#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP
+
+#include <stdio.h>
+
+#include <memory>
+#include <string>
+
+#include <opencv2/gapi/garg.hpp>
+#include <opencv2/gapi/streaming/meta.hpp>
+#include <opencv2/gapi/streaming/onevpl/onevpl_source.hpp>
+
+#ifdef HAVE_ONEVPL
+#if (MFX_VERSION >= 2000)
+#include <vpl/mfxdispatcher.h>
+#endif // MFX_VERSION
+
+#include <vpl/mfx.h>
+
+#include <vpl/mfxvideo.h>
+
+namespace cv {
+namespace gapi {
+namespace wip {
+
+struct OneVPLSource::Priv
+{
+    explicit Priv(const std::string& file_path);
+    ~Priv();
+
+    bool pull(cv::gapi::wip::Data& data);
+    GMetaArg descr_of() const;
+private:
+    Priv();
+    mfxLoader mfx_handle;
+    bool description_is_valid;
+};
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+
+#else // HAVE_ONEVPL
+
+namespace cv {
+namespace gapi {
+namespace wip {
+struct OneVPLSource::Priv final
+{
+    bool pull(cv::gapi::wip::Data&);
+    GMetaArg descr_of() const;
+};
+} // namespace wip
+} // namespace gapi
+} // namespace cv
+#endif // HAVE_ONEVPL
+#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP