Merge pull request #18213 from rgarnov:rg/rmat_api

Basic RMat implementation

* Added basic RMat implementation

* Fix typos in basic RMat implementation

Co-authored-by: Anton Potapov <anton.potapov@intel.com>

modules/gapi/include/opencv2/gapi/rmat.hpp

@@ -0,0 +1,124 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#ifndef OPENCV_GAPI_RMAT_HPP
+#define OPENCV_GAPI_RMAT_HPP
+
+#include <opencv2/gapi/gmat.hpp>
+
+namespace cv {
+
+// "Remote Mat", a general class which provides an abstraction layer over the data
+// storage and placement (host, remote device etc) and allows to access this data.
+//
+// The device specific implementation is hidden in the RMat::Adapter class
+//
+// The basic flow is the following:
+// * Backend which is aware of the remote device:
+//   - Implements own AdapterT class which is derived from RMat::Adapter
+//   - Wraps device memory into RMat via make_rmat utility function:
+//         cv::RMat rmat = cv::make_rmat<AdapterT>(args);
+//
+// * End user:
+//   - Writes the code which works with RMats without any knowledge of the remote device:
+//     void func(const cv::RMat& in_rmat, cv::RMat& out_rmat) {
+//         // Fetch input data from the device, get mapped memory for output
+//         cv::RMat::View  in_view =  in_rmat.access(Access::R);
+//         cv::RMat::View out_view = out_rmat.access(Access::W);
+//         performCalculations(in_view, out_view);
+//         // data from out_view is transferred to the device when out_view is destroyed
+//     }
+class RMat
+{
+public:
+    // A lightweight wrapper on image data:
+    // - Doesn't own the memory;
+    // - Doesn't implement copy semantics (it's assumed that a view is created each time
+    // wrapped data is being accessed);
+    // - Has an optional callback which is called when the view is destroyed.
+    class View
+    {
+    public:
+        using DestroyCallback = std::function<void()>;
+
+        View() = default;
+        View(const GMatDesc& desc, uchar* data, size_t step = 0u, DestroyCallback&& cb = nullptr)
+            : m_desc(desc), m_data(data), m_step(step == 0u ? elemSize()*cols() : step), m_cb(cb)
+        {}
+
+        View(const View&) = delete;
+        View(View&&) = default;
+        View& operator=(const View&) = delete;
+        View& operator=(View&&) = default;
+        ~View() { if (m_cb) m_cb(); }
+
+        cv::Size size() const { return m_desc.size; }
+        const std::vector<int>& dims() const { return m_desc.dims; }
+        int cols() const { return m_desc.size.width; }
+        int rows() const { return m_desc.size.height; }
+        int type() const { return CV_MAKE_TYPE(depth(), chan()); }
+        int depth() const { return m_desc.depth; }
+        int chan() const { return m_desc.chan; }
+        size_t elemSize() const { return CV_ELEM_SIZE(type()); }
+
+        template<typename T = uchar> T* ptr(int y = 0, int x = 0) {
+            return reinterpret_cast<T*>(m_data + m_step*y + x*CV_ELEM_SIZE(type()));
+        }
+        template<typename T = uchar> const T* ptr(int y = 0, int x = 0) const {
+            return reinterpret_cast<const T*>(m_data + m_step*y + x*CV_ELEM_SIZE(type()));
+        }
+        size_t step() const { return m_step; }
+
+    private:
+        GMatDesc m_desc;
+        uchar* m_data = nullptr;
+        size_t m_step = 0u;
+        DestroyCallback m_cb = nullptr;
+    };
+
+    enum class Access { R, W };
+    class Adapter
+    {
+    public:
+        virtual ~Adapter() = default;
+        virtual GMatDesc desc() const = 0;
+        // Implementation is responsible for setting the appropriate callback to
+        // the view when accessed for writing, to ensure that the data from the view
+        // is transferred to the device when the view is destroyed
+        virtual View access(Access) const = 0;
+    };
+    using AdapterP = std::shared_ptr<Adapter>;
+
+    RMat() = default;
+    RMat(AdapterP&& a) : m_adapter(std::move(a)) {}
+    GMatDesc desc() const { return m_adapter->desc(); }
+
+    // Note: When accessed for write there is no guarantee that returned view
+    // will contain actual snapshot of the mapped device memory
+    // (no guarantee that fetch from a device is performed). The only
+    // guaranty is that when the view is destroyed, its data will be
+    // transferred to the device
+    View access(Access a) const { return m_adapter->access(a); }
+
+    // Cast underlying RMat adapter to the particular adapter type,
+    // return nullptr if underlying type is different
+    template<typename T> T* get() const
+    {
+        static_assert(std::is_base_of<Adapter, T>::value, "T is not derived from Adapter!");
+        GAPI_Assert(m_adapter != nullptr);
+        return dynamic_cast<T*>(m_adapter.get());
+    }
+
+private:
+    AdapterP m_adapter = nullptr;
+};
+
+template<typename T, typename... Ts>
+RMat make_rmat(Ts&&... args) { return { std::make_shared<T>(std::forward<Ts>(args)...) }; }
+
+} //namespace cv
+
+#endif /* OPENCV_GAPI_RMAT_HPP */

modules/gapi/test/rmat/rmat_tests.cpp

@@ -0,0 +1,171 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#include "../test_precomp.hpp"
+#include <opencv2/gapi/rmat.hpp>
+
+namespace opencv_test {
+namespace {
+class RMatAdapterRef : public RMat::Adapter {
+    cv::Mat& m_mat;
+    bool& m_callbackCalled;
+public:
+    RMatAdapterRef(cv::Mat& m, bool& callbackCalled)
+        : m_mat(m), m_callbackCalled(callbackCalled)
+    {}
+    virtual RMat::View access(RMat::Access access) const override {
+        if (access == RMat::Access::W) {
+            return RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step,
+                              [this](){
+                                  EXPECT_FALSE(m_callbackCalled);
+                                  m_callbackCalled = true;
+                              });
+        } else {
+            return RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step);
+        }
+    }
+    virtual cv::GMatDesc desc() const override { return cv::descr_of(m_mat); }
+};
+
+class RMatAdapterCopy : public RMat::Adapter {
+    cv::Mat& m_deviceMat;
+    cv::Mat  m_hostMat;
+    bool& m_callbackCalled;
+
+public:
+    RMatAdapterCopy(cv::Mat& m, bool& callbackCalled)
+        : m_deviceMat(m), m_hostMat(m.clone()), m_callbackCalled(callbackCalled)
+    {}
+    virtual RMat::View access(RMat::Access access) const override {
+        if (access == RMat::Access::W) {
+            return RMat::View(cv::descr_of(m_hostMat), m_hostMat.data, m_hostMat.step,
+                              [this](){
+                                  EXPECT_FALSE(m_callbackCalled);
+                                  m_callbackCalled = true;
+                                  m_hostMat.copyTo(m_deviceMat);
+                              });
+        } else {
+            m_deviceMat.copyTo(m_hostMat);
+            return RMat::View(cv::descr_of(m_hostMat), m_hostMat.data, m_hostMat.step);
+        }
+    }
+    virtual cv::GMatDesc desc() const override { return cv::descr_of(m_hostMat); }
+};
+
+void randomizeMat(cv::Mat& m) {
+    auto ref = m.clone();
+    while (cv::norm(m, ref, cv::NORM_INF) == 0) {
+        cv::randu(m, cv::Scalar::all(127), cv::Scalar::all(40));
+    }
+}
+
+template <typename RMatAdapterT>
+struct RMatTest {
+    using AdapterT = RMatAdapterT;
+    RMatTest()
+        : m_deviceMat(8,8,CV_8UC1)
+        , m_rmat(make_rmat<RMatAdapterT>(m_deviceMat, m_callbackCalled)) {
+        randomizeMat(m_deviceMat);
+        expectNoCallbackCalled();
+    }
+
+    RMat& rmat() { return m_rmat; }
+    cv::Mat cloneDeviceMat() { return m_deviceMat.clone(); }
+    void expectCallbackCalled() { EXPECT_TRUE(m_callbackCalled); }
+    void expectNoCallbackCalled() { EXPECT_FALSE(m_callbackCalled); }
+
+    void expectDeviceDataEqual(const cv::Mat& mat) {
+        EXPECT_EQ(0, cv::norm(mat, m_deviceMat, NORM_INF));
+    }
+    void expectDeviceDataNotEqual(const cv::Mat& mat) {
+        EXPECT_NE(0, cv::norm(mat, m_deviceMat, NORM_INF));
+    }
+
+private:
+    cv::Mat m_deviceMat;
+    bool m_callbackCalled = false;
+    cv::RMat m_rmat;
+};
+} // anonymous namespace
+
+template<typename T>
+struct RMatTypedTest : public ::testing::Test, public T { using Type = T; };
+
+using RMatTestTypes = ::testing::Types< RMatTest<RMatAdapterRef>
+                                      , RMatTest<RMatAdapterCopy>
+                                      >;
+
+TYPED_TEST_CASE(RMatTypedTest, RMatTestTypes);
+
+TYPED_TEST(RMatTypedTest, Smoke) {
+    auto view = this->rmat().access(RMat::Access::R);
+    auto matFromDevice = cv::Mat(view.size(), view.type(), view.ptr());
+    EXPECT_TRUE(cv::descr_of(this->cloneDeviceMat()) == this->rmat().desc());
+    this->expectDeviceDataEqual(matFromDevice);
+}
+
+static Mat asMat(RMat::View& view) {
+    return Mat(view.size(), view.type(), view.ptr(), view.step());
+}
+
+TYPED_TEST(RMatTypedTest, BasicWorkflow) {
+    {
+        auto view = this->rmat().access(RMat::Access::R);
+        this->expectDeviceDataEqual(asMat(view));
+    }
+    this->expectNoCallbackCalled();
+
+    cv::Mat dataToWrite = this->cloneDeviceMat();
+    randomizeMat(dataToWrite);
+    this->expectDeviceDataNotEqual(dataToWrite);
+    {
+        auto view = this->rmat().access(RMat::Access::W);
+        dataToWrite.copyTo(asMat(view));
+    }
+    this->expectCallbackCalled();
+    this->expectDeviceDataEqual(dataToWrite);
+}
+
+TEST(RMat, TestEmptyAdapter) {
+    RMat rmat;
+    EXPECT_ANY_THROW(rmat.get<RMatAdapterCopy>());
+}
+
+TYPED_TEST(RMatTypedTest, CorrectAdapterCast) {
+    using T = typename TestFixture::Type::AdapterT;
+    EXPECT_NE(nullptr, this->rmat().template get<T>());
+}
+
+class DummyAdapter : public RMat::Adapter {
+    virtual RMat::View access(RMat::Access) const override { return {}; }
+    virtual cv::GMatDesc desc() const override { return {}; }
+};
+
+TYPED_TEST(RMatTypedTest, IncorrectAdapterCast) {
+    EXPECT_EQ(nullptr, this->rmat().template get<DummyAdapter>());
+}
+
+class RMatAdapterForBackend : public RMat::Adapter {
+    int m_i;
+public:
+    RMatAdapterForBackend(int i) : m_i(i) {}
+    virtual RMat::View access(RMat::Access) const override { return {}; }
+    virtual GMatDesc desc() const override { return {}; }
+    int deviceSpecificData() const { return m_i; }
+};
+
+// RMat's usage scenario in the backend:
+// we have some specific data hidden under RMat,
+// test that we can obtain it via RMat.as<T>() method
+TEST(RMat, UsageInBackend) {
+    int i = std::rand();
+    auto rmat = cv::make_rmat<RMatAdapterForBackend>(i);
+
+    auto adapter = rmat.get<RMatAdapterForBackend>();
+    EXPECT_NE(nullptr, adapter);
+    EXPECT_EQ(i, adapter->deviceSpecificData());
+}
+} // namespace opencv_test

modules/gapi/test/rmat/rmat_view_tests.cpp

@@ -0,0 +1,147 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#include "../test_precomp.hpp"
+#include <opencv2/gapi/rmat.hpp>
+
+#include <opencv2/gapi/util/compiler_hints.hpp>
+#include "../src/backends/common/gbackend.hpp"
+
+namespace opencv_test
+{
+using cv::GMatDesc;
+using View = cv::RMat::View;
+using cv::Mat;
+using namespace ::testing;
+
+static void expect_eq_desc(const View& view, const GMatDesc& desc) {
+    EXPECT_EQ(view.size(), desc.size);
+    EXPECT_EQ(view.dims(), desc.dims);
+    EXPECT_EQ(view.cols(), desc.size.width);
+    EXPECT_EQ(view.rows(), desc.size.height);
+    EXPECT_EQ(view.type(), CV_MAKE_TYPE(desc.depth,desc.chan));
+    EXPECT_EQ(view.depth(), desc.depth);
+    EXPECT_EQ(view.chan(), desc.chan);
+}
+
+TEST(RMatView, TestDefaultConstruction) {
+    View view;
+    GMatDesc desc{};
+    expect_eq_desc(view, desc);
+    EXPECT_EQ(view.ptr(), nullptr);
+    EXPECT_EQ(view.step(), 0u);
+}
+
+struct RMatViewTest : public TestWithParam<int /*dataType*/>{};
+TEST_P(RMatViewTest, ConstructionFromMat) {
+    auto type = GetParam();
+    Mat mat(8,8,type);
+    const auto desc = cv::descr_of(mat);
+    View view(cv::descr_of(mat), mat.ptr(), mat.step1());
+    expect_eq_desc(view, desc);
+    EXPECT_EQ(view.ptr(), mat.ptr());
+    EXPECT_EQ(view.step(), mat.step1());
+}
+
+TEST(RMatView, TestConstructionFromMatND) {
+    std::vector<int> dims(4, 8);
+    Mat mat(dims, CV_8UC1);
+    const auto desc = cv::descr_of(mat);
+    View view(cv::descr_of(mat), mat.ptr());
+    expect_eq_desc(view, desc);
+    EXPECT_EQ(view.ptr(), mat.ptr());
+}
+
+TEST_P(RMatViewTest, DefaultStep) {
+    auto type = GetParam();
+    GMatDesc desc;
+    desc.chan = CV_MAT_CN(type);
+    desc.depth = CV_MAT_DEPTH(type);
+    desc.size = {8,8};
+    std::vector<unsigned char> data(desc.size.width*desc.size.height*CV_ELEM_SIZE(type));
+    View view(desc, data.data());
+    EXPECT_EQ(view.step(), static_cast<size_t>(desc.size.width)*CV_ELEM_SIZE(type));
+}
+
+static Mat asMat(View& view) {
+    return Mat(view.size(), view.type(), view.ptr(), view.step());
+}
+
+TEST_P(RMatViewTest, NonDefaultStepInput) {
+    auto type = GetParam();
+    Mat bigMat(16,16,type);
+    cv::randn(bigMat, cv::Scalar::all(127), cv::Scalar::all(40));
+    Mat mat = bigMat(cv::Rect{4,4,8,8});
+    View view(cv::descr_of(mat), mat.data, mat.step);
+    const auto viewMat = asMat(view);
+    Mat ref, out;
+    cv::Size ksize{1,1};
+    cv::blur(viewMat, out, ksize);
+    cv::blur(    mat, ref, ksize);
+    EXPECT_EQ(0, cvtest::norm(ref, out, NORM_INF));
+}
+
+TEST_P(RMatViewTest, NonDefaultStepOutput) {
+    auto type = GetParam();
+    Mat mat(8,8,type);
+    cv::randn(mat, cv::Scalar::all(127), cv::Scalar::all(40));
+    Mat bigMat = Mat::zeros(16,16,type);
+    Mat out = bigMat(cv::Rect{4,4,8,8});
+    View view(cv::descr_of(out), out.ptr(), out.step);
+    auto viewMat = asMat(view);
+    Mat ref;
+    cv::Size ksize{1,1};
+    cv::blur(mat, viewMat, ksize);
+    cv::blur(mat, ref,     ksize);
+    EXPECT_EQ(0, cvtest::norm(ref, out, NORM_INF));
+}
+
+INSTANTIATE_TEST_CASE_P(Test, RMatViewTest,
+                        Values(CV_8UC1, CV_8UC3, CV_32FC1));
+
+struct RMatViewCallbackTest : public ::testing::Test {
+    RMatViewCallbackTest()
+        : mat(8,8,CV_8UC1), view(cv::descr_of(mat), mat.ptr(), mat.step1(), [this](){ callbackCalls++; }) {
+        cv::randn(mat, cv::Scalar::all(127), cv::Scalar::all(40));
+    }
+    int callbackCalls = 0;
+    Mat mat;
+    View view;
+};
+
+TEST_F(RMatViewCallbackTest, MoveCopy) {
+    {
+        View copy(std::move(view));
+        cv::util::suppress_unused_warning(copy);
+        EXPECT_EQ(callbackCalls, 0);
+    }
+    EXPECT_EQ(callbackCalls, 1);
+}
+
+static int firstElement(const View& view) { return *view.ptr(); }
+static void setFirstElement(View& view, uchar value) { *view.ptr() = value; }
+
+TEST_F(RMatViewCallbackTest, MagazineInteraction) {
+    cv::gimpl::magazine::Class<View> mag;
+    constexpr int rc = 1;
+    constexpr uchar value = 11;
+    mag.slot<View>()[rc] = std::move(view);
+    {
+        auto& mag_view = mag.slot<View>()[rc];
+        setFirstElement(mag_view, value);
+        auto mag_el = firstElement(mag_view);
+        EXPECT_EQ(mag_el, value);
+    }
+    {
+        const auto& mag_view = mag.slot<View>()[rc];
+        auto mag_el = firstElement(mag_view);
+        EXPECT_EQ(mag_el, value);
+    }
+    EXPECT_EQ(callbackCalls, 0);
+    mag.slot<View>().erase(rc);
+    EXPECT_EQ(callbackCalls, 1);
+}
+} // namespace opencv_test