mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-08-05 22:19:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #18600 from TolyaTalamanov:at/implement-render-using-stateful

Browse Source 
[G-API] Implement render using stateful kernel

* Implement render using stateful kernel

* Move freetype to backends folder

* Fix freetype compilation

* Fix freetype smoke test

* Fix comments

* Refactoring
This commit is contained in:
Anatoliy Talamanov 2020-10-26 22:55:43 +03:00 committed by GitHub
parent 93c3775927
commit 3d4563913d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 83 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/gapi/CMakeLists.txt
View File

@ -77,7 +77,6 @@ set(gapi_srcs
src/api/render.cpp
src/api/render_ocv.cpp
src/api/ginfer.cpp
src/api/ft_render.cpp
src/api/media.cpp
src/api/rmat.cpp
@ -131,8 +130,8 @@ set(gapi_srcs
src/backends/ie/giebackend/giewrapper.cpp
# Render Backend.
src/backends/render/grenderocvbackend.cpp
src/backends/render/grenderocv.cpp
src/backends/render/ft_render.cpp
#PlaidML Backend
src/backends/plaidml/gplaidmlcore.cpp

24
modules/gapi/src/api/render_ocv.cpp
View File

@ -2,7 +2,7 @@
#include <opencv2/gapi/render/render.hpp> // Kernel API's
#include "api/render_ocv.hpp"
#include "api/ft_render.hpp"
#include "backends/render/ft_render.hpp"
namespace cv
{
@ -146,12 +146,8 @@ struct EmptyConverter
template <typename ColorConverter>
void drawPrimitivesOCV(cv::Mat& in,
const cv::gapi::wip::draw::Prims& prims,
cv::gapi::wip::draw::FTTextRender* ftpr)
std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& ftpr)
{
#ifndef HAVE_FREETYPE
cv::util::suppress_unused_warning(ftpr);
#endif
using namespace cv::gapi::wip::draw;
ColorConverter converter;
@ -177,7 +173,6 @@ void drawPrimitivesOCV(cv::Mat& in,
case Prim::index_of<FText>():
{
#ifdef HAVE_FREETYPE
const auto& ftp = cv::util::get<FText>(p);
const auto color = converter.cvtColor(ftp.color);
@ -196,9 +191,6 @@ void drawPrimitivesOCV(cv::Mat& in,
cv::Point tl(ftp.org.x, ftp.org.y - mask.size().height + baseline);
blendTextMask(in, mask, tl, color);
#else
cv::util::throw_error(std::runtime_error("FreeType not found !"));
#endif
break;
}
@ -251,16 +243,16 @@ void drawPrimitivesOCV(cv::Mat& in,
}
}
void drawPrimitivesOCVBGR(cv::Mat &in,
const cv::gapi::wip::draw::Prims &prims,
cv::gapi::wip::draw::FTTextRender* ftpr)
void drawPrimitivesOCVBGR(cv::Mat &in,
const cv::gapi::wip::draw::Prims &prims,
std::shared_ptr<cv::gapi::wip::draw::FTTextRender> &ftpr)
{
drawPrimitivesOCV<EmptyConverter>(in, prims, ftpr);
}
void drawPrimitivesOCVYUV(cv::Mat &in,
const cv::gapi::wip::draw::Prims &prims,
cv::gapi::wip::draw::FTTextRender* ftpr)
void drawPrimitivesOCVYUV(cv::Mat &in,
const cv::gapi::wip::draw::Prims &prims,
std::shared_ptr<cv::gapi::wip::draw::FTTextRender> &ftpr)
{
drawPrimitivesOCV<BGR2YUVConverter>(in, prims, ftpr);
}

6
modules/gapi/src/api/render_ocv.hpp
View File

@ -1,6 +1,6 @@
#include <vector>
#include "render_priv.hpp"
#include "ft_render.hpp"
#include "backends/render/ft_render.hpp"
#ifndef OPENCV_RENDER_OCV_HPP
#define OPENCV_RENDER_OCV_HPP
@ -15,8 +15,8 @@ namespace draw
{
// FIXME only for tests
void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc);
void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc);
void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& mc);
void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& mc);
} // namespace draw
} // namespace wip

26
modules/gapi/src/api/ft_render.cpp → modules/gapi/src/backends/render/ft_render.cpp
View File

@ -5,11 +5,11 @@
// Copyright (C) 2019 Intel Corporation
#include "precomp.hpp"
#include "ft_render.hpp"
#ifdef HAVE_FREETYPE
#include "api/ft_render.hpp"
#include "api/ft_render_priv.hpp"
#include "ft_render_priv.hpp"
#include <opencv2/gapi/util/throw.hpp>
#include <opencv2/gapi/own/assert.hpp>
@ -166,6 +166,11 @@ void cv::gapi::wip::draw::FTTextRender::Priv::putText(cv::Mat& mat,
"Failed to load char");
FT_Bitmap *bitmap = &(m_face->glyph->bitmap);
// FIXME: Skip glyph, if size is 0
if (bitmap->rows == 0 || bitmap->width == 0) {
continue;
}
cv::Mat glyph(bitmap->rows, bitmap->width, CV_8UC1, bitmap->buffer, bitmap->pitch);
int left = m_face->glyph->bitmap_left;
@ -211,4 +216,21 @@ void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat& mat,
m_priv->putText(mat, text, org, fh);
}
#else
cv::Size cv::gapi::wip::draw::FTTextRender::getTextSize(const std::wstring&, int, int*)
{
cv::util::throw_error(std::runtime_error("Freetype not found"));
}
void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat&, const std::wstring&, const cv::Point&, int)
{
cv::util::throw_error(std::runtime_error("Freetype not found"));
}
cv::gapi::wip::draw::FTTextRender::FTTextRender(const std::string&)
{
cv::util::throw_error(std::runtime_error("Freetype not found"));
}
#endif // HAVE_FREETYPE

8
modules/gapi/src/api/ft_render.hpp → modules/gapi/src/backends/render/ft_render.hpp
View File

@ -23,8 +23,6 @@ namespace wip
namespace draw
{
#ifdef HAVE_FREETYPE
class GAPI_EXPORTS FTTextRender
{
public:
@ -38,12 +36,6 @@ private:
std::shared_ptr<Priv> m_priv;
};
#else
class GAPI_EXPORTS FTTextRender {};
#endif // HAVE_FREETYPE
} // namespace draw
} // namespace wip
} // namespace gapi

2
modules/gapi/src/api/ft_render_priv.hpp → modules/gapi/src/backends/render/ft_render_priv.hpp
View File

@ -10,7 +10,7 @@
#ifndef OPENCV_FT_RENDER_PRIV_HPP
#define OPENCV_FT_RENDER_PRIV_HPP
#include "api/ft_render.hpp"
#include "ft_render.hpp"
#include <ft2build.h>
#include FT_FREETYPE_H

54
modules/gapi/src/backends/render/grenderocv.cpp
View File

@ -1,16 +1,21 @@
#include <opencv2/imgproc.hpp>
#include "api/render_ocv.hpp"
#include "backends/render/grenderocv.hpp"
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include <opencv2/gapi/fluid/core.hpp>
GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR)
struct RenderOCVState
{
std::shared_ptr<cv::gapi::wip::draw::FTTextRender> ftpr;
};
GAPI_OCV_KERNEL_ST(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR, RenderOCVState)
{
static void run(const cv::Mat& in,
const cv::gapi::wip::draw::Prims& prims,
cv::gapi::wip::draw::FTTextRender* ftpr,
cv::Mat& out)
cv::Mat& out,
RenderOCVState& state)
{
// NB: If in and out cv::Mats are the same object
// we can avoid copy and render on out cv::Mat
@ -19,18 +24,33 @@ GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR)
in.copyTo(out);
}
cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, ftpr);
cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, state.ftpr);
}
static void setup(const cv::GMatDesc& /* in */,
const cv::GArrayDesc& /* prims */,
std::shared_ptr<RenderOCVState>& state,
const cv::GCompileArgs& args)
{
using namespace cv::gapi::wip::draw;
auto opt_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
state = std::make_shared<RenderOCVState>();
if (opt_freetype_font.has_value())
{
state->ftpr = std::make_shared<FTTextRender>(opt_freetype_font->path);
}
}
};
GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOCVState)
{
static void run(const cv::Mat& in_y,
const cv::Mat& in_uv,
const cv::gapi::wip::draw::Prims& prims,
cv::gapi::wip::draw::FTTextRender* ftpr,
cv::Mat& out_y,
cv::Mat& out_uv)
cv::Mat& out_uv,
RenderOCVState& state)
{
// NB: If in and out cv::Mats are the same object
// we can avoid copy and render on out cv::Mat
@ -67,7 +87,7 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR);
cv::merge(std::vector<cv::Mat>{in_y, upsample_uv}, yuv);
cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, ftpr);
cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr);
// YUV -> NV12
cv::Mat out_u, out_v, uv_plane;
@ -76,6 +96,22 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
cv::merge(std::vector<cv::Mat>{chs[1], chs[2]}, uv_plane);
cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR);
}
static void setup(const cv::GMatDesc& /* in_y */,
const cv::GMatDesc& /* in_uv */,
const cv::GArrayDesc& /* prims */,
std::shared_ptr<RenderOCVState>& state,
const cv::GCompileArgs& args)
{
using namespace cv::gapi::wip::draw;
auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
state = std::make_shared<RenderOCVState>();
if (has_freetype_font)
{
state->ftpr = std::make_shared<FTTextRender>(has_freetype_font->path);
}
}
};
cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels()

55
modules/gapi/src/backends/render/grenderocv.hpp
View File

@ -1,55 +0,0 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
#ifndef OPENCV_GAPI_GRENDEROCV_HPP
#define OPENCV_GAPI_GRENDEROCV_HPP
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include "api/render_priv.hpp"
#include "api/ft_render.hpp"
namespace cv
{
namespace gapi
{
namespace render
{
namespace ocv
{
GAPI_EXPORTS cv::gapi::GBackend backend();
template<typename, typename>
struct add_type_to_tuple;
template<typename P, typename ...Ts>
struct add_type_to_tuple<P, std::tuple<Ts...>>
{
using type = std::tuple<Ts..., P>;
};
template<class Impl, class K>
class GRenderKernelImpl: public cv::detail::OCVCallHelper<Impl, typename K::InArgs, typename K::OutArgs>,
public cv::detail::KernelTag
{
using InArgs = typename add_type_to_tuple<cv::gapi::wip::draw::FTTextRender*, typename K::InArgs>::type;
using P = detail::OCVCallHelper<Impl, InArgs, typename K::OutArgs>;
public:
using API = K;
static cv::gapi::GBackend backend() { return cv::gapi::render::ocv::backend(); }
static cv::GCPUKernel kernel() { return GCPUKernel(&P::call); }
};
#define GAPI_RENDER_OCV_KERNEL(Name, API) struct Name: public cv::gapi::render::ocv::GRenderKernelImpl<Name, API>
} // namespace ocv
} // namespace render
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_GRENDEROCV_HPP

161
modules/gapi/src/backends/render/grenderocvbackend.cpp
View File

@ -1,161 +0,0 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2018-2020 Intel Corporation
#include "precomp.hpp"
#include <functional>
#include <unordered_set>
#include <ade/util/algorithm.hpp>
#include <ade/util/range.hpp>
#include <ade/util/zip_range.hpp>
#include <ade/util/chain_range.hpp>
#include <ade/typed_graph.hpp>
#include <opencv2/gapi/gcommon.hpp>
#include <opencv2/gapi/garray.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/gapi/gtype_traits.hpp>
#include "compiler/gobjref.hpp"
#include "compiler/gmodel.hpp"
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
#include "api/render_ocv.hpp"
#include "backends/render/grenderocvbackend.hpp"
#include <opencv2/gapi/render/render.hpp>
#include "api/ocv_mask_creator.hpp"
#include "api/ft_render.hpp"
using GRenderModel = ade::TypedGraph
< cv::gimpl::render::ocv::RenderUnit
>;
// FIXME: Same issue with Typed and ConstTyped
using GConstRenderModel = ade::ConstTypedGraph
< cv::gimpl::render::ocv::RenderUnit
>;
cv::gimpl::render::ocv::GRenderExecutable::GRenderExecutable(const ade::Graph &g,
const std::vector<ade::NodeHandle> &nodes,
std::unique_ptr<cv::gapi::wip::draw::FTTextRender>&& ftpr)
: m_g(g), m_gm(m_g), m_ftpr(std::move(ftpr)) {
GConstRenderModel gcm(m_g);
auto is_op = [&](ade::NodeHandle nh) {
return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
};
auto it = ade::util::find_if(nodes, is_op);
GAPI_Assert(it != nodes.end());
this_nh = *it;
if (!std::none_of(std::next(it), nodes.end(), is_op)) {
util::throw_error(std::logic_error("Multi-node rendering is not supported!"));
}
}
void cv::gimpl::render::ocv::GRenderExecutable::run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs) {
GConstRenderModel gcm(m_g);
for (auto& it : input_objs) magazine::bindInArg (m_res, it.first, it.second);
for (auto& it : output_objs) magazine::bindOutArg(m_res, it.first, it.second);
const auto &op = m_gm.metadata(this_nh).get<Op>();
// Initialize kernel's execution context:
// - Input parameters
GCPUContext context;
context.m_args.reserve(op.args.size());
using namespace std::placeholders;
ade::util::transform(op.args,
std::back_inserter(context.m_args),
std::bind(&GRenderExecutable::packArg, this, _1));
// - Output parameters.
for (const auto &out_it : ade::util::indexed(op.outs)) {
// FIXME: Can the same GArg type resolution mechanism be reused here?
const auto out_port = ade::util::index(out_it);
const auto out_desc = ade::util::value(out_it);
context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc);
}
auto k = gcm.metadata(this_nh).get<RenderUnit>().k;
context.m_args.emplace_back(m_ftpr.get());
k.m_runF(context);
for (auto &it : output_objs) magazine::writeBack(m_res, it.first, it.second);
// In/Out args clean-up is mandatory now with RMat
for (auto &it : input_objs) magazine::unbind(m_res, it.first);
for (auto &it : output_objs) magazine::unbind(m_res, it.first);
}
cv::GArg cv::gimpl::render::ocv::GRenderExecutable::packArg(const cv::GArg &arg) {
// No API placeholders allowed at this point
// FIXME: this check has to be done somewhere in compilation stage.
GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY);
if (arg.kind != cv::detail::ArgKind::GOBJREF) {
util::throw_error(std::logic_error("Render supports G-types ONLY!"));
}
GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF);
const cv::gimpl::RcDesc &ref = arg.get<cv::gimpl::RcDesc>();
switch (ref.shape)
{
case GShape::GMAT: return GArg(m_res.slot<cv::Mat>()[ref.id]);
case GShape::GARRAY: return GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;
}
}
namespace {
class GRenderBackendImpl final: public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &gr,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override {
GRenderModel rm(gr);
auto render_impl = cv::util::any_cast<cv::GCPUKernel>(impl.opaque);
rm.metadata(op_node).set(cv::gimpl::render::ocv::RenderUnit{render_impl});
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs& args,
const std::vector<ade::NodeHandle> &nodes) const override {
using namespace cv::gapi::wip::draw;
auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
std::unique_ptr<FTTextRender> ftpr;
if (has_freetype_font)
{
#ifndef HAVE_FREETYPE
throw std::runtime_error("Freetype not found");
#else
ftpr.reset(new FTTextRender(has_freetype_font.value().path));
#endif
}
return EPtr{new cv::gimpl::render::ocv::GRenderExecutable(graph, nodes, std::move(ftpr))};
}
};
}
cv::gapi::GBackend cv::gapi::render::ocv::backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GRenderBackendImpl>());
return this_backend;
}

73
modules/gapi/src/backends/render/grenderocvbackend.hpp
View File

@ -1,73 +0,0 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2019 Intel Corporation
#ifndef OPENCV_GAPI_GRENDEROCVBACKEND_HPP
#define OPENCV_GAPI_GRENDEROCVBACKEND_HPP
#include <opencv2/gapi/garg.hpp>
#include <opencv2/gapi/gproto.hpp>
#include <opencv2/gapi/render/render.hpp>
#include "api/gorigin.hpp"
#include "backends/common/gbackend.hpp"
#include "compiler/gislandmodel.hpp"
#include "backends/render/grenderocv.hpp"
#include <opencv2/gapi/cpu/gcpukernel.hpp>
namespace cv
{
namespace gimpl
{
namespace render
{
namespace ocv
{
struct RenderUnit
{
static const char *name() { return "RenderUnit"; }
GCPUKernel k;
};
class GRenderExecutable final: public GIslandExecutable
{
const ade::Graph &m_g;
GModel::ConstGraph m_gm;
std::unique_ptr<cv::gapi::wip::draw::FTTextRender> m_ftpr;
// The only executable stuff in this graph
// (assuming it is always single-op)
ade::NodeHandle this_nh;
//// Actual data of all resources in graph (both internal and external)
Mag m_res;
//// Execution helpers
GArg packArg(const GArg &arg);
public:
GRenderExecutable(const ade::Graph &graph,
const std::vector<ade::NodeHandle> &nodes,
std::unique_ptr<cv::gapi::wip::draw::FTTextRender>&& ftpr);
virtual inline bool canReshape() const override { return false; }
virtual inline void reshape(ade::Graph&, const GCompileArgs&) override {
GAPI_Assert(false); // Not implemented yet
}
virtual void run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs) override;
};
} // namespace ocv
} // namespace render
} // namespace gimpl
} // namespace cv
#endif // OPENCV_GAPI_GRENDEROCVBACKEND_HPP

2
modules/gapi/test/render/ftp_render_test.cpp
View File

@ -13,7 +13,7 @@
#include <opencv2/core/utils/configuration.private.hpp>
#include "api/ft_render.hpp"
#include "backends/render/ft_render.hpp"
namespace opencv_test
{

1
modules/gapi/test/render/gapi_render_tests_ocv.cpp
View File

@ -95,7 +95,6 @@ TEST_P(RenderNV12OCVTestFTexts, AccuracyTest)
cv::compile_args(cv::gapi::wip::draw::freetype_font{
"/usr/share/fonts/truetype/wqy/wqy-microhei.ttc"
})));
}
static std::wstring to_wstring(const char* bytes)