// This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // // Copyright (C) 2018 Intel Corporation #include "precomp.hpp" #include // hex, dec (debug) #include "opencv2/gapi/own/convert.hpp" #include "opencv2/gapi/own/types.hpp" #include "opencv2/gapi/fluid/gfluidbuffer.hpp" #include "backends/fluid/gfluidbuffer_priv.hpp" #include "opencv2/gapi/opencv_includes.hpp" #include "backends/fluid/gfluidutils.hpp" // saturate namespace cv { namespace gapi { namespace fluid { bool operator == (const fluid::Border& b1, const fluid::Border& b2) { return b1.type == b2.type && b1.value == b2.value; } } // namespace fluid // Fluid BorderHandler implementation ///////////////////////////////////////////////// namespace { template // Expected inputs: // row - row buffer allocated with border in mind (have memory for both image and border pixels) // length - size of the buffer with left and right borders included void fillBorderReplicateRow(uint8_t* row, int length, int chan, int borderSize) { auto leftBorder = reinterpret_cast(row); auto rightBorder = leftBorder + (length - borderSize) * chan; for (int b = 0; b < borderSize; b++) { for (int c = 0; c < chan; c++) { leftBorder [b*chan + c] = leftBorder [borderSize*chan + c]; rightBorder[b*chan + c] = rightBorder[-chan + c]; } } } template void fillBorderReflectRow(uint8_t* row, int length, int chan, int borderSize) { auto leftBorder = reinterpret_cast(row); auto rightBorder = leftBorder + (length - borderSize) * chan; for (int b = 0; b < borderSize; b++) { for (int c = 0; c < chan; c++) { leftBorder [b*chan + c] = leftBorder [(2*borderSize - b)*chan + c]; rightBorder[b*chan + c] = rightBorder[(-b - 2)*chan + c]; } } } template void fillConstBorderRow(uint8_t* row, int length, int chan, int borderSize, cv::gapi::own::Scalar borderValue) { GAPI_DbgAssert(chan > 0 && chan <= 4); auto leftBorder = reinterpret_cast(row); auto rightBorder = leftBorder + (length - borderSize) * chan; for (int b = 0; b < borderSize; b++) { for (int c = 0; c < chan; c++) { leftBorder [b*chan + c] = fluid::saturate(borderValue[c], fluid::roundd); rightBorder[b*chan + c] = fluid::saturate(borderValue[c], fluid::roundd); } } } // Fills const border pixels in the whole mat void fillBorderConstant(int borderSize, cv::gapi::own::Scalar borderValue, cv::gapi::own::Mat& mat) { // cv::Scalar can contain maximum 4 chan GAPI_Assert(mat.channels() > 0 && mat.channels() <= 4); auto getFillBorderRowFunc = [&](int type) { switch(type) { case CV_8U: return &fillConstBorderRow< uint8_t>; break; case CV_16S: return &fillConstBorderRow< int16_t>; break; case CV_16U: return &fillConstBorderRow; break; case CV_32F: return &fillConstBorderRow< float >; break; default: GAPI_Assert(false); return &fillConstBorderRow; } }; auto fillBorderRow = getFillBorderRowFunc(mat.depth()); for (int y = 0; y < mat.rows; y++) { fillBorderRow(mat.ptr(y), mat.cols, mat.channels(), borderSize, borderValue); } } } // anonymous namespace fluid::BorderHandler::BorderHandler(int border_size) { GAPI_Assert(border_size > 0); m_border_size = border_size; } template fluid::BorderHandlerT::BorderHandlerT(int border_size, int data_type) : BorderHandler(border_size) { auto getFillBorderRowFunc = [&](int border, int depth) { if (border == cv::BORDER_REPLICATE) { switch(depth) { case CV_8U: return &fillBorderReplicateRow< uint8_t>; break; case CV_16S: return &fillBorderReplicateRow< int16_t>; break; case CV_16U: return &fillBorderReplicateRow; break; case CV_32F: return &fillBorderReplicateRow< float >; break; default: GAPI_Assert(!"Unsupported data type"); return &fillBorderReplicateRow; } } else if (border == cv::BORDER_REFLECT_101) { switch(depth) { case CV_8U: return &fillBorderReflectRow< uint8_t>; break; case CV_16S: return &fillBorderReflectRow< int16_t>; break; case CV_16U: return &fillBorderReflectRow; break; case CV_32F: return &fillBorderReflectRow< float >; break; default: GAPI_Assert(!"Unsupported data type"); return &fillBorderReflectRow; } } else { GAPI_Assert(!"Unsupported border type"); return &fillBorderReflectRow; } }; m_fill_border_row = getFillBorderRowFunc(BorderType, CV_MAT_DEPTH(data_type)); } namespace { template int getBorderIdx(int log_idx, int desc_height); template<> int getBorderIdx(int log_idx, int desc_height) { return log_idx < 0 ? 0 : desc_height - 1; } template<> int getBorderIdx(int log_idx, int desc_height) { return log_idx < 0 ? -log_idx : 2*(desc_height - 1) - log_idx; } } // namespace template const uint8_t* fluid::BorderHandlerT::inLineB(int log_idx, const BufferStorageWithBorder& data, int desc_height) const { auto idx = getBorderIdx(log_idx, desc_height); return data.ptr(idx); } fluid::BorderHandlerT::BorderHandlerT(int border_size, cv::gapi::own::Scalar border_value) : BorderHandler(border_size), m_border_value(border_value) { /* nothing */ } const uint8_t* fluid::BorderHandlerT::inLineB(int /*log_idx*/, const BufferStorageWithBorder& /*data*/, int /*desc_height*/) const { return m_const_border.ptr(0, m_border_size); } void fluid::BorderHandlerT::fillCompileTimeBorder(BufferStorageWithBorder& data) { m_const_border.create(1, data.cols(), data.data().type()); m_const_border = m_border_value; cv::gapi::fillBorderConstant(m_border_size, m_border_value, data.data()); } template void fluid::BorderHandlerT::updateBorderPixels(BufferStorageWithBorder &data, int startLine, int nLines) const { auto& mat = data.data(); auto length = mat.cols; auto chan = mat.channels(); for (int l = startLine; l < startLine + nLines; l++) { auto row = mat.ptr(data.physIdx(l)); m_fill_border_row(row, length, chan, m_border_size); } } std::size_t fluid::BorderHandlerT::size() const { return m_const_border.total() * m_const_border.elemSize(); } // Fluid BufferStorage implementation ////////////////////////////////////////// void fluid::BufferStorage::updateInCache(View::Cache& cache, int start_log_idx, int nLines) const { for (int i = 0; i < nLines; i++) { cache.m_linePtrs[i] = inLineB(start_log_idx + i, cache.m_desc.size.height); } } void fluid::BufferStorage::updateOutCache(Buffer::Cache& cache, int start_log_idx, int nLines) { for (int i = 0; i < nLines; i++) { cache.m_linePtrs[i] = ptr(start_log_idx + i); } } void fluid::BufferStorageWithBorder::init(int dtype, int border_size, Border border) { switch(border.type) { case cv::BORDER_CONSTANT: m_borderHandler.reset(new BorderHandlerT(border_size, border.value)); break; case cv::BORDER_REPLICATE: m_borderHandler.reset(new BorderHandlerT(border_size, dtype)); break; case cv::BORDER_REFLECT_101: m_borderHandler.reset(new BorderHandlerT(border_size, dtype)); break; default: GAPI_Assert(false); } } void fluid::BufferStorageWithBorder::create(int capacity, int desc_width, int dtype) { auto borderSize = m_borderHandler->borderSize(); auto width = (desc_width + 2*borderSize); m_data.create(capacity, width, dtype); m_borderHandler->fillCompileTimeBorder(*this); } void fluid::BufferStorageWithoutBorder::create(int capacity, int desc_width, int dtype) { auto width = desc_width; m_data.create(capacity, width, dtype); m_is_virtual = true; } const uint8_t* fluid::BufferStorageWithBorder::inLineB(int log_idx, int desc_height) const { if (log_idx < 0 || log_idx >= desc_height) { return m_borderHandler->inLineB(log_idx, *this, desc_height); } else { return ptr(log_idx); } } static void copyWithoutBorder(const cv::gapi::own::Mat& src, int src_border_size, cv::gapi::own::Mat& dst, int dst_border_size, int startSrcLine, int startDstLine, int lpi) { auto subSrc = src(cv::gapi::own::Rect{src_border_size, startSrcLine, src.cols - 2*src_border_size, lpi}); auto subDst = dst(cv::gapi::own::Rect{dst_border_size, startDstLine, dst.cols - 2*dst_border_size, lpi}); subSrc.copyTo(subDst); } void fluid::BufferStorageWithoutBorder::copyTo(BufferStorageWithBorder &dst, int startLine, int nLines) const { for (int l = startLine; l < startLine + nLines; l++) { copyWithoutBorder(m_data, 0, dst.data(), dst.borderSize(), physIdx(l), dst.physIdx(l), 1); } } void fluid::BufferStorageWithBorder::copyTo(BufferStorageWithBorder &dst, int startLine, int nLines) const { // Copy required lpi lines line by line (to avoid wrap if invoked for multiple lines) for (int l = startLine; l < startLine + nLines; l++) { copyWithoutBorder(m_data, borderSize(), dst.data(), dst.borderSize(), physIdx(l), dst.physIdx(l), 1); } } // FIXME? remember parent and remove src parameter? void fluid::BufferStorageWithBorder::updateBeforeRead(int startLine, int nLines, const BufferStorage& src) { // TODO: // Cover with tests!! // (Ensure that there are no redundant copies done // and only required (not fetched before) lines are copied) GAPI_DbgAssert(startLine >= 0); src.copyTo(*this, startLine, nLines); m_borderHandler->updateBorderPixels(*this, startLine, nLines); } void fluid::BufferStorageWithoutBorder::updateBeforeRead(int /*startLine*/, int /*lpi*/, const BufferStorage& /*src*/) { /* nothing */ } void fluid::BufferStorageWithBorder::updateAfterWrite(int startLine, int nLines) { // FIXME? // Actually startLine + nLines can be > logical height so // redundant end lines which will never be read // can be filled in the ring buffer m_borderHandler->updateBorderPixels(*this, startLine, nLines); } void fluid::BufferStorageWithoutBorder::updateAfterWrite(int /*startLine*/, int /*lpi*/) { /* nothing */ } size_t fluid::BufferStorageWithBorder::size() const { return m_data.total()*m_data.elemSize() + m_borderHandler->size(); } size_t fluid::BufferStorageWithoutBorder::size() const { return m_data.total()*m_data.elemSize(); } namespace fluid { namespace { std::unique_ptr createStorage(int capacity, int desc_width, int type, int border_size, fluid::BorderOpt border); std::unique_ptr createStorage(int capacity, int desc_width, int type, int border_size, fluid::BorderOpt border) { if (border) { std::unique_ptr storage(new BufferStorageWithBorder); storage->init(type, border_size, border.value()); storage->create(capacity, desc_width, type); return std::move(storage); } std::unique_ptr storage(new BufferStorageWithoutBorder); storage->create(capacity, desc_width, type); return std::move(storage); } std::unique_ptr createStorage(const cv::gapi::own::Mat& data, cv::gapi::own::Rect roi); std::unique_ptr createStorage(const cv::gapi::own::Mat& data, cv::gapi::own::Rect roi) { std::unique_ptr storage(new BufferStorageWithoutBorder); storage->attach(data, roi); return std::move(storage); } } // namespace } // namespace fluid // Fluid View implementation /////////////////////////////////////////////////// void fluid::View::Priv::reset(int linesForFirstIteration) { GAPI_DbgAssert(m_p); m_lines_next_iter = linesForFirstIteration; m_read_caret = m_p->priv().readStart(); } void fluid::View::Priv::readDone(int linesRead, int linesForNextIteration) { GAPI_DbgAssert(m_p); m_read_caret += linesRead; m_lines_next_iter = linesForNextIteration; } bool fluid::View::Priv::ready() const { auto lastWrittenLine = m_p->priv().writeStart() + m_p->linesReady(); // + bottom border if (lastWrittenLine == m_p->meta().size.height) lastWrittenLine += m_border_size; // + top border lastWrittenLine += m_border_size; auto lastRequiredLine = m_read_caret + m_lines_next_iter; return lastWrittenLine >= lastRequiredLine; } fluid::ViewPrivWithoutOwnBorder::ViewPrivWithoutOwnBorder(const Buffer *parent, int borderSize) { GAPI_Assert(parent); m_p = parent; m_border_size = borderSize; } void fluid::ViewPrivWithoutOwnBorder::allocate(int lineConsumption, BorderOpt) { initCache(lineConsumption); } void fluid::ViewPrivWithoutOwnBorder::prepareToRead() { const auto &storage = m_p->priv().storage(); const int start_log_idx = m_read_caret - m_border_size; storage.updateInCache(m_cache, start_log_idx, m_lines_next_iter); } fluid::ViewPrivWithOwnBorder::ViewPrivWithOwnBorder(const Buffer *parent, int borderSize) { GAPI_Assert(parent); m_p = parent; m_border_size = borderSize; } void fluid::ViewPrivWithOwnBorder::allocate(int lineConsumption, BorderOpt border) { initCache(lineConsumption); const auto& desc = m_cache.m_desc; int type = CV_MAKETYPE(desc.depth, desc.chan); m_own_storage.init(type, m_border_size, border.value()); m_own_storage.create(lineConsumption, desc.size.width, type); } void fluid::ViewPrivWithOwnBorder::prepareToRead() { int startLine = 0; int nLines = 0; if (m_read_caret == m_p->priv().readStart()) { // Need to fetch full window on the first iteration startLine = (m_read_caret > m_border_size) ? m_read_caret - m_border_size : 0; nLines = m_lines_next_iter; } else { startLine = m_read_caret + m_border_size; nLines = m_lines_next_iter - 2*m_border_size; } m_own_storage.updateBeforeRead(startLine, nLines, m_p->priv().storage()); const int start_log_idx = m_read_caret - m_border_size; m_own_storage.updateInCache(m_cache, start_log_idx, m_lines_next_iter); } std::size_t fluid::ViewPrivWithOwnBorder::size() const { GAPI_DbgAssert(m_p); return m_own_storage.size(); } bool fluid::View::ready() const { return m_priv->ready(); } int fluid::View::y() const { return m_priv->m_read_caret - m_priv->m_border_size; } fluid::View::Priv& fluid::View::priv() { return *m_priv; } const fluid::View::Priv& fluid::View::priv() const { return *m_priv; } void fluid::View::Priv::initCache(int lineConsumption) { m_cache.m_linePtrs.resize(lineConsumption); m_cache.m_desc = m_p->priv().meta(); m_cache.m_border_size = m_border_size; } // Fluid Buffer implementation ///////////////////////////////////////////////// fluid::Buffer::Priv::Priv(int read_start, cv::gapi::own::Rect roi) : m_readStart(read_start) , m_roi(roi) {} void fluid::Buffer::Priv::init(const cv::GMatDesc &desc, int writer_lpi, int readStartPos, cv::gapi::own::Rect roi) { m_writer_lpi = writer_lpi; m_desc = desc; m_readStart = readStartPos; m_roi = roi == own::Rect{} ? own::Rect{ 0, 0, desc.size.width, desc.size.height } : roi; m_cache.m_linePtrs.resize(writer_lpi); m_cache.m_desc = desc; } void fluid::Buffer::Priv::allocate(BorderOpt border, int border_size, int line_consumption, int skew) { GAPI_Assert(line_consumption > 0); // Init physical buffer // FIXME? combine line_consumption with skew? auto data_height = std::max(line_consumption, skew) + m_writer_lpi - 1; m_storage = createStorage(data_height, m_desc.size.width, CV_MAKETYPE(m_desc.depth, m_desc.chan), border_size, border); // Finally, initialize carets m_write_caret = writeStart(); m_storage->updateOutCache(m_cache, m_write_caret, m_writer_lpi); } void fluid::Buffer::Priv::bindTo(const cv::gapi::own::Mat &data, bool is_input) { // FIXME: move all these fields into a separate structure GAPI_Assert(m_desc == descr_of(data)); // Currently m_writer_lpi is obtained from metadata which is shared between islands // and this assert can trigger for slot which connects two fluid islands. // m_writer_lpi is used only in write-related functions and doesn't affect // buffer which is island's input so it's safe to skip this check. // FIXME: // Bring back this check when we move to 1 buffer <-> 1 metadata model // if (is_input) GAPI_Assert(m_writer_lpi == 1); m_storage = createStorage(data, m_roi); m_is_input = is_input; m_write_caret = is_input ? writeEnd(): writeStart(); // NB: views remain the same! m_storage->updateOutCache(m_cache, m_write_caret, m_writer_lpi); } bool fluid::Buffer::Priv::full() const { int slowest_y = writeEnd(); if (!m_views.empty()) { // reset with maximum possible value and then find minimum slowest_y = m_desc.size.height; for (const auto &v : m_views) slowest_y = std::min(slowest_y, v.y()); } return m_write_caret + lpi() - slowest_y > m_storage->rows(); } void fluid::Buffer::Priv::writeDone() { // There are possible optimizations which can be done to fill a border values // in compile time of the graph (for example border is const), // so there is no need to update border values after each write. // If such optimizations weren't applied, fill border for lines // which have been just written m_storage->updateAfterWrite(m_write_caret, m_writer_lpi); // Final write may produce less LPI, so // write caret may exceed logical buffer size m_write_caret += m_writer_lpi; // FIXME: add consistency check! m_storage->updateOutCache(m_cache, m_write_caret, m_writer_lpi); } void fluid::Buffer::Priv::reset() { m_write_caret = m_is_input ? writeEnd() : writeStart(); m_storage->updateOutCache(m_cache, m_write_caret, m_writer_lpi); } int fluid::Buffer::Priv::size() const { std::size_t view_sz = 0; for (const auto &v : m_views) view_sz += v.priv().size(); auto total = view_sz; if (m_storage) total += m_storage->size(); // FIXME: Change API to return size_t!!! return static_cast(total); } int fluid::Buffer::Priv::linesReady() const { if (m_is_input) { return m_storage->rows(); } else { const int writes = std::min(m_write_caret - writeStart(), outputLines()); return writes; } } int fluid::Buffer::Priv::lpi() const { // FIXME: // m_write_caret can be greater than m_writeRoi.y + m_writeRoi.height, so return value can be negative !!! return std::min(writeEnd() - m_write_caret, m_writer_lpi); } fluid::Buffer::Buffer() : m_priv(new Priv()) , m_cache(&m_priv->cache()) { } fluid::Buffer::Buffer(const cv::GMatDesc &desc) : m_priv(new Priv()) , m_cache(&m_priv->cache()) { int lineConsumption = 1; int border = 0, skew = 0, wlpi = 1, readStart = 0; cv::gapi::own::Rect roi = {0, 0, desc.size.width, desc.size.height}; m_priv->init(desc, wlpi, readStart, roi); m_priv->allocate({}, border, lineConsumption, skew); } fluid::Buffer::Buffer(const cv::GMatDesc &desc, int max_line_consumption, int border_size, int skew, int wlpi, BorderOpt border) : m_priv(new Priv()) , m_cache(&m_priv->cache()) { int readStart = 0; cv::gapi::own::Rect roi = {0, 0, desc.size.width, desc.size.height}; m_priv->init(desc, wlpi, readStart, roi); m_priv->allocate(border, border_size, max_line_consumption, skew); } fluid::Buffer::Buffer(const cv::gapi::own::Mat &data, bool is_input) : m_priv(new Priv()) , m_cache(&m_priv->cache()) { int wlpi = 1, readStart = 0; cv::gapi::own::Rect roi{0, 0, data.cols, data.rows}; m_priv->init(descr_of(data), wlpi, readStart, roi); m_priv->bindTo(data, is_input); } int fluid::Buffer::linesReady() const { return m_priv->linesReady(); } int fluid::Buffer::lpi() const { return m_priv->lpi(); } fluid::View::View(Priv* p) : m_priv(p), m_cache(&p->cache()) { /* nothing */ } fluid::View fluid::Buffer::mkView(int borderSize, bool ownStorage) { // FIXME: logic outside of Priv (because View takes pointer to Buffer) auto view = ownStorage ? View(new ViewPrivWithOwnBorder(this, borderSize)) : View(new ViewPrivWithoutOwnBorder(this, borderSize)); m_priv->addView(view); return view; } void fluid::debugBufferPriv(const fluid::Buffer& buffer, std::ostream &os) { // FIXME Use cv::gapi::own Size and Rect with operator<<, when merged ADE-285 const auto& p = buffer.priv(); os << "Fluid buffer " << std::hex << &buffer << std::dec << " " << p.m_desc.size.width << " x " << p.m_desc.size.height << "]" << " readStart:" << p.m_readStart << " roi:" << "[" << p.m_roi.width << " x " << p.m_roi.height << " from (" << p.m_roi.x << ", " << p.m_roi.y << ")]" <<" (phys " << "[" << p.storage().cols() << " x " << p.storage().rows() << "]" << ") :" << " w: " << p.m_write_caret << ", r: ["; for (const auto &v : p.m_views) { os << &v.priv() << ":" << v.y() << " "; } os << "], avail: " << buffer.linesReady() << std::endl; } void fluid::Buffer::debug(std::ostream &os) const { debugBufferPriv(*this, os); } fluid::Buffer::Priv& fluid::Buffer::priv() { return *m_priv; } const fluid::Buffer::Priv& fluid::Buffer::priv() const { return *m_priv; } int fluid::Buffer::y() const { return m_priv->y(); } } // namespace cv::gapi } // namespace cv