mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-27 12:40:05 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Intel Inference Engine deep learning backend (#10608)

Browse Source 
* Intel Inference Engine deep learning backend.

* OpenFace network using Inference Engine backend
This commit is contained in:
Dmitry Kurtaev 2018-02-06 11:57:35 +03:00 committed by Vadim Pisarevsky
parent 292dfc2d72
commit 10e1de74d2
26 changed files with 1379 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -223,6 +223,7 @@ OCV_OPTION(WITH_GTK "Include GTK support" ON
OCV_OPTION(WITH_GTK_2_X "Use GTK version 2" OFF IF (UNIX AND NOT APPLE AND NOT ANDROID) )
OCV_OPTION(WITH_IPP "Include Intel IPP support" (NOT MINGW AND NOT CV_DISABLE_OPTIMIZATION) IF (X86_64 OR X86) AND NOT WINRT AND NOT IOS )
OCV_OPTION(WITH_HALIDE "Include Halide support" OFF)
OCV_OPTION(WITH_INF_ENGINE "Include Intel Inference Engine support" OFF)
OCV_OPTION(WITH_JASPER "Include JPEG2K support" ON IF (NOT IOS) )
OCV_OPTION(WITH_JPEG "Include JPEG support" ON)
OCV_OPTION(WITH_WEBP "Include WebP support" ON IF (NOT WINRT) )
@ -669,6 +670,11 @@ if(WITH_HALIDE)
include(cmake/OpenCVDetectHalide.cmake)
endif()
# --- Inference Engine ---
if(WITH_INF_ENGINE)
include(cmake/OpenCVDetectInferenceEngine.cmake)
endif()
# --- DirectX ---
if(WITH_DIRECTX)
include(cmake/OpenCVDetectDirectX.cmake)
@ -1353,6 +1359,10 @@ if(WITH_HALIDE OR HAVE_HALIDE)
status(" Halide:" HAVE_HALIDE THEN "YES (${HALIDE_LIBRARIES} ${HALIDE_INCLUDE_DIRS})" ELSE NO)
endif()
if(WITH_INF_ENGINE OR HAVE_INF_ENGINE)
status(" Inference Engine:" HAVE_INF_ENGINE THEN "YES (${INF_ENGINE_LIBRARIES} ${INF_ENGINE_INCLUDE_DIRS})" ELSE NO)
endif()
if(WITH_EIGEN OR HAVE_EIGEN)
status(" Eigen:" HAVE_EIGEN THEN "YES (ver ${EIGEN_WORLD_VERSION}.${EIGEN_MAJOR_VERSION}.${EIGEN_MINOR_VERSION})" ELSE NO)
endif()

59
cmake/OpenCVDetectInferenceEngine.cmake Normal file
View File

@ -0,0 +1,59 @@
# The script detects Intel(R) Inference Engine installation
#
# Parameters:
# INTEL_CVSDK_DIR - Path to Inference Engine root folder
# IE_PLUGINS_PATH - Path to folder with Inference Engine plugins
#
# On return this will define:
#
# HAVE_INF_ENGINE - True if Intel Inference Engine was found
# INF_ENGINE_INCLUDE_DIRS - Inference Engine include folder
# INF_ENGINE_LIBRARIES - Inference Engine libraries and it's dependencies
#
macro(ie_fail)
set(HAVE_INF_ENGINE FALSE)
return()
endmacro()
if(NOT INF_ENGINE_ROOT_DIR OR NOT EXISTS "${INF_ENGINE_ROOT_DIR}/inference_engine/include/inference_engine.hpp")
set(ie_root_paths "${INF_ENGINE_ROOT_DIR}")
if(DEFINED ENV{INTEL_CVSDK_DIR})
list(APPEND ie_root_paths "$ENV{INTEL_CVSDK_DIR}")
endif()
if(WITH_INF_ENGINE AND NOT ie_root_paths)
list(APPEND ie_root_paths "/opt/intel/deeplearning_deploymenttoolkit/deployment_tools")
endif()
find_path(INF_ENGINE_ROOT_DIR inference_engine/include/inference_engine.hpp PATHS ${ie_root_paths})
endif()
set(INF_ENGINE_INCLUDE_DIRS "${INF_ENGINE_ROOT_DIR}/inference_engine/include" CACHE PATH "Path to Inference Engine include directory")
if(NOT INF_ENGINE_ROOT_DIR
OR NOT EXISTS "${INF_ENGINE_ROOT_DIR}"
OR NOT EXISTS "${INF_ENGINE_INCLUDE_DIRS}"
OR NOT EXISTS "${INF_ENGINE_INCLUDE_DIRS}/inference_engine.hpp"
)
ie_fail()
endif()
set(INF_ENGINE_LIBRARIES "")
foreach(lib inference_engine mklml_intel iomp5)
find_library(${lib}
NAMES ${lib}
HINTS ${IE_PLUGINS_PATH}
HINTS "$ENV{IE_PLUGINS_PATH}"
HINTS ${INF_ENGINE_ROOT_DIR}/external/mklml_lnx/lib
)
if(NOT ${lib})
ie_fail()
endif()
list(APPEND INF_ENGINE_LIBRARIES ${${lib}})
endforeach()
set(HAVE_INF_ENGINE TRUE)
include_directories(${INF_ENGINE_INCLUDE_DIRS})
list(APPEND OPENCV_LINKER_LIBS ${INF_ENGINE_LIBRARIES})
add_definitions(-DHAVE_INF_ENGINE)

2
modules/core/CMakeLists.txt
View File

@ -59,7 +59,7 @@ ocv_create_module(${extra_libs})
ocv_target_link_libraries(${the_module} LINK_PRIVATE
"${ZLIB_LIBRARIES}" "${OPENCL_LIBRARIES}" "${VA_LIBRARIES}"
"${LAPACK_LIBRARIES}" "${CPUFEATURES_LIBRARIES}" "${HALIDE_LIBRARIES}"
"${LAPACK_LIBRARIES}" "${CPUFEATURES_LIBRARIES}" "${HALIDE_LIBRARIES}" "${INF_ENGINE_LIBRARIES}"
"${ITT_LIBRARIES}"
"${OPENCV_HAL_LINKER_LIBS}"
)

1
modules/dnn/CMakeLists.txt
View File

@ -27,6 +27,7 @@ else()
-Wunused-parameter -Wunused-local-typedefs -Wsign-compare -Wsign-promo
-Wundef -Wtautological-undefined-compare -Wignored-qualifiers -Wextra
-Wunused-function -Wunused-const-variable -Wdeprecated-declarations
-Werror=non-virtual-dtor
)
endif()

5
modules/dnn/include/opencv2/dnn/dnn.hpp
View File

@ -70,7 +70,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
enum Backend
{
DNN_BACKEND_DEFAULT,
DNN_BACKEND_HALIDE
DNN_BACKEND_HALIDE,
DNN_BACKEND_INFERENCE_ENGINE
};
/**
@ -242,6 +243,8 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
*/
virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs);
/**
* @brief Automatic Halide scheduling based on layer hyper-parameters.
* @param[in] node Backend node with Halide functions.

75
modules/dnn/perf/perf_net.cpp
View File

@ -13,14 +13,7 @@
namespace
{
#ifdef HAVE_HALIDE
#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE
#else
#define TEST_DNN_BACKEND DNN_BACKEND_DEFAULT
#endif
#define TEST_DNN_TARGET DNN_TARGET_CPU, DNN_TARGET_OPENCL
CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE)
CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE, DNN_BACKEND_INFERENCE_ENGINE)
CV_ENUM(DNNTarget, DNN_TARGET_CPU, DNN_TARGET_OPENCL)
class DNNTestNetwork : public ::perf::TestBaseWithParam< tuple<DNNBackend, DNNTarget> >
@ -31,13 +24,16 @@ public:
dnn::Net net;
void processNet(std::string weights, std::string proto, std::string halide_scheduler,
const Mat& input, const std::string& outputLayer,
const std::string& framework)
DNNTestNetwork()
{
backend = (dnn::Backend)(int)get<0>(GetParam());
target = (dnn::Target)(int)get<1>(GetParam());
}
void processNet(std::string weights, std::string proto, std::string halide_scheduler,
const Mat& input, const std::string& outputLayer,
const std::string& framework)
{
if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL)
{
#if defined(HAVE_OPENCL)
@ -47,6 +43,8 @@ public:
throw ::SkipTestException("OpenCL is not available/disabled in OpenCV");
}
}
if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
throw SkipTestException("Skip OpenCL target of Inference Engine backend");
randu(input, 0.0f, 1.0f);
@ -117,7 +115,7 @@ PERF_TEST_P_(DNNTestNetwork, GoogLeNet)
"", Mat(cv::Size(224, 224), CV_32FC3), "prob", "caffe");
}
PERF_TEST_P_(DNNTestNetwork, ResNet50)
PERF_TEST_P_(DNNTestNetwork, ResNet_50)
{
processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
"resnet_50.yml", Mat(cv::Size(224, 224), CV_32FC3), "prob", "caffe");
@ -131,6 +129,7 @@ PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1)
PERF_TEST_P_(DNNTestNetwork, Inception_5h)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/tensorflow_inception_graph.pb", "",
"inception_5h.yml",
Mat(cv::Size(224, 224), CV_32FC3), "softmax2", "tensorflow");
@ -138,12 +137,14 @@ PERF_TEST_P_(DNNTestNetwork, Inception_5h)
PERF_TEST_P_(DNNTestNetwork, ENet)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/Enet-model-best.net", "", "enet.yml",
Mat(cv::Size(512, 256), CV_32FC3), "l367_Deconvolution", "torch");
}
PERF_TEST_P_(DNNTestNetwork, SSD)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", "disabled",
Mat(cv::Size(300, 300), CV_32FC3), "detection_out", "caffe");
}
@ -162,15 +163,53 @@ PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_Caffe)
PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_TensorFlow)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/ssd_mobilenet_v1_coco.pb", "ssd_mobilenet_v1_coco.pbtxt", "",
Mat(cv::Size(300, 300), CV_32FC3), "", "tensorflow");
}
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork,
testing::Combine(
::testing::Values(TEST_DNN_BACKEND),
DNNTarget::all()
)
);
PERF_TEST_P_(DNNTestNetwork, DenseNet_121)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", "",
Mat(cv::Size(224, 224), CV_32FC3), "", "caffe");
}
PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_coco)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", "",
Mat(cv::Size(368, 368), CV_32FC3), "", "caffe");
}
PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", "",
Mat(cv::Size(368, 368), CV_32FC3), "", "caffe");
}
PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
// The same .caffemodel but modified .prototxt
// See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp
processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", "",
Mat(cv::Size(368, 368), CV_32FC3), "", "caffe");
}
const tuple<DNNBackend, DNNTarget> testCases[] = {
#ifdef HAVE_HALIDE
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_CPU),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_OPENCL),
#endif
#ifdef HAVE_INF_ENGINE
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_CPU),
#endif
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_CPU),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL)
};
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases));
} // namespace

159
modules/dnn/src/dnn.cpp
View File

@ -41,6 +41,7 @@
#include "precomp.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "halide_scheduler.hpp"
#include <set>
#include <algorithm>
@ -471,9 +472,9 @@ public:
}
}
void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst)
void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, bool forceCreate)
{
if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS)
if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS && !forceCreate)
{
Mat bestBlob;
LayerPin bestBlobPin;
@ -518,7 +519,8 @@ public:
}
void allocateBlobsForLayer(LayerData &ld, const LayerShapes& layerShapes,
std::vector<LayerPin>& pinsForInternalBlobs)
std::vector<LayerPin>& pinsForInternalBlobs,
bool forceCreate = false)
{
CV_TRACE_FUNCTION();
@ -589,7 +591,7 @@ public:
reuse(ld.inputBlobsId[0], blobPin);
}
else
reuseOrCreate(shapes[index], blobPin, *blobs[index]);
reuseOrCreate(shapes[index], blobPin, *blobs[index], forceCreate);
}
}
}
@ -638,6 +640,13 @@ static Ptr<BackendWrapper> wrapMat(int backendId, int targetId, cv::Mat& m)
#ifdef HAVE_HALIDE
return Ptr<BackendWrapper>(new HalideBackendWrapper(targetId, m));
#endif // HAVE_HALIDE
}
else if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
{
CV_Assert(haveInfEngine());
#ifdef HAVE_INF_ENGINE
return Ptr<BackendWrapper>(new InfEngineBackendWrapper(targetId, m));
#endif // HAVE_INF_ENGINE
}
else
CV_Error(Error::StsNotImplemented, "Unknown backend identifier");
@ -710,6 +719,10 @@ struct Net::Impl
return Ptr<BackendWrapper>(new HalideBackendWrapper(baseBuffer, shape));
#endif // HAVE_HALIDE
}
else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE)
{
return wrapMat(preferableBackend, preferableTarget, host);
}
else
CV_Error(Error::StsNotImplemented, "Unknown backend identifier");
}
@ -999,12 +1012,20 @@ struct Net::Impl
void initBackend()
{
CV_TRACE_FUNCTION();
if (preferableBackend == DNN_BACKEND_DEFAULT)
{
CV_Assert(preferableTarget == DNN_TARGET_CPU || preferableTarget == DNN_TARGET_OPENCL);
return;
}
else if (preferableBackend == DNN_BACKEND_HALIDE)
initHalideBackend();
else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE)
initInfEngineBackend();
else
CV_Error(Error::StsNotImplemented, "Unknown backend identifier");
}
void initHalideBackend()
{
CV_TRACE_FUNCTION();
CV_Assert(preferableBackend == DNN_BACKEND_HALIDE, haveHalide());
// Iterator to current layer.
MapIdToLayerData::iterator it = layers.begin();
@ -1050,17 +1071,115 @@ struct Net::Impl
}
// No layers fusion.
ldTop.skip = false;
if (preferableBackend == DNN_BACKEND_HALIDE)
ldTop.backendNodes[DNN_BACKEND_HALIDE] =
layerTop->initHalide(ldTop.inputBlobsWrappers);
baseIt = it;
}
}
void initInfEngineBackend()
{
// Build Inference Engine networks from sets of layers that support this
// backend. If an internal layer isn't supported we'll use default
// implementation of it but build a new network after it.
CV_TRACE_FUNCTION();
CV_Assert(preferableBackend == DNN_BACKEND_INFERENCE_ENGINE, haveInfEngine());
#ifdef HAVE_INF_ENGINE
MapIdToLayerData::iterator it;
Ptr<InfEngineBackendNet> net;
for (it = layers.begin(); it != layers.end(); ++it)
{
LayerData &ld = it->second;
ld.skip = true;
Ptr<Layer> layer = ld.layerInstance;
if (!layer->supportBackend(preferableBackend))
{
ldTop.backendNodes[DNN_BACKEND_HALIDE] =
layerTop->initHalide(ldTop.inputBlobsWrappers);
baseIt = it;
for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i)
{
auto dataPtr = infEngineDataNode(ld.outputBlobsWrappers[i]);
dataPtr->name = ld.name;
}
ld.skip = false;
net = Ptr<InfEngineBackendNet>();
continue;
}
// Check what all inputs are from the same network or from default backend.
for (int i = 0; i < ld.inputBlobsId.size(); ++i)
{
LayerData &inpLd = layers[ld.inputBlobsId[i].lid];
Ptr<BackendNode> inpNode = inpLd.backendNodes[preferableBackend];
if (!inpNode.empty())
{
Ptr<InfEngineBackendNode> ieInpNode = inpNode.dynamicCast<InfEngineBackendNode>();
CV_Assert(!ieInpNode.empty(), net.empty() || net == ieInpNode->net);
}
}
bool fused = false;
Ptr<BackendNode> node;
if (!net.empty())
{
// Try to fuse.
bool inPlace = ld.inputBlobsId.size() == 1 && ld.outputBlobs.size() == 1 &&
ld.inputBlobs[0]->data == ld.outputBlobs[0].data;
if (inPlace)
{
node = layer->tryAttach(layers[ld.inputBlobsId[0].lid].backendNodes[preferableBackend]);
fused = !node.empty();
if (fused)
ld.inputBlobsWrappers = layers[ld.inputBlobsId[0].lid].inputBlobsWrappers;
}
}
else
net = Ptr<InfEngineBackendNet>(new InfEngineBackendNet());
if (!fused)
{
CV_Error(Error::StsNotImplemented, "Unknown backend identifier");
node = layer->initInfEngine(ld.inputBlobsWrappers);
}
CV_Assert(!node.empty());
ld.backendNodes[preferableBackend] = node;
Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
CV_Assert(!ieNode.empty());
ieNode->net = net;
ieNode->connect(ld.inputBlobsWrappers, ld.outputBlobsWrappers);
net->addBlobs(ld.inputBlobsWrappers);
net->addBlobs(ld.outputBlobsWrappers);
if (!fused)
net->addLayer(ieNode->layer);
}
// Initialize all networks.
std::set<InfEngineBackendNet> initializedNets;
for (MapIdToLayerData::reverse_iterator it = layers.rbegin(); it != layers.rend(); ++it)
{
LayerData &ld = it->second;
if (ld.backendNodes.find(preferableBackend) == ld.backendNodes.end())
continue;
Ptr<BackendNode> node = ld.backendNodes[preferableBackend];
if (node.empty())
continue;
Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
if (ieNode.empty())
continue;
CV_Assert(!ieNode->net.empty());
if (!ieNode->net->isInitialized())
{
ieNode->net->initEngine();
ld.skip = false;
}
}
#endif // HAVE_INF_ENGINE
}
void allocateLayer(int lid, const LayersShapesMap& layersShapes)
@ -1117,7 +1236,8 @@ struct Net::Impl
CV_Assert(layerShapesIt != layersShapes.end());
std::vector<LayerPin> pinsForInternalBlobs;
blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs);
blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs,
preferableBackend == DNN_BACKEND_INFERENCE_ENGINE);
ld.outputBlobsWrappers.resize(ld.outputBlobs.size());
for (int i = 0; i < ld.outputBlobs.size(); ++i)
{
@ -1564,6 +1684,10 @@ struct Net::Impl
{
forwardHalide(ld.outputBlobsWrappers, node);
}
else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE)
{
forwardInfEngine(node);
}
else
{
CV_Error(Error::StsNotImplemented, "Unknown backend identifier");
@ -2329,6 +2453,13 @@ Ptr<BackendNode> Layer::initHalide(const std::vector<Ptr<BackendWrapper> > &)
return Ptr<BackendNode>();
}
Ptr<BackendNode> Layer::initInfEngine(const std::vector<Ptr<BackendWrapper> > &)
{
CV_Error(Error::StsNotImplemented, "Inference Engine pipeline of " + type +
" layers is not defined.");
return Ptr<BackendNode>();
}
void Layer::applyHalideScheduler(Ptr<BackendNode>& node, const std::vector<Mat*> &inputs,
const std::vector<Mat> &outputs, int targetId) const
{

36
modules/dnn/src/layers/batch_norm_layer.cpp
View File

@ -11,6 +11,7 @@ Implementation of Batch Normalization layer.
#include "../precomp.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include "opencl_kernels_dnn.hpp"
@ -103,7 +104,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
#ifdef HAVE_OPENCL
@ -226,6 +228,19 @@ public:
#endif // HAVE_HALIDE
break;
}
case DNN_BACKEND_INFERENCE_ENGINE:
{
#ifdef HAVE_INF_ENGINE
auto base = node.dynamicCast<InfEngineBackendNode>();
auto conv = std::dynamic_pointer_cast<InferenceEngine::ConvolutionLayer>(base->layer);
if (conv)
{
fuseConvWeights(conv, weights_, bias_);
return base;
}
#endif // HAVE_INF_ENGINE
break;
}
}
return Ptr<BackendNode>();
}
@ -257,6 +272,25 @@ public:
}
#endif // HAVE_HALIDE
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "BatchNormalization";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::BatchNormalizationLayer> ieLayer(new InferenceEngine::BatchNormalizationLayer(lp));
size_t numChannels = weights_.total();
ieLayer->epsilon = epsilon;
ieLayer->_weights = wrapToInfEngineBlob(blobs[1], {numChannels});
ieLayer->_biases = wrapToInfEngineBlob(blobs[0], {numChannels});
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

18
modules/dnn/src/layers/concat_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencl_kernels_dnn.hpp"
namespace cv
@ -100,7 +101,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding; // By channels
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding || // By channels
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && !padding;
}
class ChannelConcatInvoker : public ParallelLoopBody
@ -295,6 +297,20 @@ public:
#endif // HAVE_HALIDE
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Concat";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::ConcatLayer> ieLayer(new InferenceEngine::ConcatLayer(lp));
ieLayer->_axis = axis;
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
};
Ptr<ConcatLayer> ConcatLayer::create(const LayerParams& params)

40
modules/dnn/src/layers/convolution_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencv2/core/hal/hal.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include <iostream>
@ -65,7 +66,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
void finalize(const std::vector<Mat*> &inputs, std::vector<Mat> &outputs)
@ -335,6 +337,42 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]);
CV_Assert(input->dims.size() == 4);
const int inpCn = input->dims[2]; // NOTE: input->dims are reversed (whcn)
const int outCn = blobs[0].size[0];
const int inpGroupCn = blobs[0].size[1];
const int group = inpCn / inpGroupCn;
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Convolution";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::ConvolutionLayer> ieLayer(new InferenceEngine::ConvolutionLayer(lp));
ieLayer->_kernel_x = kernel.width;
ieLayer->_kernel_y = kernel.height;
ieLayer->_stride_x = stride.width;
ieLayer->_stride_y = stride.height;
ieLayer->_out_depth = outCn;
ieLayer->_padding_x = pad.width;
ieLayer->_padding_y = pad.height;
ieLayer->_dilation_x = dilation.width;
ieLayer->_dilation_y = dilation.height;
ieLayer->_group = group;
ieLayer->_weights = wrapToInfEngineBlob(blobs[0]);
if (hasBias())
ieLayer->_biases = wrapToInfEngineBlob(blobs[1]);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
class ParallelConv : public cv::ParallelLoopBody
{
public:

34
modules/dnn/src/layers/detection_output_layer.cpp
View File

@ -42,6 +42,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_inf_engine.hpp"
#include <float.h>
#include <string>
#include "../nms.inl.hpp"
@ -189,6 +190,12 @@ public:
setParamsFrom(params);
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -203,10 +210,10 @@ public:
// num() and channels() are 1.
// Since the number of bboxes to be kept is unknown before nms, we manually
// set it to (fake) 1.
// set it to maximal number of detections, [keep_top_k] parameter.
// Each row is a 7 dimension std::vector, which stores
// [image_id, label, confidence, xmin, ymin, xmax, ymax]
outputs.resize(1, shape(1, 1, 1, 7));
outputs.resize(1, shape(1, 1, _keepTopK, 7));
return false;
}
@ -850,6 +857,29 @@ public:
return 0.;
}
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "DetectionOutput";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::CNNLayer> ieLayer(new InferenceEngine::CNNLayer(lp));
ieLayer->params["num_classes"] = format("%d", _numClasses);
ieLayer->params["share_location"] = _shareLocation ? "1" : "0";
ieLayer->params["background_label_id"] = format("%d", _backgroundLabelId);
ieLayer->params["nms_threshold"] = format("%f", _nmsThreshold);
ieLayer->params["top_k"] = format("%d", _topK);
ieLayer->params["keep_top_k"] = format("%d", _keepTopK);
ieLayer->params["confidence_threshold"] = format("%f", _confidenceThreshold);
ieLayer->params["variance_encoded_in_target"] = _varianceEncodedInTarget ? "1" : "0";
ieLayer->params["code_type"] = "caffe.PriorBoxParameter." + _codeType;
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
};
float util::caffe_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b)

92
modules/dnn/src/layers/elementwise_layers.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencv2/imgproc.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include "opencl_kernels_dnn.hpp"
@ -112,7 +113,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node)
@ -147,6 +149,17 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = this->name;
lp.precision = InferenceEngine::Precision::FP32;
return Ptr<BackendNode>(new InfEngineBackendNode(func.initInfEngine(lp)));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -308,6 +321,15 @@ struct ReLUFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
lp.type = "ReLU";
std::shared_ptr<InferenceEngine::ReLULayer> ieLayer(new InferenceEngine::ReLULayer(lp));
return ieLayer;
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 1; }
};
@ -372,6 +394,14 @@ struct ReLU6Functor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "ReLU6");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 2; }
};
@ -427,6 +457,14 @@ struct TanHFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "TanH");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 1; }
};
@ -462,6 +500,14 @@ struct SigmoidFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "Sigmoid");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 3; }
};
@ -499,6 +545,14 @@ struct ELUFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "ELU");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 2; }
};
@ -534,6 +588,14 @@ struct AbsValFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "Abs");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 1; }
};
@ -569,6 +631,14 @@ struct BNLLFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "BNLL");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 5; }
};
@ -658,6 +728,18 @@ struct PowerFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
lp.type = "Power";
std::shared_ptr<InferenceEngine::PowerLayer> ieLayer(new InferenceEngine::PowerLayer(lp));
ieLayer->power = power;
ieLayer->scale = scale;
ieLayer->offset = shift;
return ieLayer;
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; }
};
@ -750,6 +832,14 @@ struct ChannelsPReLUFunctor
}
#endif // HAVE_HALIDE
#ifdef HAVE_INF_ENGINE
InferenceEngine::CNNLayerPtr initInfEngine(InferenceEngine::LayerParams& lp)
{
CV_Error(Error::StsNotImplemented, "PReLU");
return InferenceEngine::CNNLayerPtr();
}
#endif // HAVE_INF_ENGINE
int64 getFLOPSPerElement() const { return 1; }
};

25
modules/dnn/src/layers/eltwise_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencl_kernels_dnn.hpp"
namespace cv
@ -93,7 +94,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
@ -402,6 +404,27 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Eltwise";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::EltwiseLayer> ieLayer(new InferenceEngine::EltwiseLayer(lp));
if (op == SUM)
ieLayer->_operation = InferenceEngine::EltwiseLayer::Sum;
else if (op == PROD)
ieLayer->_operation = InferenceEngine::EltwiseLayer::Prod;
else if (op == MAX)
ieLayer->_operation = InferenceEngine::EltwiseLayer::Max;
else
CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation");
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

22
modules/dnn/src/layers/flatten_layer.cpp
View File

@ -42,6 +42,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_inf_engine.hpp"
#include <float.h>
#include <algorithm>
#include <opencv2/dnn/shape_utils.hpp>
@ -61,6 +62,12 @@ public:
setParamsFrom(params);
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -153,6 +160,21 @@ public:
}
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Flatten";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::CNNLayer> ieLayer(new InferenceEngine::CNNLayer(lp));
ieLayer->params["axis"] = format("%d", _startAxis);
ieLayer->params["end_axis"] = format("%d", _endAxis);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
int _startAxis;
int _endAxis;
};

22
modules/dnn/src/layers/fully_connected_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencl_kernels_dnn.hpp"
#include <opencv2/dnn/shape_utils.hpp>
@ -127,7 +128,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1;
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && axis == 1;
}
virtual bool setActivation(const Ptr<ActivationLayer>& layer)
@ -395,6 +397,24 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "FullyConnected";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::FullyConnectedLayer> ieLayer(new InferenceEngine::FullyConnectedLayer(lp));
ieLayer->_out_num = blobs[0].size[0];
ieLayer->_weights = wrapToInfEngineBlob(blobs[0]);
if (blobs.size() > 1)
ieLayer->_biases = wrapToInfEngineBlob(blobs[1]);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

23
modules/dnn/src/layers/lrn_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/dnn/shape_utils.hpp"
#include "opencv2/core/hal/hal.hpp"
@ -90,7 +91,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
#ifdef HAVE_OPENCL
@ -369,6 +371,25 @@ public:
#endif // HAVE_HALIDE
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Norm";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::NormLayer> ieLayer(new InferenceEngine::NormLayer(lp));
ieLayer->_size = size;
ieLayer->_k = (int)bias;
ieLayer->_beta = beta;
ieLayer->_alpha = alpha;
ieLayer->_isAcrossMaps = (type == CHANNEL_NRM);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

26
modules/dnn/src/layers/permute_layer.cpp
View File

@ -42,6 +42,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_inf_engine.hpp"
#include <float.h>
#include <algorithm>
#include "opencl_kernels_dnn.hpp"
@ -112,6 +113,12 @@ public:
checkNeedForPermutation();
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -370,6 +377,25 @@ public:
}
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Permute";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::CNNLayer> ieLayer(new InferenceEngine::CNNLayer(lp));
CV_Assert(!_order.empty());
ieLayer->params["order"] = format("%d", _order[0]);
for (int i = 1; i < _order.size(); ++i)
ieLayer->params["order"] += format(",%d", _order[i]);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
size_t _count;
std::vector<size_t> _order;

33
modules/dnn/src/layers/pooling_layer.cpp
View File

@ -44,6 +44,7 @@
#include "layers_common.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencl_kernels_dnn.hpp"
#include <float.h>
#include <algorithm>
@ -130,7 +131,8 @@ public:
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide() &&
(type == MAX || type == AVE && !pad.width && !pad.height);
(type == MAX || type == AVE && !pad.width && !pad.height) ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && (type == MAX || type == AVE);
}
#ifdef HAVE_OPENCL
@ -222,6 +224,35 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Pooling";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::PoolingLayer> ieLayer(new InferenceEngine::PoolingLayer(lp));
ieLayer->_kernel_x = kernel.width;
ieLayer->_kernel_y = kernel.height;
ieLayer->_stride_x = stride.width;
ieLayer->_stride_y = stride.height;
ieLayer->_padding_x = pad.width;
ieLayer->_padding_y = pad.height;
ieLayer->_exclude_pad = false;
ieLayer->params["rounding-type"] = ceilMode ? "ceil" : "floor";
if (type == MAX)
ieLayer->_type = InferenceEngine::PoolingLayer::PoolType::MAX;
else if (type == AVE)
ieLayer->_type = InferenceEngine::PoolingLayer::PoolType::AVG;
else
CV_Error(Error::StsNotImplemented, "Unsupported pooling type");
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
class PoolingInvoker : public ParallelLoopBody
{
public:

44
modules/dnn/src/layers/prior_box_layer.cpp
View File

@ -42,6 +42,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_inf_engine.hpp"
#include <float.h>
#include <algorithm>
#include <cmath>
@ -248,6 +249,12 @@ public:
}
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -518,6 +525,43 @@ public:
}
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "PriorBox";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::CNNLayer> ieLayer(new InferenceEngine::CNNLayer(lp));
ieLayer->params["min_size"] = format("%f", _minSize);
ieLayer->params["max_size"] = _maxSize > 0 ? format("%f", _maxSize) : "";
CV_Assert(!_aspectRatios.empty());
ieLayer->params["aspect_ratio"] = format("%f", _aspectRatios[0]);
for (int i = 1; i < _aspectRatios.size(); ++i)
ieLayer->params["aspect_ratio"] += format(",%f", _aspectRatios[i]);
ieLayer->params["flip"] = _flip ? "1" : "0";
ieLayer->params["clip"] = _clip ? "1" : "0";
CV_Assert(!_variance.empty());
ieLayer->params["variance"] = format("%f", _variance[0]);
for (int i = 1; i < _variance.size(); ++i)
ieLayer->params["variance"] += format(",%f", _variance[i]);
ieLayer->params["step"] = "0";
ieLayer->params["step_h"] = _stepY;
ieLayer->params["step_w"] = _stepX;
CV_Assert(_offsetsX.size() == 1, _offsetsY.size() == 1, _offsetsX[0] == _offsetsY[0]);
ieLayer->params["offset"] = format("%f", _offsetsX[0]);;
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

21
modules/dnn/src/layers/reshape_layer.cpp
View File

@ -42,6 +42,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_inf_engine.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv
@ -165,6 +166,12 @@ public:
}
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@ -231,6 +238,20 @@ public:
srcBlob.reshape(1, shape(outputs[i])).copyTo(outputs[i]);
}
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "Reshape";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::ReshapeLayer> ieLayer(new InferenceEngine::ReshapeLayer(lp));
ieLayer->shape = newShapeDesc;
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
};
Ptr<ReshapeLayer> ReshapeLayer::create(const LayerParams& params)

36
modules/dnn/src/layers/scale_layer.cpp
View File

@ -12,6 +12,7 @@ Implementation of Scale layer.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv
@ -42,7 +43,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide();
backendId == DNN_BACKEND_HALIDE && haveHalide() ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr)
@ -130,6 +132,20 @@ public:
#endif // HAVE_HALIDE
break;
}
case DNN_BACKEND_INFERENCE_ENGINE:
{
#ifdef HAVE_INF_ENGINE
auto base = node.dynamicCast<InfEngineBackendNode>();
auto conv = std::dynamic_pointer_cast<InferenceEngine::ConvolutionLayer>(base->layer);
if (conv)
{
Mat bias = hasBias ? blobs[1] : Mat();
fuseConvWeights(conv, blobs[0], bias);
return base;
}
#endif // HAVE_INF_ENGINE
break;
}
}
return Ptr<BackendNode>();
}
@ -167,6 +183,24 @@ public:
}
#endif // HAVE_HALIDE
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "ScaleShift";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::ScaleShiftLayer> ieLayer(new InferenceEngine::ScaleShiftLayer(lp));
ieLayer->_weights = wrapToInfEngineBlob(blobs[0]);
if (hasBias)
ieLayer->_biases = wrapToInfEngineBlob(blobs[1]);
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

20
modules/dnn/src/layers/softmax_layer.cpp
View File

@ -43,6 +43,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "op_halide.hpp"
#include "op_inf_engine.hpp"
#include "opencl_kernels_dnn.hpp"
#include <algorithm>
#include <stdlib.h>
@ -63,7 +64,7 @@ public:
SoftMaxLayerImpl(const LayerParams& params)
{
axisRaw = params.get<int>("axis", 1);
logSoftMax = params.get<int>("log_softmax", false);
logSoftMax = params.get<bool>("log_softmax", false);
setParamsFrom(params);
}
@ -87,7 +88,8 @@ public:
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1;
backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1 ||
backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine() && !logSoftMax;
}
#ifdef HAVE_OPENCL
@ -307,6 +309,20 @@ public:
return Ptr<BackendNode>();
}
virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&)
{
#ifdef HAVE_INF_ENGINE
InferenceEngine::LayerParams lp;
lp.name = name;
lp.type = "SoftMax";
lp.precision = InferenceEngine::Precision::FP32;
std::shared_ptr<InferenceEngine::SoftMaxLayer> ieLayer(new InferenceEngine::SoftMaxLayer(lp));
ieLayer->axis = axisRaw;
return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
#endif // HAVE_INF_ENGINE
return Ptr<BackendNode>();
}
int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const
{

360
modules/dnn/src/op_inf_engine.cpp Normal file
View File

@ -0,0 +1,360 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2018, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "precomp.hpp"
#include "op_inf_engine.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv { namespace dnn {
#ifdef HAVE_INF_ENGINE
InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& _layer)
: BackendNode(DNN_BACKEND_INFERENCE_ENGINE), layer(_layer) {}
void InfEngineBackendNode::connect(std::vector<Ptr<BackendWrapper> >& inputs,
std::vector<Ptr<BackendWrapper> >& outputs)
{
layer->insData.resize(inputs.size());
for (int i = 0; i < inputs.size(); ++i)
{
InferenceEngine::DataPtr dataPtr = infEngineDataNode(inputs[i]);
layer->insData[i] = InferenceEngine::DataWeakPtr(dataPtr);
dataPtr->inputTo[layer->name] = layer;
}
CV_Assert(!outputs.empty());
layer->outData.resize(1);
InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[0]);
dataPtr->name = layer->name;
layer->outData[0] = dataPtr;
dataPtr->creatorLayer = InferenceEngine::CNNLayerWeakPtr(layer);
}
static std::vector<Ptr<InfEngineBackendWrapper> >
infEngineWrappers(const std::vector<Ptr<BackendWrapper> >& ptrs)
{
std::vector<Ptr<InfEngineBackendWrapper> > wrappers(ptrs.size());
for (int i = 0; i < ptrs.size(); ++i)
{
CV_Assert(!ptrs[i].empty());
wrappers[i] = ptrs[i].dynamicCast<InfEngineBackendWrapper>();
CV_Assert(!wrappers[i].empty());
}
return wrappers;
}
static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "")
{
std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
std::reverse(reversedShape.begin(), reversedShape.end());
return InferenceEngine::DataPtr(
new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32)
);
}
InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape)
{
return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
shape, (float*)m.data);
}
InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m)
{
std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
std::reverse(reversedShape.begin(), reversedShape.end());
return wrapToInfEngineBlob(m, reversedShape);
}
InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr)
{
CV_Assert(!ptr.empty());
Ptr<InfEngineBackendWrapper> p = ptr.dynamicCast<InfEngineBackendWrapper>();
CV_Assert(!p.empty());
return p->dataPtr;
}
InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
: BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, targetId)
{
dataPtr = wrapToInfEngineDataNode(m);
blob = wrapToInfEngineBlob(m);
}
InfEngineBackendWrapper::~InfEngineBackendWrapper()
{
}
void InfEngineBackendWrapper::copyToHost()
{
}
void InfEngineBackendWrapper::setHostDirty()
{
}
void InfEngineBackendNet::Release() noexcept
{
layers.clear();
inputs.clear();
outputs.clear();
}
InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept
{
return InferenceEngine::Precision::FP32;
}
// Assume that outputs of network is unconnected blobs.
void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept
{
if (outputs.empty())
{
for (const auto& l : layers)
{
// Add all outputs.
for (const InferenceEngine::DataPtr& out : l->outData)
{
// TODO: Replace to uniquness assertion.
if (outputs.find(out->name) == outputs.end())
outputs[out->name] = out;
}
// Remove internally connected outputs.
for (const InferenceEngine::DataWeakPtr& inp : l->insData)
{
outputs.erase(InferenceEngine::DataPtr(inp)->name);
}
}
CV_Assert(layers.empty() || !outputs.empty());
}
outBlobs.clear();
for (const auto& it : outputs)
{
CV_Assert(allBlobs.find(it.first) != allBlobs.end());
outBlobs[it.first] = allBlobs[it.first];
}
outputs_ = outputs;
}
// Returns input references that aren't connected to internal outputs.
void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) noexcept
{
if (inputs.empty())
{
std::map<std::string, InferenceEngine::DataPtr> internalOutputs;
for (const auto& l : layers)
{
for (const InferenceEngine::DataWeakPtr& ptr : l->insData)
{
InferenceEngine::DataPtr inp(ptr);
if (internalOutputs.find(inp->name) == internalOutputs.end())
{
InferenceEngine::InputInfo::Ptr inpInfo(new InferenceEngine::InputInfo());
inpInfo->setInputData(inp);
if (inputs.find(inp->name) == inputs.end())
inputs[inp->name] = inpInfo;
}
}
for (const InferenceEngine::DataPtr& out : l->outData)
{
// TODO: Replace to uniquness assertion.
if (internalOutputs.find(out->name) == internalOutputs.end())
internalOutputs[out->name] = out;
}
}
CV_Assert(layers.empty() || !inputs.empty());
}
inpBlobs.clear();
for (const auto& it : inputs)
{
CV_Assert(allBlobs.find(it.first) != allBlobs.end());
inpBlobs[it.first] = allBlobs[it.first];
}
inputs_ = inputs;
}
InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) noexcept
{
getInputsInfo(inputs);
const auto& it = inputs.find(inputName);
CV_Assert(it != inputs.end());
return it->second;
}
void InfEngineBackendNet::getName(char *pName, size_t len) noexcept
{
CV_Error(Error::StsNotImplemented, "");
}
size_t InfEngineBackendNet::layerCount() noexcept
{
return layers.size();
}
InferenceEngine::DataPtr& InfEngineBackendNet::getData(const char *dname) noexcept
{
CV_Error(Error::StsNotImplemented, "");
return outputs.begin()->second; // Just return something.
}
void InfEngineBackendNet::addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept
{
layers.push_back(layer);
inputs.clear();
outputs.clear();
}
InferenceEngine::StatusCode
InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex,
InferenceEngine::ResponseDesc *resp) noexcept
{
CV_Error(Error::StsNotImplemented, "");
return InferenceEngine::StatusCode::OK;
}
InferenceEngine::StatusCode
InfEngineBackendNet::getLayerByName(const char *layerName, InferenceEngine::CNNLayerPtr &out,
InferenceEngine::ResponseDesc *resp) noexcept
{
CV_Error(Error::StsNotImplemented, "");
return InferenceEngine::StatusCode::OK;
}
void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) noexcept
{
if (device != InferenceEngine::TargetDevice::eCPU)
CV_Error(Error::StsNotImplemented, "");
}
InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept
{
return InferenceEngine::TargetDevice::eCPU;
}
InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t size) noexcept
{
CV_Error(Error::StsNotImplemented, "");
return InferenceEngine::StatusCode::OK;
}
size_t InfEngineBackendNet::getBatchSize() const noexcept
{
CV_Error(Error::StsNotImplemented, "");
return 0;
}
void InfEngineBackendNet::initEngine()
{
CV_Assert(!isInitialized());
engine = InferenceEngine::InferenceEnginePluginPtr("libMKLDNNPlugin.so");
InferenceEngine::ResponseDesc resp;
InferenceEngine::StatusCode status = engine->LoadNetwork(*this, &resp);
if (status != InferenceEngine::StatusCode::OK)
CV_Error(Error::StsAssert, resp.msg);
}
bool InfEngineBackendNet::isInitialized()
{
return (bool)engine;
}
void InfEngineBackendNet::addBlobs(const std::vector<Ptr<BackendWrapper> >& ptrs)
{
auto wrappers = infEngineWrappers(ptrs);
for (const auto& wrapper : wrappers)
{
allBlobs[wrapper->dataPtr->name] = wrapper->blob;
}
}
void InfEngineBackendNet::forward()
{
InferenceEngine::ResponseDesc resp;
InferenceEngine::StatusCode status = engine->Infer(inpBlobs, outBlobs, &resp);
if (status != InferenceEngine::StatusCode::OK)
CV_Error(Error::StsAssert, resp.msg);
}
static inline Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob)
{
// NOTE: Inference Engine sizes are reversed.
std::vector<int> size(blob->dims().begin(), blob->dims().end());
std::reverse(size.begin(), size.end());
return Mat(size, CV_32F, (void*)blob->buffer());
}
void fuseConvWeights(const std::shared_ptr<InferenceEngine::ConvolutionLayer>& conv,
const Mat& w, const Mat& b)
{
// Get convolution's weights. Clone the data because Inference Engine can host it
// and conv->_weights->allocate() below will deallocate it.
Mat originWeights = infEngineBlobToMat(conv->_weights).clone();
// Create new weights blob.
conv->_weights = InferenceEngine::make_shared_blob<float>(
InferenceEngine::Precision::FP32, conv->_weights->dims());
conv->_weights->allocate();
// Convolution weights have OIHW data layout.
// (conv(I) + b1 ) * w + b2
// w*conv(I) + b1 * w + b2
Mat fusedWeights = infEngineBlobToMat(conv->_weights);
const int numChannels = fusedWeights.size[0];
// Mat weights = blobs[0].reshape(1, 1);
// Mat bias = hasBias ? blobs[1].reshape(1, 1) : Mat();
CV_Assert(numChannels == w.total());
CV_Assert(b.empty() || numChannels == b.total());
for (int i = 0; i < numChannels; ++i)
{
cv::multiply(slice(originWeights, i), w.at<float>(i), slice(fusedWeights, i));
}
if (conv->_biases)
{
// The same for biases.
Mat originBiases = infEngineBlobToMat(conv->_biases).clone();
conv->_biases = InferenceEngine::make_shared_blob<float>(
InferenceEngine::Precision::FP32, conv->_biases->dims());
conv->_biases->allocate();
Mat fusedBiases = infEngineBlobToMat(conv->_biases);
cv::multiply(w.reshape(1, fusedBiases.dims, &fusedBiases.size[0]), originBiases, fusedBiases);
if (!b.empty())
cv::add(fusedBiases, b.reshape(1, fusedBiases.dims, &fusedBiases.size[0]), fusedBiases);
}
else
conv->_biases = wrapToInfEngineBlob(b);
}
#endif // HAVE_INF_ENGINE
bool haveInfEngine()
{
#ifdef HAVE_INF_ENGINE
return true;
#else
return false;
#endif // HAVE_INF_ENGINE
}
void forwardInfEngine(Ptr<BackendNode>& node)
{
CV_Assert(haveInfEngine());
#ifdef HAVE_INF_ENGINE
CV_Assert(!node.empty());
Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
CV_Assert(!ieNode.empty());
ieNode->net->forward();
#endif // HAVE_INF_ENGINE
}
}} // namespace dnn, namespace cv

122
modules/dnn/src/op_inf_engine.hpp Normal file
View File

@ -0,0 +1,122 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2018, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
#ifndef __OPENCV_DNN_OP_INF_ENGINE_HPP__
#define __OPENCV_DNN_OP_INF_ENGINE_HPP__
#include "precomp.hpp"
#ifdef HAVE_INF_ENGINE
#include <inference_engine.hpp>
#endif // HAVE_INF_ENGINE
namespace cv { namespace dnn {
#ifdef HAVE_INF_ENGINE
class InfEngineBackendNet : public InferenceEngine::ICNNNetwork
{
public:
virtual void Release() noexcept;
virtual InferenceEngine::Precision getPrecision() noexcept;
virtual void getOutputsInfo(InferenceEngine::OutputsDataMap &out) noexcept;
virtual void getInputsInfo(InferenceEngine::InputsDataMap &inputs) noexcept;
virtual InferenceEngine::InputInfo::Ptr getInput(const std::string &inputName) noexcept;
virtual void getName(char *pName, size_t len) noexcept;
virtual size_t layerCount() noexcept;
virtual InferenceEngine::DataPtr& getData(const char *dname) noexcept;
virtual void addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept;
virtual InferenceEngine::StatusCode addOutput(const std::string &layerName,
size_t outputIndex = 0,
InferenceEngine::ResponseDesc *resp = nullptr) noexcept;
virtual InferenceEngine::StatusCode getLayerByName(const char *layerName,
InferenceEngine::CNNLayerPtr &out,
InferenceEngine::ResponseDesc *resp) noexcept;
virtual void setTargetDevice(InferenceEngine::TargetDevice device) noexcept;
virtual InferenceEngine::TargetDevice getTargetDevice() noexcept;
virtual InferenceEngine::StatusCode setBatchSize(const size_t size) noexcept;
virtual size_t getBatchSize() const noexcept;
void initEngine();
void addBlobs(const std::vector<Ptr<BackendWrapper> >& wrappers);
void forward();
bool isInitialized();
private:
std::vector<InferenceEngine::CNNLayerPtr> layers;
InferenceEngine::InputsDataMap inputs;
InferenceEngine::OutputsDataMap outputs;
InferenceEngine::BlobMap inpBlobs;
InferenceEngine::BlobMap outBlobs;
InferenceEngine::BlobMap allBlobs;
InferenceEngine::InferenceEnginePluginPtr engine;
};
class InfEngineBackendNode : public BackendNode
{
public:
InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& layer);
void connect(std::vector<Ptr<BackendWrapper> >& inputs,
std::vector<Ptr<BackendWrapper> >& outputs);
InferenceEngine::CNNLayerPtr layer;
// Inference Engine network object that allows to obtain the outputs of this layer.
Ptr<InfEngineBackendNet> net;
};
class InfEngineBackendWrapper : public BackendWrapper
{
public:
InfEngineBackendWrapper(int targetId, const Mat& m);
~InfEngineBackendWrapper();
virtual void copyToHost();
virtual void setHostDirty();
InferenceEngine::DataPtr dataPtr;
InferenceEngine::TBlob<float>::Ptr blob;
};
InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m);
InferenceEngine::TBlob<float>::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape);
InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr);
// Fuses convolution weights and biases with channel-wise scales and shifts.
void fuseConvWeights(const std::shared_ptr<InferenceEngine::ConvolutionLayer>& conv,
const Mat& w, const Mat& b = Mat());
#endif // HAVE_INF_ENGINE
bool haveInfEngine();
void forwardInfEngine(Ptr<BackendNode>& node);
}} // namespace dnn, namespace cv
#endif // __OPENCV_DNN_OP_INF_ENGINE_HPP__

38
modules/dnn/test/test_backends.cpp
View File

@ -14,7 +14,7 @@ using namespace cv;
using namespace dnn;
using namespace testing;
CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE)
CV_ENUM(DNNBackend, DNN_BACKEND_DEFAULT, DNN_BACKEND_HALIDE, DNN_BACKEND_INFERENCE_ENGINE)
CV_ENUM(DNNTarget, DNN_TARGET_CPU, DNN_TARGET_OPENCL)
static void loadNet(const std::string& weights, const std::string& proto,
@ -151,6 +151,7 @@ TEST_P(DNNTestNetwork, GoogLeNet)
TEST_P(DNNTestNetwork, Inception_5h)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/tensorflow_inception_graph.pb", "", Size(224, 224), "softmax2", "tensorflow",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_inception_5h.yml" :
"dnn/halide_scheduler_inception_5h.yml");
@ -158,6 +159,7 @@ TEST_P(DNNTestNetwork, Inception_5h)
TEST_P(DNNTestNetwork, ENet)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
processNet("dnn/Enet-model-best.net", "", Size(512, 512), "l367_Deconvolution", "torch",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_enet.yml" :
"dnn/halide_scheduler_enet.yml",
@ -176,16 +178,48 @@ TEST_P(DNNTestNetwork, MobileNetSSD)
TEST_P(DNNTestNetwork, SSD_VGG16)
{
if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL ||
backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU)
backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU ||
backend == DNN_BACKEND_INFERENCE_ENGINE)
throw SkipTestException("");
processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel",
"dnn/ssd_vgg16.prototxt", Size(300, 300), "detection_out", "caffe");
}
TEST_P(DNNTestNetwork, OpenPose_pose_coco)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt",
Size(368, 368), "", "caffe");
}
TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt",
Size(368, 368), "", "caffe");
}
TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
{
if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
// The same .caffemodel but modified .prototxt
// See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp
processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt",
Size(368, 368), "", "caffe");
}
TEST_P(DNNTestNetwork, OpenFace)
{
processNet("dnn/openface_nn4.small2.v1.t7", "", Size(96, 96), "", "torch");
}
const tuple<DNNBackend, DNNTarget> testCases[] = {
#ifdef HAVE_HALIDE
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_CPU),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_OPENCL),
#endif
#ifdef HAVE_INF_ENGINE
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_CPU),
#endif
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL)
};

105
samples/dnn/openpose.py Normal file
View File

@ -0,0 +1,105 @@
# To use Inference Engine backend, specify location of plugins:
# export LD_LIBRARY_PATH=/opt/intel/deeplearning_deploymenttoolkit/deployment_tools/external/mklml_lnx/lib:$LD_LIBRARY_PATH
import cv2 as cv
import numpy as np
import argparse
parser = argparse.ArgumentParser(
description='This script is used to demonstrate OpenPose human pose estimation network '
'from https://github.com/CMU-Perceptual-Computing-Lab/openpose project using OpenCV. '
'The sample and model are simplified and could be used for a single person on the frame.')
parser.add_argument('--input', help='Path to image or video. Skip to capture frames from camera')
parser.add_argument('--proto', help='Path to .prototxt')
parser.add_argument('--model', help='Path to .caffemodel')
parser.add_argument('--dataset', help='Specify what kind of model was trained. '
'It could be (COCO, MPI) depends on dataset.')
parser.add_argument('--thr', default=0.1, type=float, help='Threshold value for pose parts heat map')
parser.add_argument('--width', default=368, type=int, help='Resize input to specific width.')
parser.add_argument('--height', default=368, type=int, help='Resize input to specific height.')
parser.add_argument('--inf_engine', action='store_true',
help='Enable Intel Inference Engine computational backend. '
'Check that plugins folder is in LD_LIBRARY_PATH environment variable')
args = parser.parse_args()
if args.dataset == 'COCO':
BODY_PARTS = { "Nose": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4,
"LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9,
"RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "REye": 14,
"LEye": 15, "REar": 16, "LEar": 17, "Background": 18 }
POSE_PAIRS = [ ["Neck", "RShoulder"], ["Neck", "LShoulder"], ["RShoulder", "RElbow"],
["RElbow", "RWrist"], ["LShoulder", "LElbow"], ["LElbow", "LWrist"],
["Neck", "RHip"], ["RHip", "RKnee"], ["RKnee", "RAnkle"], ["Neck", "LHip"],
["LHip", "LKnee"], ["LKnee", "LAnkle"], ["Neck", "Nose"], ["Nose", "REye"],
["REye", "REar"], ["Nose", "LEye"], ["LEye", "LEar"] ]
else:
assert(args.dataset == 'MPI')
BODY_PARTS = { "Head": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4,
"LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9,
"RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "Chest": 14,
"Background": 15 }
POSE_PAIRS = [ ["Head", "Neck"], ["Neck", "RShoulder"], ["RShoulder", "RElbow"],
["RElbow", "RWrist"], ["Neck", "LShoulder"], ["LShoulder", "LElbow"],
["LElbow", "LWrist"], ["Neck", "Chest"], ["Chest", "RHip"], ["RHip", "RKnee"],
["RKnee", "RAnkle"], ["Chest", "LHip"], ["LHip", "LKnee"], ["LKnee", "LAnkle"] ]
inWidth = args.width
inHeight = args.height
net = cv.dnn.readNetFromCaffe(args.proto, args.model)
if args.inf_engine:
net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE)
cap = cv.VideoCapture(args.input if args.input else 0)
while cv.waitKey(1) < 0:
hasFrame, frame = cap.read()
if not hasFrame:
cv.waitKey()
break
frameWidth = frame.shape[1]
frameHeight = frame.shape[0]
inp = cv.dnn.blobFromImage(frame, 1.0 / 255, (inWidth, inHeight),
(0, 0, 0), swapRB=False, crop=False)
net.setInput(inp)
out = net.forward()
assert(len(BODY_PARTS) == out.shape[1])
points = []
for i in range(len(BODY_PARTS)):
# Slice heatmap of corresponging body's part.
heatMap = out[0, i, :, :]
# Originally, we try to find all the local maximums. To simplify a sample
# we just find a global one. However only a single pose at the same time
# could be detected this way.
_, conf, _, point = cv.minMaxLoc(heatMap)
x = (frameWidth * point[0]) / out.shape[3]
y = (frameHeight * point[1]) / out.shape[2]
# Add a point if it's confidence is higher than threshold.
points.append((x, y) if conf > args.thr else None)
for pair in POSE_PAIRS:
partFrom = pair[0]
partTo = pair[1]
assert(partFrom in BODY_PARTS)
assert(partTo in BODY_PARTS)
idFrom = BODY_PARTS[partFrom]
idTo = BODY_PARTS[partTo]
if points[idFrom] and points[idTo]:
cv.line(frame, points[idFrom], points[idTo], (0, 255, 0), 3)
cv.ellipse(frame, points[idFrom], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED)
cv.ellipse(frame, points[idTo], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED)
t, _ = net.getPerfProfile()
freq = cv.getTickFrequency() / 1000
cv.putText(frame, '%.2fms' % (t / freq), (10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0))
cv.imshow('OpenPose using OpenCV', frame)