2019-12-02 21:16:06 +08:00
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2018-2019, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
# include "precomp.hpp"
2020-05-26 20:45:55 +08:00
# include <fstream>
2019-12-02 21:16:06 +08:00
# include "ie_ngraph.hpp"
# include <opencv2/dnn/shape_utils.hpp>
# ifdef HAVE_DNN_NGRAPH
# include <ie_extension.h>
# endif // HAVE_DNN_NGRAPH
# include <opencv2/core/utils/configuration.private.hpp>
# include <opencv2/core/utils/logger.hpp>
namespace cv { namespace dnn {
# ifdef HAVE_DNN_NGRAPH
2020-05-26 20:45:55 +08:00
static bool DNN_IE_SERIALIZE = utils : : getConfigurationParameterBool ( " OPENCV_DNN_IE_SERIALIZE " , false ) ;
2019-12-02 21:16:06 +08:00
// For networks with input layer which has an empty name, IE generates a name id[some_number].
// OpenCV lets users use an empty input name and to prevent unexpected naming,
// we can use some predefined name.
2020-03-03 16:01:44 +08:00
static std : : string kDefaultInpLayerName = " opencv_ngraph_empty_inp_layer_name " ;
static constexpr const char * kOpenCVLayersType = " opencv_ngraph_layer " ;
2020-02-26 22:51:18 +08:00
static std : : string shapesToStr ( const std : : vector < Mat > & mats )
{
std : : ostringstream shapes ;
shapes < < mats . size ( ) < < " " ;
for ( const Mat & m : mats )
{
shapes < < m . dims < < " " ;
for ( int i = 0 ; i < m . dims ; + + i )
shapes < < m . size [ i ] < < " " ;
}
return shapes . str ( ) ;
}
static void strToShapes ( const std : : string & str , std : : vector < std : : vector < size_t > > & shapes )
{
std : : istringstream ss ( str ) ;
int num , dims ;
ss > > num ;
shapes . resize ( num ) ;
for ( int i = 0 ; i < num ; + + i )
{
ss > > dims ;
shapes [ i ] . resize ( dims ) ;
for ( int j = 0 ; j < dims ; + + j )
ss > > shapes [ i ] [ j ] ;
}
}
2019-12-02 21:16:06 +08:00
static std : : vector < Ptr < NgraphBackendWrapper > >
ngraphWrappers ( const std : : vector < Ptr < BackendWrapper > > & ptrs )
{
std : : vector < Ptr < NgraphBackendWrapper > > wrappers ( ptrs . size ( ) ) ;
for ( int i = 0 ; i < ptrs . size ( ) ; + + i )
{
CV_Assert ( ! ptrs [ i ] . empty ( ) ) ;
wrappers [ i ] = ptrs [ i ] . dynamicCast < NgraphBackendWrapper > ( ) ;
CV_Assert ( ! wrappers [ i ] . empty ( ) ) ;
}
return wrappers ;
}
2020-02-26 22:51:18 +08:00
class NgraphCustomOp : public ngraph : : op : : Op {
public :
const ngraph : : NodeTypeInfo & get_type_info ( ) const override
{
static constexpr ngraph : : NodeTypeInfo type_info { kOpenCVLayersType , 0 } ;
return type_info ;
}
2020-04-07 21:17:21 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_3)
2020-04-15 23:12:56 +08:00
NgraphCustomOp ( const ngraph : : OutputVector & inputs ,
# else
2020-02-26 22:51:18 +08:00
NgraphCustomOp ( const ngraph : : NodeVector & inputs ,
2020-04-15 23:12:56 +08:00
# endif
2020-02-26 22:51:18 +08:00
const std : : map < std : : string , InferenceEngine : : Parameter > & params = { } ) :
Op ( inputs ) , params ( params )
{
constructor_validate_and_infer_types ( ) ;
}
2020-03-03 16:01:44 +08:00
~ NgraphCustomOp ( )
{
// nothing
}
2020-02-26 22:51:18 +08:00
void validate_and_infer_types ( ) override
{
std : : vector < std : : vector < size_t > > shapes ;
strToShapes ( params [ " outputs " ] , shapes ) ;
set_output_size ( shapes . size ( ) ) ;
for ( size_t i = 0 ; i < shapes . size ( ) ; + + i )
{
ngraph : : Shape output_shape ( shapes [ i ] ) ;
set_output_type ( i , get_input_element_type ( 0 ) , output_shape ) ;
}
}
std : : shared_ptr < ngraph : : Node > copy_with_new_args ( const ngraph : : NodeVector & new_args ) const override
{
2020-04-07 21:17:21 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_3)
2020-04-15 23:12:56 +08:00
return std : : make_shared < NgraphCustomOp > ( ngraph : : as_output_vector ( new_args ) , params ) ;
# else
2020-02-26 22:51:18 +08:00
return std : : make_shared < NgraphCustomOp > ( new_args , params ) ;
2020-04-15 23:12:56 +08:00
# endif
2020-02-26 22:51:18 +08:00
}
bool visit_attributes ( ngraph : : AttributeVisitor & visitor ) override
{
for ( auto & attr : params )
{
if ( attr . second . is < std : : string > ( ) )
visitor . on_attribute ( attr . first , attr . second . as < std : : string > ( ) ) ;
}
return true ;
}
std : : map < std : : string , InferenceEngine : : Parameter > params ;
} ;
2020-03-03 16:01:44 +08:00
class InfEngineNgraphCustomLayer : public InferenceEngine : : ILayerExecImpl
{
public :
2020-06-23 02:35:52 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2)
explicit InfEngineNgraphCustomLayer ( const std : : shared_ptr < ngraph : : Node > & _node )
{
node = std : : dynamic_pointer_cast < NgraphCustomOp > ( _node ) ;
CV_Assert ( node ) ;
std : : string implStr = node - > params [ " impl " ] ;
std : : istringstream iss ( implStr ) ;
# else
2020-03-03 16:01:44 +08:00
explicit InfEngineNgraphCustomLayer ( const InferenceEngine : : CNNLayer & layer ) : cnnLayer ( layer )
{
std : : istringstream iss ( layer . GetParamAsString ( " impl " ) ) ;
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
size_t ptr ;
iss > > ptr ;
cvLayer = ( Layer * ) ptr ;
std : : vector < std : : vector < size_t > > shapes ;
2020-06-23 02:35:52 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2)
strToShapes ( node - > params [ " internals " ] , shapes ) ;
# else
2020-03-03 16:01:44 +08:00
strToShapes ( layer . GetParamAsString ( " internals " ) , shapes ) ;
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
internals . resize ( shapes . size ( ) ) ;
for ( int i = 0 ; i < shapes . size ( ) ; + + i )
internals [ i ] . create ( std : : vector < int > ( shapes [ i ] . begin ( ) , shapes [ i ] . end ( ) ) , CV_32F ) ;
}
~ InfEngineNgraphCustomLayer ( )
{
// nothing
}
virtual InferenceEngine : : StatusCode execute ( std : : vector < InferenceEngine : : Blob : : Ptr > & inputs ,
std : : vector < InferenceEngine : : Blob : : Ptr > & outputs ,
InferenceEngine : : ResponseDesc * resp ) noexcept
{
std : : vector < Mat > inpMats , outMats ;
infEngineBlobsToMats ( inputs , inpMats ) ;
infEngineBlobsToMats ( outputs , outMats ) ;
try
{
cvLayer - > forward ( inpMats , outMats , internals ) ;
return InferenceEngine : : StatusCode : : OK ;
}
catch ( . . . )
{
return InferenceEngine : : StatusCode : : GENERAL_ERROR ;
}
}
virtual InferenceEngine : : StatusCode
getSupportedConfigurations ( std : : vector < InferenceEngine : : LayerConfig > & conf ,
InferenceEngine : : ResponseDesc * resp ) noexcept
{
std : : vector < InferenceEngine : : DataConfig > inDataConfig ;
std : : vector < InferenceEngine : : DataConfig > outDataConfig ;
2020-06-23 02:35:52 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2)
InferenceEngine : : SizeVector order ;
size_t offset = std : : numeric_limits < size_t > : : max ( ) ;
for ( int i = 0 ; i < node - > get_input_size ( ) ; + + i )
{
InferenceEngine : : DataConfig conf ;
auto shape = node - > input_value ( i ) . get_shape ( ) ;
order . resize ( shape . size ( ) ) ;
std : : iota ( order . begin ( ) , order . end ( ) , 0 ) ;
conf . desc = InferenceEngine : : TensorDesc ( InferenceEngine : : Precision : : FP32 , shape , { shape , order , offset } ) ;
inDataConfig . push_back ( conf ) ;
}
for ( int i = 0 ; i < node - > get_output_size ( ) ; + + i )
{
InferenceEngine : : DataConfig conf ;
auto shape = node - > output ( i ) . get_shape ( ) ;
order . resize ( shape . size ( ) ) ;
std : : iota ( order . begin ( ) , order . end ( ) , 0 ) ;
conf . desc = InferenceEngine : : TensorDesc ( InferenceEngine : : Precision : : FP32 , shape , { shape , order , offset } ) ;
outDataConfig . push_back ( conf ) ;
}
# else
2020-03-03 16:01:44 +08:00
for ( auto & it : cnnLayer . insData )
{
InferenceEngine : : DataConfig conf ;
conf . desc = it . lock ( ) - > getTensorDesc ( ) ;
inDataConfig . push_back ( conf ) ;
}
for ( auto & it : cnnLayer . outData )
{
InferenceEngine : : DataConfig conf ;
conf . desc = it - > getTensorDesc ( ) ;
outDataConfig . push_back ( conf ) ;
}
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
InferenceEngine : : LayerConfig layerConfig ;
layerConfig . inConfs = inDataConfig ;
layerConfig . outConfs = outDataConfig ;
conf . push_back ( layerConfig ) ;
return InferenceEngine : : StatusCode : : OK ;
}
InferenceEngine : : StatusCode init ( InferenceEngine : : LayerConfig & config ,
InferenceEngine : : ResponseDesc * resp ) noexcept
{
return InferenceEngine : : StatusCode : : OK ;
}
private :
2020-06-23 02:35:52 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2)
std : : shared_ptr < NgraphCustomOp > node ;
# else
2020-03-03 16:01:44 +08:00
InferenceEngine : : CNNLayer cnnLayer ;
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
dnn : : Layer * cvLayer ;
std : : vector < Mat > internals ;
} ;
2020-06-23 02:35:52 +08:00
# if INF_ENGINE_VER_MAJOR_LT(INF_ENGINE_RELEASE_2020_2)
2020-03-03 16:01:44 +08:00
class InfEngineNgraphCustomLayerFactory : public InferenceEngine : : ILayerImplFactory {
public :
explicit InfEngineNgraphCustomLayerFactory ( const InferenceEngine : : CNNLayer * layer ) : cnnLayer ( * layer )
{
// nothing
}
InferenceEngine : : StatusCode
getImplementations ( std : : vector < InferenceEngine : : ILayerImpl : : Ptr > & impls ,
InferenceEngine : : ResponseDesc * resp ) noexcept override
{
impls . push_back ( std : : make_shared < InfEngineNgraphCustomLayer > ( cnnLayer ) ) ;
return InferenceEngine : : StatusCode : : OK ;
}
private :
InferenceEngine : : CNNLayer cnnLayer ;
} ;
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
class InfEngineNgraphExtension : public InferenceEngine : : IExtension
{
public :
2020-06-23 02:35:52 +08:00
void Unload ( ) noexcept override { }
void Release ( ) noexcept override { delete this ; }
void GetVersion ( const InferenceEngine : : Version * & ) const noexcept override { }
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2)
std : : vector < std : : string > getImplTypes ( const std : : shared_ptr < ngraph : : Node > & node ) override {
return { " CPU " } ;
}
InferenceEngine : : ILayerImpl : : Ptr getImplementation ( const std : : shared_ptr < ngraph : : Node > & node , const std : : string & implType ) override {
if ( std : : dynamic_pointer_cast < NgraphCustomOp > ( node ) & & implType = = " CPU " ) {
return std : : make_shared < InfEngineNgraphCustomLayer > ( node ) ;
}
return nullptr ;
}
# else
2020-03-03 16:01:44 +08:00
virtual void SetLogCallback ( InferenceEngine : : IErrorListener & ) noexcept { }
virtual InferenceEngine : : StatusCode getPrimitiveTypes ( char * * & , unsigned int & ,
InferenceEngine : : ResponseDesc * ) noexcept
{
return InferenceEngine : : StatusCode : : OK ;
}
InferenceEngine : : StatusCode getFactoryFor ( InferenceEngine : : ILayerImplFactory * & factory ,
const InferenceEngine : : CNNLayer * cnnLayer ,
InferenceEngine : : ResponseDesc * resp ) noexcept
{
if ( cnnLayer - > type ! = kOpenCVLayersType )
return InferenceEngine : : StatusCode : : NOT_IMPLEMENTED ;
factory = new InfEngineNgraphCustomLayerFactory ( cnnLayer ) ;
return InferenceEngine : : StatusCode : : OK ;
}
2020-06-23 02:35:52 +08:00
# endif
2020-03-03 16:01:44 +08:00
} ;
2019-12-02 21:16:06 +08:00
InfEngineNgraphNode : : InfEngineNgraphNode ( std : : shared_ptr < ngraph : : Node > & & _node )
: BackendNode ( DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ) , node ( std : : move ( _node ) ) { }
InfEngineNgraphNode : : InfEngineNgraphNode ( std : : shared_ptr < ngraph : : Node > & _node )
: BackendNode ( DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ) , node ( _node ) { }
2020-02-26 22:51:18 +08:00
InfEngineNgraphNode : : InfEngineNgraphNode ( const std : : vector < Ptr < BackendNode > > & nodes ,
Ptr < Layer > & cvLayer_ , std : : vector < Mat * > & inputs ,
std : : vector < Mat > & outputs , std : : vector < Mat > & internals )
: BackendNode ( DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ) , cvLayer ( cvLayer_ )
{
std : : ostringstream oss ;
oss < < ( size_t ) cvLayer . get ( ) ;
std : : map < std : : string , InferenceEngine : : Parameter > params = {
{ " impl " , oss . str ( ) } ,
{ " outputs " , shapesToStr ( outputs ) } ,
{ " internals " , shapesToStr ( internals ) }
} ;
2020-04-07 21:17:21 +08:00
# if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_3)
2020-04-15 23:12:56 +08:00
ngraph : : OutputVector inp_nodes ;
# else
2020-02-26 22:51:18 +08:00
ngraph : : NodeVector inp_nodes ;
2020-04-15 23:12:56 +08:00
# endif
2020-02-26 22:51:18 +08:00
for ( const auto & node : nodes )
inp_nodes . emplace_back ( node . dynamicCast < InfEngineNgraphNode > ( ) - > node ) ;
node = std : : make_shared < NgraphCustomOp > ( inp_nodes , params ) ;
CV_Assert ( ! cvLayer - > name . empty ( ) ) ;
setName ( cvLayer - > name ) ;
}
2019-12-02 21:16:06 +08:00
void InfEngineNgraphNode : : setName ( const std : : string & name ) {
node - > set_friendly_name ( name ) ;
}
2020-05-26 20:45:55 +08:00
InfEngineNgraphNet : : InfEngineNgraphNet ( detail : : NetImplBase & netImpl )
: netImpl_ ( netImpl )
2019-12-02 21:16:06 +08:00
{
hasNetOwner = false ;
device_name = " CPU " ;
}
2020-05-26 20:45:55 +08:00
InfEngineNgraphNet : : InfEngineNgraphNet ( detail : : NetImplBase & netImpl , InferenceEngine : : CNNNetwork & net )
: netImpl_ ( netImpl )
, cnn ( net )
2019-12-02 21:16:06 +08:00
{
hasNetOwner = true ;
device_name = " CPU " ;
}
void InfEngineNgraphNet : : addOutput ( const std : : string & name )
{
requestedOutputs . push_back ( name ) ;
}
void InfEngineNgraphNet : : setNodePtr ( std : : shared_ptr < ngraph : : Node > * ptr ) {
all_nodes . emplace ( ( * ptr ) - > get_friendly_name ( ) , ptr ) ;
}
void InfEngineNgraphNet : : release ( ) {
for ( auto & node : components . back ( ) ) {
if ( ! ( node - > is_parameter ( ) | | node - > is_output ( ) | | node - > is_constant ( ) ) ) {
auto it = all_nodes . find ( node - > get_friendly_name ( ) ) ;
if ( it ! = all_nodes . end ( ) ) {
unconnectedNodes . erase ( * ( it - > second ) ) ;
it - > second - > reset ( ) ;
all_nodes . erase ( it ) ;
}
}
}
}
void InfEngineNgraphNet : : dfs ( std : : shared_ptr < ngraph : : Node > & node ,
std : : vector < std : : shared_ptr < ngraph : : Node > > & comp ,
std : : unordered_map < std : : string , bool > & used ) {
used [ node - > get_friendly_name ( ) ] = true ;
comp . push_back ( node ) ;
auto inputs = node - > get_users ( ) ;
for ( size_t i = 0 ; i < node - > get_input_size ( ) ; + + i ) {
inputs . push_back ( node - > input_value ( i ) . get_node ( ) - > shared_from_this ( ) ) ;
}
for ( auto & to : inputs ) {
if ( ! used [ to - > get_friendly_name ( ) ] ) {
dfs ( to , comp , used ) ;
}
}
}
int InfEngineNgraphNet : : getNumComponents ( ) {
if ( ! components . empty ( ) ) {
return components . size ( ) ;
}
std : : unordered_map < std : : string , bool > used ;
auto inputs = ngraph_function - > get_ordered_ops ( ) ;
for ( auto & node : inputs ) {
used . emplace ( node - > get_friendly_name ( ) , false ) ;
}
for ( auto & node : inputs ) {
if ( ! used [ node - > get_friendly_name ( ) ] ) {
std : : vector < std : : shared_ptr < ngraph : : Node > > current_comp ;
dfs ( node , current_comp , used ) ;
components . push_back ( current_comp ) ;
}
}
return components . size ( ) ;
}
void InfEngineNgraphNet : : createNet ( Target targetId ) {
if ( ! hasNetOwner )
{
CV_Assert ( ! unconnectedNodes . empty ( ) ) ;
ngraph : : ResultVector outs ;
for ( auto & node : unconnectedNodes )
{
auto out = std : : make_shared < ngraph : : op : : Result > ( node ) ;
outs . push_back ( out ) ;
}
CV_Assert_N ( ! inputs_vec . empty ( ) , ! outs . empty ( ) ) ;
ngraph_function = std : : make_shared < ngraph : : Function > ( outs , inputs_vec ) ;
int num_comp = getNumComponents ( ) ;
if ( num_comp > 1 ) {
for ( int i = num_comp - 1 ; i > = 0 ; - - i ) {
ngraph : : ResultVector outputs ;
ngraph : : ParameterVector inps ;
for ( auto & node : components . back ( ) ) {
if ( node - > is_parameter ( ) ) {
auto parameter = std : : dynamic_pointer_cast < ngraph : : op : : Parameter > ( node ) ;
inps . push_back ( parameter ) ;
}
else if ( node - > is_output ( ) ) {
auto result = std : : dynamic_pointer_cast < ngraph : : op : : Result > ( node ) ;
outputs . push_back ( result ) ;
}
}
isInit = false ;
CV_Assert_N ( ! inps . empty ( ) , ! outputs . empty ( ) ) ;
ngraph_function = std : : make_shared < ngraph : : Function > ( outputs , inps ) ;
release ( ) ;
components . pop_back ( ) ;
init ( targetId ) ;
}
} else {
release ( ) ;
components . clear ( ) ;
init ( targetId ) ;
}
}
}
void InfEngineNgraphNet : : init ( Target targetId )
{
if ( ! hasNetOwner )
{
2019-12-24 18:34:33 +08:00
if ( targetId = = DNN_TARGET_OPENCL_FP16 )
{
2019-12-02 21:16:06 +08:00
auto nodes = ngraph_function - > get_ordered_ops ( ) ;
2019-12-24 18:34:33 +08:00
for ( auto & node : nodes )
{
2019-12-02 21:16:06 +08:00
auto parameter = std : : dynamic_pointer_cast < ngraph : : op : : Parameter > ( node ) ;
2019-12-24 18:34:33 +08:00
if ( parameter & & parameter - > get_element_type ( ) = = ngraph : : element : : f32 )
{
2019-12-02 21:16:06 +08:00
parameter - > set_element_type ( ngraph : : element : : f16 ) ;
}
auto constant = std : : dynamic_pointer_cast < ngraph : : op : : Constant > ( node ) ;
2019-12-24 18:34:33 +08:00
if ( constant & & constant - > get_element_type ( ) = = ngraph : : element : : f32 )
{
const float * floatsData = constant - > get_data_ptr < float > ( ) ;
size_t total = ngraph : : shape_size ( constant - > get_shape ( ) ) ;
Mat floats ( 1 , total , CV_32F , ( void * ) floatsData ) ;
Mat halfs ;
cv : : convertFp16 ( floats , halfs ) ;
auto new_const = std : : make_shared < ngraph : : op : : Constant > ( ngraph : : element : : f16 , constant - > get_shape ( ) , halfs . data ) ;
2019-12-02 21:16:06 +08:00
new_const - > set_friendly_name ( constant - > get_friendly_name ( ) ) ;
ngraph : : replace_node ( constant , new_const ) ;
}
}
ngraph_function - > validate_nodes_and_infer_types ( ) ;
}
cnn = InferenceEngine : : CNNNetwork ( ngraph_function ) ;
2020-05-26 20:45:55 +08:00
if ( DNN_IE_SERIALIZE )
{
# ifndef OPENCV_DNN_DISABLE_NETWORK_AUTO_DUMP
std : : string dumpFileNameBase = netImpl_ . getDumpFileNameBase ( ) ;
try
{
cnn . serialize ( dumpFileNameBase + " _ngraph.xml " , dumpFileNameBase + " _ngraph.bin " ) ;
}
catch ( const std : : exception & e )
{
std : : ofstream out ( ( dumpFileNameBase + " _ngraph.error " ) . c_str ( ) , std : : ios : : out ) ;
out < < " Exception: " < < e . what ( ) < < std : : endl ;
}
catch ( . . . )
{
std : : ofstream out ( ( dumpFileNameBase + " _ngraph.error " ) . c_str ( ) , std : : ios : : out ) ;
out < < " Can't dump: unknown exception " < < std : : endl ;
}
2019-12-02 21:16:06 +08:00
# endif
2020-05-26 20:45:55 +08:00
}
2019-12-02 21:16:06 +08:00
}
switch ( targetId )
{
case DNN_TARGET_CPU :
device_name = " CPU " ;
break ;
case DNN_TARGET_OPENCL :
case DNN_TARGET_OPENCL_FP16 :
device_name = " GPU " ;
break ;
case DNN_TARGET_MYRIAD :
device_name = " MYRIAD " ;
break ;
case DNN_TARGET_FPGA :
device_name = " FPGA " ;
break ;
default :
CV_Error ( Error : : StsNotImplemented , " Unknown target " ) ;
} ;
if ( ! hasNetOwner ) {
for ( size_t i = 0 ; i < ngraph_function - > get_output_size ( ) ; + + i ) {
auto node = ngraph_function - > output ( i ) . get_node ( ) ;
for ( size_t j = 0 ; j < node - > get_input_size ( ) ; + + j ) {
std : : string name = node - > input_value ( j ) . get_node ( ) - > get_friendly_name ( ) ;
auto iter = std : : find ( requestedOutputs . begin ( ) , requestedOutputs . end ( ) , name ) ;
if ( iter ! = requestedOutputs . end ( ) ) {
requestedOutputs . erase ( iter ) ;
cnn . addOutput ( name ) ;
}
}
}
2020-02-17 03:12:14 +08:00
}
for ( const auto & name : requestedOutputs )
{
cnn . addOutput ( name ) ;
2019-12-02 21:16:06 +08:00
}
for ( const auto & it : cnn . getInputsInfo ( ) )
{
const std : : string & name = it . first ;
auto blobIt = allBlobs . find ( name ) ;
CV_Assert ( blobIt ! = allBlobs . end ( ) ) ;
it . second - > setPrecision ( blobIt - > second - > getTensorDesc ( ) . getPrecision ( ) ) ;
}
for ( const auto & it : cnn . getOutputsInfo ( ) )
{
const std : : string & name = it . first ;
auto blobIt = allBlobs . find ( name ) ;
CV_Assert ( blobIt ! = allBlobs . end ( ) ) ;
it . second - > setPrecision ( blobIt - > second - > getTensorDesc ( ) . getPrecision ( ) ) ; // Should be always FP32
}
initPlugin ( cnn ) ;
}
ngraph : : ParameterVector InfEngineNgraphNet : : setInputs ( const std : : vector < cv : : Mat > & inputs ,
const std : : vector < std : : string > & names ) {
CV_Assert_N ( inputs . size ( ) = = names . size ( ) ) ;
ngraph : : ParameterVector current_inp ;
for ( size_t i = 0 ; i < inputs . size ( ) ; i + + )
{
std : : vector < size_t > shape = getShape < size_t > ( inputs [ i ] ) ;
auto inp = std : : make_shared < ngraph : : op : : Parameter > ( ngraph : : element : : f32 , ngraph : : Shape ( shape ) ) ;
inp - > set_friendly_name ( names [ i ] ) ;
auto it = std : : find_if ( inputs_vec . begin ( ) , inputs_vec . end ( ) ,
[ & inp ] ( const std : : shared_ptr < ngraph : : op : : Parameter > & a ) {
return a - > get_friendly_name ( ) = = inp - > get_friendly_name ( ) ;
} ) ;
if ( it = = inputs_vec . end ( ) ) {
inputs_vec . push_back ( inp ) ;
current_inp . push_back ( inp ) ;
} else {
current_inp . push_back ( * it ) ;
}
}
return current_inp ;
}
void InfEngineNgraphNet : : setUnconnectedNodes ( Ptr < InfEngineNgraphNode > & node ) {
unconnectedNodes . insert ( node - > node ) ;
}
void InfEngineNgraphNet : : initPlugin ( InferenceEngine : : CNNNetwork & net )
{
CV_Assert ( ! isInitialized ( ) ) ;
try
{
AutoLock lock ( getInitializationMutex ( ) ) ;
2020-03-13 23:33:27 +08:00
InferenceEngine : : Core & ie = getCore ( device_name ) ;
2019-12-02 21:16:06 +08:00
{
isInit = true ;
std : : vector < std : : string > candidates ;
std : : string param_pluginPath = utils : : getConfigurationParameterString ( " OPENCV_DNN_IE_EXTRA_PLUGIN_PATH " , " " ) ;
if ( ! param_pluginPath . empty ( ) )
{
candidates . push_back ( param_pluginPath ) ;
}
bool found = false ;
for ( size_t i = 0 ; i ! = candidates . size ( ) ; + + i )
{
const std : : string & libName = candidates [ i ] ;
try
{
InferenceEngine : : IExtensionPtr extension =
InferenceEngine : : make_so_pointer < InferenceEngine : : IExtension > ( libName ) ;
ie . AddExtension ( extension , " CPU " ) ;
CV_LOG_INFO ( NULL , " DNN-IE: Loaded extension plugin: " < < libName ) ;
found = true ;
break ;
}
catch ( . . . ) { }
}
if ( ! found & & ! candidates . empty ( ) )
{
CV_LOG_WARNING ( NULL , " DNN-IE: Can't load extension plugin (extra layers for some networks). Specify path via OPENCV_DNN_IE_EXTRA_PLUGIN_PATH parameter " ) ;
}
// Some of networks can work without a library of extra layers.
// OpenCV fallbacks as extensions.
2020-01-15 20:10:17 +08:00
try
{
2020-03-03 16:01:44 +08:00
ie . AddExtension ( std : : make_shared < InfEngineNgraphExtension > ( ) , " CPU " ) ;
2020-01-15 20:10:17 +08:00
}
catch ( const std : : exception & e )
{
2020-03-03 16:01:44 +08:00
CV_LOG_INFO ( NULL , " DNN-IE: Can't register OpenCV custom layers nGraph extension: " < < e . what ( ) ) ;
2020-01-15 20:10:17 +08:00
}
2019-12-02 21:16:06 +08:00
# ifndef _WIN32
// Limit the number of CPU threads.
if ( device_name = = " CPU " )
ie . SetConfig ( { {
InferenceEngine : : PluginConfigParams : : KEY_CPU_THREADS_NUM , format ( " %d " , getNumThreads ( ) ) ,
} } , device_name ) ;
# endif
}
std : : map < std : : string , std : : string > config ;
if ( device_name = = " MYRIAD " ) {
config . emplace ( " VPU_DETECT_NETWORK_BATCH " , CONFIG_VALUE ( NO ) ) ;
}
2020-02-26 22:51:18 +08:00
2020-02-27 22:45:28 +08:00
bool isHetero = device_name = = " FPGA " ;
// It is actual only for non-CPU targets and networks built in runtime using nGraph.
// We do not check IR models because they can be with version less than IRv10
if ( ! isHetero & & device_name ! = " CPU " & & ! hasNetOwner )
2020-02-26 22:51:18 +08:00
{
2020-02-27 22:45:28 +08:00
for ( auto & node : net . getFunction ( ) - > get_ops ( ) )
2020-02-26 22:51:18 +08:00
{
2020-02-27 22:45:28 +08:00
if ( node - > description ( ) = = kOpenCVLayersType )
2020-02-26 22:51:18 +08:00
{
isHetero = true ;
break ;
}
}
}
if ( isHetero )
netExec = ie . LoadNetwork ( net , " HETERO: " + device_name + " ,CPU " , config ) ;
else
netExec = ie . LoadNetwork ( net , device_name , config ) ;
2019-12-02 21:16:06 +08:00
}
catch ( const std : : exception & ex )
{
CV_Error ( Error : : StsError , format ( " Failed to initialize Inference Engine backend (device = %s): %s " , device_name . c_str ( ) , ex . what ( ) ) ) ;
}
}
bool InfEngineNgraphNet : : isInitialized ( )
{
return isInit ;
}
bool NgraphBackendLayer : : getMemoryShapes ( const std : : vector < MatShape > & inputs ,
const int requiredOutputs ,
std : : vector < MatShape > & outputs ,
std : : vector < MatShape > & internals ) const
{
InferenceEngine : : ICNNNetwork : : InputShapes inShapes = t_net . getInputShapes ( ) ;
InferenceEngine : : ICNNNetwork : : InputShapes : : iterator itr ;
bool equal_flag = true ;
size_t i = 0 ;
for ( itr = inShapes . begin ( ) ; itr ! = inShapes . end ( ) ; + + itr )
{
InferenceEngine : : SizeVector currentInShape ( inputs [ i ] . begin ( ) , inputs [ i ] . end ( ) ) ;
if ( itr - > second ! = currentInShape )
{
itr - > second = currentInShape ;
equal_flag = false ;
}
i + + ;
}
if ( ! equal_flag )
{
InferenceEngine : : CNNNetwork curr_t_net ( t_net ) ;
curr_t_net . reshape ( inShapes ) ;
}
std : : vector < size_t > dims = t_net . getOutputsInfo ( ) [ name ] - > getDims ( ) ;
outputs . push_back ( MatShape ( dims . begin ( ) , dims . end ( ) ) ) ;
return false ;
}
bool NgraphBackendLayer : : supportBackend ( int backendId )
{
CV_LOG_DEBUG ( NULL , " NgraphBackendLayer::supportBackend( " < < backendId < < " ) " ) ;
return backendId = = DNN_BACKEND_DEFAULT | |
( backendId = = DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ) ;
}
void NgraphBackendLayer : : forward ( InputArrayOfArrays inputs , OutputArrayOfArrays outputs ,
OutputArrayOfArrays internals )
{
CV_Error ( Error : : StsInternal , " Choose Inference Engine as a preferable backend. " ) ;
}
static InferenceEngine : : Layout estimateLayout ( const Mat & m )
{
if ( m . dims = = 4 )
return InferenceEngine : : Layout : : NCHW ;
else if ( m . dims = = 2 )
return InferenceEngine : : Layout : : NC ;
else
return InferenceEngine : : Layout : : ANY ;
}
static InferenceEngine : : DataPtr wrapToInfEngineDataNode ( const Mat & m , const std : : string & name = " " )
{
std : : vector < size_t > shape = getShape < size_t > ( m ) ;
if ( m . type ( ) = = CV_32F )
return InferenceEngine : : DataPtr ( new InferenceEngine : : Data ( name ,
{ InferenceEngine : : Precision : : FP32 , shape , estimateLayout ( m ) } ) ) ;
else if ( m . type ( ) = = CV_8U )
return InferenceEngine : : DataPtr ( new InferenceEngine : : Data ( name ,
{ InferenceEngine : : Precision : : U8 , shape , estimateLayout ( m ) } ) ) ;
else
CV_Error ( Error : : StsNotImplemented , format ( " Unsupported data type %s " , typeToString ( m . type ( ) ) . c_str ( ) ) ) ;
}
InferenceEngine : : Blob : : Ptr wrapToNgraphBlob ( const Mat & m , const std : : vector < size_t > & shape ,
InferenceEngine : : Layout layout )
{
if ( m . type ( ) = = CV_32F )
return InferenceEngine : : make_shared_blob < float > (
{ InferenceEngine : : Precision : : FP32 , shape , layout } , ( float * ) m . data ) ;
else if ( m . type ( ) = = CV_8U )
return InferenceEngine : : make_shared_blob < uint8_t > (
{ InferenceEngine : : Precision : : U8 , shape , layout } , ( uint8_t * ) m . data ) ;
else
CV_Error ( Error : : StsNotImplemented , format ( " Unsupported data type %s " , typeToString ( m . type ( ) ) . c_str ( ) ) ) ;
}
InferenceEngine : : Blob : : Ptr wrapToNgraphBlob ( const Mat & m , InferenceEngine : : Layout layout )
{
std : : vector < size_t > shape = getShape < size_t > ( m ) ;
return wrapToNgraphBlob ( m , shape , layout ) ;
}
NgraphBackendWrapper : : NgraphBackendWrapper ( int targetId , const cv : : Mat & m )
: BackendWrapper ( DNN_BACKEND_INFERENCE_ENGINE_NGRAPH , targetId )
{
dataPtr = wrapToInfEngineDataNode ( m ) ;
blob = wrapToNgraphBlob ( m , estimateLayout ( m ) ) ;
}
NgraphBackendWrapper : : NgraphBackendWrapper ( Ptr < BackendWrapper > wrapper )
: BackendWrapper ( DNN_BACKEND_INFERENCE_ENGINE_NGRAPH , wrapper - > targetId )
{
Ptr < NgraphBackendWrapper > ieWrapper = wrapper . dynamicCast < NgraphBackendWrapper > ( ) ;
CV_Assert ( ! ieWrapper . empty ( ) ) ;
InferenceEngine : : DataPtr srcData = ieWrapper - > dataPtr ;
dataPtr = InferenceEngine : : DataPtr ( new InferenceEngine : : Data ( srcData - > getName ( ) , srcData - > getTensorDesc ( ) ) ) ;
blob = ieWrapper - > blob ;
}
Ptr < BackendWrapper > NgraphBackendWrapper : : create ( Ptr < BackendWrapper > wrapper )
{
return Ptr < BackendWrapper > ( new NgraphBackendWrapper ( wrapper ) ) ;
}
NgraphBackendWrapper : : ~ NgraphBackendWrapper ( )
{
// nothing
}
void NgraphBackendWrapper : : copyToHost ( )
{
CV_LOG_DEBUG ( NULL , " NgraphBackendWrapper::copyToHost() " ) ;
//CV_Error(Error::StsNotImplemented, "");
}
void NgraphBackendWrapper : : setHostDirty ( )
{
CV_LOG_DEBUG ( NULL , " NgraphBackendWrapper::setHostDirty() " ) ;
//CV_Error(Error::StsNotImplemented, "");
}
InferenceEngine : : Blob : : Ptr copyBlob ( const InferenceEngine : : Blob : : Ptr & blob )
{
InferenceEngine : : Blob : : Ptr copy ;
auto description = blob - > getTensorDesc ( ) ;
InferenceEngine : : Precision precision = description . getPrecision ( ) ;
if ( precision = = InferenceEngine : : Precision : : FP32 )
{
copy = InferenceEngine : : make_shared_blob < float > ( description ) ;
}
else if ( precision = = InferenceEngine : : Precision : : U8 )
{
copy = InferenceEngine : : make_shared_blob < uint8_t > ( description ) ;
}
else
CV_Error ( Error : : StsNotImplemented , " Unsupported blob precision " ) ;
copy - > allocate ( ) ;
return copy ;
}
InferenceEngine : : DataPtr ngraphDataNode ( const Ptr < BackendWrapper > & ptr )
{
CV_Assert ( ! ptr . empty ( ) ) ;
Ptr < NgraphBackendWrapper > p = ptr . dynamicCast < NgraphBackendWrapper > ( ) ;
CV_Assert ( ! p . empty ( ) ) ;
return p - > dataPtr ;
}
void forwardNgraph ( const std : : vector < Ptr < BackendWrapper > > & outBlobsWrappers ,
Ptr < BackendNode > & node , bool isAsync )
{
CV_Assert ( ! node . empty ( ) ) ;
Ptr < InfEngineNgraphNode > ieNode = node . dynamicCast < InfEngineNgraphNode > ( ) ;
CV_Assert ( ! ieNode . empty ( ) ) ;
ieNode - > net - > forward ( outBlobsWrappers , isAsync ) ;
}
2020-04-19 00:42:48 +08:00
void InfEngineNgraphNet : : reset ( )
{
allBlobs . clear ( ) ;
infRequests . clear ( ) ;
isInit = false ;
}
2019-12-02 21:16:06 +08:00
void InfEngineNgraphNet : : addBlobs ( const std : : vector < cv : : Ptr < BackendWrapper > > & ptrs )
{
auto wrappers = ngraphWrappers ( ptrs ) ;
for ( const auto & wrapper : wrappers )
{
std : : string name = wrapper - > dataPtr - > getName ( ) ;
name = name . empty ( ) ? kDefaultInpLayerName : name ;
allBlobs . insert ( { name , wrapper - > blob } ) ;
}
}
void InfEngineNgraphNet : : NgraphReqWrapper : : makePromises ( const std : : vector < Ptr < BackendWrapper > > & outsWrappers )
{
auto outs = ngraphWrappers ( outsWrappers ) ;
outProms . clear ( ) ;
outProms . resize ( outs . size ( ) ) ;
outsNames . resize ( outs . size ( ) ) ;
for ( int i = 0 ; i < outs . size ( ) ; + + i )
{
outs [ i ] - > futureMat = outProms [ i ] . getArrayResult ( ) ;
outsNames [ i ] = outs [ i ] - > dataPtr - > getName ( ) ;
}
}
Mat ngraphBlobToMat ( const InferenceEngine : : Blob : : Ptr & blob )
{
std : : vector < size_t > dims = blob - > getTensorDesc ( ) . getDims ( ) ;
std : : vector < int > size ( dims . begin ( ) , dims . end ( ) ) ;
auto precision = blob - > getTensorDesc ( ) . getPrecision ( ) ;
int type = - 1 ;
switch ( precision )
{
case InferenceEngine : : Precision : : FP32 : type = CV_32F ; break ;
case InferenceEngine : : Precision : : U8 : type = CV_8U ; break ;
default :
CV_Error ( Error : : StsNotImplemented , " Unsupported blob precision " ) ;
}
return Mat ( size , type , ( void * ) blob - > buffer ( ) ) ;
}
void InfEngineNgraphNet : : forward ( const std : : vector < Ptr < BackendWrapper > > & outBlobsWrappers , bool isAsync )
{
CV_LOG_DEBUG ( NULL , " InfEngineNgraphNet::forward( " < < ( isAsync ? " async " : " sync " ) < < " ) " ) ;
// Look for finished requests.
Ptr < NgraphReqWrapper > reqWrapper ;
for ( auto & wrapper : infRequests )
{
if ( wrapper - > isReady )
{
reqWrapper = wrapper ;
break ;
}
}
if ( reqWrapper . empty ( ) )
{
reqWrapper = Ptr < NgraphReqWrapper > ( new NgraphReqWrapper ( ) ) ;
try
{
reqWrapper - > req = netExec . CreateInferRequest ( ) ;
}
catch ( const std : : exception & ex )
{
CV_Error ( Error : : StsAssert , format ( " Failed to initialize Inference Engine backend: %s " , ex . what ( ) ) ) ;
}
infRequests . push_back ( reqWrapper ) ;
InferenceEngine : : BlobMap inpBlobs , outBlobs ;
for ( const auto & it : cnn . getInputsInfo ( ) )
{
const std : : string & name = it . first ;
auto blobIt = allBlobs . find ( name ) ;
CV_Assert ( blobIt ! = allBlobs . end ( ) ) ;
inpBlobs [ name ] = isAsync ? copyBlob ( blobIt - > second ) : blobIt - > second ;
}
for ( const auto & it : cnn . getOutputsInfo ( ) )
{
const std : : string & name = it . first ;
auto blobIt = allBlobs . find ( name ) ;
CV_Assert ( blobIt ! = allBlobs . end ( ) ) ;
outBlobs [ name ] = isAsync ? copyBlob ( blobIt - > second ) : blobIt - > second ;
}
reqWrapper - > req . SetInput ( inpBlobs ) ;
reqWrapper - > req . SetOutput ( outBlobs ) ;
InferenceEngine : : IInferRequest : : Ptr infRequestPtr = reqWrapper - > req ;
infRequestPtr - > SetUserData ( reqWrapper . get ( ) , 0 ) ;
infRequestPtr - > SetCompletionCallback (
[ ] ( InferenceEngine : : IInferRequest : : Ptr request , InferenceEngine : : StatusCode status )
{
CV_LOG_DEBUG ( NULL , " DNN(nGraph): completionCallback( " < < ( int ) status < < " ) " ) ;
NgraphReqWrapper * wrapper ;
request - > GetUserData ( ( void * * ) & wrapper , 0 ) ;
CV_Assert ( wrapper & & " Internal error " ) ;
size_t processedOutputs = 0 ;
try
{
for ( ; processedOutputs < wrapper - > outProms . size ( ) ; + + processedOutputs )
{
const std : : string & name = wrapper - > outsNames [ processedOutputs ] ;
Mat m = ngraphBlobToMat ( wrapper - > req . GetBlob ( name ) ) ;
try
{
CV_Assert ( status = = InferenceEngine : : StatusCode : : OK ) ;
wrapper - > outProms [ processedOutputs ] . setValue ( m . clone ( ) ) ;
}
catch ( . . . )
{
try {
wrapper - > outProms [ processedOutputs ] . setException ( std : : current_exception ( ) ) ;
} catch ( . . . ) {
2019-12-26 19:45:03 +08:00
CV_LOG_ERROR ( NULL , " DNN: Exception occurred during async inference exception propagation " ) ;
2019-12-02 21:16:06 +08:00
}
}
}
}
catch ( . . . )
{
std : : exception_ptr e = std : : current_exception ( ) ;
for ( ; processedOutputs < wrapper - > outProms . size ( ) ; + + processedOutputs )
{
try {
wrapper - > outProms [ processedOutputs ] . setException ( e ) ;
} catch ( . . . ) {
2019-12-26 19:45:03 +08:00
CV_LOG_ERROR ( NULL , " DNN: Exception occurred during async inference exception propagation " ) ;
2019-12-02 21:16:06 +08:00
}
}
}
wrapper - > isReady = true ;
}
) ;
}
if ( isAsync )
{
// Copy actual data to infer request's input blobs.
for ( const auto & it : cnn . getInputsInfo ( ) )
{
const std : : string & name = it . first ;
auto blobIt = allBlobs . find ( name ) ;
Mat srcMat = ngraphBlobToMat ( blobIt - > second ) ;
Mat dstMat = ngraphBlobToMat ( reqWrapper - > req . GetBlob ( name ) ) ;
srcMat . copyTo ( dstMat ) ;
}
// Set promises to output blobs wrappers.
reqWrapper - > makePromises ( outBlobsWrappers ) ;
reqWrapper - > isReady = false ;
reqWrapper - > req . StartAsync ( ) ;
}
else
{
reqWrapper - > req . Infer ( ) ;
}
}
# else
void forwardNgraph ( const std : : vector < Ptr < BackendWrapper > > & outBlobsWrappers ,
Ptr < BackendNode > & node , bool isAsync )
{
CV_Assert ( false & & " nGraph is not enabled in this OpenCV build " ) ;
}
# endif
} }