Merge pull request #10173 from dkurt:update_priorbox_layer

modules/dnn/misc/caffe/opencv-caffe.pb.h

@@ -1537,12 +1537,53 @@ class PriorBoxParameter : public ::google::protobuf::Message /* @@protoc_inserti
   float offset() const;
   void set_offset(float value);
 
-  // optional bool additional_y_offset = 14 [default = false];
-  bool has_additional_y_offset() const;
-  void clear_additional_y_offset();
-  static const int kAdditionalYOffsetFieldNumber = 14;
-  bool additional_y_offset() const;
-  void set_additional_y_offset(bool value);
+  // repeated float offset_h = 14;
+  int offset_h_size() const;
+  void clear_offset_h();
+  static const int kOffsetHFieldNumber = 14;
+  float offset_h(int index) const;
+  void set_offset_h(int index, float value);
+  void add_offset_h(float value);
+  const ::google::protobuf::RepeatedField< float >&
+      offset_h() const;
+  ::google::protobuf::RepeatedField< float >*
+      mutable_offset_h();
+
+  // repeated float offset_w = 15;
+  int offset_w_size() const;
+  void clear_offset_w();
+  static const int kOffsetWFieldNumber = 15;
+  float offset_w(int index) const;
+  void set_offset_w(int index, float value);
+  void add_offset_w(float value);
+  const ::google::protobuf::RepeatedField< float >&
+      offset_w() const;
+  ::google::protobuf::RepeatedField< float >*
+      mutable_offset_w();
+
+  // repeated float width = 16;
+  int width_size() const;
+  void clear_width();
+  static const int kWidthFieldNumber = 16;
+  float width(int index) const;
+  void set_width(int index, float value);
+  void add_width(float value);
+  const ::google::protobuf::RepeatedField< float >&
+      width() const;
+  ::google::protobuf::RepeatedField< float >*
+      mutable_width();
+
+  // repeated float height = 17;
+  int height_size() const;
+  void clear_height();
+  static const int kHeightFieldNumber = 17;
+  float height(int index) const;
+  void set_height(int index, float value);
+  void add_height(float value);
+  const ::google::protobuf::RepeatedField< float >&
+      height() const;
+  ::google::protobuf::RepeatedField< float >*
+      mutable_height();
 
   // @@protoc_insertion_point(class_scope:opencv_caffe.PriorBoxParameter)
  private:
@@ -1568,14 +1609,16 @@ class PriorBoxParameter : public ::google::protobuf::Message /* @@protoc_inserti
   inline void clear_has_step_w();
   inline void set_has_offset();
   inline void clear_has_offset();
-  inline void set_has_additional_y_offset();
-  inline void clear_has_additional_y_offset();
 
   ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
   ::google::protobuf::internal::HasBits<1> _has_bits_;
   mutable int _cached_size_;
   ::google::protobuf::RepeatedField< float > aspect_ratio_;
   ::google::protobuf::RepeatedField< float > variance_;
+  ::google::protobuf::RepeatedField< float > offset_h_;
+  ::google::protobuf::RepeatedField< float > offset_w_;
+  ::google::protobuf::RepeatedField< float > width_;
+  ::google::protobuf::RepeatedField< float > height_;
   float min_size_;
   float max_size_;
   ::google::protobuf::uint32 img_size_;
@@ -1584,7 +1627,6 @@ class PriorBoxParameter : public ::google::protobuf::Message /* @@protoc_inserti
   float step_;
   float step_h_;
   float step_w_;
-  bool additional_y_offset_;
   bool flip_;
   bool clip_;
   float offset_;
@@ -13655,28 +13697,124 @@ inline void PriorBoxParameter::set_offset(float value) {
   // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.offset)
 }
 
-// optional bool additional_y_offset = 14 [default = false];
-inline bool PriorBoxParameter::has_additional_y_offset() const {
-  return (_has_bits_[0] & 0x00002000u) != 0;
+// repeated float offset_h = 14;
+inline int PriorBoxParameter::offset_h_size() const {
+  return offset_h_.size();
 }
-inline void PriorBoxParameter::set_has_additional_y_offset() {
-  _has_bits_[0] |= 0x00002000u;
+inline void PriorBoxParameter::clear_offset_h() {
+  offset_h_.Clear();
 }
-inline void PriorBoxParameter::clear_has_additional_y_offset() {
-  _has_bits_[0] &= ~0x00002000u;
+inline float PriorBoxParameter::offset_h(int index) const {
+  // @@protoc_insertion_point(field_get:opencv_caffe.PriorBoxParameter.offset_h)
+  return offset_h_.Get(index);
 }
-inline void PriorBoxParameter::clear_additional_y_offset() {
-  additional_y_offset_ = false;
-  clear_has_additional_y_offset();
+inline void PriorBoxParameter::set_offset_h(int index, float value) {
+  offset_h_.Set(index, value);
+  // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.offset_h)
 }
-inline bool PriorBoxParameter::additional_y_offset() const {
-  // @@protoc_insertion_point(field_get:opencv_caffe.PriorBoxParameter.additional_y_offset)
-  return additional_y_offset_;
+inline void PriorBoxParameter::add_offset_h(float value) {
+  offset_h_.Add(value);
+  // @@protoc_insertion_point(field_add:opencv_caffe.PriorBoxParameter.offset_h)
 }
-inline void PriorBoxParameter::set_additional_y_offset(bool value) {
-  set_has_additional_y_offset();
-  additional_y_offset_ = value;
-  // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.additional_y_offset)
+inline const ::google::protobuf::RepeatedField< float >&
+PriorBoxParameter::offset_h() const {
+  // @@protoc_insertion_point(field_list:opencv_caffe.PriorBoxParameter.offset_h)
+  return offset_h_;
+}
+inline ::google::protobuf::RepeatedField< float >*
+PriorBoxParameter::mutable_offset_h() {
+  // @@protoc_insertion_point(field_mutable_list:opencv_caffe.PriorBoxParameter.offset_h)
+  return &offset_h_;
+}
+
+// repeated float offset_w = 15;
+inline int PriorBoxParameter::offset_w_size() const {
+  return offset_w_.size();
+}
+inline void PriorBoxParameter::clear_offset_w() {
+  offset_w_.Clear();
+}
+inline float PriorBoxParameter::offset_w(int index) const {
+  // @@protoc_insertion_point(field_get:opencv_caffe.PriorBoxParameter.offset_w)
+  return offset_w_.Get(index);
+}
+inline void PriorBoxParameter::set_offset_w(int index, float value) {
+  offset_w_.Set(index, value);
+  // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.offset_w)
+}
+inline void PriorBoxParameter::add_offset_w(float value) {
+  offset_w_.Add(value);
+  // @@protoc_insertion_point(field_add:opencv_caffe.PriorBoxParameter.offset_w)
+}
+inline const ::google::protobuf::RepeatedField< float >&
+PriorBoxParameter::offset_w() const {
+  // @@protoc_insertion_point(field_list:opencv_caffe.PriorBoxParameter.offset_w)
+  return offset_w_;
+}
+inline ::google::protobuf::RepeatedField< float >*
+PriorBoxParameter::mutable_offset_w() {
+  // @@protoc_insertion_point(field_mutable_list:opencv_caffe.PriorBoxParameter.offset_w)
+  return &offset_w_;
+}
+
+// repeated float width = 16;
+inline int PriorBoxParameter::width_size() const {
+  return width_.size();
+}
+inline void PriorBoxParameter::clear_width() {
+  width_.Clear();
+}
+inline float PriorBoxParameter::width(int index) const {
+  // @@protoc_insertion_point(field_get:opencv_caffe.PriorBoxParameter.width)
+  return width_.Get(index);
+}
+inline void PriorBoxParameter::set_width(int index, float value) {
+  width_.Set(index, value);
+  // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.width)
+}
+inline void PriorBoxParameter::add_width(float value) {
+  width_.Add(value);
+  // @@protoc_insertion_point(field_add:opencv_caffe.PriorBoxParameter.width)
+}
+inline const ::google::protobuf::RepeatedField< float >&
+PriorBoxParameter::width() const {
+  // @@protoc_insertion_point(field_list:opencv_caffe.PriorBoxParameter.width)
+  return width_;
+}
+inline ::google::protobuf::RepeatedField< float >*
+PriorBoxParameter::mutable_width() {
+  // @@protoc_insertion_point(field_mutable_list:opencv_caffe.PriorBoxParameter.width)
+  return &width_;
+}
+
+// repeated float height = 17;
+inline int PriorBoxParameter::height_size() const {
+  return height_.size();
+}
+inline void PriorBoxParameter::clear_height() {
+  height_.Clear();
+}
+inline float PriorBoxParameter::height(int index) const {
+  // @@protoc_insertion_point(field_get:opencv_caffe.PriorBoxParameter.height)
+  return height_.Get(index);
+}
+inline void PriorBoxParameter::set_height(int index, float value) {
+  height_.Set(index, value);
+  // @@protoc_insertion_point(field_set:opencv_caffe.PriorBoxParameter.height)
+}
+inline void PriorBoxParameter::add_height(float value) {
+  height_.Add(value);
+  // @@protoc_insertion_point(field_add:opencv_caffe.PriorBoxParameter.height)
+}
+inline const ::google::protobuf::RepeatedField< float >&
+PriorBoxParameter::height() const {
+  // @@protoc_insertion_point(field_list:opencv_caffe.PriorBoxParameter.height)
+  return height_;
+}
+inline ::google::protobuf::RepeatedField< float >*
+PriorBoxParameter::mutable_height() {
+  // @@protoc_insertion_point(field_mutable_list:opencv_caffe.PriorBoxParameter.height)
+  return &height_;
 }
 
 inline const PriorBoxParameter* PriorBoxParameter::internal_default_instance() {

modules/dnn/src/caffe/opencv-caffe.proto

@@ -145,8 +145,14 @@ message PriorBoxParameter {
   optional float step_w = 12;
   // Offset to the top left corner of each cell.
   optional float offset = 13 [default = 0.5];
-  // If true, two additional boxes for each center will be generated. Their centers will be shifted by y coordinate.
-  optional bool additional_y_offset = 14 [default = false];
+  // Offset to the top corner of each cell.
+  repeated float offset_h = 14;
+  // Offset to the left corner of each cell.
+  repeated float offset_w = 15;
+  // Priox boxes width (in pixels).
+  repeated float width = 16;
+  // Priox boxes height (in pixels).
+  repeated float height = 17;
 }
 
 // Message that store parameters used by DetectionOutputLayer

modules/dnn/src/init.cpp

@@ -113,6 +113,7 @@ void initializeLayerFactory()
     CV_DNN_REGISTER_LAYER_CLASS(Eltwise,        EltwiseLayer);
     CV_DNN_REGISTER_LAYER_CLASS(Permute,        PermuteLayer);
     CV_DNN_REGISTER_LAYER_CLASS(PriorBox,       PriorBoxLayer);
+    CV_DNN_REGISTER_LAYER_CLASS(PriorBoxClustered, PriorBoxLayer);
     CV_DNN_REGISTER_LAYER_CLASS(Reorg,          ReorgLayer);
     CV_DNN_REGISTER_LAYER_CLASS(Region,         RegionLayer);
     CV_DNN_REGISTER_LAYER_CLASS(DetectionOutput, DetectionOutputLayer);

modules/dnn/src/layers/prior_box_layer.cpp

@@ -54,9 +54,9 @@ namespace dnn
 class PriorBoxLayerImpl : public PriorBoxLayer
 {
 public:
-    bool getParameterDict(const LayerParams &params,
-                          const std::string &parameterName,
-                          DictValue& result)
+    static bool getParameterDict(const LayerParams &params,
+                                 const std::string &parameterName,
+                                 DictValue& result)
     {
         if (!params.has(parameterName))
         {
@@ -98,7 +98,8 @@ public:
     {
         DictValue aspectRatioParameter;
         bool aspectRatioRetieved = getParameterDict(params, "aspect_ratio", aspectRatioParameter);
-        CV_Assert(aspectRatioRetieved);
+        if (!aspectRatioRetieved)
+            return;
 
         for (int i = 0; i < aspectRatioParameter.size(); ++i)
         {
@@ -124,18 +125,20 @@ public:
         }
     }
 
-    void getScales(const LayerParams &params)
+    static void getParams(const std::string& name, const LayerParams &params,
+                          std::vector<float>* values)
     {
-        DictValue scalesParameter;
-        bool scalesRetieved = getParameterDict(params, "scales", scalesParameter);
-        if (scalesRetieved)
+        DictValue dict;
+        if (getParameterDict(params, name, dict))
         {
-            _scales.resize(scalesParameter.size());
-            for (int i = 0; i < scalesParameter.size(); ++i)
+            values->resize(dict.size());
+            for (int i = 0; i < dict.size(); ++i)
             {
-                _scales[i] = scalesParameter.get<float>(i);
+                (*values)[i] = dict.get<float>(i);
             }
         }
+        else
+            values->clear();
     }
 
     void getVariance(const LayerParams &params)
@@ -177,20 +180,31 @@ public:
         : _boxWidth(0), _boxHeight(0)
     {
         setParamsFrom(params);
-        _minSize = getParameter<float>(params, "min_size");
-        CV_Assert(_minSize > 0);
-
-        _flip = getParameter<bool>(params, "flip");
-        _clip = getParameter<bool>(params, "clip");
+        _minSize = getParameter<float>(params, "min_size", 0, false, 0);
+        _flip = getParameter<bool>(params, "flip", 0, false, true);
+        _clip = getParameter<bool>(params, "clip", 0, false, true);
 
         _scales.clear();
         _aspectRatios.clear();
 
         getAspectRatios(params);
         getVariance(params);
-        getScales(params);
+        getParams("scales", params, &_scales);
+        getParams("width", params, &_widths);
+        getParams("height", params, &_heights);
+        _explicitSizes = !_widths.empty();
+        CV_Assert(_widths.size() == _heights.size());
 
-        _numPriors = _aspectRatios.size() + 1;  // + 1 for an aspect ratio 1.0
+        if (_explicitSizes)
+        {
+            CV_Assert(_aspectRatios.empty(), !params.has("min_size"), !params.has("max_size"));
+            _numPriors = _widths.size();
+        }
+        else
+        {
+            CV_Assert(!_aspectRatios.empty(), _minSize > 0);
+            _numPriors = _aspectRatios.size() + 1;  // + 1 for an aspect ratio 1.0
+        }
 
         _maxSize = -1;
         if (params.has("max_size"))
@@ -216,14 +230,20 @@ public:
           _stepY = 0;
           _stepX = 0;
         }
-        if(params.has("additional_y_offset"))
+        if (params.has("offset_h") || params.has("offset_w"))
         {
-          _additional_y_offset = getParameter<bool>(params, "additional_y_offset");
-          if(_additional_y_offset)
-            _numPriors *= 2;
+            CV_Assert(!params.has("offset"), params.has("offset_h"), params.has("offset_w"));
+            getParams("offset_h", params, &_offsetsY);
+            getParams("offset_w", params, &_offsetsX);
+            CV_Assert(_offsetsX.size() == _offsetsY.size());
+            _numPriors *= std::max((size_t)1, 2 * (_offsetsX.size() - 1));
         }
         else
-          _additional_y_offset = false;
+        {
+            float offset = getParameter<float>(params, "offset", 0, false, 0.5);
+            _offsetsX.assign(1, offset);
+            _offsetsY.assign(1, offset);
+        }
     }
 
     bool getMemoryShapes(const std::vector<MatShape> &inputs,
@@ -262,7 +282,7 @@ public:
         CV_TRACE_FUNCTION();
         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
 
-        size_t real_numPriors = _additional_y_offset ? _numPriors / 2 : _numPriors;
+        size_t real_numPriors = _numPriors / pow(2, _offsetsX.size() - 1);
         if (_scales.empty())
             _scales.resize(real_numPriors, 1.0f);
         else
@@ -286,93 +306,64 @@ public:
         int _outChannelSize = _layerHeight * _layerWidth * _numPriors * 4;
 
         float* outputPtr = outputs[0].ptr<float>();
-
-        // first prior: aspect_ratio = 1, size = min_size
-        int idx = 0;
         for (size_t h = 0; h < _layerHeight; ++h)
         {
             for (size_t w = 0; w < _layerWidth; ++w)
             {
-                _boxWidth = _boxHeight = _minSize * _scales[0];
-
-                float center_x = (w + 0.5) * stepX;
-                float center_y = (h + 0.5) * stepY;
-                // xmin
-                outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                // ymin
-                outputPtr[idx++] = (center_y - _boxHeight / 2.) / _imageHeight;
-                // xmax
-                outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                // ymax
-                outputPtr[idx++] = (center_y + _boxHeight / 2.) / _imageHeight;
-
-                if(_additional_y_offset)
+                // first prior: aspect_ratio = 1, size = min_size
+                if (_explicitSizes)
                 {
-                  float center_y_offset_1 = (h + 1.0) * stepY;
-                  // xmin
-                  outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                  // ymin
-                  outputPtr[idx++] = (center_y_offset_1 - _boxHeight / 2.) / _imageHeight;
-                  // xmax
-                  outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                  // ymax
-                  outputPtr[idx++] = (center_y_offset_1 + _boxHeight / 2.) / _imageHeight;
+                    _boxWidth = _widths[0] * _scales[0];
+                    _boxHeight = _heights[0] * _scales[0];
                 }
+                else
+                    _boxWidth = _boxHeight = _minSize * _scales[0];
 
+                for (int i = 0; i < _offsetsX.size(); ++i)
+                {
+                    float center_x = (w + _offsetsX[i]) * stepX;
+                    float center_y = (h + _offsetsY[i]) * stepY;
+                    outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
+                }
                 if (_maxSize > 0)
                 {
                     // second prior: aspect_ratio = 1, size = sqrt(min_size * max_size)
                     _boxWidth = _boxHeight = sqrt(_minSize * _maxSize) * _scales[1];
-                    // xmin
-                    outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                    // ymin
-                    outputPtr[idx++] = (center_y - _boxHeight / 2.) / _imageHeight;
-                    // xmax
-                    outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                    // ymax
-                    outputPtr[idx++] = (center_y + _boxHeight / 2.) / _imageHeight;
-
-                    if(_additional_y_offset)
+                    for (int i = 0; i < _offsetsX.size(); ++i)
                     {
-                      float center_y_offset_1 = (h + 1.0) * stepY;
-                      // xmin
-                      outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                      // ymin
-                      outputPtr[idx++] = (center_y_offset_1 - _boxHeight / 2.) / _imageHeight;
-                      // xmax
-                      outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                      // ymax
-                      outputPtr[idx++] = (center_y_offset_1 + _boxHeight / 2.) / _imageHeight;
+                        float center_x = (w + _offsetsX[i]) * stepX;
+                        float center_y = (h + _offsetsY[i]) * stepY;
+                        outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
                     }
                 }
 
                 // rest of priors
-                CV_Assert((_maxSize > 0 ? 2 : 1) + _aspectRatios.size() == _scales.size());
+                CV_Assert(_aspectRatios.empty() || (_maxSize > 0 ? 2 : 1) + _aspectRatios.size() == _scales.size());
                 for (size_t r = 0; r < _aspectRatios.size(); ++r)
                 {
                     float ar = _aspectRatios[r];
                     float scale = _scales[(_maxSize > 0 ? 2 : 1) + r];
                     _boxWidth = _minSize * sqrt(ar) * scale;
                     _boxHeight = _minSize / sqrt(ar) * scale;
-                    // xmin
-                    outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                    // ymin
-                    outputPtr[idx++] = (center_y - _boxHeight / 2.) / _imageHeight;
-                    // xmax
-                    outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                    // ymax
-                    outputPtr[idx++] = (center_y + _boxHeight / 2.) / _imageHeight;
-                    if(_additional_y_offset)
+                    for (int i = 0; i < _offsetsX.size(); ++i)
                     {
-                      float center_y_offset_1 = (h + 1.0) * stepY;
-                      // xmin
-                      outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
-                      // ymin
-                      outputPtr[idx++] = (center_y_offset_1 - _boxHeight / 2.) / _imageHeight;
-                      // xmax
-                      outputPtr[idx++] = (center_x + _boxWidth / 2.) / _imageWidth;
-                      // ymax
-                      outputPtr[idx++] = (center_y_offset_1 + _boxHeight / 2.) / _imageHeight;
+                        float center_x = (w + _offsetsX[i]) * stepX;
+                        float center_y = (h + _offsetsY[i]) * stepY;
+                        outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
+                    }
+                }
+
+                // rest of sizes
+                CV_Assert(_widths.empty() || _widths.size() == _scales.size());
+                for (size_t i = 1; i < _widths.size(); ++i)
+                {
+                    _boxWidth = _widths[i] * _scales[i];
+                    _boxHeight = _heights[i] * _scales[i];
+                    for (int j = 0; j < _offsetsX.size(); ++j)
+                    {
+                        float center_x = (w + _offsetsX[j]) * stepX;
+                        float center_y = (h + _offsetsY[j]) * stepY;
+                        outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
                     }
                 }
             }
@@ -426,6 +417,7 @@ public:
         return flops;
     }
 
+private:
     float _minSize;
     float _maxSize;
 
@@ -437,15 +429,29 @@ public:
     std::vector<float> _aspectRatios;
     std::vector<float> _variance;
     std::vector<float> _scales;
+    std::vector<float> _widths;
+    std::vector<float> _heights;
+    std::vector<float> _offsetsX;
+    std::vector<float> _offsetsY;
 
     bool _flip;
     bool _clip;
-    bool _additional_y_offset;
+    bool _explicitSizes;
 
     size_t _numPriors;
 
     static const size_t _numAxes = 4;
     static const std::string _layerName;
+
+    static float* addPrior(float center_x, float center_y, float width, float height,
+                           float imgWidth, float imgHeight, float* dst)
+    {
+        dst[0] = (center_x - width * 0.5f) / imgWidth;    // xmin
+        dst[1] = (center_y - height * 0.5f) / imgHeight;  // ymin
+        dst[2] = (center_x + width * 0.5f) / imgWidth;    // xmax
+        dst[3] = (center_y + height * 0.5f) / imgHeight;  // ymax
+        return dst + 4;
+    }
 };
 
 const std::string PriorBoxLayerImpl::_layerName = std::string("PriorBox");