From 384fa95680bf9a49577dae8be5cb1be686d7eb78 Mon Sep 17 00:00:00 2001
From: Harshal Nishar <34619300+harshalnishar@users.noreply.github.com>
Date: Fri, 2 Feb 2018 19:30:11 +0530
Subject: [PATCH] Fix in Canny when Sobel apertureSize is 7 (#10743)

* Fixing a bug in Canny implemetation when Sobel aperture size is 7.

* Fixing the bug in Canny accross variants and in test_canny.cpp

* Replacing a tab with white space
---
 modules/imgproc/src/canny.cpp       | 31 +++++++++++++++++++++++++----
 modules/imgproc/test/test_canny.cpp | 10 +++++-----
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/modules/imgproc/src/canny.cpp b/modules/imgproc/src/canny.cpp
index 7c13e4d32c..6d7990b4b2 100644
--- a/modules/imgproc/src/canny.cpp
+++ b/modules/imgproc/src/canny.cpp
@@ -159,6 +159,12 @@ static bool ocl_Canny(InputArray _src, const UMat& dx_, const UMat& dy_, OutputA
         lSizeY = 1;
     }
 
+    if (aperture_size == 7)
+    {
+        low_thresh = low_thresh / 16.0f;
+        high_thresh = high_thresh / 16.0f;
+    }
+
     if (L2gradient)
     {
         low_thresh = std::min(32767.0f, low_thresh);
@@ -212,11 +218,17 @@ static bool ocl_Canny(InputArray _src, const UMat& dx_, const UMat& dy_, OutputA
                 Non maxima suppression
                 Double thresholding
         */
+        double scale = 1.0;
+        if (aperture_size == 7)
+        {
+            scale = 1 / 16.0;
+        }
+
         UMat dx, dy;
         if (!useCustomDeriv)
         {
-            Sobel(_src, dx, CV_16S, 1, 0, aperture_size, 1, 0, BORDER_REPLICATE);
-            Sobel(_src, dy, CV_16S, 0, 1, aperture_size, 1, 0, BORDER_REPLICATE);
+            Sobel(_src, dx, CV_16S, 1, 0, aperture_size, scale, 0, BORDER_REPLICATE);
+            Sobel(_src, dy, CV_16S, 0, 1, aperture_size, scale, 0, BORDER_REPLICATE);
         }
         else
         {
@@ -355,12 +367,17 @@ public:
         int *_mag_p, *_mag_a, *_mag_n;
         short *_dx, *_dy, *_dx_a = NULL, *_dy_a = NULL, *_dx_n = NULL, *_dy_n = NULL;
         uchar *_pmap;
+        double scale = 1.0;
 
         CV_TRACE_REGION("gradient")
         if(needGradient)
         {
-            Sobel(src.rowRange(rowStart, rowEnd), dx, CV_16S, 1, 0, aperture_size, 1, 0, BORDER_REPLICATE);
-            Sobel(src.rowRange(rowStart, rowEnd), dy, CV_16S, 0, 1, aperture_size, 1, 0, BORDER_REPLICATE);
+            if (aperture_size == 7)
+            {
+                scale = 1 / 16.0;
+            }
+            Sobel(src.rowRange(rowStart, rowEnd), dx, CV_16S, 1, 0, aperture_size, scale, 0, BORDER_REPLICATE);
+            Sobel(src.rowRange(rowStart, rowEnd), dy, CV_16S, 0, 1, aperture_size, scale, 0, BORDER_REPLICATE);
         }
         else
         {
@@ -946,6 +963,12 @@ void Canny( InputArray _src, OutputArray _dst,
     if ((aperture_size & 1) == 0 || (aperture_size != -1 && (aperture_size < 3 || aperture_size > 7)))
         CV_Error(CV_StsBadFlag, "Aperture size should be odd between 3 and 7");
 
+    if (aperture_size == 7)
+    {
+        low_thresh = low_thresh / 16.0;
+        high_thresh = high_thresh / 16.0;
+    }
+
     if (low_thresh > high_thresh)
         std::swap(low_thresh, high_thresh);
 
diff --git a/modules/imgproc/test/test_canny.cpp b/modules/imgproc/test/test_canny.cpp
index ff7dc6a314..0dd0ee34ac 100644
--- a/modules/imgproc/test/test_canny.cpp
+++ b/modules/imgproc/test/test_canny.cpp
@@ -211,15 +211,15 @@ test_Canny( const Mat& src, Mat& dst,
     Mat dxkernel = cvtest::calcSobelKernel2D( 1, 0, m, 0 );
     Mat dykernel = cvtest::calcSobelKernel2D( 0, 1, m, 0 );
     Mat dx, dy, mag(height, width, CV_32F);
-    cvtest::filter2D(src, dx, CV_16S, dxkernel, anchor, 0, BORDER_REPLICATE);
-    cvtest::filter2D(src, dy, CV_16S, dykernel, anchor, 0, BORDER_REPLICATE);
+    cvtest::filter2D(src, dx, CV_32S, dxkernel, anchor, 0, BORDER_REPLICATE);
+    cvtest::filter2D(src, dy, CV_32S, dykernel, anchor, 0, BORDER_REPLICATE);
 
     // calc gradient magnitude
     for( y = 0; y < height; y++ )
     {
         for( x = 0; x < width; x++ )
         {
-            int dxval = dx.at<short>(y, x), dyval = dy.at<short>(y, x);
+            int dxval = dx.at<int>(y, x), dyval = dy.at<int>(y, x);
             mag.at<float>(y, x) = use_true_gradient ?
                 (float)sqrt((double)(dxval*dxval + dyval*dyval)) :
                 (float)(fabs((double)dxval) + fabs((double)dyval));
@@ -238,8 +238,8 @@ test_Canny( const Mat& src, Mat& dst,
             if( a <= lowThreshold )
                 continue;
 
-            int dxval = dx.at<short>(y, x);
-            int dyval = dy.at<short>(y, x);
+            int dxval = dx.at<int>(y, x);
+            int dyval = dy.at<int>(y, x);
 
             double tg = dxval ? (double)dyval/dxval : DBL_MAX*CV_SIGN(dyval);