From 7dd67f4997ad871029dd73653945df0cfd6ebcee Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Thu, 14 May 2015 20:33:53 +0300 Subject: [PATCH] use _mm_sad_epu8 instead of _mm_movemask_epi8 in countNonZero. Therefore, we do not need any popcount table and this solves http://code.opencv.org/issues/4333 --- modules/core/src/stat.cpp | 105 ++++++++------------------------------ 1 file changed, 20 insertions(+), 85 deletions(-) diff --git a/modules/core/src/stat.cpp b/modules/core/src/stat.cpp index 8a6bd54721..81f9a2484e 100644 --- a/modules/core/src/stat.cpp +++ b/modules/core/src/stat.cpp @@ -504,54 +504,21 @@ static int countNonZero_(const T* src, int len ) return nz; } -#if CV_SSE2 - -static const uchar * initPopcountTable() -{ - static uchar tab[256]; - static volatile bool initialized = false; - if( !initialized ) - { - // we compute inverse popcount table, - // since we pass (img[x] == 0) mask as index in the table. - unsigned int j = 0u; -#if CV_POPCNT - if (checkHardwareSupport(CV_CPU_POPCNT)) - { - for( ; j < 256u; j++ ) - tab[j] = (uchar)(8 - _mm_popcnt_u32(j)); - } -#endif - for( ; j < 256u; j++ ) - { - int val = 0; - for( int mask = 1; mask < 256; mask += mask ) - val += (j & mask) == 0; - tab[j] = (uchar)val; - } - initialized = true; - } - - return tab; -} - -#endif - static int countNonZero8u( const uchar* src, int len ) { int i=0, nz = 0; #if CV_SSE2 if(USE_SSE2)//5x-6x { - __m128i pattern = _mm_setzero_si128 (); - static const uchar * tab = initPopcountTable(); + __m128i v_zero = _mm_setzero_si128(); + __m128i sum = _mm_setzero_si128(); for (; i<=len-16; i+=16) { __m128i r0 = _mm_loadu_si128((const __m128i*)(src+i)); - int val = _mm_movemask_epi8(_mm_cmpeq_epi8(r0, pattern)); - nz += tab[val & 255] + tab[val >> 8]; + sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi8(r0, v_zero)), v_zero)); } + nz = i - _mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))); } #elif CV_NEON int len0 = len & -16, blockSize1 = (1 << 8) - 16, blockSize0 = blockSize1 << 6; @@ -598,15 +565,15 @@ static int countNonZero16u( const ushort* src, int len ) if (USE_SSE2) { __m128i v_zero = _mm_setzero_si128 (); - static const uchar * tab = initPopcountTable(); + __m128i sum = _mm_setzero_si128(); for ( ; i <= len - 8; i += 8) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + i)); - int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_cmpeq_epi16(v_src, v_zero), v_zero)); - nz += tab[val]; + __m128i r0 = _mm_loadu_si128((const __m128i*)(src + i)); + sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi16(r0, v_zero)), v_zero)); } + nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 1); src += i; } #elif CV_NEON @@ -649,20 +616,15 @@ static int countNonZero32s( const int* src, int len ) if (USE_SSE2) { __m128i v_zero = _mm_setzero_si128 (); - static const uchar * tab = initPopcountTable(); + __m128i sum = _mm_setzero_si128(); - for ( ; i <= len - 8; i += 8) + for ( ; i <= len - 4; i += 4) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + i)); - __m128i v_dst0 = _mm_cmpeq_epi32(v_src, v_zero); - - v_src = _mm_loadu_si128((const __m128i*)(src + i + 4)); - __m128i v_dst1 = _mm_cmpeq_epi32(v_src, v_zero); - - int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero)); - nz += tab[val]; + __m128i r0 = _mm_loadu_si128((const __m128i*)(src + i)); + sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_cmpeq_epi32(r0, v_zero)), v_zero)); } + nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2); src += i; } #elif CV_NEON @@ -706,19 +668,17 @@ static int countNonZero32f( const float* src, int len ) #if CV_SSE2 if (USE_SSE2) { - __m128i v_zero_i = _mm_setzero_si128(); __m128 v_zero_f = _mm_setzero_ps(); - static const uchar * tab = initPopcountTable(); + __m128i v_zero = _mm_setzero_si128 (); + __m128i sum = _mm_setzero_si128(); - for ( ; i <= len - 8; i += 8) + for ( ; i <= len - 4; i += 4) { - __m128i v_dst0 = _mm_castps_si128(_mm_cmpeq_ps(_mm_loadu_ps(src + i), v_zero_f)); - __m128i v_dst1 = _mm_castps_si128(_mm_cmpeq_ps(_mm_loadu_ps(src + i + 4), v_zero_f)); - - int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero_i)); - nz += tab[val]; + __m128 r0 = _mm_loadu_ps(src + i); + sum = _mm_add_epi32(sum, _mm_sad_epu8(_mm_sub_epi8(v_zero, _mm_castps_si128(_mm_cmpeq_ps(r0, v_zero_f))), v_zero)); } + nz = i - (_mm_cvtsi128_si32(_mm_add_epi32(sum, _mm_unpackhi_epi64(sum, sum))) >> 2); src += i; } #elif CV_NEON @@ -758,32 +718,7 @@ static int countNonZero32f( const float* src, int len ) static int countNonZero64f( const double* src, int len ) { - int i = 0, nz = 0; -#if CV_SSE2 - if (USE_SSE2) - { - __m128i v_zero_i = _mm_setzero_si128(); - __m128d v_zero_d = _mm_setzero_pd(); - static const uchar * tab = initPopcountTable(); - - for ( ; i <= len - 8; i += 8) - { - __m128i v_dst0 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i), v_zero_d)); - __m128i v_dst1 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 2), v_zero_d)); - __m128i v_dst2 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 4), v_zero_d)); - __m128i v_dst3 = _mm_castpd_si128(_mm_cmpeq_pd(_mm_loadu_pd(src + i + 6), v_zero_d)); - - v_dst0 = _mm_packs_epi32(v_dst0, v_dst1); - v_dst1 = _mm_packs_epi32(v_dst2, v_dst3); - - int val = _mm_movemask_epi8(_mm_packs_epi16(_mm_packs_epi32(v_dst0, v_dst1), v_zero_i)); - nz += tab[val]; - } - - src += i; - } -#endif - return nz + countNonZero_(src, len - i); + return countNonZero_(src, len); } typedef int (*CountNonZeroFunc)(const uchar*, int);