diff --git a/3rdparty/carotene/CMakeLists.txt b/3rdparty/carotene/CMakeLists.txt index 4319815708..bd26a2d7ef 100644 --- a/3rdparty/carotene/CMakeLists.txt +++ b/3rdparty/carotene/CMakeLists.txt @@ -27,7 +27,7 @@ if(CMAKE_COMPILER_IS_GNUCC) endif() endif() -add_library(carotene_objs OBJECT +add_library(carotene_objs OBJECT EXCLUDE_FROM_ALL ${carotene_headers} ${carotene_sources} ) @@ -41,4 +41,4 @@ if(WITH_NEON) endif() # we add dummy file to fix XCode build -add_library(carotene STATIC EXCLUDE_FROM_ALL "$" "${CAROTENE_SOURCE_DIR}/dummy.cpp") +add_library(carotene STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} "$" "${CAROTENE_SOURCE_DIR}/dummy.cpp") diff --git a/3rdparty/cpufeatures/CMakeLists.txt b/3rdparty/cpufeatures/CMakeLists.txt index 92bce6abf8..bf7af0ecde 100644 --- a/3rdparty/cpufeatures/CMakeLists.txt +++ b/3rdparty/cpufeatures/CMakeLists.txt @@ -14,7 +14,7 @@ if(NOT DEFINED CPUFEATURES_SOURCES) endif() include_directories(${CPUFEATURES_INCLUDE_DIRS}) -add_library(${OPENCV_CPUFEATURES_TARGET_NAME} STATIC ${CPUFEATURES_SOURCES}) +add_library(${OPENCV_CPUFEATURES_TARGET_NAME} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${CPUFEATURES_SOURCES}) set_target_properties(${OPENCV_CPUFEATURES_TARGET_NAME} PROPERTIES OUTPUT_NAME cpufeatures @@ -29,7 +29,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${OPENCV_CPUFEATURES_TARGET_NAME} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${OPENCV_CPUFEATURES_TARGET_NAME} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(cpufeatures LICENSE README.md) diff --git a/3rdparty/ippicv/CMakeLists.txt b/3rdparty/ippicv/CMakeLists.txt index 7931832737..43ad806dd7 100644 --- a/3rdparty/ippicv/CMakeLists.txt +++ b/3rdparty/ippicv/CMakeLists.txt @@ -17,7 +17,7 @@ file(GLOB lib_hdrs ${IPP_IW_PATH}/include/*.h ${IPP_IW_PATH}/include/iw/*.h ${IP # Define the library target: # ---------------------------------------------------------------------------------- -add_library(${IPP_IW_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs}) +add_library(${IPP_IW_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(UNIX) if(CV_GCC OR CV_CLANG OR CV_ICC) @@ -41,5 +41,5 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${IPP_IW_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${IPP_IW_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() diff --git a/3rdparty/ittnotify/CMakeLists.txt b/3rdparty/ittnotify/CMakeLists.txt index c2caf76723..a227aff88e 100644 --- a/3rdparty/ittnotify/CMakeLists.txt +++ b/3rdparty/ittnotify/CMakeLists.txt @@ -37,7 +37,7 @@ set(ITT_SRCS src/ittnotify/jitprofiling.c ) -add_library(${ITT_LIBRARY} STATIC ${ITT_SRCS} ${ITT_PUBLIC_HDRS} ${ITT_PRIVATE_HDRS}) +add_library(${ITT_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${ITT_SRCS} ${ITT_PUBLIC_HDRS} ${ITT_PRIVATE_HDRS}) if(NOT WIN32) if(HAVE_DL_LIBRARY) @@ -60,7 +60,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${ITT_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${ITT_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(ittnotify src/ittnotify/LICENSE.BSD src/ittnotify/LICENSE.GPL) diff --git a/3rdparty/libjasper/CMakeLists.txt b/3rdparty/libjasper/CMakeLists.txt index 897b6ae606..9f05d89733 100644 --- a/3rdparty/libjasper/CMakeLists.txt +++ b/3rdparty/libjasper/CMakeLists.txt @@ -17,7 +17,7 @@ file(GLOB lib_ext_hdrs jasper/*.h) # Define the library target: # ---------------------------------------------------------------------------------- -add_library(${JASPER_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs} ${lib_ext_hdrs}) +add_library(${JASPER_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs} ${lib_ext_hdrs}) if(WIN32 AND NOT MINGW) add_definitions(-DJAS_WIN_MSVC_BUILD) @@ -46,7 +46,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${JASPER_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${JASPER_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(jasper LICENSE README copyright) diff --git a/3rdparty/libjpeg-turbo/CMakeLists.txt b/3rdparty/libjpeg-turbo/CMakeLists.txt index 374d7875de..901669a4a8 100644 --- a/3rdparty/libjpeg-turbo/CMakeLists.txt +++ b/3rdparty/libjpeg-turbo/CMakeLists.txt @@ -4,9 +4,9 @@ ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wsign-compare -Wshorten-6 set(VERSION_MAJOR 2) set(VERSION_MINOR 0) -set(VERSION_REVISION 5) +set(VERSION_REVISION 6) set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_REVISION}) -set(LIBJPEG_TURBO_VERSION_NUMBER 2000005) +set(LIBJPEG_TURBO_VERSION_NUMBER 2000006) string(TIMESTAMP BUILD "opencv-${OPENCV_VERSION}-libjpeg-turbo") if(CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -106,7 +106,7 @@ set(JPEG_SOURCES ${JPEG_SOURCES} jsimd_none.c) ocv_list_add_prefix(JPEG_SOURCES src/) -add_library(${JPEG_LIBRARY} STATIC ${JPEG_SOURCES} ${SIMD_OBJS}) +add_library(${JPEG_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${JPEG_SOURCES} ${SIMD_OBJS}) set_target_properties(${JPEG_LIBRARY} PROPERTIES OUTPUT_NAME ${JPEG_LIBRARY} @@ -121,7 +121,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${JPEG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${JPEG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(libjpeg-turbo README.md LICENSE.md README.ijg) diff --git a/3rdparty/libjpeg-turbo/LICENSE.md b/3rdparty/libjpeg-turbo/LICENSE.md index 5ca512b34d..99c9aadcc4 100644 --- a/3rdparty/libjpeg-turbo/LICENSE.md +++ b/3rdparty/libjpeg-turbo/LICENSE.md @@ -91,7 +91,7 @@ best of our understanding. The Modified (3-clause) BSD License =================================== -Copyright (C)2009-2019 D. R. Commander. All Rights Reserved. +Copyright (C)2009-2020 D. R. Commander. All Rights Reserved. Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. Redistribution and use in source and binary forms, with or without diff --git a/3rdparty/libjpeg-turbo/README.ijg b/3rdparty/libjpeg-turbo/README.ijg index 2e39f965c2..d681cf1273 100644 --- a/3rdparty/libjpeg-turbo/README.ijg +++ b/3rdparty/libjpeg-turbo/README.ijg @@ -223,12 +223,12 @@ https://www.iso.org/standard/54989.html and http://www.itu.int/rec/T-REC-T.871. A PDF file of the older JFIF 1.02 specification is available at http://www.w3.org/Graphics/JPEG/jfif3.pdf. -The TIFF 6.0 file format specification can be obtained by FTP from -ftp://ftp.sgi.com/graphics/tiff/TIFF6.ps.gz. The JPEG incorporation scheme -found in the TIFF 6.0 spec of 3-June-92 has a number of serious problems. -IJG does not recommend use of the TIFF 6.0 design (TIFF Compression tag 6). -Instead, we recommend the JPEG design proposed by TIFF Technical Note #2 -(Compression tag 7). Copies of this Note can be obtained from +The TIFF 6.0 file format specification can be obtained from +http://mirrors.ctan.org/graphics/tiff/TIFF6.ps.gz. The JPEG incorporation +scheme found in the TIFF 6.0 spec of 3-June-92 has a number of serious +problems. IJG does not recommend use of the TIFF 6.0 design (TIFF Compression +tag 6). Instead, we recommend the JPEG design proposed by TIFF Technical Note +#2 (Compression tag 7). Copies of this Note can be obtained from http://www.ijg.org/files/. It is expected that the next revision of the TIFF spec will replace the 6.0 JPEG design with the Note's design. Although IJG's own code does not support TIFF/JPEG, the free libtiff library @@ -243,14 +243,8 @@ The most recent released version can always be found there in directory "files". The JPEG FAQ (Frequently Asked Questions) article is a source of some -general information about JPEG. -It is available on the World Wide Web at http://www.faqs.org/faqs/jpeg-faq/ -and other news.answers archive sites, including the official news.answers -archive at rtfm.mit.edu: ftp://rtfm.mit.edu/pub/usenet/news.answers/jpeg-faq/. -If you don't have Web or FTP access, send e-mail to mail-server@rtfm.mit.edu -with body - send usenet/news.answers/jpeg-faq/part1 - send usenet/news.answers/jpeg-faq/part2 +general information about JPEG. It is available at +http://www.faqs.org/faqs/jpeg-faq. FILE FORMAT COMPATIBILITY diff --git a/3rdparty/libjpeg-turbo/README.md b/3rdparty/libjpeg-turbo/README.md index e7ff743a47..90a4a43ee1 100644 --- a/3rdparty/libjpeg-turbo/README.md +++ b/3rdparty/libjpeg-turbo/README.md @@ -2,7 +2,7 @@ Background ========== libjpeg-turbo is a JPEG image codec that uses SIMD instructions to accelerate -baseline JPEG compression and decompression on x86, x86-64, ARM, PowerPC, and +baseline JPEG compression and decompression on x86, x86-64, Arm, PowerPC, and MIPS systems, as well as progressive JPEG compression on x86 and x86-64 systems. On such systems, libjpeg-turbo is generally 2-6x as fast as libjpeg, all else being equal. On other types of systems, libjpeg-turbo can still @@ -179,8 +179,8 @@ supported and which aren't. NOTE: As of this writing, extensive research has been conducted into the usefulness of DCT scaling as a means of data reduction and SmartScale as a -means of quality improvement. The reader is invited to peruse the research at - and draw his/her own conclusions, +means of quality improvement. Readers are invited to peruse the research at + and draw their own conclusions, but it is the general belief of our project that these features have not demonstrated sufficient usefulness to justify inclusion in libjpeg-turbo. @@ -287,12 +287,13 @@ following reasons: (and slightly faster) floating point IDCT algorithm introduced in libjpeg v8a as opposed to the algorithm used in libjpeg v6b. It should be noted, however, that this algorithm basically brings the accuracy of the floating - point IDCT in line with the accuracy of the slow integer IDCT. The floating - point DCT/IDCT algorithms are mainly a legacy feature, and they do not - produce significantly more accuracy than the slow integer algorithms (to put - numbers on this, the typical difference in PNSR between the two algorithms - is less than 0.10 dB, whereas changing the quality level by 1 in the upper - range of the quality scale is typically more like a 1.0 dB difference.) + point IDCT in line with the accuracy of the accurate integer IDCT. The + floating point DCT/IDCT algorithms are mainly a legacy feature, and they do + not produce significantly more accuracy than the accurate integer algorithms + (to put numbers on this, the typical difference in PNSR between the two + algorithms is less than 0.10 dB, whereas changing the quality level by 1 in + the upper range of the quality scale is typically more like a 1.0 dB + difference.) - If the floating point algorithms in libjpeg-turbo are not implemented using SIMD instructions on a particular platform, then the accuracy of the @@ -340,7 +341,7 @@ The algorithm used by the SIMD-accelerated quantization function cannot produce correct results whenever the fast integer forward DCT is used along with a JPEG quality of 98-100. Thus, libjpeg-turbo must use the non-SIMD quantization function in those cases. This causes performance to drop by as much as 40%. -It is therefore strongly advised that you use the slow integer forward DCT +It is therefore strongly advised that you use the accurate integer forward DCT whenever encoding images with a JPEG quality of 98 or higher. diff --git a/3rdparty/libjpeg-turbo/src/jchuff.c b/3rdparty/libjpeg-turbo/src/jchuff.c index cb05055d99..db85ce114f 100644 --- a/3rdparty/libjpeg-turbo/src/jchuff.c +++ b/3rdparty/libjpeg-turbo/src/jchuff.c @@ -34,10 +34,10 @@ * memory footprint by 64k, which is important for some mobile applications * that create many isolated instances of libjpeg-turbo (web browsers, for * instance.) This may improve performance on some mobile platforms as well. - * This feature is enabled by default only on ARM processors, because some x86 + * This feature is enabled by default only on Arm processors, because some x86 * chips have a slow implementation of bsr, and the use of clz/bsr cannot be * shown to have a significant performance impact even on the x86 chips that - * have a fast implementation of it. When building for ARMv6, you can + * have a fast implementation of it. When building for Armv6, you can * explicitly disable the use of clz/bsr by adding -mthumb to the compiler * flags (this defines __thumb__). */ diff --git a/3rdparty/libjpeg-turbo/src/jcinit.c b/3rdparty/libjpeg-turbo/src/jcinit.c index 78aa465786..157353a22e 100644 --- a/3rdparty/libjpeg-turbo/src/jcinit.c +++ b/3rdparty/libjpeg-turbo/src/jcinit.c @@ -1,8 +1,10 @@ /* * jcinit.c * + * This file was part of the Independent JPEG Group's software: * Copyright (C) 1991-1997, Thomas G. Lane. - * This file is part of the Independent JPEG Group's software. + * libjpeg-turbo Modifications: + * Copyright (C) 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -19,6 +21,7 @@ #define JPEG_INTERNALS #include "jinclude.h" #include "jpeglib.h" +#include "jpegcomp.h" /* diff --git a/3rdparty/libjpeg-turbo/src/jcphuff.c b/3rdparty/libjpeg-turbo/src/jcphuff.c index 8c4efaf16c..a8b94bed84 100644 --- a/3rdparty/libjpeg-turbo/src/jcphuff.c +++ b/3rdparty/libjpeg-turbo/src/jcphuff.c @@ -43,10 +43,10 @@ * memory footprint by 64k, which is important for some mobile applications * that create many isolated instances of libjpeg-turbo (web browsers, for * instance.) This may improve performance on some mobile platforms as well. - * This feature is enabled by default only on ARM processors, because some x86 + * This feature is enabled by default only on Arm processors, because some x86 * chips have a slow implementation of bsr, and the use of clz/bsr cannot be * shown to have a significant performance impact even on the x86 chips that - * have a fast implementation of it. When building for ARMv6, you can + * have a fast implementation of it. When building for Armv6, you can * explicitly disable the use of clz/bsr by adding -mthumb to the compiler * flags (this defines __thumb__). */ diff --git a/3rdparty/libjpeg-turbo/src/jctrans.c b/3rdparty/libjpeg-turbo/src/jctrans.c index ce70a30940..ab6a2186db 100644 --- a/3rdparty/libjpeg-turbo/src/jctrans.c +++ b/3rdparty/libjpeg-turbo/src/jctrans.c @@ -4,8 +4,8 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1995-1998, Thomas G. Lane. * Modified 2000-2009 by Guido Vollbeding. - * It was modified by The libjpeg-turbo Project to include only code relevant - * to libjpeg-turbo. + * libjpeg-turbo Modifications: + * Copyright (C) 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -17,6 +17,7 @@ #define JPEG_INTERNALS #include "jinclude.h" #include "jpeglib.h" +#include "jpegcomp.h" /* Forward declarations */ diff --git a/3rdparty/libjpeg-turbo/src/jdapistd.c b/3rdparty/libjpeg-turbo/src/jdapistd.c index 2c808fa564..38bd1110d9 100644 --- a/3rdparty/libjpeg-turbo/src/jdapistd.c +++ b/3rdparty/libjpeg-turbo/src/jdapistd.c @@ -4,7 +4,7 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1994-1996, Thomas G. Lane. * libjpeg-turbo Modifications: - * Copyright (C) 2010, 2015-2018, D. R. Commander. + * Copyright (C) 2010, 2015-2018, 2020, D. R. Commander. * Copyright (C) 2015, Google, Inc. * For conditions of distribution and use, see the accompanying README.ijg * file. @@ -21,6 +21,8 @@ #include "jinclude.h" #include "jdmainct.h" #include "jdcoefct.h" +#include "jdmaster.h" +#include "jdmerge.h" #include "jdsample.h" #include "jmemsys.h" @@ -316,6 +318,8 @@ LOCAL(void) read_and_discard_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) { JDIMENSION n; + my_master_ptr master = (my_master_ptr)cinfo->master; + JSAMPARRAY scanlines = NULL; void (*color_convert) (j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION input_row, JSAMPARRAY output_buf, int num_rows) = NULL; @@ -332,8 +336,13 @@ read_and_discard_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) cinfo->cquantize->color_quantize = noop_quantize; } + if (master->using_merged_upsample && cinfo->max_v_samp_factor == 2) { + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; + scanlines = &upsample->spare_row; + } + for (n = 0; n < num_lines; n++) - jpeg_read_scanlines(cinfo, NULL, 1); + jpeg_read_scanlines(cinfo, scanlines, 1); if (color_convert) cinfo->cconvert->color_convert = color_convert; @@ -353,6 +362,12 @@ increment_simple_rowgroup_ctr(j_decompress_ptr cinfo, JDIMENSION rows) { JDIMENSION rows_left; my_main_ptr main_ptr = (my_main_ptr)cinfo->main; + my_master_ptr master = (my_master_ptr)cinfo->master; + + if (master->using_merged_upsample && cinfo->max_v_samp_factor == 2) { + read_and_discard_scanlines(cinfo, rows); + return; + } /* Increment the counter to the next row group after the skipped rows. */ main_ptr->rowgroup_ctr += rows / cinfo->max_v_samp_factor; @@ -382,21 +397,27 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) { my_main_ptr main_ptr = (my_main_ptr)cinfo->main; my_coef_ptr coef = (my_coef_ptr)cinfo->coef; + my_master_ptr master = (my_master_ptr)cinfo->master; my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; JDIMENSION i, x; int y; JDIMENSION lines_per_iMCU_row, lines_left_in_iMCU_row, lines_after_iMCU_row; JDIMENSION lines_to_skip, lines_to_read; + /* Two-pass color quantization is not supported. */ + if (cinfo->quantize_colors && cinfo->two_pass_quantize) + ERREXIT(cinfo, JERR_NOTIMPL); + if (cinfo->global_state != DSTATE_SCANNING) ERREXIT1(cinfo, JERR_BAD_STATE, cinfo->global_state); /* Do not skip past the bottom of the image. */ if (cinfo->output_scanline + num_lines >= cinfo->output_height) { + num_lines = cinfo->output_height - cinfo->output_scanline; cinfo->output_scanline = cinfo->output_height; (*cinfo->inputctl->finish_input_pass) (cinfo); cinfo->inputctl->eoi_reached = TRUE; - return cinfo->output_height - cinfo->output_scanline; + return num_lines; } if (num_lines == 0) @@ -445,8 +466,10 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) main_ptr->buffer_full = FALSE; main_ptr->rowgroup_ctr = 0; main_ptr->context_state = CTX_PREPARE_FOR_IMCU; - upsample->next_row_out = cinfo->max_v_samp_factor; - upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + if (!master->using_merged_upsample) { + upsample->next_row_out = cinfo->max_v_samp_factor; + upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + } } /* Skipping is much simpler when context rows are not required. */ @@ -458,8 +481,10 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) cinfo->output_scanline += lines_left_in_iMCU_row; main_ptr->buffer_full = FALSE; main_ptr->rowgroup_ctr = 0; - upsample->next_row_out = cinfo->max_v_samp_factor; - upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + if (!master->using_merged_upsample) { + upsample->next_row_out = cinfo->max_v_samp_factor; + upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + } } } @@ -494,7 +519,8 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) cinfo->output_iMCU_row += lines_to_skip / lines_per_iMCU_row; increment_simple_rowgroup_ctr(cinfo, lines_to_read); } - upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + if (!master->using_merged_upsample) + upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; return num_lines; } @@ -535,7 +561,8 @@ jpeg_skip_scanlines(j_decompress_ptr cinfo, JDIMENSION num_lines) * bit odd, since "rows_to_go" seems to be redundantly keeping track of * output_scanline. */ - upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; + if (!master->using_merged_upsample) + upsample->rows_to_go = cinfo->output_height - cinfo->output_scanline; /* Always skip the requested number of lines. */ return num_lines; diff --git a/3rdparty/libjpeg-turbo/src/jdcoefct.c b/3rdparty/libjpeg-turbo/src/jdcoefct.c index 723a9ac2be..2ba6aa11e4 100644 --- a/3rdparty/libjpeg-turbo/src/jdcoefct.c +++ b/3rdparty/libjpeg-turbo/src/jdcoefct.c @@ -6,7 +6,7 @@ * libjpeg-turbo Modifications: * Copyright 2009 Pierre Ossman for Cendio AB * Copyright (C) 2010, 2015-2016, D. R. Commander. - * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, 2020, Google, Inc. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -495,11 +495,13 @@ decompress_smooth_data(j_decompress_ptr cinfo, JSAMPIMAGE output_buf) if (first_row && block_row == 0) prev_block_row = buffer_ptr; else - prev_block_row = buffer[block_row - 1]; + prev_block_row = buffer[block_row - 1] + + cinfo->master->first_MCU_col[ci]; if (last_row && block_row == block_rows - 1) next_block_row = buffer_ptr; else - next_block_row = buffer[block_row + 1]; + next_block_row = buffer[block_row + 1] + + cinfo->master->first_MCU_col[ci]; /* We fetch the surrounding DC values using a sliding-register approach. * Initialize all nine here so as to do the right thing on narrow pics. */ diff --git a/3rdparty/libjpeg-turbo/src/jdcolor.c b/3rdparty/libjpeg-turbo/src/jdcolor.c index dc0e3b6c0e..d3ae40c7da 100644 --- a/3rdparty/libjpeg-turbo/src/jdcolor.c +++ b/3rdparty/libjpeg-turbo/src/jdcolor.c @@ -571,11 +571,10 @@ ycck_cmyk_convert(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, * RGB565 conversion */ -#define PACK_SHORT_565_LE(r, g, b) ((((r) << 8) & 0xF800) | \ - (((g) << 3) & 0x7E0) | ((b) >> 3)) -#define PACK_SHORT_565_BE(r, g, b) (((r) & 0xF8) | ((g) >> 5) | \ - (((g) << 11) & 0xE000) | \ - (((b) << 5) & 0x1F00)) +#define PACK_SHORT_565_LE(r, g, b) \ + ((((r) << 8) & 0xF800) | (((g) << 3) & 0x7E0) | ((b) >> 3)) +#define PACK_SHORT_565_BE(r, g, b) \ + (((r) & 0xF8) | ((g) >> 5) | (((g) << 11) & 0xE000) | (((b) << 5) & 0x1F00)) #define PACK_TWO_PIXELS_LE(l, r) ((r << 16) | l) #define PACK_TWO_PIXELS_BE(l, r) ((l << 16) | r) diff --git a/3rdparty/libjpeg-turbo/src/jdmerge.c b/3rdparty/libjpeg-turbo/src/jdmerge.c index dff5a35087..3a456d6581 100644 --- a/3rdparty/libjpeg-turbo/src/jdmerge.c +++ b/3rdparty/libjpeg-turbo/src/jdmerge.c @@ -5,7 +5,7 @@ * Copyright (C) 1994-1996, Thomas G. Lane. * libjpeg-turbo Modifications: * Copyright 2009 Pierre Ossman for Cendio AB - * Copyright (C) 2009, 2011, 2014-2015, D. R. Commander. + * Copyright (C) 2009, 2011, 2014-2015, 2020, D. R. Commander. * Copyright (C) 2013, Linaro Limited. * For conditions of distribution and use, see the accompanying README.ijg * file. @@ -40,41 +40,13 @@ #define JPEG_INTERNALS #include "jinclude.h" #include "jpeglib.h" +#include "jdmerge.h" #include "jsimd.h" #include "jconfigint.h" #ifdef UPSAMPLE_MERGING_SUPPORTED -/* Private subobject */ - -typedef struct { - struct jpeg_upsampler pub; /* public fields */ - - /* Pointer to routine to do actual upsampling/conversion of one row group */ - void (*upmethod) (j_decompress_ptr cinfo, JSAMPIMAGE input_buf, - JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf); - - /* Private state for YCC->RGB conversion */ - int *Cr_r_tab; /* => table for Cr to R conversion */ - int *Cb_b_tab; /* => table for Cb to B conversion */ - JLONG *Cr_g_tab; /* => table for Cr to G conversion */ - JLONG *Cb_g_tab; /* => table for Cb to G conversion */ - - /* For 2:1 vertical sampling, we produce two output rows at a time. - * We need a "spare" row buffer to hold the second output row if the - * application provides just a one-row buffer; we also use the spare - * to discard the dummy last row if the image height is odd. - */ - JSAMPROW spare_row; - boolean spare_full; /* T if spare buffer is occupied */ - - JDIMENSION out_row_width; /* samples per output row */ - JDIMENSION rows_to_go; /* counts rows remaining in image */ -} my_upsampler; - -typedef my_upsampler *my_upsample_ptr; - #define SCALEBITS 16 /* speediest right-shift on some machines */ #define ONE_HALF ((JLONG)1 << (SCALEBITS - 1)) #define FIX(x) ((JLONG)((x) * (1L << SCALEBITS) + 0.5)) @@ -189,7 +161,7 @@ typedef my_upsampler *my_upsample_ptr; LOCAL(void) build_ycc_rgb_table(j_decompress_ptr cinfo) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; int i; JLONG x; SHIFT_TEMPS @@ -232,7 +204,7 @@ build_ycc_rgb_table(j_decompress_ptr cinfo) METHODDEF(void) start_pass_merged_upsample(j_decompress_ptr cinfo) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; /* Mark the spare buffer empty */ upsample->spare_full = FALSE; @@ -254,7 +226,7 @@ merged_2v_upsample(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION *out_row_ctr, JDIMENSION out_rows_avail) /* 2:1 vertical sampling case: may need a spare row. */ { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; JSAMPROW work_ptrs[2]; JDIMENSION num_rows; /* number of rows returned to caller */ @@ -305,7 +277,7 @@ merged_1v_upsample(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION *out_row_ctr, JDIMENSION out_rows_avail) /* 1:1 vertical sampling case: much easier, never need a spare row. */ { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; /* Just do the upsampling. */ (*upsample->upmethod) (cinfo, input_buf, *in_row_group_ctr, @@ -420,11 +392,10 @@ h2v2_merged_upsample(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, * RGB565 conversion */ -#define PACK_SHORT_565_LE(r, g, b) ((((r) << 8) & 0xF800) | \ - (((g) << 3) & 0x7E0) | ((b) >> 3)) -#define PACK_SHORT_565_BE(r, g, b) (((r) & 0xF8) | ((g) >> 5) | \ - (((g) << 11) & 0xE000) | \ - (((b) << 5) & 0x1F00)) +#define PACK_SHORT_565_LE(r, g, b) \ + ((((r) << 8) & 0xF800) | (((g) << 3) & 0x7E0) | ((b) >> 3)) +#define PACK_SHORT_565_BE(r, g, b) \ + (((r) & 0xF8) | ((g) >> 5) | (((g) << 11) & 0xE000) | (((b) << 5) & 0x1F00)) #define PACK_TWO_PIXELS_LE(l, r) ((r << 16) | l) #define PACK_TWO_PIXELS_BE(l, r) ((l << 16) | r) @@ -566,11 +537,11 @@ h2v2_merged_upsample_565D(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, GLOBAL(void) jinit_merged_upsampler(j_decompress_ptr cinfo) { - my_upsample_ptr upsample; + my_merged_upsample_ptr upsample; - upsample = (my_upsample_ptr) + upsample = (my_merged_upsample_ptr) (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE, - sizeof(my_upsampler)); + sizeof(my_merged_upsampler)); cinfo->upsample = (struct jpeg_upsampler *)upsample; upsample->pub.start_pass = start_pass_merged_upsample; upsample->pub.need_context_rows = FALSE; diff --git a/3rdparty/libjpeg-turbo/src/jdmerge.h b/3rdparty/libjpeg-turbo/src/jdmerge.h new file mode 100644 index 0000000000..b583396b10 --- /dev/null +++ b/3rdparty/libjpeg-turbo/src/jdmerge.h @@ -0,0 +1,47 @@ +/* + * jdmerge.h + * + * This file was part of the Independent JPEG Group's software: + * Copyright (C) 1994-1996, Thomas G. Lane. + * libjpeg-turbo Modifications: + * Copyright (C) 2020, D. R. Commander. + * For conditions of distribution and use, see the accompanying README.ijg + * file. + */ + +#define JPEG_INTERNALS +#include "jpeglib.h" + +#ifdef UPSAMPLE_MERGING_SUPPORTED + + +/* Private subobject */ + +typedef struct { + struct jpeg_upsampler pub; /* public fields */ + + /* Pointer to routine to do actual upsampling/conversion of one row group */ + void (*upmethod) (j_decompress_ptr cinfo, JSAMPIMAGE input_buf, + JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf); + + /* Private state for YCC->RGB conversion */ + int *Cr_r_tab; /* => table for Cr to R conversion */ + int *Cb_b_tab; /* => table for Cb to B conversion */ + JLONG *Cr_g_tab; /* => table for Cr to G conversion */ + JLONG *Cb_g_tab; /* => table for Cb to G conversion */ + + /* For 2:1 vertical sampling, we produce two output rows at a time. + * We need a "spare" row buffer to hold the second output row if the + * application provides just a one-row buffer; we also use the spare + * to discard the dummy last row if the image height is odd. + */ + JSAMPROW spare_row; + boolean spare_full; /* T if spare buffer is occupied */ + + JDIMENSION out_row_width; /* samples per output row */ + JDIMENSION rows_to_go; /* counts rows remaining in image */ +} my_merged_upsampler; + +typedef my_merged_upsampler *my_merged_upsample_ptr; + +#endif /* UPSAMPLE_MERGING_SUPPORTED */ diff --git a/3rdparty/libjpeg-turbo/src/jdmrg565.c b/3rdparty/libjpeg-turbo/src/jdmrg565.c index 1b87e3718d..53f1e16700 100644 --- a/3rdparty/libjpeg-turbo/src/jdmrg565.c +++ b/3rdparty/libjpeg-turbo/src/jdmrg565.c @@ -5,7 +5,7 @@ * Copyright (C) 1994-1996, Thomas G. Lane. * libjpeg-turbo Modifications: * Copyright (C) 2013, Linaro Limited. - * Copyright (C) 2014-2015, 2018, D. R. Commander. + * Copyright (C) 2014-2015, 2018, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -19,7 +19,7 @@ h2v1_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr; @@ -90,7 +90,7 @@ h2v1_merged_upsample_565D_internal(j_decompress_ptr cinfo, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr; @@ -163,7 +163,7 @@ h2v2_merged_upsample_565_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr0, outptr1; @@ -259,7 +259,7 @@ h2v2_merged_upsample_565D_internal(j_decompress_ptr cinfo, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr0, outptr1; diff --git a/3rdparty/libjpeg-turbo/src/jdmrgext.c b/3rdparty/libjpeg-turbo/src/jdmrgext.c index b1c27df56a..c9a44d8219 100644 --- a/3rdparty/libjpeg-turbo/src/jdmrgext.c +++ b/3rdparty/libjpeg-turbo/src/jdmrgext.c @@ -4,7 +4,7 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1994-1996, Thomas G. Lane. * libjpeg-turbo Modifications: - * Copyright (C) 2011, 2015, D. R. Commander. + * Copyright (C) 2011, 2015, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -25,7 +25,7 @@ h2v1_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr; @@ -97,7 +97,7 @@ h2v2_merged_upsample_internal(j_decompress_ptr cinfo, JSAMPIMAGE input_buf, JDIMENSION in_row_group_ctr, JSAMPARRAY output_buf) { - my_upsample_ptr upsample = (my_upsample_ptr)cinfo->upsample; + my_merged_upsample_ptr upsample = (my_merged_upsample_ptr)cinfo->upsample; register int y, cred, cgreen, cblue; int cb, cr; register JSAMPROW outptr0, outptr1; diff --git a/3rdparty/libjpeg-turbo/src/jdtrans.c b/3rdparty/libjpeg-turbo/src/jdtrans.c index 56713efe64..d7ec4b83b3 100644 --- a/3rdparty/libjpeg-turbo/src/jdtrans.c +++ b/3rdparty/libjpeg-turbo/src/jdtrans.c @@ -3,8 +3,8 @@ * * This file was part of the Independent JPEG Group's software: * Copyright (C) 1995-1997, Thomas G. Lane. - * It was modified by The libjpeg-turbo Project to include only code relevant - * to libjpeg-turbo. + * libjpeg-turbo Modifications: + * Copyright (C) 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -16,6 +16,7 @@ #define JPEG_INTERNALS #include "jinclude.h" #include "jpeglib.h" +#include "jpegcomp.h" /* Forward declarations */ diff --git a/3rdparty/libjpeg-turbo/src/jfdctint.c b/3rdparty/libjpeg-turbo/src/jfdctint.c index b47c3061ac..c95a3a7fb8 100644 --- a/3rdparty/libjpeg-turbo/src/jfdctint.c +++ b/3rdparty/libjpeg-turbo/src/jfdctint.c @@ -4,11 +4,11 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1991-1996, Thomas G. Lane. * libjpeg-turbo Modifications: - * Copyright (C) 2015, D. R. Commander. + * Copyright (C) 2015, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * - * This file contains a slow-but-accurate integer implementation of the + * This file contains a slower but more accurate integer implementation of the * forward DCT (Discrete Cosine Transform). * * A 2-D DCT can be done by 1-D DCT on each row followed by 1-D DCT diff --git a/3rdparty/libjpeg-turbo/src/jidctint.c b/3rdparty/libjpeg-turbo/src/jidctint.c index 98425d5fd0..50f385da33 100644 --- a/3rdparty/libjpeg-turbo/src/jidctint.c +++ b/3rdparty/libjpeg-turbo/src/jidctint.c @@ -5,11 +5,11 @@ * Copyright (C) 1991-1998, Thomas G. Lane. * Modification developed 2002-2009 by Guido Vollbeding. * libjpeg-turbo Modifications: - * Copyright (C) 2015, D. R. Commander. + * Copyright (C) 2015, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * - * This file contains a slow-but-accurate integer implementation of the + * This file contains a slower but more accurate integer implementation of the * inverse DCT (Discrete Cosine Transform). In the IJG code, this routine * must also perform dequantization of the input coefficients. * diff --git a/3rdparty/libjpeg-turbo/src/jmorecfg.h b/3rdparty/libjpeg-turbo/src/jmorecfg.h index d0b930079a..aa29f0f9f1 100644 --- a/3rdparty/libjpeg-turbo/src/jmorecfg.h +++ b/3rdparty/libjpeg-turbo/src/jmorecfg.h @@ -5,7 +5,7 @@ * Copyright (C) 1991-1997, Thomas G. Lane. * Modified 1997-2009 by Guido Vollbeding. * libjpeg-turbo Modifications: - * Copyright (C) 2009, 2011, 2014-2015, 2018, D. R. Commander. + * Copyright (C) 2009, 2011, 2014-2015, 2018, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -273,9 +273,9 @@ typedef int boolean; /* Capability options common to encoder and decoder: */ -#define DCT_ISLOW_SUPPORTED /* slow but accurate integer algorithm */ -#define DCT_IFAST_SUPPORTED /* faster, less accurate integer method */ -#define DCT_FLOAT_SUPPORTED /* floating-point: accurate, fast on fast HW */ +#define DCT_ISLOW_SUPPORTED /* accurate integer method */ +#define DCT_IFAST_SUPPORTED /* less accurate int method [legacy feature] */ +#define DCT_FLOAT_SUPPORTED /* floating-point method [legacy feature] */ /* Encoder capability options: */ diff --git a/3rdparty/libjpeg-turbo/src/jpegcomp.h b/3rdparty/libjpeg-turbo/src/jpegcomp.h index b32d544bf1..c4834ac0df 100644 --- a/3rdparty/libjpeg-turbo/src/jpegcomp.h +++ b/3rdparty/libjpeg-turbo/src/jpegcomp.h @@ -1,7 +1,7 @@ /* * jpegcomp.h * - * Copyright (C) 2010, D. R. Commander. + * Copyright (C) 2010, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -19,6 +19,7 @@ #define _min_DCT_v_scaled_size min_DCT_v_scaled_size #define _jpeg_width jpeg_width #define _jpeg_height jpeg_height +#define JERR_ARITH_NOTIMPL JERR_NOT_COMPILED #else #define _DCT_scaled_size DCT_scaled_size #define _DCT_h_scaled_size DCT_scaled_size diff --git a/3rdparty/libjpeg-turbo/src/jpeglib.h b/3rdparty/libjpeg-turbo/src/jpeglib.h index 33f8ad2791..d7664f0630 100644 --- a/3rdparty/libjpeg-turbo/src/jpeglib.h +++ b/3rdparty/libjpeg-turbo/src/jpeglib.h @@ -5,7 +5,7 @@ * Copyright (C) 1991-1998, Thomas G. Lane. * Modified 2002-2009 by Guido Vollbeding. * libjpeg-turbo Modifications: - * Copyright (C) 2009-2011, 2013-2014, 2016-2017, D. R. Commander. + * Copyright (C) 2009-2011, 2013-2014, 2016-2017, 2020, D. R. Commander. * Copyright (C) 2015, Google, Inc. * For conditions of distribution and use, see the accompanying README.ijg * file. @@ -244,9 +244,9 @@ typedef enum { /* DCT/IDCT algorithm options. */ typedef enum { - JDCT_ISLOW, /* slow but accurate integer algorithm */ - JDCT_IFAST, /* faster, less accurate integer method */ - JDCT_FLOAT /* floating-point: accurate, fast on fast HW */ + JDCT_ISLOW, /* accurate integer method */ + JDCT_IFAST, /* less accurate integer method [legacy feature] */ + JDCT_FLOAT /* floating-point method [legacy feature] */ } J_DCT_METHOD; #ifndef JDCT_DEFAULT /* may be overridden in jconfig.h */ diff --git a/3rdparty/libjpeg-turbo/src/jquant2.c b/3rdparty/libjpeg-turbo/src/jquant2.c index 0ce0ca5472..6570613bb9 100644 --- a/3rdparty/libjpeg-turbo/src/jquant2.c +++ b/3rdparty/libjpeg-turbo/src/jquant2.c @@ -4,7 +4,7 @@ * This file was part of the Independent JPEG Group's software: * Copyright (C) 1991-1996, Thomas G. Lane. * libjpeg-turbo Modifications: - * Copyright (C) 2009, 2014-2015, D. R. Commander. + * Copyright (C) 2009, 2014-2015, 2020, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -1145,7 +1145,7 @@ start_pass_2_quant(j_decompress_ptr cinfo, boolean is_pre_scan) int i; /* Only F-S dithering or no dithering is supported. */ - /* If user asks for ordered dither, give him F-S. */ + /* If user asks for ordered dither, give them F-S. */ if (cinfo->dither_mode != JDITHER_NONE) cinfo->dither_mode = JDITHER_FS; @@ -1263,7 +1263,7 @@ jinit_2pass_quantizer(j_decompress_ptr cinfo) cquantize->sv_colormap = NULL; /* Only F-S dithering or no dithering is supported. */ - /* If user asks for ordered dither, give him F-S. */ + /* If user asks for ordered dither, give them F-S. */ if (cinfo->dither_mode != JDITHER_NONE) cinfo->dither_mode = JDITHER_FS; diff --git a/3rdparty/libjpeg-turbo/src/jversion.h b/3rdparty/libjpeg-turbo/src/jversion.h index ab4a2c5703..4462b94104 100644 --- a/3rdparty/libjpeg-turbo/src/jversion.h +++ b/3rdparty/libjpeg-turbo/src/jversion.h @@ -30,23 +30,25 @@ * NOTE: It is our convention to place the authors in the following order: * - libjpeg-turbo authors (2009-) in descending order of the date of their * most recent contribution to the project, then in ascending order of the - * date of their first contribution to the project + * date of their first contribution to the project, then in alphabetical + * order * - Upstream authors in descending order of the date of the first inclusion of * their code */ #define JCOPYRIGHT \ "Copyright (C) 2009-2020 D. R. Commander\n" \ - "Copyright (C) 2011-2016 Siarhei Siamashka\n" \ + "Copyright (C) 2015, 2020 Google, Inc.\n" \ + "Copyright (C) 2019 Arm Limited\n" \ "Copyright (C) 2015-2016, 2018 Matthieu Darbois\n" \ + "Copyright (C) 2011-2016 Siarhei Siamashka\n" \ "Copyright (C) 2015 Intel Corporation\n" \ - "Copyright (C) 2015 Google, Inc.\n" \ + "Copyright (C) 2013-2014 Linaro Limited\n" \ "Copyright (C) 2013-2014 MIPS Technologies, Inc.\n" \ - "Copyright (C) 2013 Linaro Limited\n" \ + "Copyright (C) 2009, 2012 Pierre Ossman for Cendio AB\n" \ "Copyright (C) 2009-2011 Nokia Corporation and/or its subsidiary(-ies)\n" \ - "Copyright (C) 2009 Pierre Ossman for Cendio AB\n" \ "Copyright (C) 1999-2006 MIYASAKA Masaru\n" \ - "Copyright (C) 1991-2016 Thomas G. Lane, Guido Vollbeding" + "Copyright (C) 1991-2017 Thomas G. Lane, Guido Vollbeding" #define JCOPYRIGHT_SHORT \ "Copyright (C) 1991-2020 The libjpeg-turbo Project and many others" diff --git a/3rdparty/libjpeg/CMakeLists.txt b/3rdparty/libjpeg/CMakeLists.txt index b50fc09840..c0524cc38a 100644 --- a/3rdparty/libjpeg/CMakeLists.txt +++ b/3rdparty/libjpeg/CMakeLists.txt @@ -19,7 +19,7 @@ endif() # Define the library target: # ---------------------------------------------------------------------------------- -add_library(${JPEG_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs}) +add_library(${JPEG_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(CV_GCC OR CV_CLANG) set_source_files_properties(jcdctmgr.c PROPERTIES COMPILE_FLAGS "-O1") @@ -42,7 +42,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${JPEG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${JPEG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(libjpeg README) diff --git a/3rdparty/libpng/CMakeLists.txt b/3rdparty/libpng/CMakeLists.txt index 31e77676e8..efa59627eb 100644 --- a/3rdparty/libpng/CMakeLists.txt +++ b/3rdparty/libpng/CMakeLists.txt @@ -74,7 +74,7 @@ if(MSVC) add_definitions(-D_CRT_SECURE_NO_DEPRECATE) endif(MSVC) -add_library(${PNG_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs}) +add_library(${PNG_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) target_link_libraries(${PNG_LIBRARY} ${ZLIB_LIBRARIES}) ocv_warnings_disable(CMAKE_C_FLAGS -Wundef -Wcast-align -Wimplicit-fallthrough -Wunused-parameter -Wsign-compare) @@ -92,7 +92,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${PNG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${PNG_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(libpng LICENSE README) diff --git a/3rdparty/libtiff/CMakeLists.txt b/3rdparty/libtiff/CMakeLists.txt index 16cb598955..61e40b2885 100644 --- a/3rdparty/libtiff/CMakeLists.txt +++ b/3rdparty/libtiff/CMakeLists.txt @@ -462,7 +462,7 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4456 /wd4457 /wd4312) # vs2015 ocv_warnings_disable(CMAKE_C_FLAGS /wd4267 /wd4244 /wd4018 /wd4311 /wd4312) -add_library(${TIFF_LIBRARY} STATIC ${lib_srcs}) +add_library(${TIFF_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs}) target_link_libraries(${TIFF_LIBRARY} ${ZLIB_LIBRARIES}) set_target_properties(${TIFF_LIBRARY} @@ -479,7 +479,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${TIFF_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${TIFF_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(libtiff COPYRIGHT) diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 83884c9d4d..80ab0b86ab 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -34,7 +34,7 @@ endif() add_definitions(-DWEBP_USE_THREAD) -add_library(${WEBP_LIBRARY} STATIC ${lib_srcs} ${lib_hdrs}) +add_library(${WEBP_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(ANDROID) target_link_libraries(${WEBP_LIBRARY} ${CPUFEATURES_LIBRARIES}) endif() @@ -59,6 +59,6 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${WEBP_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${WEBP_LIBRARY} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() diff --git a/3rdparty/openexr/CMakeLists.txt b/3rdparty/openexr/CMakeLists.txt index 2ee5146a3d..8d10e7d968 100644 --- a/3rdparty/openexr/CMakeLists.txt +++ b/3rdparty/openexr/CMakeLists.txt @@ -109,6 +109,7 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS -Wshadow -Wunused -Wsign-compare -Wundef -W -Wmissing-prototypes # gcc/clang -Wreorder -Wunused-result + -Wimplicit-const-int-float-conversion # clang ) if(CV_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0) ocv_warnings_disable(CMAKE_CXX_FLAGS -Wclass-memaccess) @@ -125,7 +126,7 @@ if(MSVC AND CV_ICC) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Qrestrict") endif() -add_library(IlmImf STATIC ${lib_hdrs} ${lib_srcs}) +add_library(IlmImf STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_hdrs} ${lib_srcs}) target_link_libraries(IlmImf ${ZLIB_LIBRARIES}) set_target_properties(IlmImf @@ -142,7 +143,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(IlmImf EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(IlmImf EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(openexr LICENSE AUTHORS.ilmbase AUTHORS.openexr) diff --git a/3rdparty/openjpeg/CMakeLists.txt b/3rdparty/openjpeg/CMakeLists.txt index ec15bba850..b38bf28f05 100644 --- a/3rdparty/openjpeg/CMakeLists.txt +++ b/3rdparty/openjpeg/CMakeLists.txt @@ -11,6 +11,10 @@ set(OPENJPEG_LIBRARY_NAME libopenjp2) project(openjpeg C) +ocv_warnings_disable(CMAKE_C_FLAGS + -Wimplicit-const-int-float-conversion # clang +) + #----------------------------------------------------------------------------- # OPENJPEG version number, useful for packaging and doxygen doc: set(OPENJPEG_VERSION_MAJOR 2) diff --git a/3rdparty/protobuf/CMakeLists.txt b/3rdparty/protobuf/CMakeLists.txt index 26d6523988..c71bf9faff 100644 --- a/3rdparty/protobuf/CMakeLists.txt +++ b/3rdparty/protobuf/CMakeLists.txt @@ -140,7 +140,8 @@ append_if_exist(Protobuf_SRCS ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc ) -add_library(libprotobuf STATIC ${Protobuf_SRCS}) +include_directories(BEFORE "${PROTOBUF_ROOT}/src") # ensure using if own headers: https://github.com/opencv/opencv/issues/13328 +add_library(libprotobuf STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${Protobuf_SRCS}) target_include_directories(libprotobuf SYSTEM PUBLIC $) set_target_properties(libprotobuf PROPERTIES @@ -156,7 +157,7 @@ get_protobuf_version(Protobuf_VERSION "${PROTOBUF_ROOT}/src") set(Protobuf_VERSION ${Protobuf_VERSION} CACHE INTERNAL "" FORCE) if(NOT BUILD_SHARED_LIBS) - ocv_install_target(libprotobuf EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(libprotobuf EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(protobuf LICENSE README.md) diff --git a/3rdparty/quirc/CMakeLists.txt b/3rdparty/quirc/CMakeLists.txt index 7a6b2bb222..c0464c16ae 100644 --- a/3rdparty/quirc/CMakeLists.txt +++ b/3rdparty/quirc/CMakeLists.txt @@ -8,7 +8,7 @@ ocv_include_directories(${CURR_INCLUDE_DIR}) file(GLOB_RECURSE quirc_headers RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "include/*.h") file(GLOB_RECURSE quirc_sources RELATIVE "${CMAKE_CURRENT_LIST_DIR}" "src/*.c") -add_library(${PROJECT_NAME} STATIC ${quirc_headers} ${quirc_sources}) +add_library(${PROJECT_NAME} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${quirc_headers} ${quirc_sources}) ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-variable -Wshadow) set_target_properties(${PROJECT_NAME} @@ -24,7 +24,7 @@ if(ENABLE_SOLUTION_FOLDERS) endif() if(NOT BUILD_SHARED_LIBS) - ocv_install_target(${PROJECT_NAME} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev) + ocv_install_target(${PROJECT_NAME} EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev OPTIONAL) endif() ocv_install_3rdparty_licenses(${PROJECT_NAME} LICENSE) diff --git a/3rdparty/tbb/CMakeLists.txt b/3rdparty/tbb/CMakeLists.txt index 2aa9127da0..a085b0f3ca 100644 --- a/3rdparty/tbb/CMakeLists.txt +++ b/3rdparty/tbb/CMakeLists.txt @@ -108,7 +108,7 @@ set(tbb_version_file "version_string.ver") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${tbb_version_file}.cmakein" "${CMAKE_CURRENT_BINARY_DIR}/${tbb_version_file}" @ONLY) list(APPEND TBB_SOURCE_FILES "${CMAKE_CURRENT_BINARY_DIR}/${tbb_version_file}") -add_library(tbb ${TBB_SOURCE_FILES}) +add_library(tbb ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${TBB_SOURCE_FILES}) target_compile_definitions(tbb PUBLIC TBB_USE_GCC_BUILTINS=1 __TBB_GCC_BUILTIN_ATOMICS_PRESENT=1 @@ -165,6 +165,7 @@ ocv_install_target(tbb EXPORT OpenCVModules RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT libs LIBRARY DESTINATION ${OPENCV_LIB_INSTALL_PATH} COMPONENT libs ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev + OPTIONAL ) ocv_install_3rdparty_licenses(tbb "${tbb_src_dir}/LICENSE" "${tbb_src_dir}/README") diff --git a/3rdparty/zlib/CMakeLists.txt b/3rdparty/zlib/CMakeLists.txt index 553700bacc..9758861a6b 100644 --- a/3rdparty/zlib/CMakeLists.txt +++ b/3rdparty/zlib/CMakeLists.txt @@ -76,7 +76,7 @@ set(ZLIB_SRCS zutil.c ) -add_library(${ZLIB_LIBRARY} STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +add_library(${ZLIB_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) set_target_properties(${ZLIB_LIBRARY} PROPERTIES DEFINE_SYMBOL ZLIB_DLL) ocv_warnings_disable(CMAKE_C_FLAGS -Wshorten-64-to-32 -Wattributes -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wshift-negative-value diff --git a/CMakeLists.txt b/CMakeLists.txt index cd0341aea2..487efd5f7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -368,6 +368,9 @@ OCV_OPTION(WITH_MSMF_DXVA "Enable hardware acceleration in Media Foundation back OCV_OPTION(WITH_XIMEA "Include XIMEA cameras support" OFF VISIBLE_IF NOT ANDROID AND NOT WINRT VERIFY HAVE_XIMEA) +OCV_OPTION(WITH_UEYE "Include UEYE camera support" OFF + VISIBLE_IF NOT ANDROID AND NOT APPLE AND NOT WINRT + VERIFY HAVE_UEYE) OCV_OPTION(WITH_XINE "Include Xine support (GPL)" OFF VISIBLE_IF UNIX AND NOT APPLE AND NOT ANDROID VERIFY HAVE_XINE) @@ -440,6 +443,9 @@ OCV_OPTION(WITH_ANDROID_MEDIANDK "Use Android Media NDK for Video I/O (Android)" OCV_OPTION(WITH_TENGINE "Include Arm Inference Tengine support" OFF VISIBLE_IF (ARM OR AARCH64) AND (UNIX OR ANDROID) AND NOT IOS VERIFY HAVE_TENGINE) +OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF + VISIBLE_IF TRUE + VERIFY HAVE_ONNX) # OpenCV build components # =================================================== @@ -782,6 +788,11 @@ if(WITH_QUIRC) add_subdirectory(3rdparty/quirc) set(HAVE_QUIRC TRUE) endif() + +if(WITH_ONNX) + include(cmake/FindONNX.cmake) +endif() + # ---------------------------------------------------------------------------- # OpenCV HAL # ---------------------------------------------------------------------------- @@ -1383,6 +1394,10 @@ if(WITH_XIMEA OR HAVE_XIMEA) status(" XIMEA:" HAVE_XIMEA THEN YES ELSE NO) endif() +if(WITH_UEYE OR HAVE_UEYE) + status(" uEye:" HAVE_UEYE THEN YES ELSE NO) +endif() + if(WITH_XINE OR HAVE_XINE) status(" Xine:" HAVE_XINE THEN "YES (ver ${XINE_VERSION})" ELSE NO) endif() @@ -1567,6 +1582,15 @@ if(WITH_OPENCL OR HAVE_OPENCL) endif() endif() +if(WITH_ONNX OR HAVE_ONNX) + status("") + status(" ONNX:" HAVE_ONNX THEN "YES" ELSE "NO") + if(HAVE_ONNX) + status(" Include path:" ONNX_INCLUDE_DIR THEN "${ONNX_INCLUDE_DIR}" ELSE "NO") + status(" Link libraries:" ONNX_LIBRARIES THEN "${ONNX_LIBRARIES}" ELSE "NO") + endif() +endif() + # ========================== python ========================== if(BUILD_opencv_python2) status("") diff --git a/apps/interactive-calibration/parametersController.cpp b/apps/interactive-calibration/parametersController.cpp index c76b915c63..3bcf5b86e9 100644 --- a/apps/interactive-calibration/parametersController.cpp +++ b/apps/interactive-calibration/parametersController.cpp @@ -32,7 +32,7 @@ bool calib::parametersController::loadFromFile(const std::string &inputFileName) if(!reader.isOpened()) { std::cerr << "Warning: Unable to open " << inputFileName << - " Applicatioin stated with default advanced parameters" << std::endl; + " Application started with default advanced parameters" << std::endl; return true; } diff --git a/cmake/FindONNX.cmake b/cmake/FindONNX.cmake new file mode 100644 index 0000000000..56dd6d5098 --- /dev/null +++ b/cmake/FindONNX.cmake @@ -0,0 +1,36 @@ +ocv_clear_vars(HAVE_ONNX) + +set(ONNXRT_ROOT_DIR "" CACHE PATH "ONNX Runtime install directory") + +# For now, check the old name ORT_INSTALL_DIR +if(ORT_INSTALL_DIR AND NOT ONNXRT_ROOT_DIR) + set(ONNXRT_ROOT_DIR ${ORT_INSTALL_DIR}) +endif() + +if(ONNXRT_ROOT_DIR) + find_library(ORT_LIB onnxruntime + ${ONNXRT_ROOT_DIR}/lib + CMAKE_FIND_ROOT_PATH_BOTH) + find_path(ORT_INCLUDE onnxruntime_cxx_api.h + ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/session + CMAKE_FIND_ROOT_PATH_BOTH) +endif() + +if(ORT_LIB AND ORT_INCLUDE) + set(HAVE_ONNX TRUE) + # For CMake output only + set(ONNX_LIBRARIES "${ORT_LIB}" CACHE STRING "ONNX Runtime libraries") + set(ONNX_INCLUDE_DIR "${ORT_INCLUDE}" CACHE STRING "ONNX Runtime include path") + + # Link target with associated interface headers + set(ONNX_LIBRARY "onnxruntime" CACHE STRING "ONNX Link Target") + ocv_add_library(${ONNX_LIBRARY} SHARED IMPORTED) + set_target_properties(${ONNX_LIBRARY} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ORT_INCLUDE} + IMPORTED_LOCATION ${ORT_LIB} + IMPORTED_IMPLIB ${ORT_LIB}) +endif() + +if(NOT HAVE_ONNX) + ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE_DIR) +endif() diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 080c78c547..929c5b5e51 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -153,7 +153,7 @@ if(CV_GCC OR CV_CLANG) if(CV_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) add_extra_compiler_option(-Wno-missing-field-initializers) # GCC 4.x emits warnings about {}, fixed in GCC 5+ endif() - if(CV_CLANG AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0) + if(CV_CLANG AND NOT CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0) add_extra_compiler_option(-Wno-deprecated-enum-enum-conversion) add_extra_compiler_option(-Wno-deprecated-anon-enum-enum-conversion) endif() diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index a120ca0cef..d12a9e68ea 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -208,7 +208,7 @@ if(CUDA_FOUND) if(${status} EQUAL 0) # cache detected values - set(OPENCV_CACHE_CUDA_ACTIVE_CC ${${result_list}} CACHE INTERNAL "") + set(OPENCV_CACHE_CUDA_ACTIVE_CC ${${output}} CACHE INTERNAL "") set(OPENCV_CACHE_CUDA_ACTIVE_CC_check "${__cache_key_check}" CACHE INTERNAL "") endif() endif() diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 63ef8f0b8f..c838a40409 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -129,9 +129,9 @@ endif() if(INF_ENGINE_TARGET) if(NOT INF_ENGINE_RELEASE) - message(WARNING "InferenceEngine version has not been set, 2020.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + message(WARNING "InferenceEngine version has not been set, 2021.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") endif() - set(INF_ENGINE_RELEASE "2020040000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + set(INF_ENGINE_RELEASE "2021010000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) diff --git a/cmake/OpenCVFindLibsGrfmt.cmake b/cmake/OpenCVFindLibsGrfmt.cmake index 22e20b6b79..28aa47ba9c 100644 --- a/cmake/OpenCVFindLibsGrfmt.cmake +++ b/cmake/OpenCVFindLibsGrfmt.cmake @@ -15,11 +15,12 @@ else() endif() if(NOT ZLIB_FOUND) - ocv_clear_vars(ZLIB_LIBRARY ZLIB_LIBRARIES ZLIB_INCLUDE_DIRS) + ocv_clear_vars(ZLIB_LIBRARY ZLIB_LIBRARIES ZLIB_INCLUDE_DIR) - set(ZLIB_LIBRARY zlib) + set(ZLIB_LIBRARY zlib CACHE INTERNAL "") add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/zlib") - set(ZLIB_INCLUDE_DIRS "${${ZLIB_LIBRARY}_SOURCE_DIR}" "${${ZLIB_LIBRARY}_BINARY_DIR}") + set(ZLIB_INCLUDE_DIR "${${ZLIB_LIBRARY}_SOURCE_DIR}" "${${ZLIB_LIBRARY}_BINARY_DIR}" CACHE INTERNAL "") + set(ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) set(ZLIB_LIBRARIES ${ZLIB_LIBRARY}) ocv_parse_header2(ZLIB "${${ZLIB_LIBRARY}_SOURCE_DIR}/zlib.h" ZLIB_VERSION) @@ -37,16 +38,17 @@ if(WITH_JPEG) ocv_clear_vars(JPEG_LIBRARY JPEG_LIBRARIES JPEG_INCLUDE_DIR) if(NOT BUILD_JPEG_TURBO_DISABLE) - set(JPEG_LIBRARY libjpeg-turbo) + set(JPEG_LIBRARY libjpeg-turbo CACHE INTERNAL "") set(JPEG_LIBRARIES ${JPEG_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libjpeg-turbo") - set(JPEG_INCLUDE_DIR "${${JPEG_LIBRARY}_SOURCE_DIR}/src") + set(JPEG_INCLUDE_DIR "${${JPEG_LIBRARY}_SOURCE_DIR}/src" CACHE INTERNAL "") else() - set(JPEG_LIBRARY libjpeg) + set(JPEG_LIBRARY libjpeg CACHE INTERNAL "") set(JPEG_LIBRARIES ${JPEG_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libjpeg") - set(JPEG_INCLUDE_DIR "${${JPEG_LIBRARY}_SOURCE_DIR}") + set(JPEG_INCLUDE_DIR "${${JPEG_LIBRARY}_SOURCE_DIR}" CACHE INTERNAL "") endif() + set(JPEG_INCLUDE_DIRS "${JPEG_INCLUDE_DIR}") endif() macro(ocv_detect_jpeg_version header_file) @@ -83,10 +85,10 @@ if(WITH_TIFF) if(NOT TIFF_FOUND) ocv_clear_vars(TIFF_LIBRARY TIFF_LIBRARIES TIFF_INCLUDE_DIR) - set(TIFF_LIBRARY libtiff) + set(TIFF_LIBRARY libtiff CACHE INTERNAL "") set(TIFF_LIBRARIES ${TIFF_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libtiff") - set(TIFF_INCLUDE_DIR "${${TIFF_LIBRARY}_SOURCE_DIR}" "${${TIFF_LIBRARY}_BINARY_DIR}") + set(TIFF_INCLUDE_DIR "${${TIFF_LIBRARY}_SOURCE_DIR}" "${${TIFF_LIBRARY}_BINARY_DIR}" CACHE INTERNAL "") ocv_parse_header("${${TIFF_LIBRARY}_SOURCE_DIR}/tiff.h" TIFF_VERSION_LINES TIFF_VERSION_CLASSIC TIFF_VERSION_BIG TIFF_VERSION TIFF_BIGTIFF_VERSION) endif() @@ -128,12 +130,12 @@ endif() if(WITH_WEBP AND NOT WEBP_FOUND AND (NOT ANDROID OR HAVE_CPUFEATURES) ) - - set(WEBP_LIBRARY libwebp) + ocv_clear_vars(WEBP_LIBRARY WEBP_INCLUDE_DIR) + set(WEBP_LIBRARY libwebp CACHE INTERNAL "") set(WEBP_LIBRARIES ${WEBP_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libwebp") - set(WEBP_INCLUDE_DIR "${${WEBP_LIBRARY}_SOURCE_DIR}/src") + set(WEBP_INCLUDE_DIR "${${WEBP_LIBRARY}_SOURCE_DIR}/src" CACHE INTERNAL "") set(HAVE_WEBP 1) endif() @@ -192,10 +194,10 @@ if(WITH_JASPER AND NOT HAVE_OPENJPEG) if(NOT JASPER_FOUND) ocv_clear_vars(JASPER_LIBRARY JASPER_LIBRARIES JASPER_INCLUDE_DIR) - set(JASPER_LIBRARY libjasper) + set(JASPER_LIBRARY libjasper CACHE INTERNAL "") set(JASPER_LIBRARIES ${JASPER_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libjasper") - set(JASPER_INCLUDE_DIR "${${JASPER_LIBRARY}_SOURCE_DIR}") + set(JASPER_INCLUDE_DIR "${${JASPER_LIBRARY}_SOURCE_DIR}" CACHE INTERNAL "") endif() set(HAVE_JASPER YES) @@ -225,10 +227,10 @@ if(WITH_PNG) if(NOT PNG_FOUND) ocv_clear_vars(PNG_LIBRARY PNG_LIBRARIES PNG_INCLUDE_DIR PNG_PNG_INCLUDE_DIR HAVE_LIBPNG_PNG_H PNG_DEFINITIONS) - set(PNG_LIBRARY libpng) + set(PNG_LIBRARY libpng CACHE INTERNAL "") set(PNG_LIBRARIES ${PNG_LIBRARY}) add_subdirectory("${OpenCV_SOURCE_DIR}/3rdparty/libpng") - set(PNG_INCLUDE_DIR "${${PNG_LIBRARY}_SOURCE_DIR}") + set(PNG_INCLUDE_DIR "${${PNG_LIBRARY}_SOURCE_DIR}" CACHE INTERNAL "") set(PNG_DEFINITIONS "") ocv_parse_header("${PNG_INCLUDE_DIR}/png.h" PNG_VERSION_LINES PNG_LIBPNG_VER_MAJOR PNG_LIBPNG_VER_MINOR PNG_LIBPNG_VER_RELEASE) endif() @@ -270,7 +272,7 @@ if(WITH_GDAL) endif() endif() -if (WITH_GDCM) +if(WITH_GDCM) find_package(GDCM QUIET) if(NOT GDCM_FOUND) set(HAVE_GDCM NO) diff --git a/cmake/OpenCVFindLibsPerf.cmake b/cmake/OpenCVFindLibsPerf.cmake index b9b1a95799..3753084d28 100644 --- a/cmake/OpenCVFindLibsPerf.cmake +++ b/cmake/OpenCVFindLibsPerf.cmake @@ -51,7 +51,15 @@ endif(WITH_CUDA) # --- Eigen --- if(WITH_EIGEN AND NOT HAVE_EIGEN) - find_package(Eigen3 QUIET) + if((OPENCV_FORCE_EIGEN_FIND_PACKAGE_CONFIG + OR NOT (CMAKE_VERSION VERSION_LESS "3.0.0") # Eigen3Targets.cmake required CMake 3.0.0+ + ) AND NOT OPENCV_SKIP_EIGEN_FIND_PACKAGE_CONFIG + ) + find_package(Eigen3 CONFIG QUIET) # Ceres 2.0.0 CMake scripts doesn't work with CMake's FindEigen3.cmake module (due to missing EIGEN3_VERSION_STRING) + endif() + if(NOT Eigen3_FOUND) + find_package(Eigen3 QUIET) + endif() if(Eigen3_FOUND) if(TARGET Eigen3::Eigen) diff --git a/cmake/OpenCVFindOpenBLAS.cmake b/cmake/OpenCVFindOpenBLAS.cmake index 6cb486d95d..d1db034908 100644 --- a/cmake/OpenCVFindOpenBLAS.cmake +++ b/cmake/OpenCVFindOpenBLAS.cmake @@ -57,7 +57,7 @@ SET(Open_BLAS_INCLUDE_SEARCH_PATHS ) SET(Open_BLAS_LIB_SEARCH_PATHS - $ENV{OpenBLAS}cd + $ENV{OpenBLAS} $ENV{OpenBLAS}/lib $ENV{OpenBLAS_HOME} $ENV{OpenBLAS_HOME}/lib diff --git a/cmake/OpenCVGenInfoPlist.cmake b/cmake/OpenCVGenInfoPlist.cmake index 90dd85479f..105087907f 100644 --- a/cmake/OpenCVGenInfoPlist.cmake +++ b/cmake/OpenCVGenInfoPlist.cmake @@ -2,7 +2,11 @@ set(OPENCV_APPLE_BUNDLE_NAME "OpenCV") set(OPENCV_APPLE_BUNDLE_ID "org.opencv") if(IOS) - if (APPLE_FRAMEWORK AND DYNAMIC_PLIST) + if(MAC_CATALYST) + # Copy the iOS plist over to the OSX directory if building iOS library for Catalyst + configure_file("${OpenCV_SOURCE_DIR}/platforms/ios/Info.plist.in" + "${CMAKE_BINARY_DIR}/osx/Info.plist") + elseif(APPLE_FRAMEWORK AND DYNAMIC_PLIST) configure_file("${OpenCV_SOURCE_DIR}/platforms/ios/Info.Dynamic.plist.in" "${CMAKE_BINARY_DIR}/ios/Info.plist") else() diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index a63861e987..2ad380236c 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1512,10 +1512,16 @@ function(ocv_add_library target) set(CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG 1) + if(IOS AND NOT MAC_CATALYST) + set(OPENCV_APPLE_INFO_PLIST "${CMAKE_BINARY_DIR}/ios/Info.plist") + else() + set(OPENCV_APPLE_INFO_PLIST "${CMAKE_BINARY_DIR}/osx/Info.plist") + endif() + set_target_properties(${target} PROPERTIES FRAMEWORK TRUE MACOSX_FRAMEWORK_IDENTIFIER org.opencv - MACOSX_FRAMEWORK_INFO_PLIST ${CMAKE_BINARY_DIR}/ios/Info.plist + MACOSX_FRAMEWORK_INFO_PLIST ${OPENCV_APPLE_INFO_PLIST} # "current version" in semantic format in Mach-O binary file VERSION ${OPENCV_LIBVERSION} # "compatibility version" in semantic format in Mach-O binary file @@ -1882,6 +1888,13 @@ function(ocv_update_file filepath content) endif() endfunction() +if(NOT BUILD_SHARED_LIBS AND (CMAKE_VERSION VERSION_LESS "3.14.0")) + ocv_update(OPENCV_3RDPARTY_EXCLUDE_FROM_ALL "") # avoid CMake warnings: https://gitlab.kitware.com/cmake/cmake/-/issues/18938 +else() + ocv_update(OPENCV_3RDPARTY_EXCLUDE_FROM_ALL "EXCLUDE_FROM_ALL") +endif() + + # adopted from https://gist.github.com/amir-saniyan/de99cee82fa9d8d615bb69f3f53b6004 function(ocv_blob2hdr blob_filename hdr_filename cpp_variable) if(EXISTS "${hdr_filename}") diff --git a/cmake/templates/opencv_abi.xml.in b/cmake/templates/opencv_abi.xml.in index 212b6d67d4..711c4e99ee 100644 --- a/cmake/templates/opencv_abi.xml.in +++ b/cmake/templates/opencv_abi.xml.in @@ -32,6 +32,7 @@ opencv2/flann/hdf5.h opencv2/imgcodecs/imgcodecs_c.h opencv2/imgcodecs/ios.h + opencv2/imgcodecs/macosx.h opencv2/videoio/videoio_c.h opencv2/videoio/cap_ios.h opencv2/xobjdetect/private.hpp diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index e89a89daef..c19fe967b7 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -145,9 +145,23 @@ if(DOXYGEN_FOUND) set(tutorial_js_path "${CMAKE_CURRENT_SOURCE_DIR}/js_tutorials") set(example_path "${CMAKE_SOURCE_DIR}/samples") + set(doxygen_image_path + ${CMAKE_CURRENT_SOURCE_DIR}/images + ${paths_doc} + ${tutorial_path} + ${tutorial_py_path} + ${tutorial_js_path} + ${paths_tutorial} + #${OpenCV_SOURCE_DIR}/samples/data # TODO: need to resolve ambiguous conflicts first + ${OpenCV_SOURCE_DIR} + ${OpenCV_SOURCE_DIR}/modules # /modules + ${OPENCV_EXTRA_MODULES_PATH} # /modules + ${OPENCV_DOCS_EXTRA_IMAGE_PATH} # custom variable for user modules + ) + # set export variables string(REPLACE ";" " \\\n" CMAKE_DOXYGEN_INPUT_LIST "${rootfile} ; ${faqfile} ; ${paths_include} ; ${paths_hal_interface} ; ${paths_doc} ; ${tutorial_path} ; ${tutorial_py_path} ; ${tutorial_js_path} ; ${paths_tutorial} ; ${tutorial_contrib_root}") - string(REPLACE ";" " \\\n" CMAKE_DOXYGEN_IMAGE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/images ; ${paths_doc} ; ${tutorial_path} ; ${tutorial_py_path} ; ${tutorial_js_path} ; ${paths_tutorial}") + string(REPLACE ";" " \\\n" CMAKE_DOXYGEN_IMAGE_PATH "${doxygen_image_path}") string(REPLACE ";" " \\\n" CMAKE_DOXYGEN_EXCLUDE_LIST "${CMAKE_DOXYGEN_EXCLUDE_LIST}") string(REPLACE ";" " " CMAKE_DOXYGEN_ENABLED_SECTIONS "${CMAKE_DOXYGEN_ENABLED_SECTIONS}") # TODO: remove paths_doc from EXAMPLE_PATH after face module tutorials/samples moved to separate folders diff --git a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown index 87167cd219..435f06fe02 100644 --- a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown +++ b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown @@ -32,6 +32,15 @@ source ./emsdk_env.sh echo ${EMSCRIPTEN} @endcode +The version 1.39.16 of emscripten is verified for latest WebAssembly. Please check the version of emscripten to use the newest features of WebAssembly. + +For example: +@code{.bash} +./emsdk update +./emsdk install 1.39.16 +./emsdk activate 1.39.16 +@endcode + Obtaining OpenCV Source Code -------------------------- @@ -76,6 +85,31 @@ Building OpenCV.js from Source python ./platforms/js/build_js.py build_wasm --build_wasm @endcode +-# [Optional] To build the OpenCV.js loader, append `--build_loader`. + + For example: + @code{.bash} + python ./platforms/js/build_js.py build_js --build_loader + @endcode + + @note + The loader is implemented as a js file in the path `/bin/loader.js`. The loader utilizes the [WebAssembly Feature Detection](https://github.com/GoogleChromeLabs/wasm-feature-detect) to detect the features of the broswer and load corresponding OpenCV.js automatically. To use it, you need to use the UMD version of [WebAssembly Feature Detection](https://github.com/GoogleChromeLabs/wasm-feature-detect) and introduce the `loader.js` in your Web application. + + Example Code: + @code{.javascipt} + // Set paths configuration + let pathsConfig = { + wasm: "../../build_wasm/opencv.js", + threads: "../../build_mt/opencv.js", + simd: "../../build_simd/opencv.js", + threadsSimd: "../../build_mtSIMD/opencv.js", + } + + // Load OpenCV.js and use the pathsConfiguration and main function as the params. + loadOpenCV(pathsConfig, main); + @endcode + + -# [optional] To build documents, append `--build_doc` option. For example: diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index 1f90615736..a6ffc7ca7e 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -92,11 +92,11 @@ def main(): dest="square_size", type=float) parser.add_argument("-R", "--radius_rate", help="circles_radius = square_size/radius_rate", default="5.0", action="store", dest="radius_rate", type=float) - parser.add_argument("-w", "--page_width", help="page width in units", default="216", action="store", + parser.add_argument("-w", "--page_width", help="page width in units", default=argparse.SUPPRESS, action="store", dest="page_width", type=float) - parser.add_argument("-h", "--page_height", help="page height in units", default="279", action="store", - dest="page_width", type=float) - parser.add_argument("-a", "--page_size", help="page size, supersedes -h -w arguments", default="A4", action="store", + parser.add_argument("-h", "--page_height", help="page height in units", default=argparse.SUPPRESS, action="store", + dest="page_height", type=float) + parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", action="store", dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) args = parser.parse_args() @@ -111,12 +111,16 @@ def main(): units = args.units square_size = args.square_size radius_rate = args.radius_rate - page_size = args.page_size - # page size dict (ISO standard, mm) for easy lookup. format - size: [width, height] - page_sizes = {"A0": [840, 1188], "A1": [594, 840], "A2": [420, 594], "A3": [297, 420], "A4": [210, 297], - "A5": [148, 210]} - page_width = page_sizes[page_size.upper()][0] - page_height = page_sizes[page_size.upper()][1] + if 'page_width' and 'page_height' in args: + page_width = args.page_width + page_height = args.page_height + else: + page_size = args.page_size + # page size dict (ISO standard, mm) for easy lookup. format - size: [width, height] + page_sizes = {"A0": [840, 1188], "A1": [594, 840], "A2": [420, 594], "A3": [297, 420], "A4": [210, 297], + "A5": [148, 210]} + page_width = page_sizes[page_size][0] + page_height = page_sizes[page_size][1] pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height) # dict for easy lookup of pattern type mp = {"circles": pm.make_circles_pattern, "acircles": pm.make_acircles_pattern, diff --git a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown index e337999efd..bba7b90b9f 100644 --- a/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown +++ b/doc/py_tutorials/py_calib3d/py_calibration/py_calibration.markdown @@ -209,7 +209,7 @@ find the average error, we calculate the arithmetical mean of the errors calcula calibration images. @code{.py} mean_error = 0 -for i in xrange(len(objpoints)): +for i in range(len(objpoints)): imgpoints2, _ = cv.projectPoints(objpoints[i], rvecs[i], tvecs[i], mtx, dist) error = cv.norm(imgpoints[i], imgpoints2, cv.NORM_L2)/len(imgpoints2) mean_error += error diff --git a/doc/py_tutorials/py_calib3d/py_epipolar_geometry/py_epipolar_geometry.markdown b/doc/py_tutorials/py_calib3d/py_epipolar_geometry/py_epipolar_geometry.markdown index 3ed072c04d..6b8d90882a 100644 --- a/doc/py_tutorials/py_calib3d/py_epipolar_geometry/py_epipolar_geometry.markdown +++ b/doc/py_tutorials/py_calib3d/py_epipolar_geometry/py_epipolar_geometry.markdown @@ -79,7 +79,7 @@ from matplotlib import pyplot as plt img1 = cv.imread('myleft.jpg',0) #queryimage # left image img2 = cv.imread('myright.jpg',0) #trainimage # right image -sift = cv.SIFT() +sift = cv.SIFT_create() # find the keypoints and descriptors with SIFT kp1, des1 = sift.detectAndCompute(img1,None) @@ -93,14 +93,12 @@ search_params = dict(checks=50) flann = cv.FlannBasedMatcher(index_params,search_params) matches = flann.knnMatch(des1,des2,k=2) -good = [] pts1 = [] pts2 = [] # ratio test as per Lowe's paper for i,(m,n) in enumerate(matches): if m.distance < 0.8*n.distance: - good.append(m) pts2.append(kp2[m.trainIdx].pt) pts1.append(kp1[m.queryIdx].pt) @endcode diff --git a/doc/py_tutorials/py_imgproc/py_thresholding/images/threshold.jpg b/doc/py_tutorials/py_imgproc/py_thresholding/images/threshold.jpg index e203927791..c7053cc76d 100644 Binary files a/doc/py_tutorials/py_imgproc/py_thresholding/images/threshold.jpg and b/doc/py_tutorials/py_imgproc/py_thresholding/images/threshold.jpg differ diff --git a/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.markdown b/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.markdown index 285124d17c..0540098850 100644 --- a/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.markdown +++ b/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.markdown @@ -48,7 +48,7 @@ titles = ['Original Image','BINARY','BINARY_INV','TRUNC','TOZERO','TOZERO_INV'] images = [img, thresh1, thresh2, thresh3, thresh4, thresh5] for i in xrange(6): - plt.subplot(2,3,i+1),plt.imshow(images[i],'gray') + plt.subplot(2,3,i+1),plt.imshow(images[i],'gray',vmin=0,vmax=255) plt.title(titles[i]) plt.xticks([]),plt.yticks([]) diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_.markdown b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_.markdown index 8e863ec79e..80cc6c68fe 100644 --- a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_.markdown +++ b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_.markdown @@ -32,7 +32,7 @@ automatically available with the platform (e.g. APPLE GCD) but chances are that have access to a parallel framework either directly or by enabling the option in CMake and rebuild the library. The second (weak) precondition is more related to the task you want to achieve as not all computations -are suitable / can be adatapted to be run in a parallel way. To remain simple, tasks that can be split +are suitable / can be adapted to be run in a parallel way. To remain simple, tasks that can be split into multiple elementary operations with no memory dependency (no possible race condition) are easily parallelizable. Computer vision processing are often easily parallelizable as most of the time the processing of one pixel does not depend to the state of other pixels. diff --git a/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown b/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown index a5c6695f91..42f8c7c38f 100644 --- a/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown +++ b/doc/tutorials/imgproc/erosion_dilatation/erosion_dilatation.markdown @@ -84,57 +84,198 @@ This tutorial's code is shown below. You can also download it Explanation ----------- --# Most of the material shown here is trivial (if you have any doubt, please refer to the tutorials in - previous sections). Let's check the general structure of the C++ program: +@add_toggle_cpp +Most of the material shown here is trivial (if you have any doubt, please refer to the tutorials in +previous sections). Let's check the general structure of the C++ program: - - Load an image (can be BGR or grayscale) - - Create two windows (one for dilation output, the other for erosion) - - Create a set of two Trackbars for each operation: - - The first trackbar "Element" returns either **erosion_elem** or **dilation_elem** - - The second trackbar "Kernel size" return **erosion_size** or **dilation_size** for the - corresponding operation. - - Every time we move any slider, the user's function **Erosion** or **Dilation** will be - called and it will update the output image based on the current trackbar values. +@snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp main - Let's analyze these two functions: +-# Load an image (can be BGR or grayscale) +-# Create two windows (one for dilation output, the other for erosion) +-# Create a set of two Trackbars for each operation: + - The first trackbar "Element" returns either **erosion_elem** or **dilation_elem** + - The second trackbar "Kernel size" return **erosion_size** or **dilation_size** for the + corresponding operation. +-# Call once erosion and dilation to show the initial image. --# **erosion:** - @snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp erosion - - The function that performs the *erosion* operation is @ref cv::erode . As we can see, it - receives three arguments: - - *src*: The source image - - *erosion_dst*: The output image - - *element*: This is the kernel we will use to perform the operation. If we do not - specify, the default is a simple `3x3` matrix. Otherwise, we can specify its - shape. For this, we need to use the function cv::getStructuringElement : - @snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp kernel +Every time we move any slider, the user's function **Erosion** or **Dilation** will be +called and it will update the output image based on the current trackbar values. - We can choose any of three shapes for our kernel: +Let's analyze these two functions: - - Rectangular box: MORPH_RECT - - Cross: MORPH_CROSS - - Ellipse: MORPH_ELLIPSE +#### The erosion function - Then, we just have to specify the size of our kernel and the *anchor point*. If not - specified, it is assumed to be in the center. +@snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp erosion - - That is all. We are ready to perform the erosion of our image. -@note Additionally, there is another parameter that allows you to perform multiple erosions -(iterations) at once. However, We haven't used it in this simple tutorial. You can check out the -reference for more details. +The function that performs the *erosion* operation is @ref cv::erode . As we can see, it +receives three arguments: +- *src*: The source image +- *erosion_dst*: The output image +- *element*: This is the kernel we will use to perform the operation. If we do not + specify, the default is a simple `3x3` matrix. Otherwise, we can specify its + shape. For this, we need to use the function cv::getStructuringElement : + @snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp kernel --# **dilation:** + We can choose any of three shapes for our kernel: - The code is below. As you can see, it is completely similar to the snippet of code for **erosion**. - Here we also have the option of defining our kernel, its anchor point and the size of the operator - to be used. - @snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp dilation + - Rectangular box: MORPH_RECT + - Cross: MORPH_CROSS + - Ellipse: MORPH_ELLIPSE + + Then, we just have to specify the size of our kernel and the *anchor point*. If not + specified, it is assumed to be in the center. + +That is all. We are ready to perform the erosion of our image. + +#### The dilation function + +The code is below. As you can see, it is completely similar to the snippet of code for **erosion**. +Here we also have the option of defining our kernel, its anchor point and the size of the operator +to be used. +@snippet cpp/tutorial_code/ImgProc/Morphology_1.cpp dilation +@end_toggle + +@add_toggle_java +Most of the material shown here is trivial (if you have any doubt, please refer to the tutorials in +previous sections). Let's check however the general structure of the java class. There are 4 main +parts in the java class: + +- the class constructor which setups the window that will be filled with window components +- the `addComponentsToPane` method, which fills out the window +- the `update` method, which determines what happens when the user changes any value +- the `main` method, which is the entry point of the program + +In this tutorial we will focus on the `addComponentsToPane` and `update` methods. However, for completion the +steps followed in the constructor are: + +-# Load an image (can be BGR or grayscale) +-# Create a window +-# Add various control components with `addComponentsToPane` +-# show the window + +The components were added by the following method: + +@snippet java/tutorial_code/ImgProc/erosion_dilatation/MorphologyDemo1.java components + +In short we + +-# create a panel for the sliders +-# create a combo box for the element types +-# create a slider for the kernel size +-# create a combo box for the morphology function to use (erosion or dilation) + +The action and state changed listeners added call at the end the `update` method which updates +the image based on the current slider values. So every time we move any slider, the `update` method is triggered. + +#### Updating the image + +To update the image we used the following implementation: + +@snippet java/tutorial_code/ImgProc/erosion_dilatation/MorphologyDemo1.java update + +In other words we + +-# get the structuring element the user chose +-# execute the **erosion** or **dilation** function based on `doErosion` +-# reload the image with the morphology applied +-# repaint the frame + +Let's analyze the `erode` and `dilate` methods: + +#### The erosion method + +@snippet java/tutorial_code/ImgProc/erosion_dilatation/MorphologyDemo1.java erosion + +The function that performs the *erosion* operation is @ref cv::erode . As we can see, it +receives three arguments: +- *src*: The source image +- *erosion_dst*: The output image +- *element*: This is the kernel we will use to perform the operation. For specifying the shape, we need to use + the function cv::getStructuringElement : + @snippet java/tutorial_code/ImgProc/erosion_dilatation/MorphologyDemo1.java kernel + + We can choose any of three shapes for our kernel: + + - Rectangular box: CV_SHAPE_RECT + - Cross: CV_SHAPE_CROSS + - Ellipse: CV_SHAPE_ELLIPSE + + Together with the shape we specify the size of our kernel and the *anchor point*. If the anchor point is not + specified, it is assumed to be in the center. + +That is all. We are ready to perform the erosion of our image. + +#### The dilation function + +The code is below. As you can see, it is completely similar to the snippet of code for **erosion**. +Here we also have the option of defining our kernel, its anchor point and the size of the operator +to be used. +@snippet java/tutorial_code/ImgProc/erosion_dilatation/MorphologyDemo1.java dilation +@end_toggle + +@add_toggle_python +Most of the material shown here is trivial (if you have any doubt, please refer to the tutorials in +previous sections). Let's check the general structure of the python script: + +@snippet python/tutorial_code/imgProc/erosion_dilatation/morphology_1.py main + +-# Load an image (can be BGR or grayscale) +-# Create two windows (one for erosion output, the other for dilation) with a set of trackbars each + - The first trackbar "Element" returns the value for the morphological type that will be mapped + (1 = rectangle, 2 = cross, 3 = ellipse) + - The second trackbar "Kernel size" returns the size of the element for the + corresponding operation +-# Call once erosion and dilation to show the initial image + +Every time we move any slider, the user's function **erosion** or **dilation** will be +called and it will update the output image based on the current trackbar values. + +Let's analyze these two functions: + +#### The erosion function + +@snippet python/tutorial_code/imgProc/erosion_dilatation/morphology_1.py erosion + +The function that performs the *erosion* operation is @ref cv::erode . As we can see, it +receives two arguments and returns the processed image: +- *src*: The source image +- *element*: The kernel we will use to perform the operation. We can specify its + shape by using the function cv::getStructuringElement : + @snippet python/tutorial_code/imgProc/erosion_dilatation/morphology_1.py kernel + + We can choose any of three shapes for our kernel: + + - Rectangular box: MORPH_RECT + - Cross: MORPH_CROSS + - Ellipse: MORPH_ELLIPSE + +Then, we just have to specify the size of our kernel and the *anchor point*. If the anchor point not +specified, it is assumed to be in the center. + +That is all. We are ready to perform the erosion of our image. + +#### The dilation function + +The code is below. As you can see, it is completely similar to the snippet of code for **erosion**. +Here we also have the option of defining our kernel, its anchor point and the size of the operator +to be used. + +@snippet python/tutorial_code/imgProc/erosion_dilatation/morphology_1.py dilation +@end_toggle + +@note Additionally, there are further parameters that allow you to perform multiple erosions/dilations +(iterations) at once and also set the border type and value. However, We haven't used those +in this simple tutorial. You can check out the reference for more details. Results ------- -Compile the code above and execute it with an image as argument. For instance, using this image: +Compile the code above and execute it (or run the script if using python) with an image as argument. +If you do not provide an image as argument the default sample image +([LinuxLogo.jpg](https://github.com/opencv/opencv/tree/master/samples/data/LinuxLogo.jpg)) will be used. + +For instance, using this image: ![](images/Morphology_1_Tutorial_Original_Image.jpg) @@ -143,3 +284,4 @@ naturally. Try them out! You can even try to add a third Trackbar to control the iterations. ![](images/Morphology_1_Result.jpg) +(depending on the programming language the output might vary a little or be only 1 window) diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 196eb42ac0..3d65710630 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -247,15 +247,18 @@ When `WITH_` option is enabled: `WITH_CUDA` (default: _OFF_) -Many algorithms have been implemented using CUDA acceleration, these functions are located in separate modules: @ref cuda. CUDA toolkit must be installed from the official NVIDIA site as a prerequisite. For cmake versions older than 3.9 OpenCV uses own `cmake/FindCUDA.cmake` script, for newer versions - the one packaged with CMake. Additional options can be used to control build process, e.g. `CUDA_GENERATION` or `CUDA_ARCH_BIN`. These parameters are not documented yet, please consult with the `cmake/OpenCVDetectCUDA.cmake` script for details. - -Some tutorials can be found in the corresponding section: @ref tutorial_table_of_content_gpu +Many algorithms have been implemented using CUDA acceleration, these functions are located in separate modules. CUDA toolkit must be installed from the official NVIDIA site as a prerequisite. For cmake versions older than 3.9 OpenCV uses own `cmake/FindCUDA.cmake` script, for newer versions - the one packaged with CMake. Additional options can be used to control build process, e.g. `CUDA_GENERATION` or `CUDA_ARCH_BIN`. These parameters are not documented yet, please consult with the `cmake/OpenCVDetectCUDA.cmake` script for details. @note Since OpenCV version 4.0 all CUDA-accelerated algorithm implementations have been moved to the _opencv_contrib_ repository. To build _opencv_ and _opencv_contrib_ together check @ref tutorial_config_reference_general_contrib. +@cond CUDA_MODULES +@note Some tutorials can be found in the corresponding section: @ref tutorial_table_of_content_gpu +@see @ref cuda +@endcond + @see https://en.wikipedia.org/wiki/CUDA -TODO: other options: `WITH_CUFFT`, `WITH_CUBLAS`, WITH_NVCUVID`? +TODO: other options: `WITH_CUFFT`, `WITH_CUBLAS`, `WITH_NVCUVID`? ### OpenCL support diff --git a/doc/tutorials/video/background_subtraction/background_subtraction.markdown b/doc/tutorials/video/background_subtraction/background_subtraction.markdown index c95cd4173d..420286960d 100644 --- a/doc/tutorials/video/background_subtraction/background_subtraction.markdown +++ b/doc/tutorials/video/background_subtraction/background_subtraction.markdown @@ -32,8 +32,7 @@ In this tutorial you will learn how to: -# Create and update the background model by using @ref cv::BackgroundSubtractor class; -# Get and show the foreground mask by using @ref cv::imshow ; -Code ----- +### Code In the following you can find the source code. We will let the user choose to process either a video file or a sequence of images. diff --git a/doc/tutorials/videoio/intelperc.markdown b/doc/tutorials/videoio/intelperc.markdown index e27f70c7ed..6a6a5e5c9a 100644 --- a/doc/tutorials/videoio/intelperc.markdown +++ b/doc/tutorials/videoio/intelperc.markdown @@ -1,7 +1,7 @@ Using Creative Senz3D and other Intel RealSense SDK compatible depth sensors {#tutorial_intelperc} ======================================================================================= -@prev_tutorial{tutorial_kinect_openni} +@prev_tutorial{tutorial_orbbec_astra} **Note**: This tutorial is partially obsolete since PerC SDK has been replaced with RealSense SDK diff --git a/doc/tutorials/videoio/kinect_openni.markdown b/doc/tutorials/videoio/kinect_openni.markdown index dc1ee6eeaa..aadaec5e44 100644 --- a/doc/tutorials/videoio/kinect_openni.markdown +++ b/doc/tutorials/videoio/kinect_openni.markdown @@ -2,7 +2,7 @@ Using Kinect and other OpenNI compatible depth sensors {#tutorial_kinect_openni} ====================================================== @prev_tutorial{tutorial_video_write} -@next_tutorial{tutorial_intelperc} +@next_tutorial{tutorial_orbbec_astra} Depth sensors compatible with OpenNI (Kinect, XtionPRO, ...) are supported through VideoCapture diff --git a/doc/tutorials/videoio/orbbec-astra/images/astra_color.jpg b/doc/tutorials/videoio/orbbec-astra/images/astra_color.jpg new file mode 100644 index 0000000000..d37e2803df Binary files /dev/null and b/doc/tutorials/videoio/orbbec-astra/images/astra_color.jpg differ diff --git a/doc/tutorials/videoio/orbbec-astra/images/astra_depth.png b/doc/tutorials/videoio/orbbec-astra/images/astra_depth.png new file mode 100644 index 0000000000..6fe2c6cd38 Binary files /dev/null and b/doc/tutorials/videoio/orbbec-astra/images/astra_depth.png differ diff --git a/doc/tutorials/videoio/orbbec-astra/orbbec_astra.markdown b/doc/tutorials/videoio/orbbec-astra/orbbec_astra.markdown new file mode 100644 index 0000000000..664e4f6dfe --- /dev/null +++ b/doc/tutorials/videoio/orbbec-astra/orbbec_astra.markdown @@ -0,0 +1,150 @@ +Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} +====================================================== + +@prev_tutorial{tutorial_kinect_openni} +@next_tutorial{tutorial_intelperc} + + +### Introduction + +This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://orbbec3d.com/product-astra-pro/). +That cameras have a depth sensor in addition to a common color sensor. The depth sensors can be read using +the OpenNI interface with @ref cv::VideoCapture class. The video stream is provided through the regular camera +interface. + +### Installation Instructions + +In order to use a depth sensor with OpenCV you should do the following steps: + +-# Download the latest version of Orbbec OpenNI SDK (from here ). + Unzip the archive, choose the build according to your operating system and follow installation + steps provided in the Readme file. For instance, if you use 64bit GNU/Linux run: + @code{.bash} + $ cd Linux/OpenNI-Linux-x64-2.3.0.63/ + $ sudo ./install.sh + @endcode + When you are done with the installation, make sure to replug your device for udev rules to take + effect. The camera should now work as a general camera device. Note that your current user should + belong to group `video` to have access to the camera. Also, make sure to source `OpenNIDevEnvironment` file: + @code{.bash} + $ source OpenNIDevEnvironment + @endcode + +-# Run the following commands to verify that OpenNI library and header files can be found. You should see + something similar in your terminal: + @code{.bash} + $ echo $OPENNI2_INCLUDE + /home/user/OpenNI_2.3.0.63/Linux/OpenNI-Linux-x64-2.3.0.63/Include + $ echo $OPENNI2_REDIST + /home/user/OpenNI_2.3.0.63/Linux/OpenNI-Linux-x64-2.3.0.63/Redist + @endcode + If the above two variables are empty, then you need to source `OpenNIDevEnvironment` again. Now you can + configure OpenCV with OpenNI support enabled by setting the `WITH_OPENNI2` flag in CMake. + You may also like to enable the `BUILD_EXAMPLES` flag to get a code sample working with your Astra camera. + Run the following commands in the directory containing OpenCV source code to enable OpenNI support: + @code{.bash} + $ mkdir build + $ cd build + $ cmake -DWITH_OPENNI2=ON .. + @endcode + If the OpenNI library is found, OpenCV will be built with OpenNI2 support. You can see the status of OpenNI2 + support in the CMake log: + @code{.text} + -- Video I/O: + -- DC1394: YES (2.2.6) + -- FFMPEG: YES + -- avcodec: YES (58.91.100) + -- avformat: YES (58.45.100) + -- avutil: YES (56.51.100) + -- swscale: YES (5.7.100) + -- avresample: NO + -- GStreamer: YES (1.18.1) + -- OpenNI2: YES (2.3.0) + -- v4l/v4l2: YES (linux/videodev2.h) + @endcode + +-# Build OpenCV: + @code{.bash} + $ make + @endcode + +### Code + +To get both depth and color frames, two @ref cv::VideoCapture objects should be created: + +@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Open streams + +The first object will use the regular Video4Linux2 interface to access the color sensor. The second one +is using OpenNI2 API to retrieve depth data. + +Before using the created VideoCapture objects you may want to setup stream parameters by setting +objects' properties. The most important parameters are frame width, frame height and fps: + +@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Setup streams + +For setting and getting some property of sensor data generators use @ref cv::VideoCapture::set and +@ref cv::VideoCapture::get methods respectively, e.g. : + +@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Get properties + +The following properties of cameras available through OpenNI interfaces are supported for the depth +generator: + +- @ref cv::CAP_PROP_FRAME_WIDTH -- Frame width in pixels. +- @ref cv::CAP_PROP_FRAME_HEIGHT -- Frame height in pixels. +- @ref cv::CAP_PROP_FPS -- Frame rate in FPS. +- @ref cv::CAP_PROP_OPENNI_REGISTRATION -- Flag that registers the remapping depth map to image map + by changing the depth generator's viewpoint (if the flag is "on") or sets this view point to + its normal one (if the flag is "off"). The registration process’ resulting images are + pixel-aligned, which means that every pixel in the image is aligned to a pixel in the depth + image. +- @ref cv::CAP_PROP_OPENNI2_MIRROR -- Flag to enable or disable mirroring for this stream. Set to 0 + to disable mirroring + + Next properties are available for getting only: + +- @ref cv::CAP_PROP_OPENNI_FRAME_MAX_DEPTH -- A maximum supported depth of the camera in mm. +- @ref cv::CAP_PROP_OPENNI_BASELINE -- Baseline value in mm. + +After the VideoCapture objects are set up you can start reading frames from them. + +@note + OpenCV's VideoCapture provides synchronous API, so you have to grab frames in a new thread + to avoid one stream blocking while another stream is being read. VideoCapture is not a + thread-safe class, so you need to be careful to avoid any possible deadlocks or data races. + +Example implementation that gets frames from each sensor in a new thread and stores them +in a list along with their timestamps: + +@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Read streams + +VideoCapture can retrieve the following data: + +-# data given from the depth generator: + - @ref cv::CAP_OPENNI_DEPTH_MAP - depth values in mm (CV_16UC1) + - @ref cv::CAP_OPENNI_POINT_CLOUD_MAP - XYZ in meters (CV_32FC3) + - @ref cv::CAP_OPENNI_DISPARITY_MAP - disparity in pixels (CV_8UC1) + - @ref cv::CAP_OPENNI_DISPARITY_MAP_32F - disparity in pixels (CV_32FC1) + - @ref cv::CAP_OPENNI_VALID_DEPTH_MASK - mask of valid pixels (not occluded, not shaded, etc.) + (CV_8UC1) + +-# data given from the color sensor is a regular BGR image (CV_8UC3). + +When new data is available a reading thread notifies the main thread. A frame is stored in the +ordered list -- the first frame is the latest one: + +@snippetlineno samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp Show color frame + +Depth frames can be picked the same way from the `depthFrames` list. + +After that, you'll have two frames: one containing color information and another one -- depth +information. In the sample images below you can see the color frame and the depth frame showing +the same scene. Looking at the color frame it's hard to distinguish plant leaves from leaves painted +on a wall, but the depth data makes it easy. + +![Color frame](images/astra_color.jpg) +![Depth frame](images/astra_depth.png) + +The complete implementation can be found in +[orbbec_astra.cpp](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp) +in `samples/cpp/tutorial_code/videoio` directory. diff --git a/doc/tutorials/videoio/table_of_content_videoio.markdown b/doc/tutorials/videoio/table_of_content_videoio.markdown index b27726bd87..393a0fc236 100644 --- a/doc/tutorials/videoio/table_of_content_videoio.markdown +++ b/doc/tutorials/videoio/table_of_content_videoio.markdown @@ -26,6 +26,10 @@ This section contains tutorials about how to read/save your video files. *Languages:* C++ +- @subpage tutorial_orbbec_astra + + *Languages:* C++ + - @subpage tutorial_intelperc - *Languages:* C++ \ No newline at end of file + *Languages:* C++ diff --git a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown index e24f3cb71b..76cfa3751d 100644 --- a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown +++ b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown @@ -126,13 +126,12 @@ captRefrnc.set(CAP_PROP_POS_FRAMES, 10); // go to the 10th frame of the video For properties you can read and change look into the documentation of the @ref cv::VideoCapture::get and @ref cv::VideoCapture::set functions. -Image similarity - PSNR and SSIM --------------------------------- +### Image similarity - PSNR and SSIM We want to check just how imperceptible our video converting operation went, therefore we need a system to check frame by frame the similarity or differences. The most common algorithm used for this is the PSNR (aka **Peak signal-to-noise ratio**). The simplest definition of this starts out -from the *mean squad error*. Let there be two images: I1 and I2; with a two dimensional size i and +from the *mean squared error*. Let there be two images: I1 and I2; with a two dimensional size i and j, composed of c number of channels. \f[MSE = \frac{1}{c*i*j} \sum{(I_1-I_2)^2}\f] @@ -145,15 +144,15 @@ Here the \f$MAX_I\f$ is the maximum valid value for a pixel. In case of the simp per pixel per channel this is 255. When two images are the same the MSE will give zero, resulting in an invalid divide by zero operation in the PSNR formula. In this case the PSNR is undefined and as we'll need to handle this case separately. The transition to a logarithmic scale is made because the -pixel values have a very wide dynamic range. All this translated to OpenCV and a C++ function looks +pixel values have a very wide dynamic range. All this translated to OpenCV and a function looks like: @add_toggle_cpp -@include cpp/tutorial_code/videoio/video-input-psnr-ssim/video-input-psnr-ssim.cpp get-psnr +@snippet cpp/tutorial_code/videoio/video-input-psnr-ssim/video-input-psnr-ssim.cpp get-psnr @end_toggle @add_toggle_python -@include samples/python/tutorial_code/videoio/video-input-psnr-ssim.py get-psnr +@snippet samples/python/tutorial_code/videoio/video-input-psnr-ssim.py get-psnr @end_toggle Typically result values are anywhere between 30 and 50 for video compression, where higher is @@ -172,11 +171,11 @@ implementation below. Transactions on Image Processing, vol. 13, no. 4, pp. 600-612, Apr. 2004." article. @add_toggle_cpp -@include cpp/tutorial_code/videoio/video-input-psnr-ssim/video-input-psnr-ssim.cpp get-mssim +@snippet samples/cpp/tutorial_code/videoio/video-input-psnr-ssim/video-input-psnr-ssim.cpp get-mssim @end_toggle @add_toggle_python -@include samples/python/tutorial_code/videoio/video-input-psnr-ssim.py get-mssim +@snippet samples/python/tutorial_code/videoio/video-input-psnr-ssim.py get-mssim @end_toggle This will return a similarity index for each channel of the image. This value is between zero and diff --git a/doc/tutorials/videoio/video-write/video_write.markdown b/doc/tutorials/videoio/video-write/video_write.markdown index 624b2d1db6..0100f8cfc4 100644 --- a/doc/tutorials/videoio/video-write/video_write.markdown +++ b/doc/tutorials/videoio/video-write/video_write.markdown @@ -63,7 +63,7 @@ specialized video writing libraries such as *FFMpeg* or codecs as *HuffYUV*, *Co an alternative, create the video track with OpenCV and expand it with sound tracks or convert it to other formats by using video manipulation programs such as *VirtualDub* or *AviSynth*. -The *VideoWriter* class +The VideoWriter class ----------------------- The content written here builds on the assumption you diff --git a/modules/calib3d/doc/calib3d.bib b/modules/calib3d/doc/calib3d.bib index 47dcf93788..86f2277b16 100644 --- a/modules/calib3d/doc/calib3d.bib +++ b/modules/calib3d/doc/calib3d.bib @@ -40,6 +40,14 @@ publisher={IEEE} } +@inproceedings{Terzakis20, + author = {Terzakis, George and Lourakis, Manolis}, + year = {2020}, + month = {09}, + pages = {}, + title = {A Consistently Fast and Globally Optimal Solution to the Perspective-n-Point Problem} +} + @inproceedings{strobl2011iccv, title={More accurate pinhole camera calibration with imperfect planar target}, author={Strobl, Klaus H. and Hirzinger, Gerd}, diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index cc5fdbfe1c..228a4f3f44 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -471,6 +471,7 @@ enum SolvePnPMethod { //!< - point 1: [ squareLength / 2, squareLength / 2, 0] //!< - point 2: [ squareLength / 2, -squareLength / 2, 0] //!< - point 3: [-squareLength / 2, -squareLength / 2, 0] + SOLVEPNP_SQPNP = 8, //!< SQPnP: A Consistently Fast and Globally OptimalSolution to the Perspective-n-Point Problem @cite Terzakis20 #ifndef CV_DOXYGEN SOLVEPNP_MAX_COUNT //!< Used for count #endif @@ -550,17 +551,18 @@ enum NeighborSearchMethod { NEIGH_FLANN_KNN, NEIGH_GRID, NEIGH_FLANN_RADIUS }; struct CV_EXPORTS_W_SIMPLE UsacParams { // in alphabetical order - double confidence = 0.99; - bool isParallel = false; - int loIterations = 5; - LocalOptimMethod loMethod = LocalOptimMethod::LOCAL_OPTIM_INNER_LO; - int loSampleSize = 14; - int maxIterations = 5000; - NeighborSearchMethod neighborsSearch = NeighborSearchMethod::NEIGH_GRID; - int randomGeneratorState = 0; - SamplingMethod sampler = SamplingMethod::SAMPLING_UNIFORM; - ScoreMethod score = ScoreMethod::SCORE_METHOD_MSAC; - double threshold = 1.5; + CV_WRAP UsacParams(); + CV_PROP_RW double confidence; + CV_PROP_RW bool isParallel; + CV_PROP_RW int loIterations; + CV_PROP_RW LocalOptimMethod loMethod; + CV_PROP_RW int loSampleSize; + CV_PROP_RW int maxIterations; + CV_PROP_RW NeighborSearchMethod neighborsSearch; + CV_PROP_RW int randomGeneratorState; + CV_PROP_RW SamplingMethod sampler; + CV_PROP_RW ScoreMethod score; + CV_PROP_RW double threshold; }; /** @brief Converts a rotation matrix to a rotation vector or vice versa. @@ -945,6 +947,9 @@ It requires 4 coplanar object points defined in the following order: - point 1: [ squareLength / 2, squareLength / 2, 0] - point 2: [ squareLength / 2, -squareLength / 2, 0] - point 3: [-squareLength / 2, -squareLength / 2, 0] +- **SOLVEPNP_SQPNP** Method is based on the paper "A Consistently Fast and Globally Optimal Solution to the +Perspective-n-Point Problem" by G. Terzakis and M.Lourakis (@cite Terzakis20). It requires 3 or more points. + The function estimates the object pose given a set of object points, their corresponding image projections, as well as the camera intrinsic matrix and the distortion coefficients, see the figure below @@ -1068,6 +1073,7 @@ a 3D point expressed in the world frame into the camera frame: - point 1: [ squareLength / 2, squareLength / 2, 0] - point 2: [ squareLength / 2, -squareLength / 2, 0] - point 3: [-squareLength / 2, -squareLength / 2, 0] + - With **SOLVEPNP_SQPNP** input points must be >= 3 */ CV_EXPORTS_W bool solvePnP( InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs, @@ -2597,10 +2603,10 @@ CV_EXPORTS void convertPointsHomogeneous( InputArray src, OutputArray dst ); floating-point (single or double precision). @param points2 Array of the second image points of the same size and format as points1 . @param method Method for computing a fundamental matrix. -- **CV_FM_7POINT** for a 7-point algorithm. \f$N = 7\f$ -- **CV_FM_8POINT** for an 8-point algorithm. \f$N \ge 8\f$ -- **CV_FM_RANSAC** for the RANSAC algorithm. \f$N \ge 8\f$ -- **CV_FM_LMEDS** for the LMedS algorithm. \f$N \ge 8\f$ +- @ref FM_7POINT for a 7-point algorithm. \f$N = 7\f$ +- @ref FM_8POINT for an 8-point algorithm. \f$N \ge 8\f$ +- @ref FM_RANSAC for the RANSAC algorithm. \f$N \ge 8\f$ +- @ref FM_LMEDS for the LMedS algorithm. \f$N \ge 8\f$ @param ransacReprojThreshold Parameter used only for RANSAC. It is the maximum distance from a point to an epipolar line in pixels, beyond which the point is considered an outlier and is not used for computing the final fundamental matrix. It can be set to something like 1-3, depending on the accuracy of the diff --git a/modules/calib3d/src/solvepnp.cpp b/modules/calib3d/src/solvepnp.cpp index 5c04662489..0f12333eb9 100644 --- a/modules/calib3d/src/solvepnp.cpp +++ b/modules/calib3d/src/solvepnp.cpp @@ -47,6 +47,7 @@ #include "p3p.h" #include "ap3p.h" #include "ippe.hpp" +#include "sqpnp.hpp" #include "calib3d_c_api.h" #include "usac.hpp" @@ -197,6 +198,21 @@ public: Mat tvec; }; +UsacParams::UsacParams() +{ + confidence = 0.99; + isParallel = false; + loIterations = 5; + loMethod = LocalOptimMethod::LOCAL_OPTIM_INNER_LO; + loSampleSize = 14; + maxIterations = 5000; + neighborsSearch = NeighborSearchMethod::NEIGH_GRID; + randomGeneratorState = 0; + sampler = SamplingMethod::SAMPLING_UNIFORM; + score = ScoreMethod::SCORE_METHOD_MSAC; + threshold = 1.5; +} + bool solvePnPRansac(InputArray _opoints, InputArray _ipoints, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _rvec, OutputArray _tvec, bool useExtrinsicGuess, @@ -781,7 +797,8 @@ int solvePnPGeneric( InputArray _opoints, InputArray _ipoints, Mat opoints = _opoints.getMat(), ipoints = _ipoints.getMat(); int npoints = std::max(opoints.checkVector(3, CV_32F), opoints.checkVector(3, CV_64F)); - CV_Assert( ( (npoints >= 4) || (npoints == 3 && flags == SOLVEPNP_ITERATIVE && useExtrinsicGuess) ) + CV_Assert( ( (npoints >= 4) || (npoints == 3 && flags == SOLVEPNP_ITERATIVE && useExtrinsicGuess) + || (npoints >= 3 && flags == SOLVEPNP_SQPNP) ) && npoints == std::max(ipoints.checkVector(2, CV_32F), ipoints.checkVector(2, CV_64F)) ); opoints = opoints.reshape(3, npoints); @@ -966,6 +983,14 @@ int solvePnPGeneric( InputArray _opoints, InputArray _ipoints, } } catch (...) { } } + else if (flags == SOLVEPNP_SQPNP) + { + Mat undistortedPoints; + undistortPoints(ipoints, undistortedPoints, cameraMatrix, distCoeffs); + + sqpnp::PoseSolver solver; + solver.solve(opoints, undistortedPoints, vec_rvecs, vec_tvecs); + } /*else if (flags == SOLVEPNP_DLS) { Mat undistortedPoints; @@ -993,7 +1018,8 @@ int solvePnPGeneric( InputArray _opoints, InputArray _ipoints, vec_tvecs.push_back(tvec); }*/ else - CV_Error(CV_StsBadArg, "The flags argument must be one of SOLVEPNP_ITERATIVE, SOLVEPNP_P3P, SOLVEPNP_EPNP or SOLVEPNP_DLS"); + CV_Error(CV_StsBadArg, "The flags argument must be one of SOLVEPNP_ITERATIVE, SOLVEPNP_P3P, " + "SOLVEPNP_EPNP, SOLVEPNP_DLS, SOLVEPNP_UPNP, SOLVEPNP_AP3P, SOLVEPNP_IPPE, SOLVEPNP_IPPE_SQUARE or SOLVEPNP_SQPNP"); CV_Assert(vec_rvecs.size() == vec_tvecs.size()); diff --git a/modules/calib3d/src/sqpnp.cpp b/modules/calib3d/src/sqpnp.cpp new file mode 100644 index 0000000000..7117e61c96 --- /dev/null +++ b/modules/calib3d/src/sqpnp.cpp @@ -0,0 +1,775 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This file is based on file issued with the following license: + +/* +BSD 3-Clause License + +Copyright (c) 2020, George Terzakis +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "precomp.hpp" +#include "sqpnp.hpp" + +#include + +namespace cv { +namespace sqpnp { + +const double PoseSolver::RANK_TOLERANCE = 1e-7; +const double PoseSolver::SQP_SQUARED_TOLERANCE = 1e-10; +const double PoseSolver::SQP_DET_THRESHOLD = 1.001; +const double PoseSolver::ORTHOGONALITY_SQUARED_ERROR_THRESHOLD = 1e-8; +const double PoseSolver::EQUAL_VECTORS_SQUARED_DIFF = 1e-10; +const double PoseSolver::EQUAL_SQUARED_ERRORS_DIFF = 1e-6; +const double PoseSolver::POINT_VARIANCE_THRESHOLD = 1e-5; +const double PoseSolver::SQRT3 = std::sqrt(3); +const int PoseSolver::SQP_MAX_ITERATION = 15; + +//No checking done here for overflow, since this is not public all call instances +//are assumed to be valid +template + void set(int row, int col, cv::Matx& dest, + const cv::Matx& source) +{ + for (int y = 0; y < snrows; y++) + { + for (int x = 0; x < sncols; x++) + { + dest(row + y, col + x) = source(y, x); + } + } +} + +PoseSolver::PoseSolver() + : num_null_vectors_(-1), + num_solutions_(0) +{ +} + + +void PoseSolver::solve(InputArray objectPoints, InputArray imagePoints, OutputArrayOfArrays rvecs, + OutputArrayOfArrays tvecs) +{ + //Input checking + int objType = objectPoints.getMat().type(); + CV_CheckType(objType, objType == CV_32FC3 || objType == CV_64FC3, + "Type of objectPoints must be CV_32FC3 or CV_64FC3"); + + int imgType = imagePoints.getMat().type(); + CV_CheckType(imgType, imgType == CV_32FC2 || imgType == CV_64FC2, + "Type of imagePoints must be CV_32FC2 or CV_64FC2"); + + CV_Assert(objectPoints.rows() == 1 || objectPoints.cols() == 1); + CV_Assert(objectPoints.rows() >= 3 || objectPoints.cols() >= 3); + CV_Assert(imagePoints.rows() == 1 || imagePoints.cols() == 1); + CV_Assert(imagePoints.rows() * imagePoints.cols() == objectPoints.rows() * objectPoints.cols()); + + Mat _imagePoints; + if (imgType == CV_32FC2) + { + imagePoints.getMat().convertTo(_imagePoints, CV_64F); + } + else + { + _imagePoints = imagePoints.getMat(); + } + + Mat _objectPoints; + if (objType == CV_32FC3) + { + objectPoints.getMat().convertTo(_objectPoints, CV_64F); + } + else + { + _objectPoints = objectPoints.getMat(); + } + + num_null_vectors_ = -1; + num_solutions_ = 0; + + computeOmega(_objectPoints, _imagePoints); + solveInternal(); + + int depthRot = rvecs.fixedType() ? rvecs.depth() : CV_64F; + int depthTrans = tvecs.fixedType() ? tvecs.depth() : CV_64F; + + rvecs.create(num_solutions_, 1, CV_MAKETYPE(depthRot, rvecs.fixedType() && rvecs.kind() == _InputArray::STD_VECTOR ? 3 : 1)); + tvecs.create(num_solutions_, 1, CV_MAKETYPE(depthTrans, tvecs.fixedType() && tvecs.kind() == _InputArray::STD_VECTOR ? 3 : 1)); + + for (int i = 0; i < num_solutions_; i++) + { + + Mat rvec; + Mat rotation = Mat(solutions_[i].r_hat).reshape(1, 3); + Rodrigues(rotation, rvec); + + rvecs.getMatRef(i) = rvec; + tvecs.getMatRef(i) = Mat(solutions_[i].t); + } +} + +void PoseSolver::computeOmega(InputArray objectPoints, InputArray imagePoints) +{ + omega_ = cv::Matx::zeros(); + cv::Matx qa_sum = cv::Matx::zeros(); + + cv::Point2d sum_img(0, 0); + cv::Point3d sum_obj(0, 0, 0); + double sq_norm_sum = 0; + + Mat _imagePoints = imagePoints.getMat(); + Mat _objectPoints = objectPoints.getMat(); + + int n = _objectPoints.cols * _objectPoints.rows; + + for (int i = 0; i < n; i++) + { + const cv::Point2d& img_pt = _imagePoints.at(i); + const cv::Point3d& obj_pt = _objectPoints.at(i); + + sum_img += img_pt; + sum_obj += obj_pt; + + const double& x = img_pt.x, & y = img_pt.y; + const double& X = obj_pt.x, & Y = obj_pt.y, & Z = obj_pt.z; + double sq_norm = x * x + y * y; + sq_norm_sum += sq_norm; + + double X2 = X * X, + XY = X * Y, + XZ = X * Z, + Y2 = Y * Y, + YZ = Y * Z, + Z2 = Z * Z; + + omega_(0, 0) += X2; + omega_(0, 1) += XY; + omega_(0, 2) += XZ; + omega_(1, 1) += Y2; + omega_(1, 2) += YZ; + omega_(2, 2) += Z2; + + + //Populating this manually saves operations by only calculating upper triangle + omega_(0, 6) += -x * X2; omega_(0, 7) += -x * XY; omega_(0, 8) += -x * XZ; + omega_(1, 7) += -x * Y2; omega_(1, 8) += -x * YZ; + omega_(2, 8) += -x * Z2; + + omega_(3, 6) += -y * X2; omega_(3, 7) += -y * XY; omega_(3, 8) += -y * XZ; + omega_(4, 7) += -y * Y2; omega_(4, 8) += -y * YZ; + omega_(5, 8) += -y * Z2; + + + omega_(6, 6) += sq_norm * X2; omega_(6, 7) += sq_norm * XY; omega_(6, 8) += sq_norm * XZ; + omega_(7, 7) += sq_norm * Y2; omega_(7, 8) += sq_norm * YZ; + omega_(8, 8) += sq_norm * Z2; + + //Compute qa_sum + qa_sum(0, 0) += X; qa_sum(0, 1) += Y; qa_sum(0, 2) += Z; + qa_sum(1, 3) += X; qa_sum(1, 4) += Y; qa_sum(1, 5) += Z; + + qa_sum(0, 6) += -x * X; qa_sum(0, 7) += -x * Y; qa_sum(0, 8) += -x * Z; + qa_sum(1, 6) += -y * X; qa_sum(1, 7) += -y * Y; qa_sum(1, 8) += -y * Z; + + qa_sum(2, 0) += -x * X; qa_sum(2, 1) += -x * Y; qa_sum(2, 2) += -x * Z; + qa_sum(2, 3) += -y * X; qa_sum(2, 4) += -y * Y; qa_sum(2, 5) += -y * Z; + + qa_sum(2, 6) += sq_norm * X; qa_sum(2, 7) += sq_norm * Y; qa_sum(2, 8) += sq_norm * Z; + } + + + omega_(1, 6) = omega_(0, 7); omega_(2, 6) = omega_(0, 8); omega_(2, 7) = omega_(1, 8); + omega_(4, 6) = omega_(3, 7); omega_(5, 6) = omega_(3, 8); omega_(5, 7) = omega_(4, 8); + omega_(7, 6) = omega_(6, 7); omega_(8, 6) = omega_(6, 8); omega_(8, 7) = omega_(7, 8); + + + omega_(3, 3) = omega_(0, 0); omega_(3, 4) = omega_(0, 1); omega_(3, 5) = omega_(0, 2); + omega_(4, 4) = omega_(1, 1); omega_(4, 5) = omega_(1, 2); + omega_(5, 5) = omega_(2, 2); + + //Mirror upper triangle to lower triangle + for (int r = 0; r < 9; r++) + { + for (int c = 0; c < r; c++) + { + omega_(r, c) = omega_(c, r); + } + } + + cv::Matx q; + q(0, 0) = n; q(0, 1) = 0; q(0, 2) = -sum_img.x; + q(1, 0) = 0; q(1, 1) = n; q(1, 2) = -sum_img.y; + q(2, 0) = -sum_img.x; q(2, 1) = -sum_img.y; q(2, 2) = sq_norm_sum; + + double inv_n = 1.0 / n; + double detQ = n * (n * sq_norm_sum - sum_img.y * sum_img.y - sum_img.x * sum_img.x); + double point_coordinate_variance = detQ * inv_n * inv_n * inv_n; + + CV_Assert(point_coordinate_variance >= POINT_VARIANCE_THRESHOLD); + + Matx q_inv; + analyticalInverse3x3Symm(q, q_inv); + + p_ = -q_inv * qa_sum; + + omega_ += qa_sum.t() * p_; + + cv::SVD omega_svd(omega_, cv::SVD::FULL_UV); + s_ = omega_svd.w; + u_ = cv::Mat(omega_svd.vt.t()); + + CV_Assert(s_(0) >= 1e-7); + + while (s_(7 - num_null_vectors_) < RANK_TOLERANCE) num_null_vectors_++; + + CV_Assert(++num_null_vectors_ <= 6); + + point_mean_ = cv::Vec3d(sum_obj.x / n, sum_obj.y / n, sum_obj.z / n); +} + +void PoseSolver::solveInternal() +{ + double min_sq_err = std::numeric_limits::max(); + int num_eigen_points = num_null_vectors_ > 0 ? num_null_vectors_ : 1; + + for (int i = 9 - num_eigen_points; i < 9; i++) + { + const cv::Matx e = SQRT3 * u_.col(i); + double orthogonality_sq_err = orthogonalityError(e); + + SQPSolution solutions[2]; + + //If e is orthogonal, we can skip SQP + if (orthogonality_sq_err < ORTHOGONALITY_SQUARED_ERROR_THRESHOLD) + { + solutions[0].r_hat = det3x3(e) * e; + solutions[0].t = p_ * solutions[0].r_hat; + checkSolution(solutions[0], min_sq_err); + } + else + { + Matx r; + nearestRotationMatrix(e, r); + solutions[0] = runSQP(r); + solutions[0].t = p_ * solutions[0].r_hat; + checkSolution(solutions[0], min_sq_err); + + nearestRotationMatrix(-e, r); + solutions[1] = runSQP(r); + solutions[1].t = p_ * solutions[1].r_hat; + checkSolution(solutions[1], min_sq_err); + } + } + + int c = 1; + + while (min_sq_err > 3 * s_[9 - num_eigen_points - c] && 9 - num_eigen_points - c > 0) + { + int index = 9 - num_eigen_points - c; + + const cv::Matx e = u_.col(index); + SQPSolution solutions[2]; + + Matx r; + nearestRotationMatrix(e, r); + solutions[0] = runSQP(r); + solutions[0].t = p_ * solutions[0].r_hat; + checkSolution(solutions[0], min_sq_err); + + nearestRotationMatrix(-e, r); + solutions[1] = runSQP(r); + solutions[1].t = p_ * solutions[1].r_hat; + checkSolution(solutions[1], min_sq_err); + + c++; + } +} + +PoseSolver::SQPSolution PoseSolver::runSQP(const cv::Matx& r0) +{ + cv::Matx r = r0; + + double delta_squared_norm = std::numeric_limits::max(); + cv::Matx delta; + + int step = 0; + while (delta_squared_norm > SQP_SQUARED_TOLERANCE && step++ < SQP_MAX_ITERATION) + { + solveSQPSystem(r, delta); + r += delta; + delta_squared_norm = cv::norm(delta, cv::NORM_L2SQR); + } + + SQPSolution solution; + + double det_r = det3x3(r); + if (det_r < 0) + { + r = -r; + det_r = -det_r; + } + + if (det_r > SQP_DET_THRESHOLD) + { + nearestRotationMatrix(r, solution.r_hat); + } + else + { + solution.r_hat = r; + } + + return solution; +} + +void PoseSolver::solveSQPSystem(const cv::Matx& r, cv::Matx& delta) +{ + double sqnorm_r1 = r(0) * r(0) + r(1) * r(1) + r(2) * r(2), + sqnorm_r2 = r(3) * r(3) + r(4) * r(4) + r(5) * r(5), + sqnorm_r3 = r(6) * r(6) + r(7) * r(7) + r(8) * r(8); + double dot_r1r2 = r(0) * r(3) + r(1) * r(4) + r(2) * r(5), + dot_r1r3 = r(0) * r(6) + r(1) * r(7) + r(2) * r(8), + dot_r2r3 = r(3) * r(6) + r(4) * r(7) + r(5) * r(8); + + cv::Matx N; + cv::Matx H; + cv::Matx JH; + + computeRowAndNullspace(r, H, N, JH); + + cv::Matx g; + g(0) = 1 - sqnorm_r1; g(1) = 1 - sqnorm_r2; g(2) = 1 - sqnorm_r3; g(3) = -dot_r1r2; g(4) = -dot_r2r3; g(5) = -dot_r1r3; + + cv::Matx x; + x(0) = g(0) / JH(0, 0); + x(1) = g(1) / JH(1, 1); + x(2) = g(2) / JH(2, 2); + x(3) = (g(3) - JH(3, 0) * x(0) - JH(3, 1) * x(1)) / JH(3, 3); + x(4) = (g(4) - JH(4, 1) * x(1) - JH(4, 2) * x(2) - JH(4, 3) * x(3)) / JH(4, 4); + x(5) = (g(5) - JH(5, 0) * x(0) - JH(5, 2) * x(2) - JH(5, 3) * x(3) - JH(5, 4) * x(4)) / JH(5, 5); + + delta = H * x; + + + cv::Matx nt_omega = N.t() * omega_; + cv::Matx W = nt_omega * N, W_inv; + + analyticalInverse3x3Symm(W, W_inv); + + cv::Matx y = -W_inv * nt_omega * (delta + r); + delta += N * y; +} + +bool PoseSolver::analyticalInverse3x3Symm(const cv::Matx& Q, + cv::Matx& Qinv, + const double& threshold) +{ + // 1. Get the elements of the matrix + double a = Q(0, 0), + b = Q(1, 0), d = Q(1, 1), + c = Q(2, 0), e = Q(2, 1), f = Q(2, 2); + + // 2. Determinant + double t2, t4, t7, t9, t12; + t2 = e * e; + t4 = a * d; + t7 = b * b; + t9 = b * c; + t12 = c * c; + double det = -t4 * f + a * t2 + t7 * f - 2.0 * t9 * e + t12 * d; + + if (fabs(det) < threshold) return false; + + // 3. Inverse + double t15, t20, t24, t30; + t15 = 1.0 / det; + t20 = (-b * f + c * e) * t15; + t24 = (b * e - c * d) * t15; + t30 = (a * e - t9) * t15; + Qinv(0, 0) = (-d * f + t2) * t15; + Qinv(0, 1) = Qinv(1, 0) = -t20; + Qinv(0, 2) = Qinv(2, 0) = -t24; + Qinv(1, 1) = -(a * f - t12) * t15; + Qinv(1, 2) = Qinv(2, 1) = t30; + Qinv(2, 2) = -(t4 - t7) * t15; + + return true; +} + +void PoseSolver::computeRowAndNullspace(const cv::Matx& r, + cv::Matx& H, + cv::Matx& N, + cv::Matx& K, + const double& norm_threshold) +{ + H = cv::Matx::zeros(); + + // 1. q1 + double norm_r1 = sqrt(r(0) * r(0) + r(1) * r(1) + r(2) * r(2)); + double inv_norm_r1 = norm_r1 > 1e-5 ? 1.0 / norm_r1 : 0.0; + H(0, 0) = r(0) * inv_norm_r1; + H(1, 0) = r(1) * inv_norm_r1; + H(2, 0) = r(2) * inv_norm_r1; + K(0, 0) = 2 * norm_r1; + + // 2. q2 + double norm_r2 = sqrt(r(3) * r(3) + r(4) * r(4) + r(5) * r(5)); + double inv_norm_r2 = 1.0 / norm_r2; + H(3, 1) = r(3) * inv_norm_r2; + H(4, 1) = r(4) * inv_norm_r2; + H(5, 1) = r(5) * inv_norm_r2; + K(1, 0) = 0; + K(1, 1) = 2 * norm_r2; + + // 3. q3 = (r3'*q2)*q2 - (r3'*q1)*q1 ; q3 = q3/norm(q3) + double norm_r3 = sqrt(r(6) * r(6) + r(7) * r(7) + r(8) * r(8)); + double inv_norm_r3 = 1.0 / norm_r3; + H(6, 2) = r(6) * inv_norm_r3; + H(7, 2) = r(7) * inv_norm_r3; + H(8, 2) = r(8) * inv_norm_r3; + K(2, 0) = K(2, 1) = 0; + K(2, 2) = 2 * norm_r3; + + // 4. q4 + double dot_j4q1 = r(3) * H(0, 0) + r(4) * H(1, 0) + r(5) * H(2, 0), + dot_j4q2 = r(0) * H(3, 1) + r(1) * H(4, 1) + r(2) * H(5, 1); + + H(0, 3) = r(3) - dot_j4q1 * H(0, 0); + H(1, 3) = r(4) - dot_j4q1 * H(1, 0); + H(2, 3) = r(5) - dot_j4q1 * H(2, 0); + H(3, 3) = r(0) - dot_j4q2 * H(3, 1); + H(4, 3) = r(1) - dot_j4q2 * H(4, 1); + H(5, 3) = r(2) - dot_j4q2 * H(5, 1); + double inv_norm_j4 = 1.0 / sqrt(H(0, 3) * H(0, 3) + H(1, 3) * H(1, 3) + H(2, 3) * H(2, 3) + + H(3, 3) * H(3, 3) + H(4, 3) * H(4, 3) + H(5, 3) * H(5, 3)); + + H(0, 3) *= inv_norm_j4; + H(1, 3) *= inv_norm_j4; + H(2, 3) *= inv_norm_j4; + H(3, 3) *= inv_norm_j4; + H(4, 3) *= inv_norm_j4; + H(5, 3) *= inv_norm_j4; + + K(3, 0) = r(3) * H(0, 0) + r(4) * H(1, 0) + r(5) * H(2, 0); + K(3, 1) = r(0) * H(3, 1) + r(1) * H(4, 1) + r(2) * H(5, 1); + K(3, 2) = 0; + K(3, 3) = r(3) * H(0, 3) + r(4) * H(1, 3) + r(5) * H(2, 3) + r(0) * H(3, 3) + r(1) * H(4, 3) + r(2) * H(5, 3); + + // 5. q5 + double dot_j5q2 = r(6) * H(3, 1) + r(7) * H(4, 1) + r(8) * H(5, 1); + double dot_j5q3 = r(3) * H(6, 2) + r(4) * H(7, 2) + r(5) * H(8, 2); + double dot_j5q4 = r(6) * H(3, 3) + r(7) * H(4, 3) + r(8) * H(5, 3); + + H(0, 4) = -dot_j5q4 * H(0, 3); + H(1, 4) = -dot_j5q4 * H(1, 3); + H(2, 4) = -dot_j5q4 * H(2, 3); + H(3, 4) = r(6) - dot_j5q2 * H(3, 1) - dot_j5q4 * H(3, 3); + H(4, 4) = r(7) - dot_j5q2 * H(4, 1) - dot_j5q4 * H(4, 3); + H(5, 4) = r(8) - dot_j5q2 * H(5, 1) - dot_j5q4 * H(5, 3); + H(6, 4) = r(3) - dot_j5q3 * H(6, 2); H(7, 4) = r(4) - dot_j5q3 * H(7, 2); H(8, 4) = r(5) - dot_j5q3 * H(8, 2); + + Matx q4 = H.col(4); + q4 /= cv::norm(q4); + set(0, 4, H, q4); + + K(4, 0) = 0; + K(4, 1) = r(6) * H(3, 1) + r(7) * H(4, 1) + r(8) * H(5, 1); + K(4, 2) = r(3) * H(6, 2) + r(4) * H(7, 2) + r(5) * H(8, 2); + K(4, 3) = r(6) * H(3, 3) + r(7) * H(4, 3) + r(8) * H(5, 3); + K(4, 4) = r(6) * H(3, 4) + r(7) * H(4, 4) + r(8) * H(5, 4) + r(3) * H(6, 4) + r(4) * H(7, 4) + r(5) * H(8, 4); + + + // 4. q6 + double dot_j6q1 = r(6) * H(0, 0) + r(7) * H(1, 0) + r(8) * H(2, 0); + double dot_j6q3 = r(0) * H(6, 2) + r(1) * H(7, 2) + r(2) * H(8, 2); + double dot_j6q4 = r(6) * H(0, 3) + r(7) * H(1, 3) + r(8) * H(2, 3); + double dot_j6q5 = r(0) * H(6, 4) + r(1) * H(7, 4) + r(2) * H(8, 4) + r(6) * H(0, 4) + r(7) * H(1, 4) + r(8) * H(2, 4); + + H(0, 5) = r(6) - dot_j6q1 * H(0, 0) - dot_j6q4 * H(0, 3) - dot_j6q5 * H(0, 4); + H(1, 5) = r(7) - dot_j6q1 * H(1, 0) - dot_j6q4 * H(1, 3) - dot_j6q5 * H(1, 4); + H(2, 5) = r(8) - dot_j6q1 * H(2, 0) - dot_j6q4 * H(2, 3) - dot_j6q5 * H(2, 4); + + H(3, 5) = -dot_j6q5 * H(3, 4) - dot_j6q4 * H(3, 3); + H(4, 5) = -dot_j6q5 * H(4, 4) - dot_j6q4 * H(4, 3); + H(5, 5) = -dot_j6q5 * H(5, 4) - dot_j6q4 * H(5, 3); + + H(6, 5) = r(0) - dot_j6q3 * H(6, 2) - dot_j6q5 * H(6, 4); + H(7, 5) = r(1) - dot_j6q3 * H(7, 2) - dot_j6q5 * H(7, 4); + H(8, 5) = r(2) - dot_j6q3 * H(8, 2) - dot_j6q5 * H(8, 4); + + Matx q5 = H.col(5); + q5 /= cv::norm(q5); + set(0, 5, H, q5); + + K(5, 0) = r(6) * H(0, 0) + r(7) * H(1, 0) + r(8) * H(2, 0); + K(5, 1) = 0; K(5, 2) = r(0) * H(6, 2) + r(1) * H(7, 2) + r(2) * H(8, 2); + K(5, 3) = r(6) * H(0, 3) + r(7) * H(1, 3) + r(8) * H(2, 3); + K(5, 4) = r(6) * H(0, 4) + r(7) * H(1, 4) + r(8) * H(2, 4) + r(0) * H(6, 4) + r(1) * H(7, 4) + r(2) * H(8, 4); + K(5, 5) = r(6) * H(0, 5) + r(7) * H(1, 5) + r(8) * H(2, 5) + r(0) * H(6, 5) + r(1) * H(7, 5) + r(2) * H(8, 5); + + // Great! Now H is an orthogonalized, sparse basis of the Jacobian row space and K is filled. + // + // Now get a projector onto the null space H: + const cv::Matx Pn = cv::Matx::eye() - (H * H.t()); + + // Now we need to pick 3 columns of P with non-zero norm (> 0.3) and some angle between them (> 0.3). + // + // Find the 3 columns of Pn with largest norms + int index1 = 0, + index2 = 0, + index3 = 0; + double max_norm1 = std::numeric_limits::min(); + double min_dot12 = std::numeric_limits::max(); + double min_dot1323 = std::numeric_limits::max(); + + + double col_norms[9]; + for (int i = 0; i < 9; i++) + { + col_norms[i] = cv::norm(Pn.col(i)); + if (col_norms[i] >= norm_threshold) + { + if (max_norm1 < col_norms[i]) + { + max_norm1 = col_norms[i]; + index1 = i; + } + } + } + + Matx v1 = Pn.col(index1); + v1 /= max_norm1; + set(0, 0, N, v1); + + for (int i = 0; i < 9; i++) + { + if (i == index1) continue; + if (col_norms[i] >= norm_threshold) + { + double cos_v1_x_col = fabs(Pn.col(i).dot(v1) / col_norms[i]); + + if (cos_v1_x_col <= min_dot12) + { + index2 = i; + min_dot12 = cos_v1_x_col; + } + } + } + + Matx v2 = Pn.col(index2); + Matx n0 = N.col(0); + v2 -= v2.dot(n0) * n0; + v2 /= cv::norm(v2); + set(0, 1, N, v2); + + for (int i = 0; i < 9; i++) + { + if (i == index2 || i == index1) continue; + if (col_norms[i] >= norm_threshold) + { + double cos_v1_x_col = fabs(Pn.col(i).dot(v1) / col_norms[i]); + double cos_v2_x_col = fabs(Pn.col(i).dot(v2) / col_norms[i]); + + if (cos_v1_x_col + cos_v2_x_col <= min_dot1323) + { + index3 = i; + min_dot1323 = cos_v2_x_col + cos_v2_x_col; + } + } + } + + Matx v3 = Pn.col(index3); + Matx n1 = N.col(1); + v3 -= (v3.dot(n1)) * n1 - (v3.dot(n0)) * n0; + v3 /= cv::norm(v3); + set(0, 2, N, v3); + +} + +// faster nearest rotation computation based on FOAM (see: http://users.ics.forth.gr/~lourakis/publ/2018_iros.pdf ) +/* Solve the nearest orthogonal approximation problem + * i.e., given e, find R minimizing ||R-e||_F + * + * The computation borrows from Markley's FOAM algorithm + * "Attitude Determination Using Vector Observations: A Fast Optimal Matrix Algorithm", J. Astronaut. Sci. + * + * See also M. Lourakis: "An Efficient Solution to Absolute Orientation", ICPR 2016 + * + * Copyright (C) 2019 Manolis Lourakis (lourakis **at** ics forth gr) + * Institute of Computer Science, Foundation for Research & Technology - Hellas + * Heraklion, Crete, Greece. + */ +void PoseSolver::nearestRotationMatrix(const cv::Matx& e, + cv::Matx& r) +{ + int i; + double l, lprev, det_e, e_sq, adj_e_sq, adj_e[9]; + + // e's adjoint + adj_e[0] = e(4) * e(8) - e(5) * e(7); adj_e[1] = e(2) * e(7) - e(1) * e(8); adj_e[2] = e(1) * e(5) - e(2) * e(4); + adj_e[3] = e(5) * e(6) - e(3) * e(8); adj_e[4] = e(0) * e(8) - e(2) * e(6); adj_e[5] = e(2) * e(3) - e(0) * e(5); + adj_e[6] = e(3) * e(7) - e(4) * e(6); adj_e[7] = e(1) * e(6) - e(0) * e(7); adj_e[8] = e(0) * e(4) - e(1) * e(3); + + // det(e), ||e||^2, ||adj(e)||^2 + det_e = e(0) * e(4) * e(8) - e(0) * e(5) * e(7) - e(1) * e(3) * e(8) + e(2) * e(3) * e(7) + e(1) * e(6) * e(5) - e(2) * e(6) * e(4); + e_sq = e(0) * e(0) + e(1) * e(1) + e(2) * e(2) + e(3) * e(3) + e(4) * e(4) + e(5) * e(5) + e(6) * e(6) + e(7) * e(7) + e(8) * e(8); + adj_e_sq = adj_e[0] * adj_e[0] + adj_e[1] * adj_e[1] + adj_e[2] * adj_e[2] + adj_e[3] * adj_e[3] + adj_e[4] * adj_e[4] + adj_e[5] * adj_e[5] + adj_e[6] * adj_e[6] + adj_e[7] * adj_e[7] + adj_e[8] * adj_e[8]; + + // compute l_max with Newton-Raphson from FOAM's characteristic polynomial, i.e. eq.(23) - (26) + for (i = 200, l = 2.0, lprev = 0.0; fabs(l - lprev) > 1E-12 * fabs(lprev) && i > 0; --i) { + double tmp, p, pp; + + tmp = (l * l - e_sq); + p = (tmp * tmp - 8.0 * l * det_e - 4.0 * adj_e_sq); + pp = 8.0 * (0.5 * tmp * l - det_e); + + lprev = l; + l -= p / pp; + } + + // the rotation matrix equals ((l^2 + e_sq)*e + 2*l*adj(e') - 2*e*e'*e) / (l*(l*l-e_sq) - 2*det(e)), i.e. eq.(14) using (18), (19) + { + // compute (l^2 + e_sq)*e + double tmp[9], e_et[9], denom; + const double a = l * l + e_sq; + + // e_et=e*e' + e_et[0] = e(0) * e(0) + e(1) * e(1) + e(2) * e(2); + e_et[1] = e(0) * e(3) + e(1) * e(4) + e(2) * e(5); + e_et[2] = e(0) * e(6) + e(1) * e(7) + e(2) * e(8); + + e_et[3] = e_et[1]; + e_et[4] = e(3) * e(3) + e(4) * e(4) + e(5) * e(5); + e_et[5] = e(3) * e(6) + e(4) * e(7) + e(5) * e(8); + + e_et[6] = e_et[2]; + e_et[7] = e_et[5]; + e_et[8] = e(6) * e(6) + e(7) * e(7) + e(8) * e(8); + + // tmp=e_et*e + tmp[0] = e_et[0] * e(0) + e_et[1] * e(3) + e_et[2] * e(6); + tmp[1] = e_et[0] * e(1) + e_et[1] * e(4) + e_et[2] * e(7); + tmp[2] = e_et[0] * e(2) + e_et[1] * e(5) + e_et[2] * e(8); + + tmp[3] = e_et[3] * e(0) + e_et[4] * e(3) + e_et[5] * e(6); + tmp[4] = e_et[3] * e(1) + e_et[4] * e(4) + e_et[5] * e(7); + tmp[5] = e_et[3] * e(2) + e_et[4] * e(5) + e_et[5] * e(8); + + tmp[6] = e_et[6] * e(0) + e_et[7] * e(3) + e_et[8] * e(6); + tmp[7] = e_et[6] * e(1) + e_et[7] * e(4) + e_et[8] * e(7); + tmp[8] = e_et[6] * e(2) + e_et[7] * e(5) + e_et[8] * e(8); + + // compute R as (a*e + 2*(l*adj(e)' - tmp))*denom; note that adj(e')=adj(e)' + denom = l * (l * l - e_sq) - 2.0 * det_e; + denom = 1.0 / denom; + r(0) = (a * e(0) + 2.0 * (l * adj_e[0] - tmp[0])) * denom; + r(1) = (a * e(1) + 2.0 * (l * adj_e[3] - tmp[1])) * denom; + r(2) = (a * e(2) + 2.0 * (l * adj_e[6] - tmp[2])) * denom; + + r(3) = (a * e(3) + 2.0 * (l * adj_e[1] - tmp[3])) * denom; + r(4) = (a * e(4) + 2.0 * (l * adj_e[4] - tmp[4])) * denom; + r(5) = (a * e(5) + 2.0 * (l * adj_e[7] - tmp[5])) * denom; + + r(6) = (a * e(6) + 2.0 * (l * adj_e[2] - tmp[6])) * denom; + r(7) = (a * e(7) + 2.0 * (l * adj_e[5] - tmp[7])) * denom; + r(8) = (a * e(8) + 2.0 * (l * adj_e[8] - tmp[8])) * denom; + } +} + +double PoseSolver::det3x3(const cv::Matx& e) +{ + return e(0) * e(4) * e(8) + e(1) * e(5) * e(6) + e(2) * e(3) * e(7) + - e(6) * e(4) * e(2) - e(7) * e(5) * e(0) - e(8) * e(3) * e(1); +} + +inline bool PoseSolver::positiveDepth(const SQPSolution& solution) const +{ + const cv::Matx& r = solution.r_hat; + const cv::Matx& t = solution.t; + const cv::Vec3d& mean = point_mean_; + return (r(6) * mean(0) + r(7) * mean(1) + r(8) * mean(2) + t(2) > 0); +} + +void PoseSolver::checkSolution(SQPSolution& solution, double& min_error) +{ + if (positiveDepth(solution)) + { + solution.sq_error = (omega_ * solution.r_hat).ddot(solution.r_hat); + if (fabs(min_error - solution.sq_error) > EQUAL_SQUARED_ERRORS_DIFF) + { + if (min_error > solution.sq_error) + { + min_error = solution.sq_error; + solutions_[0] = solution; + num_solutions_ = 1; + } + } + else + { + bool found = false; + for (int i = 0; i < num_solutions_; i++) + { + if (cv::norm(solutions_[i].r_hat - solution.r_hat, cv::NORM_L2SQR) < EQUAL_VECTORS_SQUARED_DIFF) + { + if (solutions_[i].sq_error > solution.sq_error) + { + solutions_[i] = solution; + } + found = true; + break; + } + } + + if (!found) + { + solutions_[num_solutions_++] = solution; + } + if (min_error > solution.sq_error) min_error = solution.sq_error; + } + } +} + +double PoseSolver::orthogonalityError(const cv::Matx& e) +{ + double sq_norm_e1 = e(0) * e(0) + e(1) * e(1) + e(2) * e(2); + double sq_norm_e2 = e(3) * e(3) + e(4) * e(4) + e(5) * e(5); + double sq_norm_e3 = e(6) * e(6) + e(7) * e(7) + e(8) * e(8); + double dot_e1e2 = e(0) * e(3) + e(1) * e(4) + e(2) * e(5); + double dot_e1e3 = e(0) * e(6) + e(1) * e(7) + e(2) * e(8); + double dot_e2e3 = e(3) * e(6) + e(4) * e(7) + e(5) * e(8); + + return (sq_norm_e1 - 1) * (sq_norm_e1 - 1) + (sq_norm_e2 - 1) * (sq_norm_e2 - 1) + (sq_norm_e3 - 1) * (sq_norm_e3 - 1) + + 2 * (dot_e1e2 * dot_e1e2 + dot_e1e3 * dot_e1e3 + dot_e2e3 * dot_e2e3); +} + +} +} diff --git a/modules/calib3d/src/sqpnp.hpp b/modules/calib3d/src/sqpnp.hpp new file mode 100644 index 0000000000..f8136324c9 --- /dev/null +++ b/modules/calib3d/src/sqpnp.hpp @@ -0,0 +1,194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This file is based on file issued with the following license: + +/* +BSD 3-Clause License + +Copyright (c) 2020, George Terzakis +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef OPENCV_CALIB3D_SQPNP_HPP +#define OPENCV_CALIB3D_SQPNP_HPP + +#include + +namespace cv { +namespace sqpnp { + + +class PoseSolver { +public: + /** + * @brief PoseSolver constructor + */ + PoseSolver(); + + /** + * @brief Finds the possible poses of a camera given a set of 3D points + * and their corresponding 2D image projections. The poses are + * sorted by lowest squared error (which corresponds to lowest + * reprojection error). + * @param objectPoints Array or vector of 3 or more 3D points defined in object coordinates. + * 1xN/Nx1 3-channel (float or double) where N is the number of points. + * @param imagePoints Array or vector of corresponding 2D points, 1xN/Nx1 2-channel. + * @param rvec The output rotation solutions (up to 18 3x1 rotation vectors) + * @param tvec The output translation solutions (up to 18 3x1 vectors) + */ + void solve(InputArray objectPoints, InputArray imagePoints, OutputArrayOfArrays rvec, + OutputArrayOfArrays tvec); + +private: + struct SQPSolution + { + cv::Matx r_hat; + cv::Matx t; + double sq_error; + }; + + /* + * @brief Computes the 9x9 PSD Omega matrix and supporting matrices. + * @param objectPoints Array or vector of 3 or more 3D points defined in object coordinates. + * 1xN/Nx1 3-channel (float or double) where N is the number of points. + * @param imagePoints Array or vector of corresponding 2D points, 1xN/Nx1 2-channel. + */ + void computeOmega(InputArray objectPoints, InputArray imagePoints); + + /* + * @brief Computes the 9x9 PSD Omega matrix and supporting matrices. + */ + void solveInternal(); + + /* + * @brief Produces the distance from being orthogonal for a given 3x3 matrix + * in row-major form. + * @param e The vector to test representing a 3x3 matrix in row major form. + * @return The distance the matrix is from being orthogonal. + */ + static double orthogonalityError(const cv::Matx& e); + + /* + * @brief Processes a solution and sorts it by error. + * @param solution The solution to evaluate. + * @param min_error The current minimum error. + */ + void checkSolution(SQPSolution& solution, double& min_error); + + /* + * @brief Computes the determinant of a matrix stored in row-major format. + * @param e Vector representing a 3x3 matrix stored in row-major format. + * @return The determinant of the matrix. + */ + static double det3x3(const cv::Matx& e); + + /* + * @brief Tests the cheirality for a given solution. + * @param solution The solution to evaluate. + */ + inline bool positiveDepth(const SQPSolution& solution) const; + + /* + * @brief Determines the nearest rotation matrix to a given rotaiton matrix. + * Input and output are 9x1 vector representing a vector stored in row-major + * form. + * @param e The input 3x3 matrix stored in a vector in row-major form. + * @param r The nearest rotation matrix to the input e (again in row-major form). + */ + static void nearestRotationMatrix(const cv::Matx& e, + cv::Matx& r); + + /* + * @brief Runs the sequential quadratic programming on orthogonal matrices. + * @param r0 The start point of the solver. + */ + SQPSolution runSQP(const cv::Matx& r0); + + /* + * @brief Steps down the gradient for the given matrix r to solve the SQP system. + * @param r The current matrix step. + * @param delta The next step down the gradient. + */ + void solveSQPSystem(const cv::Matx& r, cv::Matx& delta); + + /* + * @brief Analytically computes the inverse of a symmetric 3x3 matrix using the + * lower triangle. + * @param Q The matrix to invert. + * @param Qinv The inverse of Q. + * @param threshold The threshold to determine if Q is singular and non-invertible. + */ + bool analyticalInverse3x3Symm(const cv::Matx& Q, + cv::Matx& Qinv, + const double& threshold = 1e-8); + + /* + * @brief Computes the 3D null space and 6D normal space of the constraint Jacobian + * at a 9D vector r (representing a rank-3 matrix). Note that K is lower + * triangular so upper triangle is undefined. + * @param r 9D vector representing a rank-3 matrix. + * @param H 6D row space of the constraint Jacobian at r. + * @param N 3D null space of the constraint Jacobian at r. + * @param K The constraint Jacobian at r. + * @param norm_threshold Threshold for column vector norm of Pn (the projection onto the null space + * of the constraint Jacobian). + */ + void computeRowAndNullspace(const cv::Matx& r, + cv::Matx& H, + cv::Matx& N, + cv::Matx& K, + const double& norm_threshold = 0.1); + + static const double RANK_TOLERANCE; + static const double SQP_SQUARED_TOLERANCE; + static const double SQP_DET_THRESHOLD; + static const double ORTHOGONALITY_SQUARED_ERROR_THRESHOLD; + static const double EQUAL_VECTORS_SQUARED_DIFF; + static const double EQUAL_SQUARED_ERRORS_DIFF; + static const double POINT_VARIANCE_THRESHOLD; + static const int SQP_MAX_ITERATION; + static const double SQRT3; + + cv::Matx omega_; + cv::Vec s_; + cv::Matx u_; + cv::Matx p_; + cv::Vec3d point_mean_; + int num_null_vectors_; + + SQPSolution solutions_[18]; + int num_solutions_; + +}; + +} +} + +#endif diff --git a/modules/calib3d/src/usac.hpp b/modules/calib3d/src/usac.hpp index c18de92479..06a0ff2056 100644 --- a/modules/calib3d/src/usac.hpp +++ b/modules/calib3d/src/usac.hpp @@ -421,7 +421,7 @@ struct SPRT_history { double epsilon, delta, A; // number of samples processed by test int tested_samples; // k - SPRT_history () { + SPRT_history () : epsilon(0), delta(0), A(0) { tested_samples = 0; } }; diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index 65fa2d3b9f..0c7637d582 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -286,7 +286,7 @@ public: current_score = quality->getScore(models[i]); } else { if (is_magsac && iters % repeat_magsac == 0) { - if (!local_optimization->refineModel + if (local_optimization && !local_optimization->refineModel (models[i], best_score_thread, models[i], current_score)) continue; } else if (model_verifier->isModelGood(models[i])) { @@ -1028,4 +1028,4 @@ bool run (const Ptr ¶ms, InputArray points1, InputArray points2 } return false; } -}} \ No newline at end of file +}} diff --git a/modules/calib3d/test/test_solvepnp_ransac.cpp b/modules/calib3d/test/test_solvepnp_ransac.cpp index 0d35fa7126..fb0e2965e6 100644 --- a/modules/calib3d/test/test_solvepnp_ransac.cpp +++ b/modules/calib3d/test/test_solvepnp_ransac.cpp @@ -190,6 +190,8 @@ static std::string printMethod(int method) return "SOLVEPNP_IPPE"; case 7: return "SOLVEPNP_IPPE_SQUARE"; + case 8: + return "SOLVEPNP_SQPNP"; default: return "Unknown value"; } @@ -206,6 +208,7 @@ public: eps[SOLVEPNP_AP3P] = 1.0e-2; eps[SOLVEPNP_DLS] = 1.0e-2; eps[SOLVEPNP_UPNP] = 1.0e-2; + eps[SOLVEPNP_SQPNP] = 1.0e-2; totalTestsCount = 10; pointsCount = 500; } @@ -436,6 +439,7 @@ public: eps[SOLVEPNP_UPNP] = 1.0e-6; //UPnP is remapped to EPnP, so we use the same threshold eps[SOLVEPNP_IPPE] = 1.0e-6; eps[SOLVEPNP_IPPE_SQUARE] = 1.0e-6; + eps[SOLVEPNP_SQPNP] = 1.0e-6; totalTestsCount = 1000; diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index adbe3727a4..50af505968 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -203,6 +203,9 @@ enum CovarFlags { COVAR_COLS = 16 }; +//! @addtogroup core_cluster +//! @{ + //! k-Means flags enum KmeansFlags { /** Select random initial centers in each attempt.*/ @@ -216,12 +219,18 @@ enum KmeansFlags { KMEANS_USE_INITIAL_LABELS = 1 }; +//! @} core_cluster + +//! @addtogroup core_array +//! @{ + enum ReduceTypes { REDUCE_SUM = 0, //!< the output is the sum of all rows/columns of the matrix. REDUCE_AVG = 1, //!< the output is the mean vector of all rows/columns of the matrix. REDUCE_MAX = 2, //!< the output is the maximum (column/row-wise) of all rows/columns of the matrix. REDUCE_MIN = 3 //!< the output is the minimum (column/row-wise) of all rows/columns of the matrix. }; +//! @} core_array /** @brief Swaps two matrices */ diff --git a/modules/core/include/opencv2/core/affine.hpp b/modules/core/include/opencv2/core/affine.hpp index 7e2ed30785..1806382e99 100644 --- a/modules/core/include/opencv2/core/affine.hpp +++ b/modules/core/include/opencv2/core/affine.hpp @@ -499,7 +499,7 @@ typename cv::Affine3::Vec3 cv::Affine3::rvec() const double s = std::sqrt((rx*rx + ry*ry + rz*rz)*0.25); double c = (R.val[0] + R.val[4] + R.val[8] - 1) * 0.5; c = c > 1.0 ? 1.0 : c < -1.0 ? -1.0 : c; - double theta = acos(c); + double theta = std::acos(c); if( s < 1e-5 ) { diff --git a/modules/core/include/opencv2/core/cv_cpu_dispatch.h b/modules/core/include/opencv2/core/cv_cpu_dispatch.h index eb3f8693c2..ef2b31ac18 100644 --- a/modules/core/include/opencv2/core/cv_cpu_dispatch.h +++ b/modules/core/include/opencv2/core/cv_cpu_dispatch.h @@ -220,6 +220,11 @@ struct VZeroUpperGuard { # define CV_VSX 1 #endif +#ifdef __F16C__ +# include +# define CV_FP16 1 +#endif + #endif // !__OPENCV_BUILD && !__CUDACC (Compatibility code) diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index f7a11a29fd..1bfaa82c34 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -90,7 +90,7 @@ namespace cv { namespace debug_build_guard { } using namespace debug_build_guard // keep current value (through OpenCV port file) #elif defined __GNUC__ || (defined (__cpluscplus) && (__cpluscplus >= 201103)) #define CV_Func __func__ -#elif defined __clang__ && (__clang_minor__ * 100 + __clang_major >= 305) +#elif defined __clang__ && (__clang_minor__ * 100 + __clang_major__ >= 305) #define CV_Func __func__ #elif defined(__STDC_VERSION__) && (__STDC_VERSION >= 199901) #define CV_Func __func__ @@ -844,7 +844,7 @@ protected: float16_t() : w(0) {} explicit float16_t(float x) { - #if CV_AVX2 + #if CV_FP16 __m128 v = _mm_load_ss(&x); w = (ushort)_mm_cvtsi128_si32(_mm_cvtps_ph(v, 0)); #else @@ -875,7 +875,7 @@ protected: operator float() const { - #if CV_AVX2 + #if CV_FP16 float f; _mm_store_ss(&f, _mm_cvtph_ps(_mm_cvtsi32_si128(w))); return f; diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 5dc5bb567d..54e8927192 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -3121,18 +3121,39 @@ OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_float32x8, float, f32, v_uint32x8, un OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_int64x4, int64, s64, v_uint64x4, uint64, u64) OPENCV_HAL_IMPL_AVX_LOADSTORE_INTERLEAVE(v_float64x4, double, f64, v_uint64x4, uint64, u64) +// // FP16 +// + inline v_float32x8 v256_load_expand(const float16_t* ptr) { +#if CV_FP16 return v_float32x8(_mm256_cvtph_ps(_mm_loadu_si128((const __m128i*)ptr))); +#else + float CV_DECL_ALIGNED(32) buf[8]; + for (int i = 0; i < 8; i++) + buf[i] = (float)ptr[i]; + return v256_load_aligned(buf); +#endif } inline void v_pack_store(float16_t* ptr, const v_float32x8& a) { +#if CV_FP16 __m128i ah = _mm256_cvtps_ph(a.val, 0); _mm_storeu_si128((__m128i*)ptr, ah); +#else + float CV_DECL_ALIGNED(32) buf[8]; + v_store_aligned(buf, a); + for (int i = 0; i < 8; i++) + ptr[i] = float16_t(buf[i]); +#endif } +// +// end of FP16 +// + inline void v256_cleanup() { _mm256_zeroall(); } CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END diff --git a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp index d1bfb6da6d..ef928f6a5c 100644 --- a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp @@ -207,13 +207,7 @@ struct v_uint64x2 uint64 get0() const { -#ifdef __wasm_unimplemented_simd128__ return (uint64)wasm_i64x2_extract_lane(val, 0); -#else - uint64 des[2]; - wasm_v128_store(des, val); - return des[0]; -#endif } v128_t val; @@ -235,13 +229,7 @@ struct v_int64x2 int64 get0() const { -#ifdef __wasm_unimplemented_simd128__ return wasm_i64x2_extract_lane(val, 0); -#else - int64 des[2]; - wasm_v128_store(des, val); - return des[0]; -#endif } v128_t val; @@ -263,13 +251,7 @@ struct v_float64x2 double get0() const { -#ifdef __wasm_unimplemented_simd128__ return wasm_f64x2_extract_lane(val, 0); -#else - double des[2]; - wasm_v128_store(des, val); - return des[0]; -#endif } v128_t val; @@ -1797,22 +1779,9 @@ OPENCV_HAL_IMPL_WASM_INITVEC(v_int16x8, short, s16, i16x8, short) OPENCV_HAL_IMPL_WASM_INITVEC(v_uint32x4, unsigned, u32, i32x4, int) OPENCV_HAL_IMPL_WASM_INITVEC(v_int32x4, int, s32, i32x4, int) OPENCV_HAL_IMPL_WASM_INITVEC(v_float32x4, float, f32, f32x4, float) - -#ifdef __wasm_unimplemented_simd128__ OPENCV_HAL_IMPL_WASM_INITVEC(v_uint64x2, uint64, u64, i64x2, int64) OPENCV_HAL_IMPL_WASM_INITVEC(v_int64x2, int64, s64, i64x2, int64) OPENCV_HAL_IMPL_WASM_INITVEC(v_float64x2, double, f64, f64x2, double) -#else -#define OPENCV_HAL_IMPL_FALLBACK_INITVEC(_Tpvec, _Tp, suffix, _Tps) \ -inline _Tpvec v_setzero_##suffix() { return _Tpvec((_Tps)0, (_Tps)0); } \ -inline _Tpvec v_setall_##suffix(_Tp v) { return _Tpvec((_Tps)v, (_Tps)v); } \ -template inline _Tpvec v_reinterpret_as_##suffix(const _Tpvec0& a) \ -{ return _Tpvec(a.val); } - -OPENCV_HAL_IMPL_FALLBACK_INITVEC(v_uint64x2, uint64, u64, int64) -OPENCV_HAL_IMPL_FALLBACK_INITVEC(v_int64x2, int64, s64, int64) -OPENCV_HAL_IMPL_FALLBACK_INITVEC(v_float64x2, double, f64, double) -#endif //////////////// PACK /////////////// inline v_uint8x16 v_pack(const v_uint16x8& a, const v_uint16x8& b) @@ -1931,28 +1900,18 @@ inline v_int16x8 v_rshr_pack(const v_int32x4& a, const v_int32x4& b) template inline v_uint32x4 v_rshr_pack(const v_uint64x2& a, const v_uint64x2& b) { -#ifdef __wasm_unimplemented_simd128__ v128_t delta = wasm_i64x2_splat(((int64)1 << (n-1))); v128_t a1 = wasm_u64x2_shr(wasm_i64x2_add(a.val, delta), n); v128_t b1 = wasm_u64x2_shr(wasm_i64x2_add(b.val, delta), n); return v_uint32x4(wasm_v8x16_shuffle(a1, b1, 0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27)); -#else - fallback::v_uint64x2 a_(a), b_(b); - return fallback::v_rshr_pack(a_, b_); -#endif } template inline v_int32x4 v_rshr_pack(const v_int64x2& a, const v_int64x2& b) { -#ifdef __wasm_unimplemented_simd128__ v128_t delta = wasm_i64x2_splat(((int64)1 << (n-1))); v128_t a1 = wasm_i64x2_shr(wasm_i64x2_add(a.val, delta), n); v128_t b1 = wasm_i64x2_shr(wasm_i64x2_add(b.val, delta), n); return v_int32x4(wasm_v8x16_shuffle(a1, b1, 0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27)); -#else - fallback::v_int64x2 a_(a), b_(b); - return fallback::v_rshr_pack(a_, b_); -#endif } template inline v_uint8x16 v_rshr_pack_u(const v_int16x8& a, const v_int16x8& b) @@ -2139,7 +2098,6 @@ inline void v_rshr_pack_store(short* ptr, const v_int32x4& a) template inline void v_rshr_pack_store(unsigned* ptr, const v_uint64x2& a) { -#ifdef __wasm_unimplemented_simd128__ v128_t delta = wasm_i64x2_splat(((int64)1 << (n-1))); v128_t a1 = wasm_u64x2_shr(wasm_i64x2_add(a.val, delta), n); v128_t r = wasm_v8x16_shuffle(a1, a1, 0,1,2,3,8,9,10,11,0,1,2,3,8,9,10,11); @@ -2148,15 +2106,10 @@ inline void v_rshr_pack_store(unsigned* ptr, const v_uint64x2& a) for (int i=0; i<2; ++i) { ptr[i] = t_ptr[i]; } -#else - fallback::v_uint64x2 _a(a); - fallback::v_rshr_pack_store(ptr, _a); -#endif } template inline void v_rshr_pack_store(int* ptr, const v_int64x2& a) { -#ifdef __wasm_unimplemented_simd128__ v128_t delta = wasm_i64x2_splat(((int64)1 << (n-1))); v128_t a1 = wasm_i64x2_shr(wasm_i64x2_add(a.val, delta), n); v128_t r = wasm_v8x16_shuffle(a1, a1, 0,1,2,3,8,9,10,11,0,1,2,3,8,9,10,11); @@ -2165,10 +2118,6 @@ inline void v_rshr_pack_store(int* ptr, const v_int64x2& a) for (int i=0; i<2; ++i) { ptr[i] = t_ptr[i]; } -#else - fallback::v_int64x2 _a(a); - fallback::v_rshr_pack_store(ptr, _a); -#endif } template inline void v_rshr_pack_u_store(uchar* ptr, const v_int16x8& a) @@ -2228,7 +2177,6 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, const v_uint64x2& g, const v_uint64x2& h) { -#ifdef __wasm_unimplemented_simd128__ v128_t maxval = wasm_i32x4_splat(255); v128_t a1 = wasm_v128_bitselect(maxval, a.val, ((__u64x2)(a.val) > (__u64x2)maxval)); v128_t b1 = wasm_v128_bitselect(maxval, b.val, ((__u64x2)(b.val) > (__u64x2)maxval)); @@ -2245,10 +2193,6 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin v128_t abcd = wasm_v8x16_shuffle(ab, cd, 0,1,2,3,16,17,18,19,0,1,2,3,16,17,18,19); v128_t efgh = wasm_v8x16_shuffle(ef, gh, 0,1,2,3,16,17,18,19,0,1,2,3,16,17,18,19); return v_uint8x16(wasm_v8x16_shuffle(abcd, efgh, 0,1,2,3,4,5,6,7,16,17,18,19,20,21,22,23)); -#else - fallback::v_uint64x2 a_(a), b_(b), c_(c), d_(d), e_(e), f_(f), g_(g), h_(h); - return fallback::v_pack_b(a_, b_, c_, d_, e_, f_, g_, h_); -#endif } inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, @@ -2310,8 +2254,6 @@ OPENCV_HAL_IMPL_WASM_BIN_OP(+, v_float32x4, wasm_f32x4_add) OPENCV_HAL_IMPL_WASM_BIN_OP(-, v_float32x4, wasm_f32x4_sub) OPENCV_HAL_IMPL_WASM_BIN_OP(*, v_float32x4, wasm_f32x4_mul) OPENCV_HAL_IMPL_WASM_BIN_OP(/, v_float32x4, wasm_f32x4_div) - -#ifdef __wasm_unimplemented_simd128__ OPENCV_HAL_IMPL_WASM_BIN_OP(+, v_uint64x2, wasm_i64x2_add) OPENCV_HAL_IMPL_WASM_BIN_OP(-, v_uint64x2, wasm_i64x2_sub) OPENCV_HAL_IMPL_WASM_BIN_OP(+, v_int64x2, wasm_i64x2_add) @@ -2320,30 +2262,6 @@ OPENCV_HAL_IMPL_WASM_BIN_OP(+, v_float64x2, wasm_f64x2_add) OPENCV_HAL_IMPL_WASM_BIN_OP(-, v_float64x2, wasm_f64x2_sub) OPENCV_HAL_IMPL_WASM_BIN_OP(*, v_float64x2, wasm_f64x2_mul) OPENCV_HAL_IMPL_WASM_BIN_OP(/, v_float64x2, wasm_f64x2_div) -#else -#define OPENCV_HAL_IMPL_FALLBACK_BIN_OP(bin_op, _Tpvec) \ -inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ -{ \ - fallback::_Tpvec a_(a), b_(b); \ - return _Tpvec((a_) bin_op (b_)); \ -} \ -inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ -{ \ - fallback::_Tpvec a_(a), b_(b); \ - a_ bin_op##= b_; \ - a = _Tpvec(a_); \ - return a; \ -} - -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(+, v_uint64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(-, v_uint64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(+, v_int64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(-, v_int64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(+, v_float64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(-, v_float64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(*, v_float64x2) -OPENCV_HAL_IMPL_FALLBACK_BIN_OP(/, v_float64x2) -#endif // saturating multiply 8-bit, 16-bit #define OPENCV_HAL_IMPL_WASM_MUL_SAT(_Tpvec, _Tpwvec) \ @@ -2405,19 +2323,11 @@ inline void v_mul_expand(const v_uint16x8& a, const v_uint16x8& b, inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, v_uint64x2& c, v_uint64x2& d) { -#ifdef __wasm_unimplemented_simd128__ v_uint64x2 a0, a1, b0, b1; v_expand(a, a0, a1); v_expand(b, b0, b1); c.val = ((__u64x2)(a0.val) * (__u64x2)(b0.val)); d.val = ((__u64x2)(a1.val) * (__u64x2)(b1.val)); -#else - fallback::v_uint32x4 a_(a), b_(b); - fallback::v_uint64x2 c_, d_; - fallback::v_mul_expand(a_, b_, c_, d_); - c = v_uint64x2(c_); - d = v_uint64x2(d_); -#endif } inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) @@ -2457,7 +2367,6 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { -#ifdef __wasm_unimplemented_simd128__ v128_t a0 = wasm_i64x2_shr(wasm_i64x2_shl(a.val, 32), 32); v128_t a1 = wasm_i64x2_shr(a.val, 32); v128_t b0 = wasm_i64x2_shr(wasm_i64x2_shl(b.val, 32), 32); @@ -2465,22 +2374,10 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) v128_t c = (v128_t)((__i64x2)a0 * (__i64x2)b0); v128_t d = (v128_t)((__i64x2)a1 * (__i64x2)b1); return v_int64x2(wasm_i64x2_add(c, d)); -#else - fallback::v_int32x4 a_(a); - fallback::v_int32x4 b_(b); - return fallback::v_dotprod(a_, b_); -#endif } inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { -#ifdef __wasm_unimplemented_simd128__ return v_dotprod(a, b) + c; -#else - fallback::v_int32x4 a_(a); - fallback::v_int32x4 b_(b); - fallback::v_int64x2 c_(c); - return fallback::v_dotprod(a_, b_, c_); -#endif } // 8 >> 32 @@ -2515,32 +2412,32 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const // 16 >> 64 inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { - fallback::v_uint16x8 a_(a); - fallback::v_uint16x8 b_(b); - return fallback::v_dotprod_expand(a_, b_); + v128_t a0 = wasm_u32x4_shr(wasm_i32x4_shl(a.val, 16), 16); + v128_t a1 = wasm_u32x4_shr(a.val, 16); + v128_t b0 = wasm_u32x4_shr(wasm_i32x4_shl(b.val, 16), 16); + v128_t b1 = wasm_u32x4_shr(b.val, 16); + return v_uint64x2(( + v_dotprod(v_int32x4(a0), v_int32x4(b0)) + + v_dotprod(v_int32x4(a1), v_int32x4(b1))).val + ); } inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) -{ - fallback::v_uint16x8 a_(a); - fallback::v_uint16x8 b_(b); - fallback::v_uint64x2 c_(c); - return fallback::v_dotprod_expand(a_, b_, c_); -} +{ return v_dotprod_expand(a, b) + c; } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { - fallback::v_int16x8 a_(a); - fallback::v_int16x8 b_(b); - return fallback::v_dotprod_expand(a_, b_); + v128_t a0 = wasm_i32x4_shr(wasm_i32x4_shl(a.val, 16), 16); + v128_t a1 = wasm_i32x4_shr(a.val, 16); + v128_t b0 = wasm_i32x4_shr(wasm_i32x4_shl(b.val, 16), 16); + v128_t b1 = wasm_i32x4_shr(b.val, 16); + return v_int64x2(( + v_dotprod(v_int32x4(a0), v_int32x4(b0)) + + v_dotprod(v_int32x4(a1), v_int32x4(b1))) + ); } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) -{ - fallback::v_int16x8 a_(a); - fallback::v_int16x8 b_(b); - fallback::v_int64x2 c_(c); - return fallback::v_dotprod_expand(a_, b_, c_); -} +{ return v_dotprod_expand(a, b) + c; } // 32 >> 64f inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b) @@ -2610,44 +2507,24 @@ OPENCV_HAL_IMPL_WASM_LOGIC_OP(v_float64x2) inline v_float32x4 v_sqrt(const v_float32x4& x) { -#ifdef __wasm_unimplemented_simd128__ return v_float32x4(wasm_f32x4_sqrt(x.val)); -#else - fallback::v_float32x4 x_(x); - return fallback::v_sqrt(x_); -#endif } inline v_float32x4 v_invsqrt(const v_float32x4& x) { -#ifdef __wasm_unimplemented_simd128__ const v128_t _1_0 = wasm_f32x4_splat(1.0); return v_float32x4(wasm_f32x4_div(_1_0, wasm_f32x4_sqrt(x.val))); -#else - fallback::v_float32x4 x_(x); - return fallback::v_invsqrt(x_); -#endif } inline v_float64x2 v_sqrt(const v_float64x2& x) { -#ifdef __wasm_unimplemented_simd128__ return v_float64x2(wasm_f64x2_sqrt(x.val)); -#else - fallback::v_float64x2 x_(x); - return fallback::v_sqrt(x_); -#endif } inline v_float64x2 v_invsqrt(const v_float64x2& x) { -#ifdef __wasm_unimplemented_simd128__ const v128_t _1_0 = wasm_f64x2_splat(1.0); return v_float64x2(wasm_f64x2_div(_1_0, wasm_f64x2_sqrt(x.val))); -#else - fallback::v_float64x2 x_(x); - return fallback::v_invsqrt(x_); -#endif } #define OPENCV_HAL_IMPL_WASM_ABS_INT_FUNC(_Tpuvec, _Tpsvec, suffix, zsuffix, shiftWidth) \ @@ -2666,12 +2543,7 @@ inline v_float32x4 v_abs(const v_float32x4& x) { return v_float32x4(wasm_f32x4_abs(x.val)); } inline v_float64x2 v_abs(const v_float64x2& x) { -#ifdef __wasm_unimplemented_simd128__ return v_float64x2(wasm_f64x2_abs(x.val)); -#else - fallback::v_float64x2 x_(x); - return fallback::v_abs(x_); -#endif } // TODO: exp, log, sin, cos @@ -2684,21 +2556,8 @@ inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ OPENCV_HAL_IMPL_WASM_BIN_FUNC(v_float32x4, v_min, wasm_f32x4_min) OPENCV_HAL_IMPL_WASM_BIN_FUNC(v_float32x4, v_max, wasm_f32x4_max) - -#ifdef __wasm_unimplemented_simd128__ OPENCV_HAL_IMPL_WASM_BIN_FUNC(v_float64x2, v_min, wasm_f64x2_min) OPENCV_HAL_IMPL_WASM_BIN_FUNC(v_float64x2, v_max, wasm_f64x2_max) -#else -#define OPENCV_HAL_IMPL_WASM_MINMAX_64f_FUNC(func) \ -inline v_float64x2 func(const v_float64x2& a, const v_float64x2& b) \ -{ \ - fallback::v_float64x2 a_(a), b_(b); \ - return fallback::func(a_, b_); \ -} - -OPENCV_HAL_IMPL_WASM_MINMAX_64f_FUNC(v_min) -OPENCV_HAL_IMPL_WASM_MINMAX_64f_FUNC(v_max) -#endif #define OPENCV_HAL_IMPL_WASM_MINMAX_S_INIT_FUNC(_Tpvec, suffix) \ inline _Tpvec v_min(const _Tpvec& a, const _Tpvec& b) \ @@ -2753,24 +2612,7 @@ OPENCV_HAL_IMPL_WASM_INIT_CMP_OP(v_int16x8, i16x8, i16x8) OPENCV_HAL_IMPL_WASM_INIT_CMP_OP(v_uint32x4, u32x4, i32x4) OPENCV_HAL_IMPL_WASM_INIT_CMP_OP(v_int32x4, i32x4, i32x4) OPENCV_HAL_IMPL_WASM_INIT_CMP_OP(v_float32x4, f32x4, f32x4) - -#ifdef __wasm_unimplemented_simd128__ OPENCV_HAL_IMPL_WASM_INIT_CMP_OP(v_float64x2, f64x2, f64x2) -#else -#define OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(_Tpvec, bin_op) \ -inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ -{ \ - fallback::_Tpvec a_(a), b_(b); \ - return _Tpvec((a_) bin_op (b_));\ -} \ - -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, ==) -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, !=) -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, <) -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, >) -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, <=) -OPENCV_HAL_IMPL_INIT_FALLBACK_CMP_OP(v_float64x2, >=) -#endif #define OPENCV_HAL_IMPL_WASM_64BIT_CMP_OP(_Tpvec, cast) \ inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ @@ -2789,14 +2631,9 @@ inline v_float32x4 v_not_nan(const v_float32x4& a) } inline v_float64x2 v_not_nan(const v_float64x2& a) { -#ifdef __wasm_unimplemented_simd128__ v128_t z = wasm_i64x2_splat(0x7fffffffffffffff); v128_t t = wasm_i64x2_splat(0x7ff0000000000000); return v_float64x2((__u64x2)(wasm_v128_and(a.val, z)) < (__u64x2)t); -#else - fallback::v_float64x2 a_(a); - return fallback::v_not_nan(a_); -#endif } OPENCV_HAL_IMPL_WASM_BIN_FUNC(v_uint8x16, v_add_wrap, wasm_i8x16_add) @@ -2877,32 +2714,30 @@ inline v_float32x4 v_absdiff(const v_float32x4& a, const v_float32x4& b) } inline v_float64x2 v_absdiff(const v_float64x2& a, const v_float64x2& b) { -#ifdef __wasm_unimplemented_simd128__ v128_t absmask_vec = wasm_u64x2_shr(wasm_i32x4_splat(-1), 1); return v_float64x2(wasm_v128_and(wasm_f64x2_sub(a.val, b.val), absmask_vec)); -#else - fallback::v_float64x2 a_(a), b_(b); - return fallback::v_absdiff(a_, b_); -#endif } -#define OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(_Tpvec) \ +#define OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(_Tpvec, suffix) \ inline _Tpvec v_magnitude(const _Tpvec& a, const _Tpvec& b) \ { \ - fallback::_Tpvec a_(a), b_(b); \ - return fallback::v_magnitude(a_, b_); \ + v128_t a_Square = wasm_##suffix##_mul(a.val, a.val); \ + v128_t b_Square = wasm_##suffix##_mul(b.val, b.val); \ + return _Tpvec(wasm_##suffix##_sqrt(wasm_##suffix##_add(a_Square, b_Square))); \ } \ inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ { \ - return v_fma(a, a, b*b); \ + v128_t a_Square = wasm_##suffix##_mul(a.val, a.val); \ + v128_t b_Square = wasm_##suffix##_mul(b.val, b.val); \ + return _Tpvec(wasm_##suffix##_add(a_Square, b_Square)); \ } \ inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ { \ - return v_fma(a, b, c); \ + return _Tpvec(wasm_##suffix##_add(wasm_##suffix##_mul(a.val, b.val), c.val)); \ } -OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(v_float32x4) -OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(v_float64x2) +OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(v_float32x4, f32x4) +OPENCV_HAL_IMPL_WASM_MISC_FLT_OP(v_float64x2, f64x2) #define OPENCV_HAL_IMPL_WASM_SHIFT_OP(_Tpuvec, _Tpsvec, suffix, ssuffix) \ inline _Tpuvec operator << (const _Tpuvec& a, int imm) \ @@ -2945,37 +2780,7 @@ inline _Tpsvec v_shr(const _Tpsvec& a) \ OPENCV_HAL_IMPL_WASM_SHIFT_OP(v_uint8x16, v_int8x16, i8x16, u8x16) OPENCV_HAL_IMPL_WASM_SHIFT_OP(v_uint16x8, v_int16x8, i16x8, u16x8) OPENCV_HAL_IMPL_WASM_SHIFT_OP(v_uint32x4, v_int32x4, i32x4, u32x4) - -#ifdef __wasm_unimplemented_simd128__ OPENCV_HAL_IMPL_WASM_SHIFT_OP(v_uint64x2, v_int64x2, i64x2, u64x2) -#else -#define OPENCV_HAL_IMPL_FALLBACK_SHIFT_OP(_Tpvec) \ -inline _Tpvec operator << (const _Tpvec& a, int imm) \ -{ \ - fallback::_Tpvec a_(a); \ - return a_ << imm; \ -} \ -inline _Tpvec operator >> (const _Tpvec& a, int imm) \ -{ \ - fallback::_Tpvec a_(a); \ - return a_ >> imm; \ -} \ -template \ -inline _Tpvec v_shl(const _Tpvec& a) \ -{ \ - fallback::_Tpvec a_(a); \ - return fallback::v_shl(a_); \ -} \ -template \ -inline _Tpvec v_shr(const _Tpvec& a) \ -{ \ - fallback::_Tpvec a_(a); \ - return fallback::v_shr(a_); \ -} \ - -OPENCV_HAL_IMPL_FALLBACK_SHIFT_OP(v_uint64x2) -OPENCV_HAL_IMPL_FALLBACK_SHIFT_OP(v_int64x2) -#endif namespace hal_wasm_internal { @@ -3180,9 +2985,18 @@ OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_uint8x16, unsigned) OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_int8x16, int) OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_uint16x8, unsigned) OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_int16x8, int) -OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_uint64x2, uint64) -OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_int64x2, int64) -OPENCV_HAL_IMPL_FALLBACK_REDUCE_OP_SUM(v_float64x2, double) + + +#define OPENCV_HAL_IMPL_WASM_REDUCE_OP_2_SUM(_Tpvec, scalartype, regtype, suffix, esuffix) \ +inline scalartype v_reduce_sum(const _Tpvec& a) \ +{ \ + regtype val = a.val; \ + val = wasm_##suffix##_add(val, wasm_v8x16_shuffle(val, val, 8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7)); \ + return (scalartype)wasm_##esuffix##_extract_lane(val, 0); \ +} +OPENCV_HAL_IMPL_WASM_REDUCE_OP_2_SUM(v_uint64x2, uint64, v128_t, i64x2, i64x2) +OPENCV_HAL_IMPL_WASM_REDUCE_OP_2_SUM(v_int64x2, int64, v128_t, i64x2, i64x2) +OPENCV_HAL_IMPL_WASM_REDUCE_OP_2_SUM(v_float64x2, double, v128_t, f64x2,f64x2) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c, const v_float32x4& d) @@ -3318,30 +3132,27 @@ OPENCV_HAL_IMPL_WASM_CHECK_SIGNS(v_int16x8, i16x8, short) OPENCV_HAL_IMPL_WASM_CHECK_SIGNS(v_uint32x4, i32x4, int) OPENCV_HAL_IMPL_WASM_CHECK_SIGNS(v_int32x4, i32x4, int) OPENCV_HAL_IMPL_WASM_CHECK_SIGNS(v_float32x4, i32x4, float) +OPENCV_HAL_IMPL_WASM_CHECK_SIGNS(v_float64x2, f64x2, double) + +#define OPENCV_HAL_IMPL_WASM_CHECK_ALL_ANY(_Tpvec, suffix, esuffix) \ +inline bool v_check_all(const _Tpvec& a) \ +{ \ + v128_t masked = v_reinterpret_as_##esuffix(a).val; \ + masked = wasm_i32x4_replace_lane(masked, 0, 0xffffffff); \ + masked = wasm_i32x4_replace_lane(masked, 2, 0xffffffff); \ + return wasm_i8x16_all_true(wasm_##suffix##_lt(masked, wasm_##suffix##_splat(0))); \ +} \ +inline bool v_check_any(const _Tpvec& a) \ +{ \ + v128_t masked = v_reinterpret_as_##esuffix(a).val; \ + masked = wasm_i32x4_replace_lane(masked, 0, 0x0); \ + masked = wasm_i32x4_replace_lane(masked, 2, 0x0); \ + return wasm_i8x16_any_true(wasm_##suffix##_lt(masked, wasm_##suffix##_splat(0))); \ +} \ + +OPENCV_HAL_IMPL_WASM_CHECK_ALL_ANY(v_int64x2, i32x4, s32) +OPENCV_HAL_IMPL_WASM_CHECK_ALL_ANY(v_uint64x2, i32x4, u32) -inline int v_signmask(const v_float64x2& a) -{ - fallback::v_float64x2 a_(a); - return fallback::v_signmask(a_); -} -inline bool v_check_all(const v_float64x2& a) -{ -#ifdef __wasm_unimplemented_simd128__ - return wasm_i8x16_all_true((__i64x2)(a.val) < (__i64x2)(wasm_i64x2_splat(0))); -#else - fallback::v_float64x2 a_(a); - return fallback::v_check_all(a_); -#endif -} -inline bool v_check_any(const v_float64x2& a) -{ -#ifdef __wasm_unimplemented_simd128__ - return wasm_i8x16_any_true((__i64x2)(a.val) < (__i64x2)(wasm_i64x2_splat(0)));; -#else - fallback::v_float64x2 a_(a); - return fallback::v_check_any(a_); -#endif -} inline int v_scan_forward(const v_int8x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } inline int v_scan_forward(const v_uint8x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } @@ -3366,8 +3177,8 @@ OPENCV_HAL_IMPL_WASM_SELECT(v_uint16x8) OPENCV_HAL_IMPL_WASM_SELECT(v_int16x8) OPENCV_HAL_IMPL_WASM_SELECT(v_uint32x4) OPENCV_HAL_IMPL_WASM_SELECT(v_int32x4) -// OPENCV_HAL_IMPL_WASM_SELECT(v_uint64x2) -// OPENCV_HAL_IMPL_WASM_SELECT(v_int64x2) +OPENCV_HAL_IMPL_WASM_SELECT(v_uint64x2) +OPENCV_HAL_IMPL_WASM_SELECT(v_int64x2) OPENCV_HAL_IMPL_WASM_SELECT(v_float32x4) OPENCV_HAL_IMPL_WASM_SELECT(v_float64x2) diff --git a/modules/core/include/opencv2/core/mat.inl.hpp b/modules/core/include/opencv2/core/mat.inl.hpp index 8a7eebbe22..d6296f8e2e 100644 --- a/modules/core/include/opencv2/core/mat.inl.hpp +++ b/modules/core/include/opencv2/core/mat.inl.hpp @@ -458,158 +458,6 @@ CV__DEBUG_NS_END //////////////////////////////////////////// Mat ////////////////////////////////////////// -inline -Mat::Mat() - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{} - -inline -Mat::Mat(int _rows, int _cols, int _type) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_rows, _cols, _type); -} - -inline -Mat::Mat(int _rows, int _cols, int _type, const Scalar& _s) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_rows, _cols, _type); - *this = _s; -} - -inline -Mat::Mat(Size _sz, int _type) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create( _sz.height, _sz.width, _type ); -} - -inline -Mat::Mat(Size _sz, int _type, const Scalar& _s) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_sz.height, _sz.width, _type); - *this = _s; -} - -inline -Mat::Mat(int _dims, const int* _sz, int _type) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_dims, _sz, _type); -} - -inline -Mat::Mat(int _dims, const int* _sz, int _type, const Scalar& _s) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_dims, _sz, _type); - *this = _s; -} - -inline -Mat::Mat(const std::vector& _sz, int _type) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_sz, _type); -} - -inline -Mat::Mat(const std::vector& _sz, int _type, const Scalar& _s) - : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), - datalimit(0), allocator(0), u(0), size(&rows), step(0) -{ - create(_sz, _type); - *this = _s; -} - -inline -Mat::Mat(const Mat& m) - : flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), data(m.data), - datastart(m.datastart), dataend(m.dataend), datalimit(m.datalimit), allocator(m.allocator), - u(m.u), size(&rows), step(0) -{ - if( u ) - CV_XADD(&u->refcount, 1); - if( m.dims <= 2 ) - { - step[0] = m.step[0]; step[1] = m.step[1]; - } - else - { - dims = 0; - copySize(m); - } -} - -inline -Mat::Mat(int _rows, int _cols, int _type, void* _data, size_t _step) - : flags(MAGIC_VAL + (_type & TYPE_MASK)), dims(2), rows(_rows), cols(_cols), - data((uchar*)_data), datastart((uchar*)_data), dataend(0), datalimit(0), - allocator(0), u(0), size(&rows) -{ - CV_Assert(total() == 0 || data != NULL); - - size_t esz = CV_ELEM_SIZE(_type), esz1 = CV_ELEM_SIZE1(_type); - size_t minstep = cols * esz; - if( _step == AUTO_STEP ) - { - _step = minstep; - } - else - { - CV_DbgAssert( _step >= minstep ); - if (_step % esz1 != 0) - { - CV_Error(Error::BadStep, "Step must be a multiple of esz1"); - } - } - step[0] = _step; - step[1] = esz; - datalimit = datastart + _step * rows; - dataend = datalimit - _step + minstep; - updateContinuityFlag(); -} - -inline -Mat::Mat(Size _sz, int _type, void* _data, size_t _step) - : flags(MAGIC_VAL + (_type & TYPE_MASK)), dims(2), rows(_sz.height), cols(_sz.width), - data((uchar*)_data), datastart((uchar*)_data), dataend(0), datalimit(0), - allocator(0), u(0), size(&rows) -{ - CV_Assert(total() == 0 || data != NULL); - - size_t esz = CV_ELEM_SIZE(_type), esz1 = CV_ELEM_SIZE1(_type); - size_t minstep = cols*esz; - if( _step == AUTO_STEP ) - { - _step = minstep; - } - else - { - CV_DbgAssert( _step >= minstep ); - - if (_step % esz1 != 0) - { - CV_Error(Error::BadStep, "Step must be a multiple of esz1"); - } - } - step[0] = _step; - step[1] = esz; - datalimit = datastart + _step*rows; - dataend = datalimit - _step + minstep; - updateContinuityFlag(); -} - template inline Mat::Mat(const std::vector<_Tp>& vec, bool copyData) : flags(MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(2), rows((int)vec.size()), @@ -743,43 +591,6 @@ Mat::Mat(const MatCommaInitializer_<_Tp>& commaInitializer) *this = commaInitializer.operator Mat_<_Tp>(); } -inline -Mat::~Mat() -{ - release(); - if( step.p != step.buf ) - fastFree(step.p); -} - -inline -Mat& Mat::operator = (const Mat& m) -{ - if( this != &m ) - { - if( m.u ) - CV_XADD(&m.u->refcount, 1); - release(); - flags = m.flags; - if( dims <= 2 && m.dims <= 2 ) - { - dims = m.dims; - rows = m.rows; - cols = m.cols; - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - copySize(m); - data = m.data; - datastart = m.datastart; - dataend = m.dataend; - datalimit = m.datalimit; - allocator = m.allocator; - u = m.u; - } - return *this; -} - inline Mat Mat::row(int y) const { @@ -816,67 +627,6 @@ Mat Mat::colRange(const Range& r) const return Mat(*this, Range::all(), r); } -inline -Mat Mat::clone() const -{ - Mat m; - copyTo(m); - return m; -} - -inline -void Mat::assignTo( Mat& m, int _type ) const -{ - if( _type < 0 ) - m = *this; - else - convertTo(m, _type); -} - -inline -void Mat::create(int _rows, int _cols, int _type) -{ - _type &= TYPE_MASK; - if( dims <= 2 && rows == _rows && cols == _cols && type() == _type && data ) - return; - int sz[] = {_rows, _cols}; - create(2, sz, _type); -} - -inline -void Mat::create(Size _sz, int _type) -{ - create(_sz.height, _sz.width, _type); -} - -inline -void Mat::addref() -{ - if( u ) - CV_XADD(&u->refcount, 1); -} - -inline -void Mat::release() -{ - if( u && CV_XADD(&u->refcount, -1) == 1 ) - deallocate(); - u = NULL; - datastart = dataend = datalimit = data = 0; - for(int i = 0; i < dims; i++) - size.p[i] = 0; -#ifdef _DEBUG - flags = MAGIC_VAL; - dims = rows = cols = 0; - if(step.p != step.buf) - { - fastFree(step.p); - step.p = step.buf; - size.p = &rows; - } -#endif -} - inline Mat Mat::operator()( Range _rowRange, Range _colRange ) const { @@ -945,40 +695,6 @@ int Mat::channels() const return CV_MAT_CN(flags); } -inline -size_t Mat::step1(int i) const -{ - return step.p[i] / elemSize1(); -} - -inline -bool Mat::empty() const -{ - return data == 0 || total() == 0 || dims == 0; -} - -inline -size_t Mat::total() const -{ - if( dims <= 2 ) - return (size_t)rows * cols; - size_t p = 1; - for( int i = 0; i < dims; i++ ) - p *= size[i]; - return p; -} - -inline -size_t Mat::total(int startDim, int endDim) const -{ - CV_Assert( 0 <= startDim && startDim <= endDim); - size_t p = 1; - int endDim_ = endDim <= dims ? endDim : dims; - for( int i = startDim; i < endDim_; i++ ) - p *= size[i]; - return p; -} - inline uchar* Mat::ptr(int y) { @@ -1396,67 +1112,6 @@ void Mat::push_back(const std::vector<_Tp>& v) push_back(Mat(v)); } -inline -Mat::Mat(Mat&& m) - : flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), data(m.data), - datastart(m.datastart), dataend(m.dataend), datalimit(m.datalimit), allocator(m.allocator), - u(m.u), size(&rows) -{ - if (m.dims <= 2) // move new step/size info - { - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - { - CV_DbgAssert(m.step.p != m.step.buf); - step.p = m.step.p; - size.p = m.size.p; - m.step.p = m.step.buf; - m.size.p = &m.rows; - } - m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; - m.data = NULL; m.datastart = NULL; m.dataend = NULL; m.datalimit = NULL; - m.allocator = NULL; - m.u = NULL; -} - -inline -Mat& Mat::operator = (Mat&& m) -{ - if (this == &m) - return *this; - - release(); - flags = m.flags; dims = m.dims; rows = m.rows; cols = m.cols; data = m.data; - datastart = m.datastart; dataend = m.dataend; datalimit = m.datalimit; allocator = m.allocator; - u = m.u; - if (step.p != step.buf) // release self step/size - { - fastFree(step.p); - step.p = step.buf; - size.p = &rows; - } - if (m.dims <= 2) // move new step/size info - { - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - { - CV_DbgAssert(m.step.p != m.step.buf); - step.p = m.step.p; - size.p = m.size.p; - m.step.p = m.step.buf; - m.size.p = &m.rows; - } - m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; - m.data = NULL; m.datastart = NULL; m.dataend = NULL; m.datalimit = NULL; - m.allocator = NULL; - m.u = NULL; - return *this; -} - ///////////////////////////// MatSize //////////////////////////// @@ -1503,22 +1158,6 @@ MatSize::operator const int*() const return p; } -inline -bool MatSize::operator == (const MatSize& sz) const -{ - int d = dims(); - int dsz = sz.dims(); - if( d != dsz ) - return false; - if( d == 2 ) - return p[0] == sz.p[0] && p[1] == sz.p[1]; - - for( int i = 0; i < d; i++ ) - if( p[i] != sz.p[i] ) - return false; - return true; -} - inline bool MatSize::operator != (const MatSize& sz) const { @@ -1775,9 +1414,7 @@ template inline void Mat_<_Tp>::release() { Mat::release(); -#ifdef _DEBUG flags = (flags & ~CV_MAT_TYPE_MASK) + traits::Type<_Tp>::value; -#endif } template inline @@ -2132,51 +1769,6 @@ Mat_<_Tp>::Mat_(MatExpr&& e) ///////////////////////////// SparseMat ///////////////////////////// -inline -SparseMat::SparseMat() - : flags(MAGIC_VAL), hdr(0) -{} - -inline -SparseMat::SparseMat(int _dims, const int* _sizes, int _type) - : flags(MAGIC_VAL), hdr(0) -{ - create(_dims, _sizes, _type); -} - -inline -SparseMat::SparseMat(const SparseMat& m) - : flags(m.flags), hdr(m.hdr) -{ - addref(); -} - -inline -SparseMat::~SparseMat() -{ - release(); -} - -inline -SparseMat& SparseMat::operator = (const SparseMat& m) -{ - if( this != &m ) - { - if( m.hdr ) - CV_XADD(&m.hdr->refcount, 1); - release(); - flags = m.flags; - hdr = m.hdr; - } - return *this; -} - -inline -SparseMat& SparseMat::operator = (const Mat& m) -{ - return (*this = SparseMat(m)); -} - inline SparseMat SparseMat::clone() const { @@ -2185,30 +1777,6 @@ SparseMat SparseMat::clone() const return temp; } -inline -void SparseMat::assignTo( SparseMat& m, int _type ) const -{ - if( _type < 0 ) - m = *this; - else - convertTo(m, _type); -} - -inline -void SparseMat::addref() -{ - if( hdr ) - CV_XADD(&hdr->refcount, 1); -} - -inline -void SparseMat::release() -{ - if( hdr && CV_XADD(&hdr->refcount, -1) == 1 ) - delete hdr; - hdr = 0; -} - inline size_t SparseMat::elemSize() const { @@ -2268,36 +1836,6 @@ size_t SparseMat::nzcount() const return hdr ? hdr->nodeCount : 0; } -inline -size_t SparseMat::hash(int i0) const -{ - return (size_t)i0; -} - -inline -size_t SparseMat::hash(int i0, int i1) const -{ - return (size_t)(unsigned)i0 * HASH_SCALE + (unsigned)i1; -} - -inline -size_t SparseMat::hash(int i0, int i1, int i2) const -{ - return ((size_t)(unsigned)i0 * HASH_SCALE + (unsigned)i1) * HASH_SCALE + (unsigned)i2; -} - -inline -size_t SparseMat::hash(const int* idx) const -{ - size_t h = (unsigned)idx[0]; - if( !hdr ) - return 0; - int d = hdr->dims; - for(int i = 1; i < d; i++ ) - h = h * HASH_SCALE + (unsigned)idx[i]; - return h; -} - template inline _Tp& SparseMat::ref(int i0, size_t* hashval) { @@ -3617,74 +3155,6 @@ const Mat_<_Tp>& operator /= (const Mat_<_Tp>& a, const MatExpr& b) //////////////////////////////// UMat //////////////////////////////// -inline -UMat::UMat(UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{} - -inline -UMat::UMat(int _rows, int _cols, int _type, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create(_rows, _cols, _type); -} - -inline -UMat::UMat(int _rows, int _cols, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create(_rows, _cols, _type); - *this = _s; -} - -inline -UMat::UMat(Size _sz, int _type, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create( _sz.height, _sz.width, _type ); -} - -inline -UMat::UMat(Size _sz, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create(_sz.height, _sz.width, _type); - *this = _s; -} - -inline -UMat::UMat(int _dims, const int* _sz, int _type, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create(_dims, _sz, _type); -} - -inline -UMat::UMat(int _dims, const int* _sz, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) -: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) -{ - create(_dims, _sz, _type); - *this = _s; -} - -inline -UMat::UMat(const UMat& m) -: flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), allocator(m.allocator), - usageFlags(m.usageFlags), u(m.u), offset(m.offset), size(&rows) -{ - addref(); - if( m.dims <= 2 ) - { - step[0] = m.step[0]; step[1] = m.step[1]; - } - else - { - dims = 0; - copySize(m); - } -} - - template inline UMat::UMat(const std::vector<_Tp>& vec, bool copyData) : flags(MAGIC_VAL + traits::Type<_Tp>::value + CV_MAT_CONT_FLAG), dims(2), rows((int)vec.size()), @@ -3701,33 +3171,6 @@ cols(1), allocator(0), usageFlags(USAGE_DEFAULT), u(0), offset(0), size(&rows) Mat((int)vec.size(), 1, traits::Type<_Tp>::value, (uchar*)&vec[0]).copyTo(*this); } -inline -UMat& UMat::operator = (const UMat& m) -{ - if( this != &m ) - { - const_cast(m).addref(); - release(); - flags = m.flags; - if( dims <= 2 && m.dims <= 2 ) - { - dims = m.dims; - rows = m.rows; - cols = m.cols; - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - copySize(m); - allocator = m.allocator; - if (usageFlags == USAGE_DEFAULT) - usageFlags = m.usageFlags; - u = m.u; - offset = m.offset; - } - return *this; -} - inline UMat UMat::row(int y) const { @@ -3764,55 +3207,6 @@ UMat UMat::colRange(const Range& r) const return UMat(*this, Range::all(), r); } -inline -UMat UMat::clone() const -{ - UMat m; - copyTo(m); - return m; -} - -inline -void UMat::assignTo( UMat& m, int _type ) const -{ - if( _type < 0 ) - m = *this; - else - convertTo(m, _type); -} - -inline -void UMat::create(int _rows, int _cols, int _type, UMatUsageFlags _usageFlags) -{ - _type &= TYPE_MASK; - if( dims <= 2 && rows == _rows && cols == _cols && type() == _type && u ) - return; - int sz[] = {_rows, _cols}; - create(2, sz, _type, _usageFlags); -} - -inline -void UMat::create(Size _sz, int _type, UMatUsageFlags _usageFlags) -{ - create(_sz.height, _sz.width, _type, _usageFlags); -} - -inline -void UMat::addref() -{ - if( u ) - CV_XADD(&(u->urefcount), 1); -} - -inline void UMat::release() -{ - if( u && CV_XADD(&(u->urefcount), -1) == 1 ) - deallocate(); - for(int i = 0; i < dims; i++) - size.p[i] = 0; - u = 0; -} - inline UMat UMat::operator()( Range _rowRange, Range _colRange ) const { @@ -3887,83 +3281,6 @@ size_t UMat::step1(int i) const return step.p[i] / elemSize1(); } -inline -bool UMat::empty() const -{ - return u == 0 || total() == 0 || dims == 0; -} - -inline -size_t UMat::total() const -{ - if( dims <= 2 ) - return (size_t)rows * cols; - size_t p = 1; - for( int i = 0; i < dims; i++ ) - p *= size[i]; - return p; -} - -inline -UMat::UMat(UMat&& m) -: flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), allocator(m.allocator), - usageFlags(m.usageFlags), u(m.u), offset(m.offset), size(&rows) -{ - if (m.dims <= 2) // move new step/size info - { - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - { - CV_DbgAssert(m.step.p != m.step.buf); - step.p = m.step.p; - size.p = m.size.p; - m.step.p = m.step.buf; - m.size.p = &m.rows; - } - m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; - m.allocator = NULL; - m.u = NULL; - m.offset = 0; -} - -inline -UMat& UMat::operator = (UMat&& m) -{ - if (this == &m) - return *this; - release(); - flags = m.flags; dims = m.dims; rows = m.rows; cols = m.cols; - allocator = m.allocator; usageFlags = m.usageFlags; - u = m.u; - offset = m.offset; - if (step.p != step.buf) // release self step/size - { - fastFree(step.p); - step.p = step.buf; - size.p = &rows; - } - if (m.dims <= 2) // move new step/size info - { - step[0] = m.step[0]; - step[1] = m.step[1]; - } - else - { - CV_DbgAssert(m.step.p != m.step.buf); - step.p = m.step.p; - size.p = m.size.p; - m.step.p = m.step.buf; - m.size.p = &m.rows; - } - m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; - m.allocator = NULL; - m.u = NULL; - m.offset = 0; - return *this; -} - inline bool UMatData::hostCopyObsolete() const { return (flags & HOST_COPY_OBSOLETE) != 0; } inline bool UMatData::deviceCopyObsolete() const { return (flags & DEVICE_COPY_OBSOLETE) != 0; } diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 9e3f8f6914..276f640323 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -403,8 +403,8 @@ public: /** * @brief Simplified writing API to use with bindings. - * @param name Name of the written object - * @param val Value of the written object + * @param name Name of the written object. When writing to sequences (a.k.a. "arrays"), pass an empty string. + * @param val Value of the written object. */ CV_WRAP void write(const String& name, int val); /// @overload @@ -437,9 +437,10 @@ public: CV_WRAP void writeComment(const String& comment, bool append = false); /** @brief Starts to write a nested structure (sequence or a mapping). - @param name name of the structure (if it's a member of parent mapping, otherwise it should be empty + @param name name of the structure. When writing to sequences (a.k.a. "arrays"), pass an empty string. @param flags type of the structure (FileNode::MAP or FileNode::SEQ (both with optional FileNode::FLOW)). - @param typeName usually an empty string + @param typeName optional name of the type you store. The effect of setting this depends on the storage format. + I.e. if the format has a specification for storing type information, this parameter is used. */ CV_WRAP void startWriteStruct(const String& name, int flags, const String& typeName=String()); diff --git a/modules/core/include/opencv2/core/quaternion.hpp b/modules/core/include/opencv2/core/quaternion.hpp new file mode 100644 index 0000000000..c72ee8c37f --- /dev/null +++ b/modules/core/include/opencv2/core/quaternion.hpp @@ -0,0 +1,1194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2020, Huawei Technologies Co., Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Liangqian Kong +// Longbu Wang +#ifndef OPENCV_CORE_QUATERNION_HPP +#define OPENCV_CORE_QUATERNION_HPP + +#include +#include +namespace cv +{ +//! @addtogroup core +//! @{ + +//! Unit quaternion flag +enum QuatAssumeType +{ + /** + * This flag is specified by default. + * If this flag is specified, the input quaternions are assumed to be not unit quaternions. + * It can guarantee the correctness of the calculations, + * although the calculation speed will be slower than the flag QUAT_ASSUME_UNIT. + */ + QUAT_ASSUME_NOT_UNIT, + /** + * If this flag is specified, the input quaternions are assumed to be unit quaternions which + * will save some computations. However, if this flag is specified without unit quaternion, + * the program correctness of the result will not be guaranteed. + */ + QUAT_ASSUME_UNIT +}; + +template class Quat; +template std::ostream& operator<<(std::ostream&, const Quat<_Tp>&); + +/** + * Quaternion is a number system that extends the complex numbers. It can be expressed as a + * rotation in three-dimensional space. + * A quaternion is generally represented in the form: + * \f[q = w + x\boldsymbol{i} + y\boldsymbol{j} + z\boldsymbol{k}\f] + * \f[q = [w, x, y, z]\f] + * \f[q = [w, \boldsymbol{v}] \f] + * \f[q = ||q||[\cos\psi, u_x\sin\psi,u_y\sin\psi, u_z\sin\psi].\f] + * \f[q = ||q||[\cos\psi, \boldsymbol{u}\sin\psi]\f] + * where \f$\psi = \frac{\theta}{2}\f$, \f$\theta\f$ represents rotation angle, + * \f$\boldsymbol{u} = [u_x, u_y, u_z]\f$ represents normalized rotation axis, + * and \f$||q||\f$ represents the norm of \f$q\f$. + * + * A unit quaternion is usually represents rotation, which has the form: + * \f[q = [\cos\psi, u_x\sin\psi,u_y\sin\psi, u_z\sin\psi].\f] + * + * To create a quaternion representing the rotation around the axis \f$\boldsymbol{u}\f$ + * with angle \f$\theta\f$, you can use + * ``` + * using namespace cv; + * double angle = CV_PI; + * Vec3d axis = {0, 0, 1}; + * Quatd q = Quatd::createFromAngleAxis(angle, axis); + * ``` + * + * You can simply use four same type number to create a quaternion + * ``` + * Quatd q(1, 2, 3, 4); + * ``` + * Or use a Vec4d or Vec4f vector. + * ``` + * Vec4d vec{1, 2, 3, 4}; + * Quatd q(vec); + * ``` + * + * ``` + * Vec4f vec{1, 2, 3, 4}; + * Quatf q(vec); + * ``` + * + * If you already have a 3x3 rotation matrix R, then you can use + * ``` + * Quatd q = Quatd::createFromRotMat(R); + * ``` + * + * If you already have a rotation vector rvec which has the form of `angle * axis`, then you can use + * ``` + * Quatd q = Quatd::createFromRvec(rvec); + * ``` + * + * To extract the rotation matrix from quaternion, see toRotMat3x3() + * + * To extract the Vec4d or Vec4f, see toVec() + * + * To extract the rotation vector, see toRotVec() + * + * If there are two quaternions \f$q_0, q_1\f$ are needed to interpolate, you can use nlerp(), slerp() or spline() + * ``` + * Quatd::nlerp(q0, q1, t) + * + * Quatd::slerp(q0, q1, t) + * + * Quatd::spline(q0, q0, q1, q1, t) + * ``` + * spline can smoothly connect rotations of multiple quaternions + * + * Three ways to get an element in Quaternion + * ``` + * Quatf q(1,2,3,4); + * std::cout << q.w << std::endl; // w=1, x=2, y=3, z=4 + * std::cout << q[0] << std::endl; // q[0]=1, q[1]=2, q[2]=3, q[3]=4 + * std::cout << q.at(0) << std::endl; + * ``` + */ +template +class Quat +{ + static_assert(std::is_floating_point<_Tp>::value, "Quaternion only make sense with type of float or double"); + using value_type = _Tp; + +public: + static constexpr _Tp CV_QUAT_EPS = (_Tp)1.e-6; + + Quat(); + + /** + * @brief From Vec4d or Vec4f. + */ + explicit Quat(const Vec<_Tp, 4> &coeff); + + /** + * @brief from four numbers. + */ + Quat(_Tp w, _Tp x, _Tp y, _Tp z); + + /** + * @brief from an angle, axis. Axis will be normalized in this function. And + * it generates + * \f[q = [\cos\psi, u_x\sin\psi,u_y\sin\psi, u_z\sin\psi].\f] + * where \f$\psi = \frac{\theta}{2}\f$, \f$\theta\f$ is the rotation angle. + */ + static Quat<_Tp> createFromAngleAxis(const _Tp angle, const Vec<_Tp, 3> &axis); + + /** + * @brief from a 3x3 rotation matrix. + */ + static Quat<_Tp> createFromRotMat(InputArray R); + + /** + * @brief from a rotation vector + * \f$r\f$ has the form \f$\theta \cdot \boldsymbol{u}\f$, where \f$\theta\f$ + * represents rotation angle and \f$\boldsymbol{u}\f$ represents normalized rotation axis. + * + * Angle and axis could be easily derived as: + * \f[ + * \begin{equation} + * \begin{split} + * \psi &= ||r||\\ + * \boldsymbol{u} &= \frac{r}{\theta} + * \end{split} + * \end{equation} + * \f] + * Then a quaternion can be calculated by + * \f[q = [\cos\psi, \boldsymbol{u}\sin\psi]\f] + * where \f$\psi = \theta / 2 \f$ + */ + static Quat<_Tp> createFromRvec(InputArray rvec); + + /** + * @brief a way to get element. + * @param index over a range [0, 3]. + * + * A quaternion q + * + * q.at(0) is equivalent to q.w, + * + * q.at(1) is equivalent to q.x, + * + * q.at(2) is equivalent to q.y, + * + * q.at(3) is equivalent to q.z. + */ + _Tp at(size_t index) const; + + /** + * @brief return the conjugate of this quaternion. + * \f[q.conjugate() = (w, -x, -y, -z).\f] + */ + Quat<_Tp> conjugate() const; + + /** + * + * @brief return the value of exponential value. + * \f[\exp(q) = e^w (\cos||\boldsymbol{v}||+ \frac{v}{||\boldsymbol{v}||})\sin||\boldsymbol{v}||\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example: + * ``` + * Quatd q{1,2,3,4}; + * cout << exp(q) << endl; + * ``` + */ + template + friend Quat exp(const Quat &q); + + /** + * @brief return the value of exponential value. + * \f[\exp(q) = e^w (\cos||\boldsymbol{v}||+ \frac{v}{||\boldsymbol{v}||}\sin||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q{1,2,3,4}; + * cout << q.exp() << endl; + * ``` + */ + Quat<_Tp> exp() const; + + /** + * @brief return the value of logarithm function. + * \f[\ln(q) = \ln||q|| + \frac{\boldsymbol{v}}{||\boldsymbol{v}||}\arccos\frac{w}{||q||}.\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * @param assumeUnit if QUAT_ASSUME_UNIT, q assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q1{1,2,3,4}; + * cout << log(q1) << endl; + * ``` + */ + template + friend Quat log(const Quat &q, QuatAssumeType assumeUnit); + + /** + * @brief return the value of logarithm function. + * \f[\ln(q) = \ln||q|| + \frac{\boldsymbol{v}}{||\boldsymbol{v}||}\arccos\frac{w}{||q||}\f]. + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.log(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * Quatd q1(1,2,3,4); + * q1.normalize().log(assumeUnit); + * ``` + */ + Quat<_Tp> log(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return the value of power function with index \f$x\f$. + * \f[q^x = ||q||(cos(x\theta) + \boldsymbol{u}sin(x\theta))).\f] + * @param q a quaternion. + * @param x index of exponentiation. + * @param assumeUnit if QUAT_ASSUME_UNIT, quaternion q assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * power(q, 2); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * double angle = CV_PI; + * Vec3d axis{0, 0, 1}; + * Quatd q1 = Quatd::createFromAngleAxis(angle, axis); //generate a unit quat by axis and angle + * power(q1, 2, assumeUnit);//This assumeUnit means q1 is a unit quaternion. + * ``` + */ + template + friend Quat power(const Quat &q, _T x, QuatAssumeType assumeUnit); + + /** + * @brief return the value of power function with index \f$x\f$. + * \f[q^x = ||q||(\cos(x\theta) + \boldsymbol{u}\sin(x\theta))).\f] + * @param x index of exponentiation. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.power(2); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * double angle = CV_PI; + * Vec3d axis{0, 0, 1}; + * Quatd q1 = Quatd::createFromAngleAxis(angle, axis); //generate a unit quat by axis and angle + * q1.power(2, assumeUnit); //This assumeUnt means q1 is a unit quaternion + * ``` + */ + template + Quat<_Tp> power(_T x, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return \f$\sqrt{q}\f$. + * @param q a quaternion. + * @param assumeUnit if QUAT_ASSUME_UNIT, quaternion q assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatf q(1,2,3,4); + * sqrt(q); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q = {1,0,0,0}; + * sqrt(q, assumeUnit); //This assumeUnit means q is a unit quaternion. + * ``` + */ + template + friend Quat sqrt(const Quat &q, QuatAssumeType assumeUnit); + + /** + * @brief return \f$\sqrt{q}\f$. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatf q(1,2,3,4); + * q.sqrt(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q = {1,0,0,0}; + * q.sqrt(assumeUnit); //This assumeUnit means q is a unit quaternion + * ``` + */ + Quat<_Tp> sqrt(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return the value of power function with quaternion \f$q\f$. + * \f[p^q = e^{q\ln(p)}.\f] + * @param p base quaternion of power function. + * @param q index quaternion of power function. + * @param assumeUnit if QUAT_ASSUME_UNIT, quaternion \f$p\f$ assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd p(1,2,3,4); + * Quatd q(5,6,7,8); + * power(p, q); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * p = p.normalize(); + * power(p, q, assumeUnit); //This assumeUnit means p is a unit quaternion + * ``` + */ + template + friend Quat power(const Quat &p, const Quat &q, QuatAssumeType assumeUnit); + + /** + * @brief return the value of power function with quaternion \f$q\f$. + * \f[p^q = e^{q\ln(p)}.\f] + * @param q index quaternion of power function. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd p(1,2,3,4); + * Quatd q(5,6,7,8); + * p.power(q); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * p = p.normalize(); + * p.power(q, assumeUnit); //This assumeUnit means p is a unit quaternion + * ``` + */ + Quat<_Tp> power(const Quat<_Tp> &q, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return the crossProduct between \f$p = (a, b, c, d) = (a, \boldsymbol{u})\f$ and \f$q = (w, x, y, z) = (w, \boldsymbol{v})\f$. + * \f[p \times q = \frac{pq- qp}{2}\f] + * \f[p \times q = \boldsymbol{u} \times \boldsymbol{v}\f] + * \f[p \times q = (cz-dy)i + (dx-bz)j + (by-xc)k \f] + * + * For example + * ``` + * Quatd q{1,2,3,4}; + * Quatd p{5,6,7,8}; + * crossProduct(p, q); + * ``` + */ + template + friend Quat crossProduct(const Quat &p, const Quat &q); + + /** + * @brief return the crossProduct between \f$p = (a, b, c, d) = (a, \boldsymbol{u})\f$ and \f$q = (w, x, y, z) = (w, \boldsymbol{v})\f$. + * \f[p \times q = \frac{pq- qp}{2}.\f] + * \f[p \times q = \boldsymbol{u} \times \boldsymbol{v}.\f] + * \f[p \times q = (cz-dy)i + (dx-bz)j + (by-xc)k. \f] + * + * For example + * ``` + * Quatd q{1,2,3,4}; + * Quatd p{5,6,7,8}; + * p.crossProduct(q) + * ``` + */ + Quat<_Tp> crossProduct(const Quat<_Tp> &q) const; + + /** + * @brief return the norm of quaternion. + * \f[||q|| = \sqrt{w^2 + x^2 + y^2 + z^2}.\f] + */ + _Tp norm() const; + + /** + * @brief return a normalized \f$p\f$. + * \f[p = \frac{q}{||q||}\f] + * where \f$p\f$ satisfies \f$(p.x)^2 + (p.y)^2 + (p.z)^2 + (p.w)^2 = 1.\f$ + */ + Quat<_Tp> normalize() const; + + /** + * @brief return \f$q^{-1}\f$ which is an inverse of \f$q\f$ + * which satisfies \f$q * q^{-1} = 1\f$. + * @param q a quaternion. + * @param assumeUnit if QUAT_ASSUME_UNIT, quaternion q assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * inv(q); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q = q.normalize(); + * inv(q, assumeUnit);//This assumeUnit means p is a unit quaternion + * ``` + */ + template + friend Quat inv(const Quat &q, QuatAssumeType assumeUnit); + + /** + * @brief return \f$q^{-1}\f$ which is an inverse of \f$q\f$ + * satisfying \f$q * q^{-1} = 1\f$. + * @param assumeUnit if QUAT_ASSUME_UNIT, quaternion q assume to be a unit quaternion and this function will save some computations. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.inv(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q = q.normalize(); + * q.inv(assumeUnit); //assumeUnit means p is a unit quaternion + * ``` + */ + Quat<_Tp> inv(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return sinh value of quaternion q, sinh could be calculated as: + * \f[\sinh(p) = \sin(w)\cos(||\boldsymbol{v}||) + \cosh(w)\frac{v}{||\boldsymbol{v}||}\sin||\boldsymbol{v}||\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * sinh(q); + * ``` + */ + template + friend Quat sinh(const Quat &q); + + /** + * @brief return sinh value of this quaternion, sinh could be calculated as: + * \f$\sinh(p) = \sin(w)\cos(||\boldsymbol{v}||) + \cosh(w)\frac{v}{||\boldsymbol{v}||}\sin||\boldsymbol{v}||\f$ + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.sinh(); + * ``` + */ + Quat<_Tp> sinh() const; + + /** + * @brief return cosh value of quaternion q, cosh could be calculated as: + * \f[\cosh(p) = \cosh(w) * \cos(||\boldsymbol{v}||) + \sinh(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}\sin(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * cosh(q); + * ``` + */ + template + friend Quat cosh(const Quat &q); + + /** + * @brief return cosh value of this quaternion, cosh could be calculated as: + * \f[\cosh(p) = \cosh(w) * \cos(||\boldsymbol{v}||) + \sinh(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}sin(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.cosh(); + * ``` + */ + Quat<_Tp> cosh() const; + + /** + * @brief return tanh value of quaternion q, tanh could be calculated as: + * \f[ \tanh(q) = \frac{\sinh(q)}{\cosh(q)}.\f] + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * tanh(q); + * ``` + * @sa sinh, cosh + */ + template + friend Quat tanh(const Quat &q); + + /** + * @brief return tanh value of this quaternion, tanh could be calculated as: + * \f[ \tanh(q) = \frac{\sinh(q)}{\cosh(q)}.\f] + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.tanh(); + * ``` + * @sa sinh, cosh + */ + Quat<_Tp> tanh() const; + + /** + * @brief return tanh value of quaternion q, sin could be calculated as: + * \f[\sin(p) = \sin(w) * \cosh(||\boldsymbol{v}||) + \cos(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}\sinh(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * sin(q); + * ``` + */ + template + friend Quat sin(const Quat &q); + + /** + * @brief return sin value of this quaternion, sin could be calculated as: + * \f[\sin(p) = \sin(w) * \cosh(||\boldsymbol{v}||) + \cos(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}\sinh(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.sin(); + * ``` + */ + Quat<_Tp> sin() const; + + /** + * @brief return sin value of quaternion q, cos could be calculated as: + * \f[\cos(p) = \cos(w) * \cosh(||\boldsymbol{v}||) - \sin(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}\sinh(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * cos(q); + * ``` + */ + template + friend Quat cos(const Quat &q); + + /** + * @brief return cos value of this quaternion, cos could be calculated as: + * \f[\cos(p) = \cos(w) * \cosh(||\boldsymbol{v}||) - \sin(w)\frac{\boldsymbol{v}}{||\boldsymbol{v}||}\sinh(||\boldsymbol{v}||)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.cos(); + * ``` + */ + Quat<_Tp> cos() const; + + /** + * @brief return tan value of quaternion q, tan could be calculated as: + * \f[\tan(q) = \frac{\sin(q)}{\cos(q)}.\f] + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * tan(q); + * ``` + */ + template + friend Quat tan(const Quat &q); + + /** + * @brief return tan value of this quaternion, tan could be calculated as: + * \f[\tan(q) = \frac{\sin(q)}{\cos(q)}.\f] + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.tan(); + * ``` + */ + Quat<_Tp> tan() const; + + /** + * @brief return arcsin value of quaternion q, arcsin could be calculated as: + * \f[\arcsin(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arcsinh(q\frac{\boldsymbol{v}}{||\boldsymbol{v}||})\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * asin(q); + * ``` + */ + template + friend Quat asin(const Quat &q); + + /** + * @brief return arcsin value of this quaternion, arcsin could be calculated as: + * \f[\arcsin(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arcsinh(q\frac{\boldsymbol{v}}{||\boldsymbol{v}||})\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.asin(); + * ``` + */ + Quat<_Tp> asin() const; + + /** + * @brief return arccos value of quaternion q, arccos could be calculated as: + * \f[\arccos(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arccosh(q)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * acos(q); + * ``` + */ + template + friend Quat acos(const Quat &q); + + /** + * @brief return arccos value of this quaternion, arccos could be calculated as: + * \f[\arccos(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arccosh(q)\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.acos(); + * ``` + */ + Quat<_Tp> acos() const; + + /** + * @brief return arctan value of quaternion q, arctan could be calculated as: + * \f[\arctan(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arctanh(q\frac{\boldsymbol{v}}{||\boldsymbol{v}||})\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * atan(q); + * ``` + */ + template + friend Quat atan(const Quat &q); + + /** + * @brief return arctan value of this quaternion, arctan could be calculated as: + * \f[\arctan(q) = -\frac{\boldsymbol{v}}{||\boldsymbol{v}||}arctanh(q\frac{\boldsymbol{v}}{||\boldsymbol{v}||})\f] + * where \f$\boldsymbol{v} = [x, y, z].\f$ + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.atan(); + * ``` + */ + Quat<_Tp> atan() const; + + /** + * @brief return arcsinh value of quaternion q, arcsinh could be calculated as: + * \f[arcsinh(q) = \ln(q + \sqrt{q^2 + 1})\f]. + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * asinh(q); + * ``` + */ + template + friend Quat asinh(const Quat &q); + + /** + * @brief return arcsinh value of this quaternion, arcsinh could be calculated as: + * \f[arcsinh(q) = \ln(q + \sqrt{q^2 + 1})\f]. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.asinh(); + * ``` + */ + Quat<_Tp> asinh() const; + + /** + * @brief return arccosh value of quaternion q, arccosh could be calculated as: + * \f[arccosh(q) = \ln(q + \sqrt{q^2 - 1})\f]. + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * acosh(q); + * ``` + */ + template + friend Quat acosh(const Quat &q); + + /** + * @brief return arccosh value of this quaternion, arccosh could be calculated as: + * \f[arcosh(q) = \ln(q + \sqrt{q^2 - 1})\f]. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.acosh(); + * ``` + */ + Quat<_Tp> acosh() const; + + /** + * @brief return arctanh value of quaternion q, arctanh could be calculated as: + * \f[arctanh(q) = \frac{\ln(q + 1) - \ln(1 - q)}{2}\f]. + * @param q a quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * atanh(q); + * ``` + */ + template + friend Quat atanh(const Quat &q); + + /** + * @brief return arctanh value of this quaternion, arctanh could be calculated as: + * \f[arcsinh(q) = \frac{\ln(q + 1) - \ln(1 - q)}{2}\f]. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.atanh(); + * ``` + */ + Quat<_Tp> atanh() const; + + /** + * @brief return true if this quaternion is a unit quaternion. + * @param eps tolerance scope of normalization. The eps could be defined as + * + * \f[eps = |1 - dotValue|\f] where \f[dotValue = (this.w^2 + this.x^2 + this,y^2 + this.z^2).\f] + * And this function will consider it is normalized when the dotValue over a range \f$[1-eps, 1+eps]\f$. + */ + bool isNormal(_Tp eps=CV_QUAT_EPS) const; + + /** + * @brief to throw an error if this quaternion is not a unit quaternion. + * @param eps tolerance scope of normalization. + * @sa isNormal + */ + void assertNormal(_Tp eps=CV_QUAT_EPS) const; + + /** + * @brief transform a quaternion to a 3x3 rotation matrix. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and + * this function will save some computations. Otherwise, this function will normalized this + * quaternion at first then to do the transformation. + * + * @note Matrix A which is to be rotated should have the form + * \f[\begin{bmatrix} + * x_0& x_1& x_2&...&x_n\\ + * y_0& y_1& y_2&...&y_n\\ + * z_0& z_1& z_2&...&z_n + * \end{bmatrix}\f] + * where the same subscript represents a point. The shape of A assume to be [3, n] + * The points matrix A can be rotated by toRotMat3x3() * A. + * The result has 3 rows and n columns too. + + * For example + * ``` + * double angle = CV_PI; + * Vec3d axis{0,0,1}; + * Quatd q_unit = Quatd::createFromAngleAxis(angle, axis); //quaternion could also be get by interpolation by two or more quaternions. + * + * //assume there is two points (1,0,0) and (1,0,1) to be rotated + * Mat pointsA = (Mat_(2, 3) << 1,0,0,1,0,1); + * //change the shape + * pointsA = pointsA.t(); + * // rotate 180 degrees around the z axis + * Mat new_point = q_unit.toRotMat3x3() * pointsA; + * // print two points + * cout << new_point << endl; + * ``` + */ + Matx<_Tp, 3, 3> toRotMat3x3(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief transform a quaternion to a 4x4 rotation matrix. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and + * this function will save some computations. Otherwise, this function will normalized this + * quaternion at first then to do the transformation. + * + * The operations is similar as toRotMat3x3 + * except that the points matrix should have the form + * \f[\begin{bmatrix} + * x_0& x_1& x_2&...&x_n\\ + * y_0& y_1& y_2&...&y_n\\ + * z_0& z_1& z_2&...&z_n\\ + * 0&0&0&...&0 + * \end{bmatrix}\f] + * + * @sa toRotMat3x3 + */ + Matx<_Tp, 4, 4> toRotMat4x4(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief transform the this quaternion to a Vec. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.toVec(); + * ``` + */ + Vec<_Tp, 4> toVec() const; + + /** + * @brief transform this quaternion to a Rotation vector. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and + * this function will save some computations. + * Rotation vector rVec is defined as: + * \f[ rVec = [\theta v_x, \theta v_y, \theta v_z]\f] + * where \f$\theta\f$ represents rotation angle, and \f$\boldsymbol{v}\f$ represents the normalized rotation axis. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.toRotVec(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q.normalize().toRotVec(assumeUnit); //answer is same as q.toRotVec(). + * ``` + */ + Vec<_Tp, 3> toRotVec(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief get the angle of quaternion, it returns the rotation angle. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and + * this function will save some computations. + * \f[\psi = 2 *arccos(\frac{w}{||q||})\f] + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.getAngle(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q.normalize().getAngle(assumeUnit);//same as q.getAngle(). + * ``` + * @note It always return the value between \f$[0, 2\pi]\f$. + */ + _Tp getAngle(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief get the axis of quaternion, it returns a vector of length 3. + * @param assumeUnit if QUAT_ASSUME_UNIT, this quaternion assume to be a unit quaternion and + * this function will save some computations. + * + * the unit axis \f$\boldsymbol{u}\f$ is defined by + * \f[\begin{equation} + * \begin{split} + * \boldsymbol{v} + * &= \boldsymbol{u} ||\boldsymbol{v}||\\ + * &= \boldsymbol{u}||q||sin(\frac{\theta}{2}) + * \end{split} + * \end{equation}\f] + * where \f$v=[x, y ,z]\f$ and \f$\theta\f$ represents rotation angle. + * + * + * For example + * ``` + * Quatd q(1,2,3,4); + * q.getAxis(); + * + * QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + * q.normalize().getAxis(assumeUnit);//same as q.getAxis() + * ``` + */ + Vec<_Tp, 3> getAxis(QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT) const; + + /** + * @brief return the dot between quaternion \f$q\f$ and this quaternion. + * + * dot(p, q) is a good metric of how close the quaternions are. + * Indeed, consider the unit quaternion difference \f$p^{-1} * q\f$, its real part is dot(p, q). + * At the same time its real part is equal to \f$\cos(\beta/2)\f$ where \f$\beta\f$ is + * an angle of rotation between p and q, i.e., + * Therefore, the closer dot(p, q) to 1, + * the smaller rotation between them. + * \f[p \cdot q = p.w \cdot q.w + p.x \cdot q.x + p.y \cdot q.y + p.z \cdot q.z\f] + * @param q the other quaternion. + * + * For example + * ``` + * Quatd q(1,2,3,4); + * Quatd p(5,6,7,8); + * p.dot(q); + * ``` + */ + _Tp dot(Quat<_Tp> q) const; + + /** + * @brief To calculate the interpolation from \f$q_0\f$ to \f$q_1\f$ by Linear Interpolation(Nlerp) + * For two quaternions, this interpolation curve can be displayed as: + * \f[Lerp(q_0, q_1, t) = (1 - t)q_0 + tq_1.\f] + * Obviously, the lerp will interpolate along a straight line if we think of \f$q_0\f$ and \f$q_1\f$ as a vector + * in a two-dimensional space. When \f$t = 0\f$, it returns \f$q_0\f$ and when \f$t= 1\f$, it returns \f$q_1\f$. + * \f$t\f$ should to be ranged in \f$[0, 1]\f$ normally. + * @param q0 a quaternion used in linear interpolation. + * @param q1 a quaternion used in linear interpolation. + * @param t percent of vector \f$\overrightarrow{q_0q_1}\f$ over a range [0, 1]. + * @note it returns a non-unit quaternion. + */ + static Quat<_Tp> lerp(const Quat<_Tp> &q0, const Quat &q1, const _Tp t); + + /** + * @brief To calculate the interpolation from \f$q_0\f$ to \f$q_1\f$ by Normalized Linear Interpolation(Nlerp). + * it returns a normalized quaternion of Linear Interpolation(Lerp). + * \f[ Nlerp(q_0, q_1, t) = \frac{(1 - t)q_0 + tq_1}{||(1 - t)q_0 + tq_1||}.\f] + * The interpolation will always choose the shortest path but the constant speed is not guaranteed. + * @param q0 a quaternion used in normalized linear interpolation. + * @param q1 a quaternion used in normalized linear interpolation. + * @param t percent of vector \f$\overrightarrow{q_0q_1}\f$ over a range [0, 1]. + * @param assumeUnit if QUAT_ASSUME_UNIT, all input quaternions assume to be unit quaternion. Otherwise, all inputs + quaternion will be normalized inside the function. + * @sa lerp + */ + static Quat<_Tp> nlerp(const Quat<_Tp> &q0, const Quat &q1, const _Tp t, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + + /** + @brief To calculate the interpolation between \f$q_0\f$ and \f$q_1\f$ by Spherical Linear + Interpolation(Slerp), which can be defined as: + \f[ Slerp(q_0, q_1, t) = \frac{\sin((1-t)\theta)}{\sin(\theta)}q_0 + \frac{\sin(t\theta)}{\sin(\theta)}q_1\f] + where \f$\theta\f$ can be calculated as: + \f[\theta=cos^{-1}(q_0\cdot q_1)\f] + resulting from the both of their norm is unit. + @param q0 a quaternion used in Slerp. + @param q1 a quaternion used in Slerp. + @param t percent of angle between \f$q_0\f$ and \f$q_1\f$ over a range [0, 1]. + @param assumeUnit if QUAT_ASSUME_UNIT, all input quaternions assume to be unit quaternions. Otherwise, all input + quaternions will be normalized inside the function. + @param directChange if QUAT_ASSUME_UNIT, the interpolation will choose the nearest path. + @note If the interpolation angle is small, the error between Nlerp and Slerp is not so large. To improve efficiency and + avoid zero division error, we use Nlerp instead of Slerp. + */ + static Quat<_Tp> slerp(const Quat<_Tp> &q0, const Quat &q1, const _Tp t, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT, bool directChange=true); + + /** + * @brief To calculate the interpolation between \f$q_0\f$,\f$q_1\f$,\f$q_2\f$,\f$q_3\f$ by Spherical and quadrangle(Squad). This could be defined as: + * \f[Squad(q_i, s_i, s_{i+1}, q_{i+1}, t) = Slerp(Slerp(q_i, q_{i+1}, t), Slerp(s_i, s_{i+1}, t), 2t(1-t))\f] + * where + * \f[s_i = q_i\exp(-\frac{\log(q^*_iq_{i+1}) + \log(q^*_iq_{i-1})}{4})\f] + * + * The Squad expression is analogous to the \f$B\acute{e}zier\f$ curve, but involves spherical linear + * interpolation instead of simple linear interpolation. Each \f$s_i\f$ needs to be calculated by three + * quaternions. + * + * @param q0 the first quaternion. + * @param s0 the second quaternion. + * @param s1 the third quaternion. + * @param q1 thr fourth quaternion. + * @param t interpolation parameter of quadratic and linear interpolation over a range \f$[0, 1]\f$. + * @param assumeUnit if QUAT_ASSUME_UNIT, all input quaternions assume to be unit quaternion. Otherwise, all input + * quaternions will be normalized inside the function. + * @param directChange if QUAT_ASSUME_UNIT, squad will find the nearest path to interpolate. + * @sa interPoint, spline + */ + static Quat<_Tp> squad(const Quat<_Tp> &q0, const Quat<_Tp> &s0, + const Quat<_Tp> &s1, const Quat<_Tp> &q1, + const _Tp t, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT, + bool directChange=true); + + /** + * @brief This is the part calculation of squad. + * To calculate the intermedia quaternion \f$s_i\f$ between each three quaternion + * \f[s_i = q_i\exp(-\frac{\log(q^*_iq_{i+1}) + \log(q^*_iq_{i-1})}{4}).\f] + * @param q0 the first quaternion. + * @param q1 the second quaternion. + * @param q2 the third quaternion. + * @param assumeUnit if QUAT_ASSUME_UNIT, all input quaternions assume to be unit quaternion. Otherwise, all input + * quaternions will be normalized inside the function. + * @sa squad + */ + static Quat<_Tp> interPoint(const Quat<_Tp> &q0, const Quat<_Tp> &q1, + const Quat<_Tp> &q2, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + + /** + * @brief to calculate a quaternion which is the result of a \f$C^1\f$ continuous + * spline curve constructed by squad at the ratio t. Here, the interpolation values are + * between \f$q_1\f$ and \f$q_2\f$. \f$q_0\f$ and \f$q_2\f$ are used to ensure the \f$C^1\f$ + * continuity. if t = 0, it returns \f$q_1\f$, if t = 1, it returns \f$q_2\f$. + * @param q0 the first input quaternion to ensure \f$C^1\f$ continuity. + * @param q1 the second input quaternion. + * @param q2 the third input quaternion. + * @param q3 the fourth input quaternion the same use of \f$q1\f$. + * @param t ratio over a range [0, 1]. + * @param assumeUnit if QUAT_ASSUME_UNIT, \f$q_0, q_1, q_2, q_3\f$ assume to be unit quaternion. Otherwise, all input + * quaternions will be normalized inside the function. + * + * For example: + * + * If there are three double quaternions \f$v_0, v_1, v_2\f$ waiting to be interpolated. + * + * Interpolation between \f$v_0\f$ and \f$v_1\f$ with a ratio \f$t_0\f$ could be calculated as + * ``` + * Quatd::spline(v0, v0, v1, v2, t0); + * ``` + * Interpolation between \f$v_1\f$ and \f$v_2\f$ with a ratio \f$t_0\f$ could be calculated as + * ``` + * Quatd::spline(v0, v1, v2, v2, t0); + * ``` + * @sa squad, slerp + */ + static Quat<_Tp> spline(const Quat<_Tp> &q0, const Quat<_Tp> &q1, + const Quat<_Tp> &q2, const Quat<_Tp> &q3, + const _Tp t, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + + + Quat<_Tp> operator-() const; + + bool operator==(const Quat<_Tp>&) const; + + Quat<_Tp> operator+(const Quat<_Tp>&) const; + + Quat<_Tp>& operator+=(const Quat<_Tp>&); + + Quat<_Tp> operator-(const Quat<_Tp>&) const; + + Quat<_Tp>& operator-=(const Quat<_Tp>&); + + Quat<_Tp>& operator*=(const Quat<_Tp>&); + + Quat<_Tp>& operator*=(const _Tp&); + + Quat<_Tp> operator*(const Quat<_Tp>&) const; + + Quat<_Tp> operator/(const _Tp&) const; + + Quat<_Tp> operator/(const Quat<_Tp>&) const; + + Quat<_Tp>& operator/=(const _Tp&); + + Quat<_Tp>& operator/=(const Quat<_Tp>&); + + _Tp& operator[](std::size_t n); + + const _Tp& operator[](std::size_t n) const; + + template + friend Quat cv::operator*(const T, const Quat&); + + template + friend Quat cv::operator*(const Quat&, const T); + + template + friend std::ostream& cv::operator<<(std::ostream&, const Quat&); + + _Tp w, x, y, z; + +}; + +template +Quat inv(const Quat &q, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + +template +Quat sinh(const Quat &q); + +template +Quat cosh(const Quat &q); + +template +Quat tanh(const Quat &q); + +template +Quat sin(const Quat &q); + +template +Quat cos(const Quat &q); + +template +Quat tan(const Quat &q); + +template +Quat asinh(const Quat &q); + +template +Quat acosh(const Quat &q); + +template +Quat atanh(const Quat &q); + +template +Quat asin(const Quat &q); + +template +Quat acos(const Quat &q); + +template +Quat atan(const Quat &q); + +template +Quat power(const Quat &q, const Quat &p, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + +template +Quat exp(const Quat &q); + +template +Quat log(const Quat &q, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + +template +Quat power(const Quat& q, _T x, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + +template +Quat crossProduct(const Quat &p, const Quat &q); + +template +Quat sqrt(const Quat &q, QuatAssumeType assumeUnit=QUAT_ASSUME_NOT_UNIT); + +template +Quat operator*(const T, const Quat&); + +template +Quat operator*(const Quat&, const T); + +template +std::ostream& operator<<(std::ostream&, const Quat&); + +using Quatd = Quat; +using Quatf = Quat; + +//! @} core +} + +#include "opencv2/core/quaternion.inl.hpp" + +#endif /* OPENCV_CORE_QUATERNION_HPP */ diff --git a/modules/core/include/opencv2/core/quaternion.inl.hpp b/modules/core/include/opencv2/core/quaternion.inl.hpp new file mode 100644 index 0000000000..769f53ed4b --- /dev/null +++ b/modules/core/include/opencv2/core/quaternion.inl.hpp @@ -0,0 +1,849 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2020, Huawei Technologies Co., Ltd. All rights reserved. +// Third party copyrights are property of their respective owners. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Liangqian Kong +// Longbu Wang + +#ifndef OPENCV_CORE_QUATERNION_INL_HPP +#define OPENCV_CORE_QUATERNION_INL_HPP + +#ifndef OPENCV_CORE_QUATERNION_HPP +#erorr This is not a standalone header. Include quaternion.hpp instead. +#endif + +//@cond IGNORE +/////////////////////////////////////////////////////////////////////////////////////// +//Implementation +namespace cv { + +template +Quat::Quat() : w(0), x(0), y(0), z(0) {} + +template +Quat::Quat(const Vec &coeff):w(coeff[0]), x(coeff[1]), y(coeff[2]), z(coeff[3]){} + +template +Quat::Quat(const T qw, const T qx, const T qy, const T qz):w(qw), x(qx), y(qy), z(qz){} + +template +Quat Quat::createFromAngleAxis(const T angle, const Vec &axis) +{ + T w, x, y, z; + T vNorm = std::sqrt(axis.dot(axis)); + if (vNorm < CV_QUAT_EPS) + { + CV_Error(Error::StsBadArg, "this quaternion does not represent a rotation"); + } + const T angle_half = angle * 0.5; + w = std::cos(angle_half); + const T sin_v = std::sin(angle_half); + const T sin_norm = sin_v / vNorm; + x = sin_norm * axis[0]; + y = sin_norm * axis[1]; + z = sin_norm * axis[2]; + return Quat(w, x, y, z); +} + +template +Quat Quat::createFromRotMat(InputArray _R) +{ + CV_CheckTypeEQ(_R.type(), cv::traits::Type::value, ""); + if (_R.rows() != 3 || _R.cols() != 3) + { + CV_Error(Error::StsBadArg, "Cannot convert matrix to quaternion: rotation matrix should be a 3x3 matrix"); + } + Matx R; + _R.copyTo(R); + + T S, w, x, y, z; + T trace = R(0, 0) + R(1, 1) + R(2, 2); + if (trace > 0) + { + S = std::sqrt(trace + 1) * 2; + x = (R(1, 2) - R(2, 1)) / S; + y = (R(2, 0) - R(0, 2)) / S; + z = (R(0, 1) - R(1, 0)) / S; + w = -0.25 * S; + } + else if (R(0, 0) > R(1, 1) && R(0, 0) > R(2, 2)) + { + + S = std::sqrt(1.0 + R(0, 0) - R(1, 1) - R(2, 2)) * 2; + x = -0.25 * S; + y = -(R(1, 0) + R(0, 1)) / S; + z = -(R(0, 2) + R(2, 0)) / S; + w = (R(1, 2) - R(2, 1)) / S; + } + else if (R(1, 1) > R(2, 2)) + { + S = std::sqrt(1.0 - R(0, 0) + R(1, 1) - R(2, 2)) * 2; + x = (R(0, 1) + R(1, 0)) / S; + y = 0.25 * S; + z = (R(1, 2) + R(2, 1)) / S; + w = (R(0, 2) - R(2, 0)) / S; + } + else + { + S = std::sqrt(1.0 - R(0, 0) - R(1, 1) + R(2, 2)) * 2; + x = (R(0, 2) + R(2, 0)) / S; + y = (R(1, 2) + R(2, 1)) / S; + z = 0.25 * S; + w = -(R(0, 1) - R(1, 0)) / S; + } + return Quat (w, x, y, z); +} + +template +Quat Quat::createFromRvec(InputArray _rvec) +{ + if (!((_rvec.cols() == 1 && _rvec.rows() == 3) || (_rvec.cols() == 3 && _rvec.rows() == 1))) { + CV_Error(Error::StsBadArg, "Cannot convert rotation vector to quaternion: The length of rotation vector should be 3"); + } + Vec rvec; + _rvec.copyTo(rvec); + T psi = std::sqrt(rvec.dot(rvec)); + if (abs(psi) < CV_QUAT_EPS) { + return Quat (1, 0, 0, 0); + } + Vec axis = rvec / psi; + return createFromAngleAxis(psi, axis); +} + +template +inline Quat Quat::operator-() const +{ + return Quat(-w, -x, -y, -z); +} + + +template +inline bool Quat::operator==(const Quat &q) const +{ + return (abs(w - q.w) < CV_QUAT_EPS && abs(x - q.x) < CV_QUAT_EPS && abs(y - q.y) < CV_QUAT_EPS && abs(z - q.z) < CV_QUAT_EPS); +} + +template +inline Quat Quat::operator+(const Quat &q1) const +{ + return Quat(w + q1.w, x + q1.x, y + q1.y, z + q1.z); +} + +template +inline Quat Quat::operator-(const Quat &q1) const +{ + return Quat(w - q1.w, x - q1.x, y - q1.y, z - q1.z); +} + +template +inline Quat& Quat::operator+=(const Quat &q1) +{ + w += q1.w; + x += q1.x; + y += q1.y; + z += q1.z; + return *this; +} + +template +inline Quat& Quat::operator-=(const Quat &q1) +{ + w -= q1.w; + x -= q1.x; + y -= q1.y; + z -= q1.z; + return *this; +} + +template +inline Quat Quat::operator*(const Quat &q1) const +{ + Vec q{w, x, y, z}; + Vec q2{q1.w, q1.x, q1.y, q1.z}; + return Quat(q * q2); +} + + +template +Quat operator*(const Quat &q1, const S a) +{ + return Quat(a * q1.w, a * q1.x, a * q1.y, a * q1.z); +} + +template +Quat operator*(const S a, const Quat &q1) +{ + return Quat(a * q1.w, a * q1.x, a * q1.y, a * q1.z); +} + +template +inline Quat& Quat::operator*=(const Quat &q1) +{ + T qw, qx, qy, qz; + qw = w * q1.w - x * q1.x - y * q1.y - z * q1.z; + qx = x * q1.w + w * q1.x + y * q1.z - z * q1.y; + qy = y * q1.w + w * q1.y + z * q1.x - x * q1.z; + qz = z * q1.w + w * q1.z + x * q1.y - y * q1.x; + w = qw; + x = qx; + y = qy; + z = qz; + return *this; +} + +template +inline Quat& Quat::operator/=(const Quat &q1) +{ + Quat q(*this * q1.inv()); + w = q.w; + x = q.x; + y = q.y; + z = q.z; + return *this; +} +template +Quat& Quat::operator*=(const T &q1) +{ + w *= q1; + x *= q1; + y *= q1; + z *= q1; + return *this; +} + +template +inline Quat& Quat::operator/=(const T &a) +{ + const T a_inv = 1.0 / a; + w *= a_inv; + x *= a_inv; + y *= a_inv; + z *= a_inv; + return *this; +} + +template +inline Quat Quat::operator/(const T &a) const +{ + const T a_inv = 1.0 / a; + return Quat(w * a_inv, x * a_inv, y * a_inv, z * a_inv); +} + +template +inline Quat Quat::operator/(const Quat &q) const +{ + return *this * q.inv(); +} + +template +inline const T& Quat::operator[](std::size_t n) const +{ + switch (n) { + case 0: + return w; + case 1: + return x; + case 2: + return y; + case 3: + return z; + default: + CV_Error(Error::StsOutOfRange, "subscript exceeds the index range"); + } +} + +template +inline T& Quat::operator[](std::size_t n) +{ + switch (n) { + case 0: + return w; + case 1: + return x; + case 2: + return y; + case 3: + return z; + default: + CV_Error(Error::StsOutOfRange, "subscript exceeds the index range"); + } +} + +template +std::ostream & operator<<(std::ostream &os, const Quat &q) +{ + os << "Quat " << Vec{q.w, q.x, q.y, q.z}; + return os; +} + +template +inline T Quat::at(size_t index) const +{ + return (*this)[index]; +} + +template +inline Quat Quat::conjugate() const +{ + return Quat(w, -x, -y, -z); +} + +template +inline T Quat::norm() const +{ + return std::sqrt(dot(*this)); +} + +template +Quat exp(const Quat &q) +{ + return q.exp(); +} + +template +Quat Quat::exp() const +{ + Vec v{x, y, z}; + T normV = std::sqrt(v.dot(v)); + T k = normV < CV_QUAT_EPS ? 1 : std::sin(normV) / normV; + return std::exp(w) * Quat(std::cos(normV), v[0] * k, v[1] * k, v[2] * k); +} + +template +Quat log(const Quat &q, QuatAssumeType assumeUnit) +{ + return q.log(assumeUnit); +} + +template +Quat Quat::log(QuatAssumeType assumeUnit) const +{ + Vec v{x, y, z}; + T vNorm = std::sqrt(v.dot(v)); + if (assumeUnit) + { + T k = vNorm < CV_QUAT_EPS ? 1 : std::acos(w) / vNorm; + return Quat(0, v[0] * k, v[1] * k, v[2] * k); + } + T qNorm = norm(); + if (qNorm < CV_QUAT_EPS) + { + CV_Error(Error::StsBadArg, "Cannot apply this quaternion to log function: undefined"); + } + T k = vNorm < CV_QUAT_EPS ? 1 : std::acos(w / qNorm) / vNorm; + return Quat(std::log(qNorm), v[0] * k, v[1] * k, v[2] *k); +} + +template +inline Quat power(const Quat &q1, _T alpha, QuatAssumeType assumeUnit) +{ + return q1.power(alpha, assumeUnit); +} + +template +template +inline Quat Quat::power(_T alpha, QuatAssumeType assumeUnit) const +{ + if (x * x + y * y + z * z > CV_QUAT_EPS) + { + T angle = getAngle(assumeUnit); + Vec axis = getAxis(assumeUnit); + if (assumeUnit) + { + return createFromAngleAxis(alpha * angle, axis); + } + return std::pow(norm(), alpha) * createFromAngleAxis(alpha * angle, axis); + } + else + { + return std::pow(norm(), alpha) * Quat(w, x, y, z); + } +} + + +template +inline Quat sqrt(const Quat &q, QuatAssumeType assumeUnit) +{ + return q.sqrt(assumeUnit); +} + +template +inline Quat Quat::sqrt(QuatAssumeType assumeUnit) const +{ + return power(0.5, assumeUnit); +} + + +template +inline Quat power(const Quat &p, const Quat &q, QuatAssumeType assumeUnit) +{ + return p.power(q, assumeUnit); +} + + +template +inline Quat Quat::power(const Quat &q, QuatAssumeType assumeUnit) const +{ + return cv::exp(q * log(assumeUnit)); +} + +template +inline T Quat::dot(Quat q1) const +{ + return w * q1.w + x * q1.x + y * q1.y + z * q1.z; +} + + +template +inline Quat crossProduct(const Quat &p, const Quat &q) +{ + return p.crossProduct(q); +} + + +template +inline Quat Quat::crossProduct(const Quat &q) const +{ + return Quat (0, y * q.z - z * q.y, z * q.x - x * q.z, x * q.y - q.x * y); +} + +template +inline Quat Quat::normalize() const +{ + T normVal = norm(); + if (normVal < CV_QUAT_EPS) + { + CV_Error(Error::StsBadArg, "Cannot normalize this quaternion: the norm is too small."); + } + return Quat(w / normVal, x / normVal, y / normVal, z / normVal) ; +} + +template +inline Quat inv(const Quat &q, QuatAssumeType assumeUnit) +{ + return q.inv(assumeUnit); +} + + +template +inline Quat Quat::inv(QuatAssumeType assumeUnit) const +{ + if (assumeUnit) + { + return conjugate(); + } + T norm2 = dot(*this); + if (norm2 < CV_QUAT_EPS) + { + CV_Error(Error::StsBadArg, "This quaternion do not have inverse quaternion"); + } + return conjugate() / norm2; +} + +template +inline Quat sinh(const Quat &q) +{ + return q.sinh(); +} + + +template +inline Quat Quat::sinh() const +{ + Vec v{x, y ,z}; + T vNorm = std::sqrt(v.dot(v)); + T k = vNorm < CV_QUAT_EPS ? 1 : std::cosh(w) * std::sin(vNorm) / vNorm; + return Quat(std::sinh(w) * std::cos(vNorm), v[0] * k, v[1] * k, v[2] * k); +} + + +template +inline Quat cosh(const Quat &q) +{ + return q.cosh(); +} + + +template +inline Quat Quat::cosh() const +{ + Vec v{x, y ,z}; + T vNorm = std::sqrt(v.dot(v)); + T k = vNorm < CV_QUAT_EPS ? 1 : std::sinh(w) * std::sin(vNorm) / vNorm; + return Quat(std::cosh(w) * std::cos(vNorm), v[0] * k, v[1] * k, v[2] * k); +} + +template +inline Quat tanh(const Quat &q) +{ + return q.tanh(); +} + +template +inline Quat Quat::tanh() const +{ + return sinh() * cosh().inv(); +} + + +template +inline Quat sin(const Quat &q) +{ + return q.sin(); +} + + +template +inline Quat Quat::sin() const +{ + Vec v{x, y ,z}; + T vNorm = std::sqrt(v.dot(v)); + T k = vNorm < CV_QUAT_EPS ? 1 : std::cos(w) * std::sinh(vNorm) / vNorm; + return Quat(std::sin(w) * std::cosh(vNorm), v[0] * k, v[1] * k, v[2] * k); +} + +template +inline Quat cos(const Quat &q) +{ + return q.cos(); +} + +template +inline Quat Quat::cos() const +{ + Vec v{x, y ,z}; + T vNorm = std::sqrt(v.dot(v)); + T k = vNorm < CV_QUAT_EPS ? 1 : std::sin(w) * std::sinh(vNorm) / vNorm; + return Quat(std::cos(w) * std::cosh(vNorm), -v[0] * k, -v[1] * k, -v[2] * k); +} + +template +inline Quat tan(const Quat &q) +{ + return q.tan(); +} + +template +inline Quat Quat::tan() const +{ + return sin() * cos().inv(); +} + +template +inline Quat asinh(const Quat &q) +{ + return q.asinh(); +} + +template +inline Quat Quat::asinh() const +{ + return cv::log(*this + cv::power(*this * *this + Quat(1, 0, 0, 0), 0.5)); +} + +template +inline Quat acosh(const Quat &q) +{ + return q.acosh(); +} + +template +inline Quat Quat::acosh() const +{ + return cv::log(*this + cv::power(*this * *this - Quat(1,0,0,0), 0.5)); +} + +template +inline Quat atanh(const Quat &q) +{ + return q.atanh(); +} + +template +inline Quat Quat::atanh() const +{ + Quat ident(1, 0, 0, 0); + Quat c1 = (ident + *this).log(); + Quat c2 = (ident - *this).log(); + return 0.5 * (c1 - c2); +} + +template +inline Quat asin(const Quat &q) +{ + return q.asin(); +} + +template +inline Quat Quat::asin() const +{ + Quat v(0, x, y, z); + T vNorm = v.norm(); + T k = vNorm < CV_QUAT_EPS ? 1 : vNorm; + return -v / k * (*this * v / k).asinh(); +} + +template +inline Quat acos(const Quat &q) +{ + return q.acos(); +} + +template +inline Quat Quat::acos() const +{ + Quat v(0, x, y, z); + T vNorm = v.norm(); + T k = vNorm < CV_QUAT_EPS ? 1 : vNorm; + return -v / k * acosh(); +} + +template +inline Quat atan(const Quat &q) +{ + return q.atan(); +} + +template +inline Quat Quat::atan() const +{ + Quat v(0, x, y, z); + T vNorm = v.norm(); + T k = vNorm < CV_QUAT_EPS ? 1 : vNorm; + return -v / k * (*this * v / k).atanh(); +} + +template +inline T Quat::getAngle(QuatAssumeType assumeUnit) const +{ + if (assumeUnit) + { + return 2 * std::acos(w); + } + if (norm() < CV_QUAT_EPS) + { + CV_Error(Error::StsBadArg, "This quaternion does not represent a rotation"); + } + return 2 * std::acos(w / norm()); +} + +template +inline Vec Quat::getAxis(QuatAssumeType assumeUnit) const +{ + T angle = getAngle(assumeUnit); + const T sin_v = std::sin(angle * 0.5); + if (assumeUnit) + { + return Vec{x, y, z} / sin_v; + } + return Vec {x, y, z} / (norm() * sin_v); +} + +template +Matx Quat::toRotMat4x4(QuatAssumeType assumeUnit) const +{ + T a = w, b = x, c = y, d = z; + if (!assumeUnit) + { + Quat qTemp = normalize(); + a = qTemp.w; + b = qTemp.x; + c = qTemp.y; + d = qTemp.z; + } + Matx R{ + 1 - 2 * (c * c + d * d), 2 * (b * c - a * d) , 2 * (b * d + a * c) , 0, + 2 * (b * c + a * d) , 1 - 2 * (b * b + d * d), 2 * (c * d - a * b) , 0, + 2 * (b * d - a * c) , 2 * (c * d + a * b) , 1 - 2 * (b * b + c * c), 0, + 0 , 0 , 0 , 1, + }; + return R; +} + +template +Matx Quat::toRotMat3x3(QuatAssumeType assumeUnit) const +{ + T a = w, b = x, c = y, d = z; + if (!assumeUnit) + { + Quat qTemp = normalize(); + a = qTemp.w; + b = qTemp.x; + c = qTemp.y; + d = qTemp.z; + } + Matx R{ + 1 - 2 * (c * c + d * d), 2 * (b * c - a * d) , 2 * (b * d + a * c), + 2 * (b * c + a * d) , 1 - 2 * (b * b + d * d), 2 * (c * d - a * b), + 2 * (b * d - a * c) , 2 * (c * d + a * b) , 1 - 2 * (b * b + c * c) + }; + return R; +} + +template +Vec Quat::toRotVec(QuatAssumeType assumeUnit) const +{ + T angle = getAngle(assumeUnit); + Vec axis = getAxis(assumeUnit); + return angle * axis; +} + +template +Vec Quat::toVec() const +{ + return Vec{w, x, y, z}; +} + +template +Quat Quat::lerp(const Quat &q0, const Quat &q1, const T t) +{ + return (1 - t) * q0 + t * q1; +} + +template +Quat Quat::slerp(const Quat &q0, const Quat &q1, const T t, QuatAssumeType assumeUnit, bool directChange) +{ + Quatd v0(q0); + Quatd v1(q1); + if (!assumeUnit) + { + v0 = v0.normalize(); + v1 = v1.normalize(); + } + T cosTheta = v0.dot(v1); + constexpr T DOT_THRESHOLD = 0.995; + if (cosTheta > DOT_THRESHOLD) + { + return nlerp(v0, v1, t, QUAT_ASSUME_UNIT); + } + + if (directChange && cosTheta < 0) + { + v0 = -v0; + cosTheta = -cosTheta; + } + T sinTheta = std::sqrt(1 - cosTheta * cosTheta); + T angle = atan2(sinTheta, cosTheta); + return (std::sin((1 - t) * angle) / (sinTheta) * v0 + std::sin(t * angle) / (sinTheta) * v1).normalize(); +} + + +template +inline Quat Quat::nlerp(const Quat &q0, const Quat &q1, const T t, QuatAssumeType assumeUnit) +{ + Quat v0(q0), v1(q1); + if (v1.dot(v0) < 0) + { + v0 = -v0; + } + if (assumeUnit) + { + return ((1 - t) * v0 + t * v1).normalize(); + } + v0 = v0.normalize(); + v1 = v1.normalize(); + return ((1 - t) * v0 + t * v1).normalize(); +} + + +template +inline bool Quat::isNormal(T eps) const +{ + + double normVar = norm(); + if ((normVar > 1 - eps) && (normVar < 1 + eps)) + return true; + return false; +} + +template +inline void Quat::assertNormal(T eps) const +{ + if (!isNormal(eps)) + CV_Error(Error::StsBadArg, "Quaternion should be normalized"); +} + + +template +inline Quat Quat::squad(const Quat &q0, const Quat &q1, + const Quat &q2, const Quat &q3, + const T t, QuatAssumeType assumeUnit, + bool directChange) +{ + Quat v0(q0), v1(q1), v2(q2), v3(q3); + if (!assumeUnit) + { + v0 = v0.normalize(); + v1 = v1.normalize(); + v2 = v2.normalize(); + v3 = v3.normalize(); + } + + Quat c0 = slerp(v0, v3, t, assumeUnit, directChange); + Quat c1 = slerp(v1, v2, t, assumeUnit, directChange); + return slerp(c0, c1, 2 * t * (1 - t), assumeUnit, directChange); +} + +template +Quat Quat::interPoint(const Quat &q0, const Quat &q1, + const Quat &q2, QuatAssumeType assumeUnit) +{ + Quat v0(q0), v1(q1), v2(q2); + if (!assumeUnit) + { + v0 = v0.normalize(); + v1 = v1.normalize(); + v2 = v2.normalize(); + } + return v1 * cv::exp(-(cv::log(v1.conjugate() * v0, assumeUnit) + (cv::log(v1.conjugate() * v2, assumeUnit))) / 4); +} + +template +Quat Quat::spline(const Quat &q0, const Quat &q1, const Quat &q2, const Quat &q3, const T t, QuatAssumeType assumeUnit) +{ + Quatd v0(q0), v1(q1), v2(q2), v3(q3); + if (!assumeUnit) + { + v0 = v0.normalize(); + v1 = v1.normalize(); + v2 = v2.normalize(); + v3 = v3.normalize(); + } + T cosTheta; + std::vector> vec{v0, v1, v2, v3}; + for (size_t i = 0; i < 3; ++i) + { + cosTheta = vec[i].dot(vec[i + 1]); + if (cosTheta < 0) + { + vec[i + 1] = -vec[i + 1]; + } + } + Quat s1 = interPoint(vec[0], vec[1], vec[2], QUAT_ASSUME_UNIT); + Quat s2 = interPoint(vec[1], vec[2], vec[3], QUAT_ASSUME_UNIT); + return squad(vec[1], s1, s2, vec[2], t, assumeUnit, QUAT_ASSUME_NOT_UNIT); +} + +} // namepsace +//! @endcond + +#endif /*OPENCV_CORE_QUATERNION_INL_HPP*/ diff --git a/modules/core/misc/objc/common/Converters.h b/modules/core/misc/objc/common/Converters.h index 9a238deb82..29d1b91eb5 100755 --- a/modules/core/misc/objc/common/Converters.h +++ b/modules/core/misc/objc/common/Converters.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import +#import #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/CvType.h b/modules/core/misc/objc/common/CvType.h index fb6f86aa48..b1fd71d487 100644 --- a/modules/core/misc/objc/common/CvType.h +++ b/modules/core/misc/objc/common/CvType.h @@ -5,7 +5,7 @@ // #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/DMatch.h b/modules/core/misc/objc/common/DMatch.h index 51bed493b8..91c2c59bfa 100644 --- a/modules/core/misc/objc/common/DMatch.h +++ b/modules/core/misc/objc/common/DMatch.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Double2.h b/modules/core/misc/objc/common/Double2.h index 2162acb6d0..8e46c883d0 100644 --- a/modules/core/misc/objc/common/Double2.h +++ b/modules/core/misc/objc/common/Double2.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Double3.h b/modules/core/misc/objc/common/Double3.h index 2aaba9af80..5c741648f7 100644 --- a/modules/core/misc/objc/common/Double3.h +++ b/modules/core/misc/objc/common/Double3.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Float4.h b/modules/core/misc/objc/common/Float4.h index 2a89278040..c78e88b72e 100644 --- a/modules/core/misc/objc/common/Float4.h +++ b/modules/core/misc/objc/common/Float4.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Float6.h b/modules/core/misc/objc/common/Float6.h index d2ec19a60e..7e09772c5c 100644 --- a/modules/core/misc/objc/common/Float6.h +++ b/modules/core/misc/objc/common/Float6.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Int4.h b/modules/core/misc/objc/common/Int4.h index 1a17266572..11cc12db14 100644 --- a/modules/core/misc/objc/common/Int4.h +++ b/modules/core/misc/objc/common/Int4.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/KeyPoint.h b/modules/core/misc/objc/common/KeyPoint.h index 547960dc9d..096a1089c9 100644 --- a/modules/core/misc/objc/common/KeyPoint.h +++ b/modules/core/misc/objc/common/KeyPoint.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Mat.h b/modules/core/misc/objc/common/Mat.h index 72f81dd9b7..fd1dce27ba 100644 --- a/modules/core/misc/objc/common/Mat.h +++ b/modules/core/misc/objc/common/Mat.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif @@ -97,6 +97,7 @@ CV_EXPORTS @interface Mat : NSObject - (void)createEx:(NSArray*)sizes type:(int)type NS_SWIFT_NAME(create(sizes:type:)); - (void)copySize:(Mat*)mat; - (Mat*)cross:(Mat*)mat; +- (unsigned char*)dataPtr NS_SWIFT_NAME(dataPointer()); - (int)depth; - (Mat*)diag:(int)diagonal; - (Mat*)diag; diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index c075e26046..5d41a3622e 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -286,6 +286,10 @@ static bool updateIdx(cv::Mat* mat, std::vector& indices, int inc) { return [[Mat alloc] initWithNativeMat:new cv::Mat(_nativePtr->cross(*(cv::Mat*)mat.nativePtr))]; } +- (unsigned char*)dataPtr { + return _nativePtr->data; +} + - (int)depth { return _nativePtr->depth(); } diff --git a/modules/core/misc/objc/common/MinMaxLocResult.h b/modules/core/misc/objc/common/MinMaxLocResult.h index e8daed4cc3..5ec6029e31 100644 --- a/modules/core/misc/objc/common/MinMaxLocResult.h +++ b/modules/core/misc/objc/common/MinMaxLocResult.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point2d.h b/modules/core/misc/objc/common/Point2d.h index dbb8d55efa..0426b11d9a 100644 --- a/modules/core/misc/objc/common/Point2d.h +++ b/modules/core/misc/objc/common/Point2d.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point2f.h b/modules/core/misc/objc/common/Point2f.h index 0da4fba5d8..6d13c774d8 100644 --- a/modules/core/misc/objc/common/Point2f.h +++ b/modules/core/misc/objc/common/Point2f.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point2i.h b/modules/core/misc/objc/common/Point2i.h index 9e5d74624a..e43ee3a8ec 100644 --- a/modules/core/misc/objc/common/Point2i.h +++ b/modules/core/misc/objc/common/Point2i.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point3d.h b/modules/core/misc/objc/common/Point3d.h index 72b0d39ea8..618ded35fa 100644 --- a/modules/core/misc/objc/common/Point3d.h +++ b/modules/core/misc/objc/common/Point3d.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point3f.h b/modules/core/misc/objc/common/Point3f.h index 2370fffeaa..c98add1cec 100644 --- a/modules/core/misc/objc/common/Point3f.h +++ b/modules/core/misc/objc/common/Point3f.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Point3i.h b/modules/core/misc/objc/common/Point3i.h index b0edeaa470..9eab2ee0ea 100644 --- a/modules/core/misc/objc/common/Point3i.h +++ b/modules/core/misc/objc/common/Point3i.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Range.h b/modules/core/misc/objc/common/Range.h index 91e76393d2..fae138c6f9 100644 --- a/modules/core/misc/objc/common/Range.h +++ b/modules/core/misc/objc/common/Range.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Rect2d.h b/modules/core/misc/objc/common/Rect2d.h index ba91509b77..0ffcae9ab6 100644 --- a/modules/core/misc/objc/common/Rect2d.h +++ b/modules/core/misc/objc/common/Rect2d.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Rect2f.h b/modules/core/misc/objc/common/Rect2f.h index 6a8863800f..1f44f56263 100644 --- a/modules/core/misc/objc/common/Rect2f.h +++ b/modules/core/misc/objc/common/Rect2f.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Rect2i.h b/modules/core/misc/objc/common/Rect2i.h index 2e4e55cf30..6ed86d50bd 100644 --- a/modules/core/misc/objc/common/Rect2i.h +++ b/modules/core/misc/objc/common/Rect2i.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/RotatedRect.h b/modules/core/misc/objc/common/RotatedRect.h index c94053b6c1..a2049e6bf0 100644 --- a/modules/core/misc/objc/common/RotatedRect.h +++ b/modules/core/misc/objc/common/RotatedRect.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Scalar.h b/modules/core/misc/objc/common/Scalar.h index 63c3d1de58..d565155010 100644 --- a/modules/core/misc/objc/common/Scalar.h +++ b/modules/core/misc/objc/common/Scalar.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Size2d.h b/modules/core/misc/objc/common/Size2d.h index 11c6c50a02..cd2e4e4bc0 100644 --- a/modules/core/misc/objc/common/Size2d.h +++ b/modules/core/misc/objc/common/Size2d.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Size2f.h b/modules/core/misc/objc/common/Size2f.h index 2d1f2865c3..73ae9a2da0 100644 --- a/modules/core/misc/objc/common/Size2f.h +++ b/modules/core/misc/objc/common/Size2f.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/Size2i.h b/modules/core/misc/objc/common/Size2i.h index 61aa8da885..cd74e2c84a 100644 --- a/modules/core/misc/objc/common/Size2i.h +++ b/modules/core/misc/objc/common/Size2i.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/common/TermCriteria.h b/modules/core/misc/objc/common/TermCriteria.h index c7396582b2..ff6bfd565c 100644 --- a/modules/core/misc/objc/common/TermCriteria.h +++ b/modules/core/misc/objc/common/TermCriteria.h @@ -7,7 +7,7 @@ #pragma once #ifdef __cplusplus -#import "opencv.hpp" +#import "opencv2/core.hpp" #else #define CV_EXPORTS #endif diff --git a/modules/core/misc/objc/gen_dict.json b/modules/core/misc/objc/gen_dict.json index 25521c9326..c2ee554eba 100644 --- a/modules/core/misc/objc/gen_dict.json +++ b/modules/core/misc/objc/gen_dict.json @@ -217,6 +217,13 @@ "from_cpp": "[NSString stringWithUTF8String:%(n)s.c_str()]", "swift_type": "String" }, + "string": { + "cast_to": "std::string", + "objc_type": "NSString*", + "to_cpp": "std::string(%(n)s.UTF8String)", + "from_cpp": "[NSString stringWithUTF8String:%(n)s.c_str()]", + "swift_type": "String" + }, "TermCriteria": { "objc_type": "TermCriteria*", "to_cpp": "%(n)s.nativeRef", diff --git a/modules/core/src/convert.simd.hpp b/modules/core/src/convert.simd.hpp index 001fc78264..5154041b6d 100644 --- a/modules/core/src/convert.simd.hpp +++ b/modules/core/src/convert.simd.hpp @@ -5,6 +5,11 @@ #include "precomp.hpp" #include "convert.hpp" +#if !defined(OPENCV_SUPRESS_WARNING_AVX2_WITHOUT_FP16C) && \ + (defined(__GNUC__) && defined(__AVX2__) && !defined(__F16C__)) +#warning "Non-optimal compiler flags: AVX2 without FP16. Generated code is very slow. Consider adding '-mf16c' compiler option." +#endif + namespace cv { namespace hal { CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN diff --git a/modules/core/src/copy.cpp b/modules/core/src/copy.cpp index 7f4329df78..dcd585d834 100644 --- a/modules/core/src/copy.cpp +++ b/modules/core/src/copy.cpp @@ -1032,8 +1032,7 @@ void flip( InputArray _src, OutputArray _dst, int flip_mode ) } if ((size.width == 1 && flip_mode > 0) || - (size.height == 1 && flip_mode == 0) || - (size.height == 1 && size.width == 1 && flip_mode < 0)) + (size.height == 1 && flip_mode == 0)) { return _src.copyTo(_dst); } diff --git a/modules/core/src/directx.cpp b/modules/core/src/directx.cpp index c9bd1a4fa1..f028702d7f 100644 --- a/modules/core/src/directx.cpp +++ b/modules/core/src/directx.cpp @@ -49,6 +49,7 @@ #ifdef HAVE_DIRECTX #include #include "directx.inc.hpp" +#include "directx.hpp" #else // HAVE_DIRECTX #define NO_DIRECTX_SUPPORT_ERROR CV_Error(cv::Error::StsBadFunc, "OpenCV was build without DirectX support") #endif @@ -234,11 +235,191 @@ int getTypeFromD3DFORMAT(const int iD3DFORMAT) #endif } -namespace ocl { - #if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) -static bool g_isDirect3DDevice9Ex = false; // Direct3DDevice9Ex or Direct3DDevice9 was used +namespace internal { +struct OpenCLDirectXImpl +{ + cl_platform_id platform_; + + cl_platform_id initializedPlatform9 = NULL; + cl_platform_id initializedPlatform10 = NULL; + cl_platform_id initializedPlatform11 = NULL; +public: + OpenCLDirectXImpl() + : platform_(0) + { + } + + bool isDirect3DDevice9Ex = false; // Direct3DDevice9Ex or Direct3DDevice9 was used + +#ifdef HAVE_OPENCL_D3D11_NV + clCreateFromD3D11Texture2DNV_fn clCreateFromD3D11Texture2DNV = NULL; + clEnqueueAcquireD3D11ObjectsNV_fn clEnqueueAcquireD3D11ObjectsNV = NULL; + clEnqueueReleaseD3D11ObjectsNV_fn clEnqueueReleaseD3D11ObjectsNV = NULL; #endif + clCreateFromD3D11Texture2DKHR_fn clCreateFromD3D11Texture2DKHR = NULL; + clEnqueueAcquireD3D11ObjectsKHR_fn clEnqueueAcquireD3D11ObjectsKHR = NULL; + clEnqueueReleaseD3D11ObjectsKHR_fn clEnqueueReleaseD3D11ObjectsKHR = NULL; + + clCreateFromD3D10Texture2DKHR_fn clCreateFromD3D10Texture2DKHR = NULL; + clEnqueueAcquireD3D10ObjectsKHR_fn clEnqueueAcquireD3D10ObjectsKHR = NULL; + clEnqueueReleaseD3D10ObjectsKHR_fn clEnqueueReleaseD3D10ObjectsKHR = NULL; + + clCreateFromDX9MediaSurfaceKHR_fn clCreateFromDX9MediaSurfaceKHR = NULL; + clEnqueueAcquireDX9MediaSurfacesKHR_fn clEnqueueAcquireDX9MediaSurfacesKHR = NULL; + clEnqueueReleaseDX9MediaSurfacesKHR_fn clEnqueueReleaseDX9MediaSurfacesKHR = NULL; + + cl_platform_id getPlatform() + { + if (!platform_) + { + CV_Assert(cv::ocl::haveOpenCL()); + + cl_device_id device = (cl_device_id)ocl::Device::getDefault().ptr(); + CV_Assert(device); + cl_int status = clGetDeviceInfo(device, CL_DEVICE_PLATFORM, sizeof(platform_), &platform_, NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't get platform corresponding to device"); + } + + return platform_; + } + + + bool initializeD3D11() + { + using namespace cv::ocl; + cl_platform_id platform = getPlatform(); + + bool useCLNVEXT = false; + size_t exts_len; + cl_int status = clGetPlatformInfo(platform, CL_PLATFORM_EXTENSIONS, 0, NULL, &exts_len); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't get length of CL_PLATFORM_EXTENSIONS"); + cv::AutoBuffer extensions(exts_len); + status = clGetPlatformInfo(platform, CL_PLATFORM_EXTENSIONS, exts_len, static_cast(extensions.data()), NULL); + if (status != CL_SUCCESS) + CV_Error(cv::Error::OpenCLInitError, "OpenCL: No available CL_PLATFORM_EXTENSIONS"); + bool is_support_cl_khr_d3d11_sharing = false; + if (strstr(extensions.data(), "cl_khr_d3d11_sharing")) + is_support_cl_khr_d3d11_sharing = true; +#ifdef HAVE_OPENCL_D3D11_NV + bool is_support_cl_nv_d3d11_sharing = false; + if (strstr(extensions.data(), "cl_nv_d3d11_sharing")) + is_support_cl_nv_d3d11_sharing = true; + if (!is_support_cl_nv_d3d11_sharing && !is_support_cl_khr_d3d11_sharing) + CV_Error(cv::Error::OpenCLInitError, "OpenCL: No supported extensions"); +#else + if (!is_support_cl_khr_d3d11_sharing) + CV_Error(cv::Error::OpenCLInitError, "OpenCL: No supported extensions"); +#endif + +#ifdef HAVE_OPENCL_D3D11_NV + if (is_support_cl_nv_d3d11_sharing) + { + if (initializedPlatform11 != platform) + { + clCreateFromD3D11Texture2DNV = (clCreateFromD3D11Texture2DNV_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D11Texture2DNV"); + clEnqueueAcquireD3D11ObjectsNV = (clEnqueueAcquireD3D11ObjectsNV_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D11ObjectsNV"); + clEnqueueReleaseD3D11ObjectsNV = (clEnqueueReleaseD3D11ObjectsNV_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D11ObjectsNV"); + initializedPlatform11 = platform; + } + if (clCreateFromD3D11Texture2DNV && clEnqueueAcquireD3D11ObjectsNV && clEnqueueReleaseD3D11ObjectsNV) + { + useCLNVEXT = true; + } + } + else +#endif + { + if (is_support_cl_khr_d3d11_sharing) + { + if (initializedPlatform11 != platform) + { + clCreateFromD3D11Texture2DKHR = (clCreateFromD3D11Texture2DKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D11Texture2DKHR"); + clEnqueueAcquireD3D11ObjectsKHR = (clEnqueueAcquireD3D11ObjectsKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D11ObjectsKHR"); + clEnqueueReleaseD3D11ObjectsKHR = (clEnqueueReleaseD3D11ObjectsKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D11ObjectsKHR"); + initializedPlatform11 = platform; + } + if (!clCreateFromD3D11Texture2DKHR || !clEnqueueAcquireD3D11ObjectsKHR || !clEnqueueReleaseD3D11ObjectsKHR) + { + CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D11"); + } + } + } + return useCLNVEXT; + } + + void initializeD3D9() + { + using namespace cv::ocl; + cl_platform_id platform = getPlatform(); + if (initializedPlatform9 != platform) + { + clCreateFromDX9MediaSurfaceKHR = (clCreateFromDX9MediaSurfaceKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromDX9MediaSurfaceKHR"); + clEnqueueAcquireDX9MediaSurfacesKHR = (clEnqueueAcquireDX9MediaSurfacesKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireDX9MediaSurfacesKHR"); + clEnqueueReleaseDX9MediaSurfacesKHR = (clEnqueueReleaseDX9MediaSurfacesKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseDX9MediaSurfacesKHR"); + initializedPlatform9 = platform; + } + if (!clCreateFromDX9MediaSurfaceKHR || !clEnqueueAcquireDX9MediaSurfacesKHR || !clEnqueueReleaseDX9MediaSurfacesKHR) + { + CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D9"); + } + } + + void initializeD3D10() + { + using namespace cv::ocl; + cl_platform_id platform = getPlatform(); + if (initializedPlatform10 != platform) + { + clCreateFromD3D10Texture2DKHR = (clCreateFromD3D10Texture2DKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D10Texture2DKHR"); + clEnqueueAcquireD3D10ObjectsKHR = (clEnqueueAcquireD3D10ObjectsKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D10ObjectsKHR"); + clEnqueueReleaseD3D10ObjectsKHR = (clEnqueueReleaseD3D10ObjectsKHR_fn) + clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D10ObjectsKHR"); + initializedPlatform10 = platform; + } + if (!clCreateFromD3D10Texture2DKHR || !clEnqueueAcquireD3D10ObjectsKHR || !clEnqueueReleaseD3D10ObjectsKHR) + { + CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D10"); + } + } +}; + +OpenCLDirectXImpl* createDirectXImpl() +{ + return new OpenCLDirectXImpl(); +} +void deleteDirectXImpl(OpenCLDirectXImpl** p) +{ + if (*p) + { + delete (*p); + *p = NULL; + } +} +OpenCLDirectXImpl& getImpl() +{ + OpenCLDirectXImpl* i = getDirectXImpl(ocl::Context::getDefault()); + CV_Assert(i); + return *i; +} +} +using namespace internal; +#endif + +namespace ocl { Context& initializeContextFromD3D11Device(ID3D11Device* pD3D11Device) { @@ -715,7 +896,7 @@ Context& initializeContextFromDirect3DDevice9Ex(IDirect3DDevice9Ex* pDirect3DDev throw; } clExecCtx.bind(); - g_isDirect3DDevice9Ex = true; + getImpl().isDirect3DDevice9Ex = true; return const_cast(clExecCtx.getContext()); #endif } @@ -838,96 +1019,13 @@ Context& initializeContextFromDirect3DDevice9(IDirect3DDevice9* pDirect3DDevice9 throw; } clExecCtx.bind(); - g_isDirect3DDevice9Ex = false; + getImpl().isDirect3DDevice9Ex = false; return const_cast(clExecCtx.getContext()); #endif } } // namespace cv::ocl -#if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) - -#ifdef HAVE_OPENCL_D3D11_NV -clCreateFromD3D11Texture2DNV_fn clCreateFromD3D11Texture2DNV = NULL; -clEnqueueAcquireD3D11ObjectsNV_fn clEnqueueAcquireD3D11ObjectsNV = NULL; -clEnqueueReleaseD3D11ObjectsNV_fn clEnqueueReleaseD3D11ObjectsNV = NULL; -#endif -clCreateFromD3D11Texture2DKHR_fn clCreateFromD3D11Texture2DKHR = NULL; -clEnqueueAcquireD3D11ObjectsKHR_fn clEnqueueAcquireD3D11ObjectsKHR = NULL; -clEnqueueReleaseD3D11ObjectsKHR_fn clEnqueueReleaseD3D11ObjectsKHR = NULL; - -static bool __OpenCLinitializeD3D11() -{ - using namespace cv::ocl; - static cl_platform_id initializedPlatform = NULL; - cl_platform_id platform = (cl_platform_id)Platform::getDefault().ptr(); - - bool useCLNVEXT = false; - size_t exts_len; - cl_int status = clGetPlatformInfo(platform, CL_PLATFORM_EXTENSIONS, 0, NULL, &exts_len); - if (status != CL_SUCCESS) - CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't get length of CL_PLATFORM_EXTENSIONS"); - cv::AutoBuffer extensions(exts_len); - status = clGetPlatformInfo(platform, CL_PLATFORM_EXTENSIONS, exts_len, static_cast(extensions.data()), NULL); - if (status != CL_SUCCESS) - CV_Error(cv::Error::OpenCLInitError, "OpenCL: No available CL_PLATFORM_EXTENSIONS"); - bool is_support_cl_khr_d3d11_sharing = false; - if (strstr(extensions.data(), "cl_khr_d3d11_sharing")) - is_support_cl_khr_d3d11_sharing = true; -#ifdef HAVE_OPENCL_D3D11_NV - bool is_support_cl_nv_d3d11_sharing = false; - if (strstr(extensions.data(), "cl_nv_d3d11_sharing")) - is_support_cl_nv_d3d11_sharing = true; - if (!is_support_cl_nv_d3d11_sharing && !is_support_cl_khr_d3d11_sharing) - CV_Error(cv::Error::OpenCLInitError, "OpenCL: No supported extensions"); -#else - if (!is_support_cl_khr_d3d11_sharing) - CV_Error(cv::Error::OpenCLInitError, "OpenCL: No supported extensions"); -#endif - -#ifdef HAVE_OPENCL_D3D11_NV - if (is_support_cl_nv_d3d11_sharing) - { - if (initializedPlatform != platform) - { - clCreateFromD3D11Texture2DNV = (clCreateFromD3D11Texture2DNV_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D11Texture2DNV"); - clEnqueueAcquireD3D11ObjectsNV = (clEnqueueAcquireD3D11ObjectsNV_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D11ObjectsNV"); - clEnqueueReleaseD3D11ObjectsNV = (clEnqueueReleaseD3D11ObjectsNV_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D11ObjectsNV"); - initializedPlatform = platform; - } - if (clCreateFromD3D11Texture2DNV && clEnqueueAcquireD3D11ObjectsNV && clEnqueueReleaseD3D11ObjectsNV) - { - useCLNVEXT = true; - } - } - else -#endif - { - if (is_support_cl_khr_d3d11_sharing) - { - if (initializedPlatform != platform) - { - clCreateFromD3D11Texture2DKHR = (clCreateFromD3D11Texture2DKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D11Texture2DKHR"); - clEnqueueAcquireD3D11ObjectsKHR = (clEnqueueAcquireD3D11ObjectsKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D11ObjectsKHR"); - clEnqueueReleaseD3D11ObjectsKHR = (clEnqueueReleaseD3D11ObjectsKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D11ObjectsKHR"); - initializedPlatform = platform; - } - if (!clCreateFromD3D11Texture2DKHR || !clEnqueueAcquireD3D11ObjectsKHR || !clEnqueueReleaseD3D11ObjectsKHR) - { - CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D11"); - } - } - } - return useCLNVEXT; -} -#endif // defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) - } // namespace directx @@ -1009,20 +1107,21 @@ static void __convertToD3D11Texture2DKHR(InputArray src, ID3D11Texture2D* pD3D11 using namespace cv::ocl; Context& ctx = Context::getDefault(); cl_context context = (cl_context)ctx.ptr(); + OpenCLDirectXImpl& impl = getImpl(); cl_int status = 0; cl_mem clImage = 0; #ifdef HAVE_DIRECTX_NV12 cl_mem clImageUV = 0; #endif - clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); + clImage = impl.clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); #ifdef HAVE_DIRECTX_NV12 if(DXGI_FORMAT_NV12 == desc.Format) { - clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status); + clImageUV = impl.clCreateFromD3D11Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); } @@ -1030,21 +1129,21 @@ static void __convertToD3D11Texture2DKHR(InputArray src, ID3D11Texture2D* pD3D11 cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); #ifdef HAVE_DIRECTX_NV12 if(DXGI_FORMAT_NV12 == desc.Format) { - status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); if(!ocl::ocl_convert_bgr_to_nv12(clBuffer, (int)u.step[0], u.cols, u.rows, clImage, clImageUV)) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_bgr_to_nv12 failed"); - status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); } @@ -1060,7 +1159,7 @@ static void __convertToD3D11Texture2DKHR(InputArray src, ID3D11Texture2D* pD3D11 CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); } - status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); @@ -1107,40 +1206,41 @@ static void __convertToD3D11Texture2DNV(InputArray src, ID3D11Texture2D* pD3D11T using namespace cv::ocl; Context& ctx = Context::getDefault(); cl_context context = (cl_context)ctx.ptr(); + OpenCLDirectXImpl& impl = getImpl(); cl_int status = 0; cl_mem clImage = 0; #ifdef HAVE_DIRECTX_NV12 cl_mem clImageUV = 0; #endif - clImage = clCreateFromD3D11Texture2DNV(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); + clImage = impl.clCreateFromD3D11Texture2DNV(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DNV failed"); #ifdef HAVE_DIRECTX_NV12 if (DXGI_FORMAT_NV12 == desc.Format) { - clImageUV = clCreateFromD3D11Texture2DNV(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status); + clImageUV = impl.clCreateFromD3D11Texture2DNV(context, CL_MEM_WRITE_ONLY, pD3D11Texture2D, 1, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DNV failed"); } #endif cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsNV failed"); #ifdef HAVE_DIRECTX_NV12 if(DXGI_FORMAT_NV12 == desc.Format) { - status = clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsNV failed"); if(!ocl::ocl_convert_bgr_to_nv12(clBuffer, (int)u.step[0], u.cols, u.rows, clImage, clImageUV)) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_bgr_to_nv12 failed"); - status = clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsNV failed"); } @@ -1156,7 +1256,7 @@ static void __convertToD3D11Texture2DNV(InputArray src, ID3D11Texture2D* pD3D11T CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); } - status = clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsNV failed"); @@ -1201,11 +1301,12 @@ static void __convertFromD3D11Texture2DKHR(ID3D11Texture2D* pD3D11Texture2D, Out using namespace cv::ocl; Context& ctx = Context::getDefault(); cl_context context = (cl_context)ctx.ptr(); + OpenCLDirectXImpl& impl = getImpl(); cl_int status = 0; cl_mem clImage = 0; - clImage = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); + clImage = impl.clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); @@ -1213,7 +1314,7 @@ static void __convertFromD3D11Texture2DKHR(ID3D11Texture2D* pD3D11Texture2D, Out cl_mem clImageUV = 0; if(DXGI_FORMAT_NV12 == desc.Format) { - clImageUV = clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status); + clImageUV = impl.clCreateFromD3D11Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DKHR failed"); } @@ -1221,21 +1322,21 @@ static void __convertFromD3D11Texture2DKHR(ID3D11Texture2D* pD3D11Texture2D, Out cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); #ifdef HAVE_DIRECTX_NV12 if(DXGI_FORMAT_NV12 == desc.Format) { - status = clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsKHR failed"); if(!ocl::ocl_convert_nv12_to_bgr(clImage, clImageUV, clBuffer, (int)u.step[0], u.cols, u.rows)) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_nv12_to_bgr failed"); - status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); } @@ -1251,7 +1352,7 @@ static void __convertFromD3D11Texture2DKHR(ID3D11Texture2D* pD3D11Texture2D, Out CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); } - status = clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsKHR failed"); @@ -1296,11 +1397,12 @@ static void __convertFromD3D11Texture2DNV(ID3D11Texture2D* pD3D11Texture2D, Outp using namespace cv::ocl; Context& ctx = Context::getDefault(); cl_context context = (cl_context)ctx.ptr(); + OpenCLDirectXImpl& impl = getImpl(); cl_int status = 0; cl_mem clImage = 0; - clImage = clCreateFromD3D11Texture2DNV(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); + clImage = impl.clCreateFromD3D11Texture2DNV(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DNV failed"); @@ -1308,28 +1410,28 @@ static void __convertFromD3D11Texture2DNV(ID3D11Texture2D* pD3D11Texture2D, Outp cl_mem clImageUV = 0; if(DXGI_FORMAT_NV12 == desc.Format) { - clImageUV = clCreateFromD3D11Texture2DNV(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status); + clImageUV = impl.clCreateFromD3D11Texture2DNV(context, CL_MEM_READ_ONLY, pD3D11Texture2D, 1, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D11Texture2DNV failed"); } #endif cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsNV failed"); #ifdef HAVE_DIRECTX_NV12 if (DXGI_FORMAT::DXGI_FORMAT_NV12 == desc.Format) { - status = clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D11ObjectsNV failed"); if (!ocl::ocl_convert_nv12_to_bgr(clImage, clImageUV, clBuffer, (int)u.step[0], u.cols, u.rows)) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: ocl_convert_nv12_to_bgr failed"); - status = clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImageUV, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsNV failed"); } @@ -1345,7 +1447,7 @@ static void __convertFromD3D11Texture2DNV(ID3D11Texture2D* pD3D11Texture2D, Outp CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); } - status = clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D11ObjectsNV(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D11ObjectsNV failed"); @@ -1377,7 +1479,7 @@ void convertToD3D11Texture2D(InputArray src, ID3D11Texture2D* pD3D11Texture2D) NO_OPENCL_SUPPORT_ERROR; #else - bool useCLNVEXT = __OpenCLinitializeD3D11(); + bool useCLNVEXT = getImpl().initializeD3D11(); if(!useCLNVEXT){ __convertToD3D11Texture2DKHR(src,pD3D11Texture2D); } @@ -1399,7 +1501,7 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst NO_OPENCL_SUPPORT_ERROR; #else - bool useCLNVEXT = __OpenCLinitializeD3D11(); + bool useCLNVEXT = getImpl().initializeD3D11(); if(!useCLNVEXT){ __convertFromD3D11Texture2DKHR(pD3D11Texture2D,dst); } @@ -1412,40 +1514,14 @@ void convertFromD3D11Texture2D(ID3D11Texture2D* pD3D11Texture2D, OutputArray dst #endif } -#if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) -clCreateFromD3D10Texture2DKHR_fn clCreateFromD3D10Texture2DKHR = NULL; -clEnqueueAcquireD3D10ObjectsKHR_fn clEnqueueAcquireD3D10ObjectsKHR = NULL; -clEnqueueReleaseD3D10ObjectsKHR_fn clEnqueueReleaseD3D10ObjectsKHR = NULL; - -static void __OpenCLinitializeD3D10() -{ - using namespace cv::ocl; - static cl_platform_id initializedPlatform = NULL; - cl_platform_id platform = (cl_platform_id)Platform::getDefault().ptr(); - if (initializedPlatform != platform) - { - clCreateFromD3D10Texture2DKHR = (clCreateFromD3D10Texture2DKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromD3D10Texture2DKHR"); - clEnqueueAcquireD3D10ObjectsKHR = (clEnqueueAcquireD3D10ObjectsKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireD3D10ObjectsKHR"); - clEnqueueReleaseD3D10ObjectsKHR = (clEnqueueReleaseD3D10ObjectsKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseD3D10ObjectsKHR"); - initializedPlatform = platform; - } - if (!clCreateFromD3D10Texture2DKHR || !clEnqueueAcquireD3D10ObjectsKHR || !clEnqueueReleaseD3D10ObjectsKHR) - { - CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D10"); - } -} -#endif // defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) - void convertToD3D10Texture2D(InputArray src, ID3D10Texture2D* pD3D10Texture2D) { CV_UNUSED(src); CV_UNUSED(pD3D10Texture2D); #if !defined(HAVE_DIRECTX) NO_DIRECTX_SUPPORT_ERROR; #elif defined(HAVE_OPENCL) - __OpenCLinitializeD3D10(); + OpenCLDirectXImpl& impl = getImpl(); + impl.initializeD3D10(); D3D10_TEXTURE2D_DESC desc = { 0 }; pD3D10Texture2D->GetDesc(&desc); @@ -1468,14 +1544,14 @@ void convertToD3D10Texture2D(InputArray src, ID3D10Texture2D* pD3D10Texture2D) CV_Assert(u.isContinuous()); cl_int status = 0; - cl_mem clImage = clCreateFromD3D10Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D10Texture2D, 0, &status); + cl_mem clImage = impl.clCreateFromD3D10Texture2DKHR(context, CL_MEM_WRITE_ONLY, pD3D10Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D10Texture2DKHR failed"); cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D10ObjectsKHR failed"); size_t offset = 0; // TODO @@ -1484,7 +1560,7 @@ void convertToD3D10Texture2D(InputArray src, ID3D10Texture2D* pD3D10Texture2D) status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, dst_origin, region, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); - status = clEnqueueReleaseD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D10ObjectsKHR failed"); @@ -1506,7 +1582,8 @@ void convertFromD3D10Texture2D(ID3D10Texture2D* pD3D10Texture2D, OutputArray dst #if !defined(HAVE_DIRECTX) NO_DIRECTX_SUPPORT_ERROR; #elif defined(HAVE_OPENCL) - __OpenCLinitializeD3D10(); + OpenCLDirectXImpl& impl = getImpl(); + impl.initializeD3D10(); D3D10_TEXTURE2D_DESC desc = { 0 }; pD3D10Texture2D->GetDesc(&desc); @@ -1527,14 +1604,14 @@ void convertFromD3D10Texture2D(ID3D10Texture2D* pD3D10Texture2D, OutputArray dst CV_Assert(u.isContinuous()); cl_int status = 0; - cl_mem clImage = clCreateFromD3D10Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D10Texture2D, 0, &status); + cl_mem clImage = impl.clCreateFromD3D10Texture2DKHR(context, CL_MEM_READ_ONLY, pD3D10Texture2D, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromD3D10Texture2DKHR failed"); cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireD3D10ObjectsKHR failed"); size_t offset = 0; // TODO @@ -1543,7 +1620,7 @@ void convertFromD3D10Texture2D(ID3D10Texture2D* pD3D10Texture2D, OutputArray dst status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, src_origin, region, offset, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); - status = clEnqueueReleaseD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseD3D10ObjectsKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseD3D10ObjectsKHR failed"); @@ -1560,32 +1637,6 @@ void convertFromD3D10Texture2D(ID3D10Texture2D* pD3D10Texture2D, OutputArray dst #endif } -#if defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) -clCreateFromDX9MediaSurfaceKHR_fn clCreateFromDX9MediaSurfaceKHR = NULL; -clEnqueueAcquireDX9MediaSurfacesKHR_fn clEnqueueAcquireDX9MediaSurfacesKHR = NULL; -clEnqueueReleaseDX9MediaSurfacesKHR_fn clEnqueueReleaseDX9MediaSurfacesKHR = NULL; - -static void __OpenCLinitializeD3D9() -{ - using namespace cv::ocl; - static cl_platform_id initializedPlatform = NULL; - cl_platform_id platform = (cl_platform_id)Platform::getDefault().ptr(); - if (initializedPlatform != platform) - { - clCreateFromDX9MediaSurfaceKHR = (clCreateFromDX9MediaSurfaceKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clCreateFromDX9MediaSurfaceKHR"); - clEnqueueAcquireDX9MediaSurfacesKHR = (clEnqueueAcquireDX9MediaSurfacesKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueAcquireDX9MediaSurfacesKHR"); - clEnqueueReleaseDX9MediaSurfacesKHR = (clEnqueueReleaseDX9MediaSurfacesKHR_fn) - clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueReleaseDX9MediaSurfacesKHR"); - initializedPlatform = platform; - } - if (!clCreateFromDX9MediaSurfaceKHR || !clEnqueueAcquireDX9MediaSurfacesKHR || !clEnqueueReleaseDX9MediaSurfacesKHR) - { - CV_Error(cv::Error::OpenCLInitError, "OpenCL: Can't find functions for D3D9"); - } -} -#endif // defined(HAVE_DIRECTX) && defined(HAVE_OPENCL) void convertToDirect3DSurface9(InputArray src, IDirect3DSurface9* pDirect3DSurface9, void* surfaceSharedHandle) { @@ -1593,7 +1644,8 @@ void convertToDirect3DSurface9(InputArray src, IDirect3DSurface9* pDirect3DSurfa #if !defined(HAVE_DIRECTX) NO_DIRECTX_SUPPORT_ERROR; #elif defined(HAVE_OPENCL) - __OpenCLinitializeD3D9(); + OpenCLDirectXImpl& impl = getImpl(); + impl.initializeD3D9(); D3DSURFACE_DESC desc; if (FAILED(pDirect3DSurface9->GetDesc(&desc))) @@ -1620,8 +1672,8 @@ void convertToDirect3DSurface9(InputArray src, IDirect3DSurface9* pDirect3DSurfa cl_int status = 0; cl_dx9_surface_info_khr surfaceInfo = {pDirect3DSurface9, (HANDLE)surfaceSharedHandle}; - cl_mem clImage = clCreateFromDX9MediaSurfaceKHR(context, CL_MEM_WRITE_ONLY, - ocl::g_isDirect3DDevice9Ex ? CL_ADAPTER_D3D9EX_KHR : CL_ADAPTER_D3D9_KHR, + cl_mem clImage = impl.clCreateFromDX9MediaSurfaceKHR(context, CL_MEM_WRITE_ONLY, + impl.isDirect3DDevice9Ex ? CL_ADAPTER_D3D9EX_KHR : CL_ADAPTER_D3D9_KHR, &surfaceInfo, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromDX9MediaSurfaceKHR failed"); @@ -1629,7 +1681,7 @@ void convertToDirect3DSurface9(InputArray src, IDirect3DSurface9* pDirect3DSurfa cl_mem clBuffer = (cl_mem)u.handle(ACCESS_READ); cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireDX9MediaSurfacesKHR failed"); size_t offset = 0; // TODO @@ -1638,7 +1690,7 @@ void convertToDirect3DSurface9(InputArray src, IDirect3DSurface9* pDirect3DSurfa status = clEnqueueCopyBufferToImage(q, clBuffer, clImage, offset, dst_origin, region, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyBufferToImage failed"); - status = clEnqueueReleaseDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseDX9MediaSurfacesKHR failed"); @@ -1661,7 +1713,8 @@ void convertFromDirect3DSurface9(IDirect3DSurface9* pDirect3DSurface9, OutputArr #if !defined(HAVE_DIRECTX) NO_DIRECTX_SUPPORT_ERROR; #elif defined(HAVE_OPENCL) - __OpenCLinitializeD3D9(); + OpenCLDirectXImpl& impl = getImpl(); + impl.initializeD3D9(); D3DSURFACE_DESC desc; if (FAILED(pDirect3DSurface9->GetDesc(&desc))) @@ -1686,8 +1739,8 @@ void convertFromDirect3DSurface9(IDirect3DSurface9* pDirect3DSurface9, OutputArr cl_int status = 0; cl_dx9_surface_info_khr surfaceInfo = {pDirect3DSurface9, (HANDLE)surfaceSharedHandle}; - cl_mem clImage = clCreateFromDX9MediaSurfaceKHR(context, CL_MEM_READ_ONLY, - ocl::g_isDirect3DDevice9Ex ? CL_ADAPTER_D3D9EX_KHR : CL_ADAPTER_D3D9_KHR, + cl_mem clImage = impl.clCreateFromDX9MediaSurfaceKHR(context, CL_MEM_READ_ONLY, + impl.isDirect3DDevice9Ex ? CL_ADAPTER_D3D9EX_KHR : CL_ADAPTER_D3D9_KHR, &surfaceInfo, 0, &status); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clCreateFromDX9MediaSurfaceKHR failed"); @@ -1695,7 +1748,7 @@ void convertFromDirect3DSurface9(IDirect3DSurface9* pDirect3DSurface9, OutputArr cl_mem clBuffer = (cl_mem)u.handle(ACCESS_WRITE); cl_command_queue q = (cl_command_queue)Queue::getDefault().ptr(); - status = clEnqueueAcquireDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueAcquireDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueAcquireDX9MediaSurfacesKHR failed"); size_t offset = 0; // TODO @@ -1704,7 +1757,7 @@ void convertFromDirect3DSurface9(IDirect3DSurface9* pDirect3DSurface9, OutputArr status = clEnqueueCopyImageToBuffer(q, clImage, clBuffer, src_origin, region, offset, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueCopyImageToBuffer failed"); - status = clEnqueueReleaseDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); + status = impl.clEnqueueReleaseDX9MediaSurfacesKHR(q, 1, &clImage, 0, NULL, NULL); if (status != CL_SUCCESS) CV_Error(cv::Error::OpenCLApiCallError, "OpenCL: clEnqueueReleaseDX9MediaSurfacesKHR failed"); diff --git a/modules/core/src/directx.hpp b/modules/core/src/directx.hpp new file mode 100644 index 0000000000..9f23352d4d --- /dev/null +++ b/modules/core/src/directx.hpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_CORE_SRC_DIRECTX_HPP +#define OPENCV_CORE_SRC_DIRECTX_HPP + +#ifndef HAVE_DIRECTX +#error Invalid build configuration +#endif + +namespace cv { +namespace directx { +namespace internal { + +struct OpenCLDirectXImpl; +OpenCLDirectXImpl* createDirectXImpl(); +void deleteDirectXImpl(OpenCLDirectXImpl**); +OpenCLDirectXImpl* getDirectXImpl(ocl::Context& ctx); + +}}} // namespace internal + +#endif // OPENCV_CORE_SRC_DIRECTX_HPP diff --git a/modules/core/src/dxt.cpp b/modules/core/src/dxt.cpp index bfa61d0502..fcdb2a202f 100644 --- a/modules/core/src/dxt.cpp +++ b/modules/core/src/dxt.cpp @@ -122,6 +122,33 @@ static const double DFTTab[][2] = { 1.00000000000000000, 0.00000000292583616 } }; +namespace { +template +struct Constants { + static const T sin_120; + static const T fft5_2; + static const T fft5_3; + static const T fft5_4; + static const T fft5_5; +}; + +template +const T Constants::sin_120 = (T)0.86602540378443864676372317075294; + +template +const T Constants::fft5_2 = (T)0.559016994374947424102293417182819; + +template +const T Constants::fft5_3 = (T)-0.951056516295153572116439333379382; + +template +const T Constants::fft5_4 = (T)-1.538841768587626701285145288018455; + +template +const T Constants::fft5_5 = (T)0.363271264002680442947733378740309; + +} //namespace + #define BitRev(i,shift) \ ((int)((((unsigned)bitrevTab[(i)&255] << 24)+ \ ((unsigned)bitrevTab[((i)>> 8)&255] << 16)+ \ @@ -372,6 +399,149 @@ DFTInit( int n0, int nf, const int* factors, int* itab, int elem_size, void* _wa } } +// Reference radix-2 implementation. +template struct DFT_R2 +{ + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + const int nx = n/2; + for(int i = 0 ; i < c_n; i += n) + { + Complex* v = dst + i; + T r0 = v[0].re + v[nx].re; + T i0 = v[0].im + v[nx].im; + T r1 = v[0].re - v[nx].re; + T i1 = v[0].im - v[nx].im; + v[0].re = r0; v[0].im = i0; + v[nx].re = r1; v[nx].im = i1; + + for( int j = 1, dw = dw0; j < nx; j++, dw += dw0 ) + { + v = dst + i + j; + r1 = v[nx].re*wave[dw].re - v[nx].im*wave[dw].im; + i1 = v[nx].im*wave[dw].re + v[nx].re*wave[dw].im; + r0 = v[0].re; i0 = v[0].im; + + v[0].re = r0 + r1; v[0].im = i0 + i1; + v[nx].re = r0 - r1; v[nx].im = i0 - i1; + } + } + } +}; + +// Reference radix-3 implementation. +template struct DFT_R3 +{ + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + const int nx = n / 3; + for(int i = 0; i < c_n; i += n ) + { + { + Complex* v = dst + i; + T r1 = v[nx].re + v[nx*2].re; + T i1 = v[nx].im + v[nx*2].im; + T r0 = v[0].re; + T i0 = v[0].im; + T r2 = Constants::sin_120*(v[nx].im - v[nx*2].im); + T i2 = Constants::sin_120*(v[nx*2].re - v[nx].re); + v[0].re = r0 + r1; v[0].im = i0 + i1; + r0 -= (T)0.5*r1; i0 -= (T)0.5*i1; + v[nx].re = r0 + r2; v[nx].im = i0 + i2; + v[nx*2].re = r0 - r2; v[nx*2].im = i0 - i2; + } + + for(int j = 1, dw = dw0; j < nx; j++, dw += dw0 ) + { + Complex* v = dst + i + j; + T r0 = v[nx].re*wave[dw].re - v[nx].im*wave[dw].im; + T i0 = v[nx].re*wave[dw].im + v[nx].im*wave[dw].re; + T i2 = v[nx*2].re*wave[dw*2].re - v[nx*2].im*wave[dw*2].im; + T r2 = v[nx*2].re*wave[dw*2].im + v[nx*2].im*wave[dw*2].re; + T r1 = r0 + i2; T i1 = i0 + r2; + + r2 = Constants::sin_120*(i0 - r2); i2 = Constants::sin_120*(i2 - r0); + r0 = v[0].re; i0 = v[0].im; + v[0].re = r0 + r1; v[0].im = i0 + i1; + r0 -= (T)0.5*r1; i0 -= (T)0.5*i1; + v[nx].re = r0 + r2; v[nx].im = i0 + i2; + v[nx*2].re = r0 - r2; v[nx*2].im = i0 - i2; + } + } + } +}; + +// Reference radix-5 implementation. +template struct DFT_R5 +{ + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + const int nx = n / 5; + for(int i = 0; i < c_n; i += n ) + { + for(int j = 0, dw = 0; j < nx; j++, dw += dw0 ) + { + Complex* v0 = dst + i + j; + Complex* v1 = v0 + nx*2; + Complex* v2 = v1 + nx*2; + + T r0, i0, r1, i1, r2, i2, r3, i3, r4, i4, r5, i5; + + r3 = v0[nx].re*wave[dw].re - v0[nx].im*wave[dw].im; + i3 = v0[nx].re*wave[dw].im + v0[nx].im*wave[dw].re; + r2 = v2[0].re*wave[dw*4].re - v2[0].im*wave[dw*4].im; + i2 = v2[0].re*wave[dw*4].im + v2[0].im*wave[dw*4].re; + + r1 = r3 + r2; i1 = i3 + i2; + r3 -= r2; i3 -= i2; + + r4 = v1[nx].re*wave[dw*3].re - v1[nx].im*wave[dw*3].im; + i4 = v1[nx].re*wave[dw*3].im + v1[nx].im*wave[dw*3].re; + r0 = v1[0].re*wave[dw*2].re - v1[0].im*wave[dw*2].im; + i0 = v1[0].re*wave[dw*2].im + v1[0].im*wave[dw*2].re; + + r2 = r4 + r0; i2 = i4 + i0; + r4 -= r0; i4 -= i0; + + r0 = v0[0].re; i0 = v0[0].im; + r5 = r1 + r2; i5 = i1 + i2; + + v0[0].re = r0 + r5; v0[0].im = i0 + i5; + + r0 -= (T)0.25*r5; i0 -= (T)0.25*i5; + r1 = Constants::fft5_2*(r1 - r2); i1 = Constants::fft5_2*(i1 - i2); + r2 = -Constants::fft5_3*(i3 + i4); i2 = Constants::fft5_3*(r3 + r4); + + i3 *= -Constants::fft5_5; r3 *= Constants::fft5_5; + i4 *= -Constants::fft5_4; r4 *= Constants::fft5_4; + + r5 = r2 + i3; i5 = i2 + r3; + r2 -= i4; i2 -= r4; + + r3 = r0 + r1; i3 = i0 + i1; + r0 -= r1; i0 -= i1; + + v0[nx].re = r3 + r2; v0[nx].im = i3 + i2; + v2[0].re = r3 - r2; v2[0].im = i3 - i2; + + v1[0].re = r0 + r5; v1[0].im = i0 + i5; + v1[nx].re = r0 - r5; v1[nx].im = i0 - i5; + } + } + } +}; + +template struct DFT_VecR2 +{ + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + DFT_R2()(dst, c_n, n, dw0, wave); + } +}; + +template struct DFT_VecR3 +{ + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + DFT_R3()(dst, c_n, n, dw0, wave); + } +}; + template struct DFT_VecR4 { int operator()(Complex*, int, int, int&, const Complex*) const { return 1; } @@ -379,6 +549,98 @@ template struct DFT_VecR4 #if CV_SSE3 +// multiplies *a and *b: +// r_re + i*r_im = (a_re + i*a_im)*(b_re + i*b_im) +// r_re and r_im are placed respectively in bits 31:0 and 63:32 of the resulting +// vector register. +inline __m128 complexMul(const Complex* const a, const Complex* const b) { + const __m128 z = _mm_setzero_ps(); + const __m128 neg_elem0 = _mm_set_ps(0.0f,0.0f,0.0f,-0.0f); + // v_a[31:0] is a->re and v_a[63:32] is a->im. + const __m128 v_a = _mm_loadl_pi(z, (const __m64*)a); + const __m128 v_b = _mm_loadl_pi(z, (const __m64*)b); + // x_1 = v[nx] * wave[dw]. + const __m128 v_a_riri = _mm_shuffle_ps(v_a, v_a, _MM_SHUFFLE(0, 1, 0, 1)); + const __m128 v_b_irri = _mm_shuffle_ps(v_b, v_b, _MM_SHUFFLE(1, 0, 0, 1)); + const __m128 mul = _mm_mul_ps(v_a_riri, v_b_irri); + const __m128 xored = _mm_xor_ps(mul, neg_elem0); + return _mm_hadd_ps(xored, z); +} + +// optimized radix-2 transform +template<> struct DFT_VecR2 { + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + const __m128 z = _mm_setzero_ps(); + const int nx = n/2; + for(int i = 0 ; i < c_n; i += n) + { + { + Complex* v = dst + i; + float r0 = v[0].re + v[nx].re; + float i0 = v[0].im + v[nx].im; + float r1 = v[0].re - v[nx].re; + float i1 = v[0].im - v[nx].im; + v[0].re = r0; v[0].im = i0; + v[nx].re = r1; v[nx].im = i1; + } + + for( int j = 1, dw = dw0; j < nx; j++, dw += dw0 ) + { + Complex* v = dst + i + j; + const __m128 x_1 = complexMul(&v[nx], &wave[dw]); + const __m128 v_0 = _mm_loadl_pi(z, (const __m64*)&v[0]); + _mm_storel_pi((__m64*)&v[0], _mm_add_ps(v_0, x_1)); + _mm_storel_pi((__m64*)&v[nx], _mm_sub_ps(v_0, x_1)); + } + } + } +}; + +// Optimized radix-3 implementation. +template<> struct DFT_VecR3 { + void operator()(Complex* dst, const int c_n, const int n, const int dw0, const Complex* wave) const { + const int nx = n / 3; + const __m128 z = _mm_setzero_ps(); + const __m128 neg_elem1 = _mm_set_ps(0.0f,0.0f,-0.0f,0.0f); + const __m128 sin_120 = _mm_set1_ps(Constants::sin_120); + const __m128 one_half = _mm_set1_ps(0.5f); + for(int i = 0; i < c_n; i += n ) + { + { + Complex* v = dst + i; + + float r1 = v[nx].re + v[nx*2].re; + float i1 = v[nx].im + v[nx*2].im; + float r0 = v[0].re; + float i0 = v[0].im; + float r2 = Constants::sin_120*(v[nx].im - v[nx*2].im); + float i2 = Constants::sin_120*(v[nx*2].re - v[nx].re); + v[0].re = r0 + r1; v[0].im = i0 + i1; + r0 -= (float)0.5*r1; i0 -= (float)0.5*i1; + v[nx].re = r0 + r2; v[nx].im = i0 + i2; + v[nx*2].re = r0 - r2; v[nx*2].im = i0 - i2; + } + + for(int j = 1, dw = dw0; j < nx; j++, dw += dw0 ) + { + Complex* v = dst + i + j; + const __m128 x_0 = complexMul(&v[nx], &wave[dw]); + const __m128 x_2 = complexMul(&v[nx*2], &wave[dw*2]); + const __m128 x_1 = _mm_add_ps(x_0, x_2); + + const __m128 v_0 = _mm_loadl_pi(z, (const __m64*)&v[0]); + _mm_storel_pi((__m64*)&v[0], _mm_add_ps(v_0, x_1)); + + const __m128 x_3 = _mm_mul_ps(sin_120, _mm_xor_ps(_mm_sub_ps(x_2, x_0), neg_elem1)); + const __m128 x_3s = _mm_shuffle_ps(x_3, x_3, _MM_SHUFFLE(0, 1, 0, 1)); + const __m128 x_4 = _mm_sub_ps(v_0, _mm_mul_ps(one_half, x_1)); + _mm_storel_pi((__m64*)&v[nx], _mm_add_ps(x_4, x_3s)); + _mm_storel_pi((__m64*)&v[nx*2], _mm_sub_ps(x_4, x_3s)); + } + } + } +}; + // optimized radix-4 transform template<> struct DFT_VecR4 { @@ -573,12 +835,6 @@ struct OcvDftOptions { template static void DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) { - static const T sin_120 = (T)0.86602540378443864676372317075294; - static const T fft5_2 = (T)0.559016994374947424102293417182819; - static const T fft5_3 = (T)-0.951056516295153572116439333379382; - static const T fft5_4 = (T)-1.538841768587626701285145288018455; - static const T fft5_5 = (T)0.363271264002680442947733378740309; - const Complex* wave = (Complex*)c.wave; const int * itab = c.itab; @@ -775,30 +1031,18 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) for( ; n < c.factors[0]; ) { // do the remaining radix-2 transform - nx = n; n *= 2; dw0 /= 2; - for( i = 0; i < c.n; i += n ) + if(c.haveSSE3) { - Complex* v = dst + i; - T r0 = v[0].re + v[nx].re; - T i0 = v[0].im + v[nx].im; - T r1 = v[0].re - v[nx].re; - T i1 = v[0].im - v[nx].im; - v[0].re = r0; v[0].im = i0; - v[nx].re = r1; v[nx].im = i1; - - for( j = 1, dw = dw0; j < nx; j++, dw += dw0 ) - { - v = dst + i + j; - r1 = v[nx].re*wave[dw].re - v[nx].im*wave[dw].im; - i1 = v[nx].im*wave[dw].re + v[nx].re*wave[dw].im; - r0 = v[0].re; i0 = v[0].im; - - v[0].re = r0 + r1; v[0].im = i0 + i1; - v[nx].re = r0 - r1; v[nx].im = i0 - i1; - } + DFT_VecR2 vr2; + vr2(dst, c.n, n, dw0, wave); + } + else + { + DFT_R2 vr2; + vr2(dst, c.n, n, dw0, wave); } } } @@ -813,94 +1057,21 @@ DFT(const OcvDftOptions & c, const Complex* src, Complex* dst) if( factor == 3 ) { - // radix-3 - for( i = 0; i < c.n; i += n ) + if(c.haveSSE3) { - Complex* v = dst + i; - - T r1 = v[nx].re + v[nx*2].re; - T i1 = v[nx].im + v[nx*2].im; - T r0 = v[0].re; - T i0 = v[0].im; - T r2 = sin_120*(v[nx].im - v[nx*2].im); - T i2 = sin_120*(v[nx*2].re - v[nx].re); - v[0].re = r0 + r1; v[0].im = i0 + i1; - r0 -= (T)0.5*r1; i0 -= (T)0.5*i1; - v[nx].re = r0 + r2; v[nx].im = i0 + i2; - v[nx*2].re = r0 - r2; v[nx*2].im = i0 - i2; - - for( j = 1, dw = dw0; j < nx; j++, dw += dw0 ) - { - v = dst + i + j; - r0 = v[nx].re*wave[dw].re - v[nx].im*wave[dw].im; - i0 = v[nx].re*wave[dw].im + v[nx].im*wave[dw].re; - i2 = v[nx*2].re*wave[dw*2].re - v[nx*2].im*wave[dw*2].im; - r2 = v[nx*2].re*wave[dw*2].im + v[nx*2].im*wave[dw*2].re; - r1 = r0 + i2; i1 = i0 + r2; - - r2 = sin_120*(i0 - r2); i2 = sin_120*(i2 - r0); - r0 = v[0].re; i0 = v[0].im; - v[0].re = r0 + r1; v[0].im = i0 + i1; - r0 -= (T)0.5*r1; i0 -= (T)0.5*i1; - v[nx].re = r0 + r2; v[nx].im = i0 + i2; - v[nx*2].re = r0 - r2; v[nx*2].im = i0 - i2; - } + DFT_VecR3 vr3; + vr3(dst, c.n, n, dw0, wave); + } + else + { + DFT_R3 vr3; + vr3(dst, c.n, n, dw0, wave); } } else if( factor == 5 ) { - // radix-5 - for( i = 0; i < c.n; i += n ) - { - for( j = 0, dw = 0; j < nx; j++, dw += dw0 ) - { - Complex* v0 = dst + i + j; - Complex* v1 = v0 + nx*2; - Complex* v2 = v1 + nx*2; - - T r0, i0, r1, i1, r2, i2, r3, i3, r4, i4, r5, i5; - - r3 = v0[nx].re*wave[dw].re - v0[nx].im*wave[dw].im; - i3 = v0[nx].re*wave[dw].im + v0[nx].im*wave[dw].re; - r2 = v2[0].re*wave[dw*4].re - v2[0].im*wave[dw*4].im; - i2 = v2[0].re*wave[dw*4].im + v2[0].im*wave[dw*4].re; - - r1 = r3 + r2; i1 = i3 + i2; - r3 -= r2; i3 -= i2; - - r4 = v1[nx].re*wave[dw*3].re - v1[nx].im*wave[dw*3].im; - i4 = v1[nx].re*wave[dw*3].im + v1[nx].im*wave[dw*3].re; - r0 = v1[0].re*wave[dw*2].re - v1[0].im*wave[dw*2].im; - i0 = v1[0].re*wave[dw*2].im + v1[0].im*wave[dw*2].re; - - r2 = r4 + r0; i2 = i4 + i0; - r4 -= r0; i4 -= i0; - - r0 = v0[0].re; i0 = v0[0].im; - r5 = r1 + r2; i5 = i1 + i2; - - v0[0].re = r0 + r5; v0[0].im = i0 + i5; - - r0 -= (T)0.25*r5; i0 -= (T)0.25*i5; - r1 = fft5_2*(r1 - r2); i1 = fft5_2*(i1 - i2); - r2 = -fft5_3*(i3 + i4); i2 = fft5_3*(r3 + r4); - - i3 *= -fft5_5; r3 *= fft5_5; - i4 *= -fft5_4; r4 *= fft5_4; - - r5 = r2 + i3; i5 = i2 + r3; - r2 -= i4; i2 -= r4; - - r3 = r0 + r1; i3 = i0 + i1; - r0 -= r1; i0 -= i1; - - v0[nx].re = r3 + r2; v0[nx].im = i3 + i2; - v2[0].re = r3 - r2; v2[0].im = i3 - i2; - - v1[0].re = r0 + r5; v1[0].im = i0 + i5; - v1[nx].re = r0 - r5; v1[nx].im = i0 - i5; - } - } + DFT_R5 vr5; + vr5(dst, c.n, n, dw0, wave); } else { diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 59544f8458..122b383379 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -204,6 +204,21 @@ MatAllocator* Mat::getStdAllocator() //================================================================================================== +bool MatSize::operator==(const MatSize& sz) const +{ + int d = dims(); + int dsz = sz.dims(); + if( d != dsz ) + return false; + if( d == 2 ) + return p[0] == sz.p[0] && p[1] == sz.p[1]; + + for( int i = 0; i < d; i++ ) + if( p[i] != sz.p[i] ) + return false; + return true; +} + void setSize( Mat& m, int _dims, const int* _sz, const size_t* _steps, bool autoSteps) { CV_Assert( 0 <= _dims && _dims <= CV_MAX_DIM ); @@ -320,7 +335,330 @@ void finalizeHdr(Mat& m) m.dataend = m.datalimit = 0; } -//================================================================================================== +//======================================= Mat ====================================================== + +Mat::Mat() + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{} + +Mat::Mat(int _rows, int _cols, int _type) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_rows, _cols, _type); +} + +Mat::Mat(int _rows, int _cols, int _type, const Scalar& _s) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_rows, _cols, _type); + *this = _s; +} + +Mat::Mat(Size _sz, int _type) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create( _sz.height, _sz.width, _type ); +} + +Mat::Mat(Size _sz, int _type, const Scalar& _s) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_sz.height, _sz.width, _type); + *this = _s; +} + +Mat::Mat(int _dims, const int* _sz, int _type) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_dims, _sz, _type); +} + +Mat::Mat(int _dims, const int* _sz, int _type, const Scalar& _s) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_dims, _sz, _type); + *this = _s; +} + +Mat::Mat(const std::vector& _sz, int _type) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_sz, _type); +} + +Mat::Mat(const std::vector& _sz, int _type, const Scalar& _s) + : flags(MAGIC_VAL), dims(0), rows(0), cols(0), data(0), datastart(0), dataend(0), + datalimit(0), allocator(0), u(0), size(&rows), step(0) +{ + create(_sz, _type); + *this = _s; +} + +Mat::Mat(const Mat& m) + : flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), data(m.data), + datastart(m.datastart), dataend(m.dataend), datalimit(m.datalimit), allocator(m.allocator), + u(m.u), size(&rows), step(0) +{ + if( u ) + CV_XADD(&u->refcount, 1); + if( m.dims <= 2 ) + { + step[0] = m.step[0]; step[1] = m.step[1]; + } + else + { + dims = 0; + copySize(m); + } +} + +Mat::Mat(int _rows, int _cols, int _type, void* _data, size_t _step) + : flags(MAGIC_VAL + (_type & TYPE_MASK)), dims(2), rows(_rows), cols(_cols), + data((uchar*)_data), datastart((uchar*)_data), dataend(0), datalimit(0), + allocator(0), u(0), size(&rows) +{ + CV_Assert(total() == 0 || data != NULL); + + size_t esz = CV_ELEM_SIZE(_type), esz1 = CV_ELEM_SIZE1(_type); + size_t minstep = cols * esz; + if( _step == AUTO_STEP ) + { + _step = minstep; + } + else + { + CV_Assert( _step >= minstep ); + if (_step % esz1 != 0) + { + CV_Error(Error::BadStep, "Step must be a multiple of esz1"); + } + } + step[0] = _step; + step[1] = esz; + datalimit = datastart + _step * rows; + dataend = datalimit - _step + minstep; + updateContinuityFlag(); +} + +Mat::Mat(Size _sz, int _type, void* _data, size_t _step) + : flags(MAGIC_VAL + (_type & TYPE_MASK)), dims(2), rows(_sz.height), cols(_sz.width), + data((uchar*)_data), datastart((uchar*)_data), dataend(0), datalimit(0), + allocator(0), u(0), size(&rows) +{ + CV_Assert(total() == 0 || data != NULL); + + size_t esz = CV_ELEM_SIZE(_type), esz1 = CV_ELEM_SIZE1(_type); + size_t minstep = cols*esz; + if( _step == AUTO_STEP ) + { + _step = minstep; + } + else + { + CV_Assert(_step >= minstep); + + if (_step % esz1 != 0) + { + CV_Error(Error::BadStep, "Step must be a multiple of esz1"); + } + } + step[0] = _step; + step[1] = esz; + datalimit = datastart + _step*rows; + dataend = datalimit - _step + minstep; + updateContinuityFlag(); +} + + +Mat::~Mat() +{ + release(); + if( step.p != step.buf ) + fastFree(step.p); +} + +Mat& Mat::operator=(const Mat& m) +{ + if( this != &m ) + { + if( m.u ) + CV_XADD(&m.u->refcount, 1); + release(); + flags = m.flags; + if( dims <= 2 && m.dims <= 2 ) + { + dims = m.dims; + rows = m.rows; + cols = m.cols; + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + copySize(m); + data = m.data; + datastart = m.datastart; + dataend = m.dataend; + datalimit = m.datalimit; + allocator = m.allocator; + u = m.u; + } + return *this; +} + +Mat Mat::clone() const +{ + Mat m; + copyTo(m); + return m; +} + +void Mat::assignTo( Mat& m, int _type ) const +{ + if( _type < 0 ) + m = *this; + else + convertTo(m, _type); +} + +void Mat::create(int _rows, int _cols, int _type) +{ + _type &= TYPE_MASK; + if( dims <= 2 && rows == _rows && cols == _cols && type() == _type && data ) + return; + int sz[] = {_rows, _cols}; + create(2, sz, _type); +} + +void Mat::create(Size _sz, int _type) +{ + create(_sz.height, _sz.width, _type); +} + +void Mat::addref() +{ + if( u ) + CV_XADD(&u->refcount, 1); +} + +void Mat::release() +{ + if( u && CV_XADD(&u->refcount, -1) == 1 ) + deallocate(); + u = NULL; + datastart = dataend = datalimit = data = 0; + for(int i = 0; i < dims; i++) + size.p[i] = 0; +#ifdef _DEBUG + flags = MAGIC_VAL; + dims = rows = cols = 0; + if(step.p != step.buf) + { + fastFree(step.p); + step.p = step.buf; + size.p = &rows; + } +#endif +} + +size_t Mat::step1(int i) const +{ + return step.p[i] / elemSize1(); +} + +bool Mat::empty() const +{ + return data == 0 || total() == 0 || dims == 0; +} + +size_t Mat::total() const +{ + if( dims <= 2 ) + return (size_t)rows * cols; + size_t p = 1; + for( int i = 0; i < dims; i++ ) + p *= size[i]; + return p; +} + +size_t Mat::total(int startDim, int endDim) const +{ + CV_Assert( 0 <= startDim && startDim <= endDim); + size_t p = 1; + int endDim_ = endDim <= dims ? endDim : dims; + for( int i = startDim; i < endDim_; i++ ) + p *= size[i]; + return p; +} + + +Mat::Mat(Mat&& m) + : flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), data(m.data), + datastart(m.datastart), dataend(m.dataend), datalimit(m.datalimit), allocator(m.allocator), + u(m.u), size(&rows) +{ + if (m.dims <= 2) // move new step/size info + { + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + { + CV_Assert(m.step.p != m.step.buf); + step.p = m.step.p; + size.p = m.size.p; + m.step.p = m.step.buf; + m.size.p = &m.rows; + } + m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; + m.data = NULL; m.datastart = NULL; m.dataend = NULL; m.datalimit = NULL; + m.allocator = NULL; + m.u = NULL; +} + + +Mat& Mat::operator=(Mat&& m) +{ + if (this == &m) + return *this; + + release(); + flags = m.flags; dims = m.dims; rows = m.rows; cols = m.cols; data = m.data; + datastart = m.datastart; dataend = m.dataend; datalimit = m.datalimit; allocator = m.allocator; + u = m.u; + if (step.p != step.buf) // release self step/size + { + fastFree(step.p); + step.p = step.buf; + size.p = &rows; + } + if (m.dims <= 2) // move new step/size info + { + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + { + CV_Assert(m.step.p != m.step.buf); + step.p = m.step.p; + size.p = m.size.p; + m.step.p = m.step.buf; + m.size.p = &m.rows; + } + m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; + m.data = NULL; m.datastart = NULL; m.dataend = NULL; m.datalimit = NULL; + m.allocator = NULL; + m.u = NULL; + return *this; +} + void Mat::create(int d, const int* _sizes, int _type) { diff --git a/modules/core/src/matrix_sparse.cpp b/modules/core/src/matrix_sparse.cpp index 61e7e90a56..05d16d706e 100644 --- a/modules/core/src/matrix_sparse.cpp +++ b/modules/core/src/matrix_sparse.cpp @@ -176,6 +176,94 @@ void SparseMat::Hdr::clear() nodeCount = freeList = 0; } +///////////////////////////// SparseMat ///////////////////////////// + +SparseMat::SparseMat() + : flags(MAGIC_VAL), hdr(0) +{} + +SparseMat::SparseMat(int _dims, const int* _sizes, int _type) + : flags(MAGIC_VAL), hdr(0) +{ + create(_dims, _sizes, _type); +} + +SparseMat::SparseMat(const SparseMat& m) + : flags(m.flags), hdr(m.hdr) +{ + addref(); +} + +SparseMat::~SparseMat() +{ + release(); +} + +SparseMat& SparseMat::operator = (const SparseMat& m) +{ + if( this != &m ) + { + if( m.hdr ) + CV_XADD(&m.hdr->refcount, 1); + release(); + flags = m.flags; + hdr = m.hdr; + } + return *this; +} + +SparseMat& SparseMat::operator=(const Mat& m) +{ + return (*this = SparseMat(m)); +} + +void SparseMat::assignTo(SparseMat& m, int _type) const +{ + if( _type < 0 ) + m = *this; + else + convertTo(m, _type); +} + +void SparseMat::addref() +{ + if( hdr ) + CV_XADD(&hdr->refcount, 1); +} + +void SparseMat::release() +{ + if( hdr && CV_XADD(&hdr->refcount, -1) == 1 ) + delete hdr; + hdr = 0; +} + +size_t SparseMat::hash(int i0) const +{ + return (size_t)i0; +} + +size_t SparseMat::hash(int i0, int i1) const +{ + return (size_t)(unsigned)i0 * HASH_SCALE + (unsigned)i1; +} + +size_t SparseMat::hash(int i0, int i1, int i2) const +{ + return ((size_t)(unsigned)i0 * HASH_SCALE + (unsigned)i1) * HASH_SCALE + (unsigned)i2; +} + +size_t SparseMat::hash(const int* idx) const +{ + size_t h = (unsigned)idx[0]; + if( !hdr ) + return 0; + int d = hdr->dims; + for(int i = 1; i < d; i++ ) + h = h * HASH_SCALE + (unsigned)idx[i]; + return h; +} + SparseMat::SparseMat(const Mat& m) : flags(MAGIC_VAL), hdr(0) diff --git a/modules/core/src/matrix_wrap.cpp b/modules/core/src/matrix_wrap.cpp index 421c51febc..68a674f6f1 100644 --- a/modules/core/src/matrix_wrap.cpp +++ b/modules/core/src/matrix_wrap.cpp @@ -915,7 +915,7 @@ bool _InputArray::isContinuous(int i) const if( k == STD_ARRAY_MAT ) { const Mat* vv = (const Mat*)obj; - CV_Assert(i > 0 && i < sz.height); + CV_Assert(i >= 0 && i < sz.height); return vv[i].isContinuous(); } @@ -949,21 +949,21 @@ bool _InputArray::isSubmatrix(int i) const if( k == STD_VECTOR_MAT ) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return vv[i].isSubmatrix(); } if( k == STD_ARRAY_MAT ) { const Mat* vv = (const Mat*)obj; - CV_Assert(i < sz.height); + CV_Assert(i >= 0 && i < sz.height); return vv[i].isSubmatrix(); } if( k == STD_VECTOR_UMAT ) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return vv[i].isSubmatrix(); } @@ -994,9 +994,7 @@ size_t _InputArray::offset(int i) const if( k == STD_VECTOR_MAT ) { const std::vector& vv = *(const std::vector*)obj; - if( i < 0 ) - return 1; - CV_Assert( i < (int)vv.size() ); + CV_Assert( i >= 0 && i < (int)vv.size() ); return (size_t)(vv[i].ptr() - vv[i].datastart); } @@ -1004,16 +1002,14 @@ size_t _InputArray::offset(int i) const if( k == STD_ARRAY_MAT ) { const Mat* vv = (const Mat*)obj; - if( i < 0 ) - return 1; - CV_Assert( i < sz.height ); + CV_Assert( i >= 0 && i < sz.height ); return (size_t)(vv[i].ptr() - vv[i].datastart); } if( k == STD_VECTOR_UMAT ) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return vv[i].offset; } @@ -1027,7 +1023,7 @@ size_t _InputArray::offset(int i) const if (k == STD_VECTOR_CUDA_GPU_MAT) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return (size_t)(vv[i].data - vv[i].datastart); } @@ -1057,25 +1053,21 @@ size_t _InputArray::step(int i) const if( k == STD_VECTOR_MAT ) { const std::vector& vv = *(const std::vector*)obj; - if( i < 0 ) - return 1; - CV_Assert( i < (int)vv.size() ); + CV_Assert( i >= 0 && i < (int)vv.size() ); return vv[i].step; } if( k == STD_ARRAY_MAT ) { const Mat* vv = (const Mat*)obj; - if( i < 0 ) - return 1; - CV_Assert( i < sz.height ); + CV_Assert( i >= 0 && i < sz.height ); return vv[i].step; } if( k == STD_VECTOR_UMAT ) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return vv[i].step; } @@ -1087,7 +1079,7 @@ size_t _InputArray::step(int i) const if (k == STD_VECTOR_CUDA_GPU_MAT) { const std::vector& vv = *(const std::vector*)obj; - CV_Assert((size_t)i < vv.size()); + CV_Assert(i >= 0 && (size_t)i < vv.size()); return vv[i].step; } diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0a82424ba1..44ee8f9c59 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -113,6 +113,10 @@ #include "opencv2/core/opencl/runtime/opencl_core.hpp" +#ifdef HAVE_DIRECTX +#include "directx.hpp" +#endif + #ifdef HAVE_OPENCL_SVM #include "opencv2/core/opencl/runtime/opencl_svm_20.hpp" #include "opencv2/core/opencl/runtime/opencl_svm_hsa_extension.hpp" @@ -2327,6 +2331,9 @@ protected: , contextId(CV_XADD(&g_contextId, 1)) , configuration(configuration_) , handle(0) +#ifdef HAVE_DIRECTX + , p_directx_impl(0) +#endif #ifdef HAVE_OPENCL_SVM , svmInitialized(false) #endif @@ -2352,6 +2359,9 @@ protected: handle = NULL; } devices.clear(); +#ifdef HAVE_DIRECTX + directx::internal::deleteDirectXImpl(&p_directx_impl); +#endif } { @@ -2427,6 +2437,7 @@ public: if (impl) { CV_LOG_INFO(NULL, "OpenCL: reuse context@" << impl->contextId << " for configuration: " << configuration) + impl->addref(); return impl; } @@ -2658,6 +2669,19 @@ public: return *bufferPoolHostPtr_.get(); } +#ifdef HAVE_DIRECTX + directx::internal::OpenCLDirectXImpl* p_directx_impl; + + directx::internal::OpenCLDirectXImpl* getDirectXImpl() + { + if (!p_directx_impl) + { + p_directx_impl = directx::internal::createDirectXImpl(); + } + return p_directx_impl; + } +#endif + #ifdef HAVE_OPENCL_SVM bool svmInitialized; bool svmAvailable; @@ -7286,4 +7310,15 @@ uint64 Timer::durationNS() const }} // namespace +#ifdef HAVE_DIRECTX +namespace cv { namespace directx { namespace internal { +OpenCLDirectXImpl* getDirectXImpl(ocl::Context& ctx) +{ + ocl::Context::Impl* i = ctx.getImpl(); + CV_Assert(i); + return i->getDirectXImpl(); +} +}}} // namespace cv::directx::internal +#endif + #endif // HAVE_OPENCL diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index d5942d1edc..0ec6270a70 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -228,6 +228,211 @@ UMatDataAutoLock::~UMatDataAutoLock() getUMatDataAutoLocker().release(u1, u2); } +//////////////////////////////// UMat //////////////////////////////// + +UMat::UMat(UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{} + +UMat::UMat(int _rows, int _cols, int _type, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create(_rows, _cols, _type); +} + +UMat::UMat(int _rows, int _cols, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create(_rows, _cols, _type); + *this = _s; +} + +UMat::UMat(Size _sz, int _type, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create( _sz.height, _sz.width, _type ); +} + +UMat::UMat(Size _sz, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create(_sz.height, _sz.width, _type); + *this = _s; +} + +UMat::UMat(int _dims, const int* _sz, int _type, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create(_dims, _sz, _type); +} + +UMat::UMat(int _dims, const int* _sz, int _type, const Scalar& _s, UMatUsageFlags _usageFlags) +: flags(MAGIC_VAL), dims(0), rows(0), cols(0), allocator(0), usageFlags(_usageFlags), u(0), offset(0), size(&rows) +{ + create(_dims, _sz, _type); + *this = _s; +} + +UMat::UMat(const UMat& m) +: flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), allocator(m.allocator), + usageFlags(m.usageFlags), u(m.u), offset(m.offset), size(&rows) +{ + addref(); + if( m.dims <= 2 ) + { + step[0] = m.step[0]; step[1] = m.step[1]; + } + else + { + dims = 0; + copySize(m); + } +} + +UMat& UMat::operator=(const UMat& m) +{ + if( this != &m ) + { + const_cast(m).addref(); + release(); + flags = m.flags; + if( dims <= 2 && m.dims <= 2 ) + { + dims = m.dims; + rows = m.rows; + cols = m.cols; + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + copySize(m); + allocator = m.allocator; + if (usageFlags == USAGE_DEFAULT) + usageFlags = m.usageFlags; + u = m.u; + offset = m.offset; + } + return *this; +} + +UMat UMat::clone() const +{ + UMat m; + copyTo(m); + return m; +} + +void UMat::assignTo(UMat& m, int _type) const +{ + if( _type < 0 ) + m = *this; + else + convertTo(m, _type); +} + +void UMat::create(int _rows, int _cols, int _type, UMatUsageFlags _usageFlags) +{ + _type &= TYPE_MASK; + if( dims <= 2 && rows == _rows && cols == _cols && type() == _type && u ) + return; + int sz[] = {_rows, _cols}; + create(2, sz, _type, _usageFlags); +} + +void UMat::create(Size _sz, int _type, UMatUsageFlags _usageFlags) +{ + create(_sz.height, _sz.width, _type, _usageFlags); +} + +void UMat::addref() +{ + if( u ) + CV_XADD(&(u->urefcount), 1); +} + +void UMat::release() +{ + if( u && CV_XADD(&(u->urefcount), -1) == 1 ) + deallocate(); + for(int i = 0; i < dims; i++) + size.p[i] = 0; + u = 0; +} + +bool UMat::empty() const +{ + return u == 0 || total() == 0 || dims == 0; +} + +size_t UMat::total() const +{ + if( dims <= 2 ) + return (size_t)rows * cols; + size_t p = 1; + for( int i = 0; i < dims; i++ ) + p *= size[i]; + return p; +} + + +UMat::UMat(UMat&& m) +: flags(m.flags), dims(m.dims), rows(m.rows), cols(m.cols), allocator(m.allocator), + usageFlags(m.usageFlags), u(m.u), offset(m.offset), size(&rows) +{ + if (m.dims <= 2) // move new step/size info + { + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + { + CV_DbgAssert(m.step.p != m.step.buf); + step.p = m.step.p; + size.p = m.size.p; + m.step.p = m.step.buf; + m.size.p = &m.rows; + } + m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; + m.allocator = NULL; + m.u = NULL; + m.offset = 0; +} + +UMat& UMat::operator=(UMat&& m) +{ + if (this == &m) + return *this; + release(); + flags = m.flags; dims = m.dims; rows = m.rows; cols = m.cols; + allocator = m.allocator; usageFlags = m.usageFlags; + u = m.u; + offset = m.offset; + if (step.p != step.buf) // release self step/size + { + fastFree(step.p); + step.p = step.buf; + size.p = &rows; + } + if (m.dims <= 2) // move new step/size info + { + step[0] = m.step[0]; + step[1] = m.step[1]; + } + else + { + CV_DbgAssert(m.step.p != m.step.buf); + step.p = m.step.p; + size.p = m.size.p; + m.step.p = m.step.buf; + m.size.p = &m.rows; + } + m.flags = MAGIC_VAL; m.dims = m.rows = m.cols = 0; + m.allocator = NULL; + m.u = NULL; + m.offset = 0; + return *this; +} + MatAllocator* UMat::getStdAllocator() { diff --git a/modules/core/test/test_intrin.cpp b/modules/core/test/test_intrin.cpp index 321fa64264..71d61e14e0 100644 --- a/modules/core/test/test_intrin.cpp +++ b/modules/core/test/test_intrin.cpp @@ -126,9 +126,11 @@ DEFINE_SIMD_TESTS(256, AVX512_SKX) TEST(hal_intrin256, float16x16_FP16) { +#if CV_TRY_FP16 //CV_CPU_CALL_FP16_(test_hal_intrin_float16, ()); CV_CPU_CALL_AVX2_(test_hal_intrin_float16, ()); - throw SkipTestException("Unsupported hardware: FP16 is not available"); +#endif + throw SkipTestException("Unsupported: FP16 is not available"); } @@ -142,8 +144,10 @@ namespace intrin512 { TEST(hal_intrin512, float16x32_FP16) { +#if CV_TRY_FP16 CV_CPU_CALL_AVX512_SKX_(test_hal_intrin_float16, ()); - throw SkipTestException("Unsupported hardware: FP16 is not available"); +#endif + throw SkipTestException("Unsupported: FP16 is not available"); } diff --git a/modules/core/test/test_intrin_utils.hpp b/modules/core/test/test_intrin_utils.hpp index 6731091463..84da496b42 100644 --- a/modules/core/test/test_intrin_utils.hpp +++ b/modules/core/test/test_intrin_utils.hpp @@ -1902,21 +1902,21 @@ void test_hal_intrin_float64() #endif } -#if CV_FP16 void test_hal_intrin_float16() { DUMP_ENTRY(v_float16); #if CV_FP16 TheTest() .test_loadstore_fp16_f32() -#endif #if CV_SIMD_FP16 .test_loadstore_fp16() .test_float_cvt_fp16() #endif ; -} +#else + std::cout << "SKIP: CV_FP16 is not available" << std::endl; #endif +} /*#if defined(CV_CPU_DISPATCH_MODE_FP16) && CV_CPU_DISPATCH_MODE == FP16 void test_hal_intrin_float16() diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index e695300d4b..d30c485368 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -1640,6 +1640,32 @@ TEST(Core_InputOutput, FileStorage_free_file_after_exception) ASSERT_EQ(0, std::remove(fileName.c_str())); } +TEST(Core_InputOutput, FileStorage_write_to_sequence) +{ + const std::vector formatExts = { ".yml", ".json", ".xml" }; + const std::string fileName = "FileStorage_write_to_sequence"; + + for (const auto& ext : formatExts) + { + FileStorage fs(fileName + ext, FileStorage::WRITE); + std::vector in = { 23, 42 }; + fs.startWriteStruct("some_sequence", cv::FileNode::SEQ); + for (int i : in) + fs.write("", i); + fs.endWriteStruct(); + fs.release(); + + FileStorage fsIn(fileName + ext, FileStorage::READ); + FileNode seq = fsIn["some_sequence"]; + FileNodeIterator it = seq.begin(), it_end = seq.end(); + std::vector out; + for (; it != it_end; ++it) + out.push_back((int)*it); + + EXPECT_EQ(in, out); + } +} + TEST(Core_InputOutput, FileStorage_YAML_parse_multiple_documents) { const std::string filename = "FileStorage_YAML_parse_multiple_documents.yml"; diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 578f693dbf..74ee167c54 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -9,6 +9,8 @@ #include "opencv2/core/eigen.hpp" #endif +#include "opencv2/core/cuda.hpp" + namespace opencv_test { namespace { class Core_ReduceTest : public cvtest::BaseTest @@ -1974,6 +1976,157 @@ TEST(Core_InputArray, fetch_MatExpr) } +#ifdef CV_CXX11 +class TestInputArrayRangeChecking { + static const char *kind2str(cv::_InputArray ia) + { + switch (ia.kind()) + { + #define C(x) case cv::_InputArray::x: return #x + C(MAT); + C(UMAT); + C(EXPR); + C(MATX); + C(STD_VECTOR); + C(STD_ARRAY); + C(NONE); + C(STD_VECTOR_VECTOR); + C(STD_BOOL_VECTOR); + C(STD_VECTOR_MAT); + C(STD_ARRAY_MAT); + C(STD_VECTOR_UMAT); + C(CUDA_GPU_MAT); + C(STD_VECTOR_CUDA_GPU_MAT); + #undef C + default: + return ""; + } + } + + static void banner(cv::_InputArray ia, const char *label, const char *name) + { + std::cout << std::endl + << label << " = " << name << ", Kind: " << kind2str(ia) + << std::endl; + } + + template + static void testA(I ia, F f, const char *mfname) + { + banner(ia, "f", mfname); + EXPECT_THROW(f(ia, -1), cv::Exception) + << "f(ia, " << -1 << ") should throw cv::Exception"; + for (int i = 0; i < int(ia.size()); i++) + { + EXPECT_NO_THROW(f(ia, i)) + << "f(ia, " << i << ") should not throw an exception"; + } + EXPECT_THROW(f(ia, int(ia.size())), cv::Exception) + << "f(ia, " << ia.size() << ") should throw cv::Exception"; + } + + template + static void testB(I ia, F f, const char *mfname) + { + banner(ia, "f", mfname); + EXPECT_THROW(f(ia, -1), cv::Exception) + << "f(ia, " << -1 << ") should throw cv::Exception"; + for (int i = 0; i < int(ia.size()); i++) + { + EXPECT_NO_THROW(f(ia, i)) + << "f(ia, " << i << ") should not throw an exception"; + } + EXPECT_THROW(f(ia, int(ia.size())), cv::Exception) + << "f(ia, " << ia.size() << ") should throw cv::Exception"; + } + + static void test_isContinuous() + { + auto f = [](cv::_InputArray ia, int i) { (void)ia.isContinuous(i); }; + + cv::Mat M; + cv::UMat uM; + + std::vector vec = {M, M}; + std::array arr = {M, M}; + std::vector uvec = {uM, uM}; + + testA(vec, f, "isContinuous"); + testA(arr, f, "isContinuous"); + testA(uvec, f, "isContinuous"); + } + + static void test_isSubmatrix() + { + auto f = [](cv::_InputArray ia, int i) { (void)ia.isSubmatrix(i); }; + + cv::Mat M; + cv::UMat uM; + + std::vector vec = {M, M}; + std::array arr = {M, M}; + std::vector uvec = {uM, uM}; + + testA(vec, f, "isSubmatrix"); + testA(arr, f, "isSubmatrix"); + testA(uvec, f, "isSubmatrix"); + } + + static void test_offset() + { + auto f = [](cv::_InputArray ia, int i) { return ia.offset(i); }; + + cv::Mat M; + cv::UMat uM; + cv::cuda::GpuMat gM; + + std::vector vec = {M, M}; + std::array arr = {M, M}; + std::vector uvec = {uM, uM}; + std::vector gvec = {gM, gM}; + + testB(vec, f, "offset"); + testB(arr, f, "offset"); + testB(uvec, f, "offset"); + testB(gvec, f, "offset"); + } + + static void test_step() + { + auto f = [](cv::_InputArray ia, int i) { return ia.step(i); }; + + cv::Mat M; + cv::UMat uM; + cv::cuda::GpuMat gM; + + std::vector vec = {M, M}; + std::array arr = {M, M}; + std::vector uvec = {uM, uM}; + std::vector gvec = {gM, gM}; + + testB(vec, f, "step"); + testB(arr, f, "step"); + testB(uvec, f, "step"); + testB(gvec, f, "step"); + } + +public: + static void run() + { + test_isContinuous(); + test_isSubmatrix(); + test_offset(); + test_step(); + } +}; + +TEST(Core_InputArray, range_checking) +{ + TestInputArrayRangeChecking::run(); +} +#endif + + TEST(Core_Vectors, issue_13078) { float floats_[] = { 1, 2, 3, 4, 5, 6, 7, 8 }; diff --git a/modules/core/test/test_math.cpp b/modules/core/test/test_math.cpp index 0b9469ee83..ab0d52778d 100644 --- a/modules/core/test/test_math.cpp +++ b/modules/core/test/test_math.cpp @@ -2584,7 +2584,7 @@ TEST(Core_CheckRange_INT_MAX, accuracy) TEST(Core_CheckRange_INT_MAX1, accuracy) { cv::Mat m(3, 3, CV_32SC1, cv::Scalar(INT_MAX)); - ASSERT_TRUE( cv::checkRange(m, true, 0, 0, INT_MAX+1.0f) ); + ASSERT_TRUE( cv::checkRange(m, true, 0, 0, (float)((double)INT_MAX+1.0f)) ); ASSERT_TRUE( cv::checkRange(m) ); } diff --git a/modules/core/test/test_quaternion.cpp b/modules/core/test/test_quaternion.cpp new file mode 100644 index 0000000000..0025674ec7 --- /dev/null +++ b/modules/core/test/test_quaternion.cpp @@ -0,0 +1,255 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include +#include +using namespace cv; +namespace opencv_test{ namespace { +class QuatTest: public ::testing::Test { +protected: + void SetUp() override + { + q1 = {1,2,3,4}; + q2 = {2.5,-2,3.5,4}; + q1Unit = {1 / sqrt(30), sqrt(2) /sqrt(15), sqrt(3) / sqrt(10), 2 * sqrt(2) / sqrt(15)}; + q1Inv = {1.0 / 30, -1.0 / 15, -1.0 / 10, -2.0 / 15}; + } + double scalar = 2.5; + double angle = CV_PI; + int qNorm2 = 2; + Vec axis{1, 1, 1}; + Vec unAxis{0, 0, 0}; + Vec unitAxis{1.0 / sqrt(3), 1.0 / sqrt(3), 1.0 / sqrt(3)}; + Quatd q3 = Quatd::createFromAngleAxis(angle, axis); + Quatd q3UnitAxis = Quatd::createFromAngleAxis(angle, unitAxis); + Quat q3Norm2 = q3 * qNorm2; + + Quat q1Inv; + Quat q1; + Quat q2; + Quat q1Unit; + + Quatd qNull{0, 0, 0, 0}; + Quatd qIdentity{1, 0, 0, 0}; + QuatAssumeType assumeUnit = QUAT_ASSUME_UNIT; + +}; + +TEST_F(QuatTest, constructor){ + Vec coeff{1, 2, 3, 4}; + EXPECT_EQ(Quat (coeff), q1); + EXPECT_EQ(q3, q3UnitAxis); + EXPECT_ANY_THROW(Quatd::createFromAngleAxis(angle, unAxis)); + Matx33d R1{ + -1.0 / 3, 2.0 / 3 , 2.0 / 3, + 2.0 / 3 , -1.0 / 3, 2.0 / 3, + 2.0 / 3 , 2.0 / 3 , -1.0 / 3 + }; + Matx33d R2{ + -2.0 / 3, -2.0 / 3, -1.0 / 3, + -2.0 / 3, 1.0 / 3, 2.0 / 3, + -1.0 / 3, 2.0 / 3, -2.0 / 3 + }; + Matx33d R3{ + 0.818181818181, 0.181818181818, 0.54545455454, + 0.545454545545, -0.54545454545, -0.6363636364, + 0.181818181818, 0.818181818182, -0.5454545455 + }; + Matx33d R4{ + 0.818181818181, -0.181818181818, 0.54545455454, + 0.545454545545, 0.54545454545, -0.6363636364, + -0.181818181818, 0.818181818182, 0.5454545455 + }; + Quatd qMat = Quatd::createFromRotMat(R1); + Quatd qMat2 = Quatd::createFromRotMat(R2); + Quatd qMat3 = Quatd::createFromRotMat(R3); + Quatd qMat4 = Quatd::createFromRotMat(R4); + EXPECT_EQ(qMat2, Quatd(0, -0.408248290463, 0.816496580927, 0.408248904638)); + EXPECT_EQ(qMat3, Quatd(-0.426401432711,-0.852802865422, -0.213200716355, -0.2132007163)); + EXPECT_EQ(qMat, q3); + EXPECT_EQ(qMat4, -Quatd(0.852802865422, 0.426401432711221, 0.2132007163556, 0.2132007163)); + + Vec3d rot{angle / sqrt(3),angle / sqrt(3), angle / sqrt(3)}; + Quatd rotQuad{0, 1.0 / sqrt(3), 1. / sqrt(3), 1. / sqrt(3)}; + Quatd qRot = Quatd::createFromRvec(rot); + EXPECT_EQ(qRot, rotQuad); + EXPECT_EQ(Quatd::createFromRvec(Vec3d(0, 0, 0)), qIdentity); +} + +TEST_F(QuatTest, basicfuns){ + Quat q1Conj{1, -2, -3, -4}; + EXPECT_EQ(q3Norm2.normalize(), q3); + EXPECT_EQ(q1.norm(), sqrt(30)); + EXPECT_EQ(q1.normalize(), q1Unit); + EXPECT_ANY_THROW(qNull.normalize()); + EXPECT_EQ(q1.conjugate(), q1Conj); + EXPECT_EQ(q1.inv(), q1Inv); + EXPECT_EQ(inv(q1), q1Inv); + EXPECT_EQ(q3.inv(assumeUnit) * q3, qIdentity); + EXPECT_EQ(q1.inv() * q1, qIdentity); + EXPECT_ANY_THROW(inv(qNull)); + EXPECT_NO_THROW(q1.at(0)); + EXPECT_ANY_THROW(q1.at(4)); + + Matx33d R{ + -2.0 / 3, 2.0 / 15 , 11.0 / 15, + 2.0 / 3 , -1.0 / 3 , 2.0 / 3 , + 1.0 / 3 , 14.0 / 15, 2.0 / 15 + }; + Matx33d q1RotMat = q1.toRotMat3x3(); + EXPECT_MAT_NEAR(q1RotMat, R, 1e-6); + Vec3d z_axis{0,0,1}; + Quatd q_unit1 = Quatd::createFromAngleAxis(angle, z_axis); + Mat pointsA = (Mat_(2, 3) << 1,0,0,1,0,1); + pointsA = pointsA.t(); + Mat new_point = q_unit1.toRotMat3x3() * pointsA; + Mat afterRo = (Mat_(3, 2) << -1,-1,0,0,0,1); + EXPECT_MAT_NEAR(afterRo, new_point, 1e-6); + EXPECT_ANY_THROW(qNull.toRotVec()); + Vec3d rodVec{CV_PI/sqrt(3), CV_PI/sqrt(3), CV_PI/sqrt(3)}; + Vec3d q3Rod = q3.toRotVec(); + EXPECT_NEAR(q3Rod[0], rodVec[0], 1e-6); + EXPECT_NEAR(q3Rod[1], rodVec[1], 1e-6); + EXPECT_NEAR(q3Rod[2], rodVec[2], 1e-6); + + EXPECT_EQ(log(q1Unit, assumeUnit), log(q1Unit)); + EXPECT_EQ(log(qIdentity, assumeUnit), qNull); + EXPECT_EQ(log(q3), Quatd(0, angle * unitAxis[0] / 2, angle * unitAxis[1] / 2, angle * unitAxis[2] / 2)); + EXPECT_ANY_THROW(log(qNull)); + EXPECT_EQ(log(Quatd(exp(1), 0, 0, 0)), qIdentity); + + EXPECT_EQ(exp(qIdentity), Quatd(exp(1), 0, 0, 0)); + EXPECT_EQ(exp(qNull), qIdentity); + EXPECT_EQ(exp(Quatd(0, angle * unitAxis[0] / 2, angle * unitAxis[1] / 2, angle * unitAxis[2] / 2)), q3); + + EXPECT_EQ(power(q3, 2), Quatd::createFromAngleAxis(2*angle, axis)); + EXPECT_EQ(power(Quatd(0.5, 0.5, 0.5, 0.5), 2.0, assumeUnit), Quatd(-0.5,0.5,0.5,0.5)); + EXPECT_EQ(power(Quatd(0.5, 0.5, 0.5, 0.5), -2.0), Quatd(-0.5,-0.5,-0.5,-0.5)); + EXPECT_EQ(sqrt(q1), power(q1, 0.5)); + EXPECT_EQ(exp(q3 * log(q1)), power(q1, q3)); + EXPECT_EQ(exp(q1 * log(q3)), power(q3, q1, assumeUnit)); + EXPECT_EQ(crossProduct(q1, q3), (q1 * q3 - q3 * q1) / 2); + EXPECT_EQ(sinh(qNull), qNull); + EXPECT_EQ(sinh(q1), (exp(q1) - exp(-q1)) / 2); + EXPECT_EQ(sinh(qIdentity), Quatd(sinh(1), 0, 0, 0)); + EXPECT_EQ(sinh(q1), Quatd(0.73233760604, -0.44820744998, -0.67231117497, -0.8964148999610843)); + EXPECT_EQ(cosh(qNull), qIdentity); + EXPECT_EQ(cosh(q1), Quatd(0.961585117636, -0.34135217456, -0.51202826184, -0.682704349122)); + EXPECT_EQ(tanh(q1), sinh(q1) * inv(cosh(q1))); + EXPECT_EQ(sin(qNull), qNull); + EXPECT_EQ(sin(q1), Quatd(91.78371578403, 21.88648685303, 32.829730279543, 43.772973706058)); + EXPECT_EQ(cos(qNull), qIdentity); + EXPECT_EQ(cos(q1), Quatd(58.9336461679, -34.0861836904, -51.12927553569, -68.17236738093)); + EXPECT_EQ(tan(q1), sin(q1)/cos(q1)); + EXPECT_EQ(sinh(asinh(q1)), q1); + Quatd c1 = asinh(sinh(q1)); + EXPECT_EQ(sinh(c1), sinh(q1)); + EXPECT_EQ(cosh(acosh(q1)), q1); + c1 = acosh(cosh(q1)); + EXPECT_EQ(cosh(c1), cosh(q1)); + EXPECT_EQ(tanh(atanh(q1)), q1); + c1 = atanh(tanh(q1)); + EXPECT_EQ(tanh(q1), tanh(c1)); + EXPECT_EQ(asin(sin(q1)), q1); + EXPECT_EQ(sin(asin(q1)), q1); + EXPECT_EQ(acos(cos(q1)), q1); + EXPECT_EQ(cos(acos(q1)), q1); + EXPECT_EQ(atan(tan(q3)), q3); + EXPECT_EQ(tan(atan(q1)), q1); +} + +TEST_F(QuatTest, opeartor){ + Quatd minusQ{-1, -2, -3, -4}; + Quatd qAdd{3.5, 0, 6.5, 8}; + Quatd qMinus{-1.5, 4, -0.5, 0}; + Quatd qMultq{-20, 1, -5, 27}; + Quatd qMults{2.5, 5.0, 7.5, 10.0}; + Quatd qDvss{1.0 / 2.5, 2.0 / 2.5, 3.0 / 2.5, 4.0 / 2.5}; + Quatd qOrigin(q1); + + EXPECT_EQ(-q1, minusQ); + EXPECT_EQ(q1 + q2, qAdd); + EXPECT_EQ(q1 - q2, qMinus); + EXPECT_EQ(q1 * q2, qMultq); + EXPECT_EQ(q1 * scalar, qMults); + EXPECT_EQ(scalar * q1, qMults); + EXPECT_EQ(q1 / q1, qIdentity); + EXPECT_EQ(q1 / scalar, qDvss); + q1 += q2; + EXPECT_EQ(q1, qAdd); + q1 -= q2; + EXPECT_EQ(q1, qOrigin); + q1 *= q2; + EXPECT_EQ(q1, qMultq); + q1 /= q2; + EXPECT_EQ(q1, qOrigin); + q1 *= scalar; + EXPECT_EQ(q1, qMults); + q1 /= scalar; + EXPECT_EQ(q1, qOrigin); + EXPECT_NO_THROW(q1[0]); + EXPECT_NO_THROW(q1.at(0)); + EXPECT_ANY_THROW(q1[4]); + EXPECT_ANY_THROW(q1.at(4)); +} + +TEST_F(QuatTest, quatAttrs){ + double angleQ1 = 2 * acos(1.0 / sqrt(30)); + Vec3d axis1{0.3713906763541037, 0.557086014, 0.742781352}; + Vec q1axis1 = q1.getAxis(); + + EXPECT_EQ(angleQ1, q1.getAngle()); + EXPECT_EQ(angleQ1, q1Unit.getAngle()); + EXPECT_EQ(angleQ1, q1Unit.getAngle(assumeUnit)); + EXPECT_EQ(0, qIdentity.getAngle()); + EXPECT_ANY_THROW(qNull.getAxis()); + EXPECT_NEAR(axis1[0], q1axis1[0], 1e-6); + EXPECT_NEAR(axis1[1], q1axis1[1], 1e-6); + EXPECT_NEAR(axis1[2], q1axis1[2], 1e-6); + EXPECT_NEAR(q3Norm2.norm(), qNorm2, 1e-6); + EXPECT_EQ(q3Norm2.getAngle(), angle); + EXPECT_NEAR(axis1[0], axis1[0], 1e-6); + EXPECT_NEAR(axis1[1], axis1[1], 1e-6); + EXPECT_NEAR(axis1[2], axis1[2], 1e-6); +} + +TEST_F(QuatTest, interpolation){ + Quatd qNoRot = Quatd::createFromAngleAxis(0, axis); + Quatd qLerpInter(1.0 / 2, sqrt(3) / 6, sqrt(3) / 6, sqrt(3) / 6); + EXPECT_EQ(Quatd::lerp(qNoRot, q3, 0), qNoRot); + EXPECT_EQ(Quatd::lerp(qNoRot, q3, 1), q3); + EXPECT_EQ(Quatd::lerp(qNoRot, q3, 0.5), qLerpInter); + Quatd q3NrNn2 = qNoRot * qNorm2; + EXPECT_EQ(Quatd::nlerp(q3NrNn2, q3Norm2, 0), qNoRot); + EXPECT_EQ(Quatd::nlerp(q3NrNn2, q3Norm2, 1), q3); + EXPECT_EQ(Quatd::nlerp(q3NrNn2, q3Norm2, 0.5), qLerpInter.normalize()); + EXPECT_EQ(Quatd::nlerp(qNoRot, q3, 0, assumeUnit), qNoRot); + EXPECT_EQ(Quatd::nlerp(qNoRot, q3, 1, assumeUnit), q3); + EXPECT_EQ(Quatd::nlerp(qNoRot, q3, 0.5, assumeUnit), qLerpInter.normalize()); + Quatd q3Minus(-q3); + EXPECT_EQ(Quatd::nlerp(qNoRot, q3, 0.4), -Quatd::nlerp(qNoRot, q3Minus, 0.4)); + EXPECT_EQ(Quatd::slerp(qNoRot, q3, 0, assumeUnit), qNoRot); + EXPECT_EQ(Quatd::slerp(qNoRot, q3, 1, assumeUnit), q3); + EXPECT_EQ(Quatd::slerp(qNoRot, q3, 0.5, assumeUnit), -Quatd::nlerp(qNoRot, -q3, 0.5, assumeUnit)); + EXPECT_EQ(Quatd::slerp(qNoRot, q1, 0.5), Quatd(0.76895194, 0.2374325, 0.35614876, 0.47486501)); + EXPECT_EQ(Quatd::slerp(-qNoRot, q1, 0.5), Quatd(0.76895194, 0.2374325, 0.35614876, 0.47486501)); + EXPECT_EQ(Quatd::slerp(qNoRot, -q1, 0.5), -Quatd::slerp(-qNoRot, q1, 0.5)); + + Quat tr1 = Quatd::createFromAngleAxis(0, axis); + Quat tr2 = Quatd::createFromAngleAxis(angle / 2, axis); + Quat tr3 = Quatd::createFromAngleAxis(angle, axis); + Quat tr4 = Quatd::createFromAngleAxis(angle, Vec3d{-1/sqrt(2),0,1/(sqrt(2))}); + EXPECT_ANY_THROW(Quatd::spline(qNull, tr1, tr2, tr3, 0)); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr4, 0), tr2); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr4, 1), tr3); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr4, 0.6, assumeUnit), Quatd::spline(tr1, tr2, tr3, tr4, 0.6)); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr3, 0.5), Quatd::spline(tr1, -tr2, tr3, tr3, 0.5)); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr3, 0.5), -Quatd::spline(-tr1, -tr2, -tr3, tr3, 0.5)); + EXPECT_EQ(Quatd::spline(tr1, tr2, tr3, tr3, 0.5), Quatd(0.336889853392, 0.543600719487, 0.543600719487, 0.543600719487)); +} + +} // namespace + +}// opencv_test \ No newline at end of file diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 5d8fbc8b84..39aaa1edb4 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -248,8 +248,6 @@ CV__DNN_INLINE_NS_BEGIN int type; std::vector kernel_size, strides; std::vector pads_begin, pads_end; - CV_DEPRECATED_EXTERNAL Size kernel, stride, pad; - CV_DEPRECATED_EXTERNAL int pad_l, pad_t, pad_r, pad_b; bool globalPooling; //!< Flag is true if at least one of the axes is global pooled. std::vector isGlobalPooling; bool computeMaxIdx; diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 3b12508c74..69b71f90ce 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -93,7 +93,8 @@ CV__DNN_INLINE_NS_BEGIN DNN_TARGET_VULKAN, DNN_TARGET_FPGA, //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin. DNN_TARGET_CUDA, - DNN_TARGET_CUDA_FP16 + DNN_TARGET_CUDA_FP16, + DNN_TARGET_HDDL }; CV_EXPORTS std::vector< std::pair > getAvailableBackends(); @@ -364,9 +365,12 @@ CV__DNN_INLINE_NS_BEGIN const int requiredOutputs, std::vector &outputs, std::vector &internals) const; + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;} + virtual bool updateMemoryShapes(const std::vector &inputs); + CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes. CV_PROP String type; //!< Type name which was used for creating layer by layer factory. CV_PROP int preferableTarget; //!< prefer target for layer forwarding @@ -571,6 +575,7 @@ CV__DNN_INLINE_NS_BEGIN * | DNN_TARGET_FPGA | | + | | | * | DNN_TARGET_CUDA | | | | + | * | DNN_TARGET_CUDA_FP16 | | | | + | + * | DNN_TARGET_HDDL | | + | | | */ CV_WRAP void setPreferableTarget(int targetId); @@ -1072,14 +1077,17 @@ CV__DNN_INLINE_NS_BEGIN * Model creates net from file with trained weights and config, * sets preprocessing input and runs forward pass. */ - class CV_EXPORTS_W_SIMPLE Model : public Net + class CV_EXPORTS_W_SIMPLE Model { public: - /** - * @brief Default constructor. - */ + CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first) Model(); + Model(const Model&) = default; + Model(Model&&) = default; + Model& operator=(const Model&) = default; + Model& operator=(Model&&) = default; + /** * @brief Create model from deep learning network represented in one of the supported formats. * An order of @p model and @p config arguments does not matter. @@ -1100,13 +1108,12 @@ CV__DNN_INLINE_NS_BEGIN */ CV_WRAP Model& setInputSize(const Size& size); - /** @brief Set input size for frame. + /** @overload * @param[in] width New input width. * @param[in] height New input height. - * @note If shape of the new blob less than 0, - * then frame size not change. */ - CV_WRAP Model& setInputSize(int width, int height); + CV_WRAP inline + Model& setInputSize(int width, int height) { return setInputSize(Size(width, height)); } /** @brief Set mean value for frame. * @param[in] mean Scalar with mean values which are subtracted from channels. @@ -1143,10 +1150,31 @@ CV__DNN_INLINE_NS_BEGIN * @param[in] frame The input image. * @param[out] outs Allocated output blobs, which will store results of the computation. */ - CV_WRAP void predict(InputArray frame, OutputArrayOfArrays outs); + CV_WRAP void predict(InputArray frame, OutputArrayOfArrays outs) const; + + + // ============================== Net proxy methods ============================== + // Never expose methods with network implementation details, like: + // - addLayer, addLayerToPrev, connect, setInputsNames, setInputShape, setParam, getParam + // - getLayer*, getUnconnectedOutLayers, getUnconnectedOutLayersNames, getLayersShapes + // - forward* methods, setInput + + /// @sa Net::setPreferableBackend + CV_WRAP Model& setPreferableBackend(dnn::Backend backendId); + /// @sa Net::setPreferableTarget + CV_WRAP Model& setPreferableTarget(dnn::Target targetId); + + CV_DEPRECATED_EXTERNAL + operator Net&() const { return getNetwork_(); } + + //protected: - internal/tests usage only + Net& getNetwork_() const; + inline Net& getNetwork_() { return const_cast(this)->getNetwork_(); } - protected: struct Impl; + inline Impl* getImpl() const { return impl.get(); } + inline Impl& getImplRef() const { CV_DbgAssert(impl); return *impl.get(); } + protected: Ptr impl; }; diff --git a/modules/dnn/include/opencv2/dnn/utils/inference_engine.hpp b/modules/dnn/include/opencv2/dnn/utils/inference_engine.hpp index 7db93a916d..29882b92b0 100644 --- a/modules/dnn/include/opencv2/dnn/utils/inference_engine.hpp +++ b/modules/dnn/include/opencv2/dnn/utils/inference_engine.hpp @@ -58,6 +58,11 @@ CV_EXPORTS_W void resetMyriadDevice(); CV_EXPORTS_W cv::String getInferenceEngineVPUType(); +/** @brief Release a HDDL plugin. + */ +CV_EXPORTS_W void releaseHDDLPlugin(); + + CV__DNN_INLINE_NS_END }} // namespace diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 87c2a8e3bc..7dc2786906 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20200908 +#define OPENCV_DNN_API_VERSION 20201117 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn5_v, OPENCV_DNN_API_VERSION) diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index 7d51cd300f..c2a3a66ab9 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -533,7 +533,7 @@ struct ConvParamID CONV_100 = 100, CONV_LAST = sizeof(testConvolutionConfigs) / sizeof(testConvolutionConfigs[0]) }; - int val_; \ + int val_; ConvParamID(int val = 0) : val_(val) {} operator int() const { return val_; } static ::testing::internal::ParamGenerator all() @@ -546,7 +546,7 @@ struct ConvParamID ConvParamID v_[NUM]; for (int i = 0; i < NUM; ++i) { v_[i] = ConvParamID(i); } // reduce generated code size return ::testing::ValuesIn(v_, v_ + NUM); } -}; \ +}; static inline void PrintTo(const ConvParamID& v, std::ostream* os) { CV_Assert((int)v >= 0); CV_Assert((int)v < ConvParamID::CONV_LAST); diff --git a/modules/dnn/perf/perf_convolution1d.cpp b/modules/dnn/perf/perf_convolution1d.cpp new file mode 100644 index 0000000000..c35cbd503f --- /dev/null +++ b/modules/dnn/perf/perf_convolution1d.cpp @@ -0,0 +1,163 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" +#include + +namespace opencv_test { + +struct Conv1DParam_t { + int kernel; + struct BlobShape { int dims[3]; } shapeIn; + int outCN; + int groups; + int stride; + int dilation; + int pad[2]; + const char* padMode; + bool hasBias; + double declared_flops; +}; +// Details: #12142 +static const Conv1DParam_t testConvolution1DConfigs[] = { + {3, {{1, 6, 10}}, 6, 1, 1, 1, {0, 0}, "VALID", true, 1776.}, + {3, {{1, 2, 19}}, 2, 2, 2, 1, {1, 1}, "", true, 260.}, + {3, {{1, 2, 25}}, 2, 2, 1, 1, {2, 2}, "SAME", false, 650.}, +}; + +struct Conv1DParamID +{ + enum { + CONV_0 = 0, + CONV_LAST = sizeof(testConvolution1DConfigs) / sizeof(testConvolution1DConfigs[0]) + }; + int val_; + Conv1DParamID(int val = 0) : val_(val) {} + operator int() const { return val_; } + static ::testing::internal::ParamGenerator all() + { + enum { NUM = (int)CONV_LAST }; + Conv1DParamID v_[NUM]; for (int i = 0; i < NUM; ++i) { v_[i] = Conv1DParamID(i); } // reduce generated code size + return ::testing::ValuesIn(v_, v_ + NUM); + } +}; +static inline void PrintTo(const Conv1DParamID& v, std::ostream* os) +{ + CV_Assert((int)v >= 0); CV_Assert((int)v < Conv1DParamID::CONV_LAST); + const Conv1DParam_t& p = testConvolution1DConfigs[(int)v]; + + *os << "GFLOPS=" << cv::format("%.3f", p.declared_flops * 1e-9) + << ", K=[" << p.kernel << "]" + << ", IN={" << p.shapeIn.dims[0] << ", " << p.shapeIn.dims[1] << ", " << p.shapeIn.dims[2] << "}" + << ", OCN=" << p.outCN; + if (p.groups > 1) + *os << ", G=" << p.groups; + if (p.stride != 1) + *os << ", S=" << p.stride; + if (p.dilation != 1) + *os << ", D=" << p.dilation; + if (p.pad[0] != 0 && p.pad[1] != 0 ) + *os << ", P=(" << p.pad[0] << ", " << p.pad[1] << ")"; + if (!((std::string)p.padMode).empty()) + *os << ", PM=" << ((std::string)p.padMode); + if (p.hasBias) + *os << ", BIAS"; +} + + +typedef tuple > Conv1DTestParam_t; +typedef TestBaseWithParam Conv1D; + +PERF_TEST_P_(Conv1D, conv1d) +{ + int test_id = (int)get<0>(GetParam()); + ASSERT_GE(test_id, 0); ASSERT_LT(test_id, Conv1DParamID::CONV_LAST); + const Conv1DParam_t& params = testConvolution1DConfigs[test_id]; + double declared_flops = params.declared_flops; + + DictValue kernel = DictValue::arrayInt(¶ms.kernel, 1); + DictValue stride = DictValue::arrayInt(¶ms.stride, 1); + DictValue pad = DictValue::arrayInt(¶ms.pad[0], 2); + DictValue dilation = DictValue::arrayInt(¶ms.dilation, 1); + + MatShape inputShape = MatShape(params.shapeIn.dims, params.shapeIn.dims + 3); + int outChannels = params.outCN; + int groups = params.groups; + std::string padMode(params.padMode); + + bool hasBias = params.hasBias; + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + + if (targetId != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + + int inChannels = inputShape[1]; + + int sz[] = {outChannels, inChannels / groups, params.kernel}; + Mat weights(3, &sz[0], CV_32F); + randu(weights, -1.0f, 1.0f); + + LayerParams lp; + lp.set("kernel_size", kernel); + lp.set("pad", pad); + if (!padMode.empty()) + lp.set("pad_mode", padMode); + + lp.set("stride", stride); + lp.set("dilation", dilation); + lp.set("num_output", outChannels); + lp.set("group", groups); + lp.set("bias_term", hasBias); + lp.type = "Convolution"; + lp.name = "testLayer"; + lp.blobs.push_back(weights); + + if (hasBias) + { + Mat bias(1, outChannels, CV_32F); + randu(bias, -1.0f, 1.0f); + lp.blobs.push_back(bias); + } + + int inpSz[] = {1, inChannels, inputShape[2]}; + Mat input(3, &inpSz[0], CV_32F); + randu(input, -1.0f, 1.0f); + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + + net.setInput(input); + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + + // warmup + Mat output = net.forward(); + + MatShape netInputShape = shape(input); + size_t weightsMemory = 0, blobsMemory = 0; + net.getMemoryConsumption(netInputShape, weightsMemory, blobsMemory); + int64 flops = net.getFLOPS(netInputShape); + CV_Assert(flops > 0); + + std::cout + << "IN=" << divUp(input.total() * input.elemSize(), 1u<<10) << " Kb " << netInputShape + << " OUT=" << divUp(output.total() * output.elemSize(), 1u<<10) << " Kb " << shape(output) + << " Weights(parameters): " << divUp(weightsMemory, 1u<<10) << " Kb" + << " MFLOPS=" << flops * 1e-6 << std::endl; + + TEST_CYCLE() + { + Mat res = net.forward(); + } + EXPECT_NEAR(flops, declared_flops, declared_flops * 1e-6); + SANITY_CHECK_NOTHING(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Conv1D, Combine( + Conv1DParamID::all(), + dnnBackendsAndTargets(false, false) // defined in ../test/test_common.hpp +)); + +} // namespace diff --git a/modules/dnn/perf/perf_convolution3d.cpp b/modules/dnn/perf/perf_convolution3d.cpp index e81a4bfc5b..22f61b3b9c 100644 --- a/modules/dnn/perf/perf_convolution3d.cpp +++ b/modules/dnn/perf/perf_convolution3d.cpp @@ -46,7 +46,7 @@ struct Conv3DParamID CONV_100 = 16, CONV_LAST = sizeof(testConvolution3DConfigs) / sizeof(testConvolution3DConfigs[0]) }; - int val_; \ + int val_; Conv3DParamID(int val = 0) : val_(val) {} operator int() const { return val_; } static ::testing::internal::ParamGenerator all() @@ -59,7 +59,7 @@ struct Conv3DParamID Conv3DParamID v_[NUM]; for (int i = 0; i < NUM; ++i) { v_[i] = Conv3DParamID(i); } // reduce generated code size return ::testing::ValuesIn(v_, v_ + NUM); } -}; \ +}; static inline void PrintTo(const Conv3DParamID& v, std::ostream* os) { CV_Assert((int)v >= 0); CV_Assert((int)v < Conv3DParamID::CONV_LAST); diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index 23ece025e7..aef3bc2c31 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -111,6 +111,10 @@ PERF_TEST_P_(DNNTestNetwork, ENet) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) || (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)) throw SkipTestException(""); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + throw SkipTestException(""); +#endif processNet("dnn/Enet-model-best.net", "", "enet.yml", Mat(cv::Size(512, 256), CV_32FC3)); } @@ -126,7 +130,7 @@ PERF_TEST_P_(DNNTestNetwork, OpenFace) if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL)) throw SkipTestException(""); #endif processNet("dnn/openface_nn4.small2.v1.t7", "", "", @@ -168,7 +172,7 @@ PERF_TEST_P_(DNNTestNetwork, DenseNet_121) PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages) { if (backend == DNN_BACKEND_HALIDE || - (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD)) + (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL))) throw SkipTestException(""); // The same .caffemodel but modified .prototxt // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp @@ -202,6 +206,10 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv3) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) throw SkipTestException("Test is disabled in OpenVINO 2020.4"); #endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021010000) // nGraph compilation failure + if (target == DNN_TARGET_MYRIAD) + throw SkipTestException(""); +#endif Mat sample = imread(findDataFile("dnn/dog416.png")); cvtColor(sample, sample, COLOR_BGR2RGB); @@ -214,7 +222,7 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv4) { if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); - if (target == DNN_TARGET_MYRIAD) + if (target == DNN_TARGET_MYRIAD) // not enough resources throw SkipTestException(""); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) @@ -233,6 +241,10 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv4_tiny) { if (backend == DNN_BACKEND_HALIDE) throw SkipTestException(""); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021010000) // nGraph compilation failure + if (target == DNN_TARGET_MYRIAD) + throw SkipTestException(""); +#endif Mat sample = imread(findDataFile("dnn/dog416.png")); cvtColor(sample, sample, COLOR_BGR2RGB); Mat inp; @@ -263,6 +275,10 @@ PERF_TEST_P_(DNNTestNetwork, Inception_v2_Faster_RCNN) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) throw SkipTestException("Test is disabled in OpenVINO 2019R2"); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + throw SkipTestException("Test is disabled in OpenVINO 2021.1 / MYRIAD"); #endif if (backend == DNN_BACKEND_HALIDE || (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) || diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index f39305eef4..0f60a393a5 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -122,6 +122,8 @@ public: { if (std::string::npos != i->find("MYRIAD") && target == DNN_TARGET_MYRIAD) return true; + if (std::string::npos != i->find("HDDL") && target == DNN_TARGET_HDDL) + return true; else if (std::string::npos != i->find("FPGA") && target == DNN_TARGET_FPGA) return true; else if (std::string::npos != i->find("CPU") && target == DNN_TARGET_CPU) @@ -184,6 +186,14 @@ private: #endif #ifdef HAVE_DNN_NGRAPH backends.push_back(std::make_pair(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, DNN_TARGET_MYRIAD)); +#endif + } + if (checkIETarget(DNN_TARGET_HDDL)) { +#ifdef HAVE_DNN_IE_NN_BUILDER_2019 + backends.push_back(std::make_pair(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, DNN_TARGET_HDDL)); +#endif +#ifdef HAVE_DNN_NGRAPH + backends.push_back(std::make_pair(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, DNN_TARGET_HDDL)); #endif } #ifdef HAVE_DNN_IE_NN_BUILDER_2019 @@ -1172,6 +1182,7 @@ struct Net::Impl : public detail::NetImplBase preferableBackend = DNN_BACKEND_DEFAULT; preferableTarget = DNN_TARGET_CPU; skipInfEngineInit = false; + hasDynamicShapes = false; } Ptr netInputLayer; @@ -1183,6 +1194,7 @@ struct Net::Impl : public detail::NetImplBase int preferableTarget; String halideConfigFile; bool skipInfEngineInit; + bool hasDynamicShapes; // Map host data to backend specific wrapper. std::map > backendWrappers; @@ -1379,6 +1391,7 @@ struct Net::Impl : public detail::NetImplBase preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16 || preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_HDDL || preferableTarget == DNN_TARGET_FPGA ); } @@ -1813,7 +1826,7 @@ struct Net::Impl : public detail::NetImplBase INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R2) && supportsCPUFallback; // TODO: there is a bug in Myriad plugin with custom layers shape infer. - if (preferableTarget == DNN_TARGET_MYRIAD) + if (preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL) { for (int i = 0; customizable && i < ld.inputBlobs.size(); ++i) { @@ -1823,6 +1836,7 @@ struct Net::Impl : public detail::NetImplBase // TODO: fix these workarounds if (preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_HDDL || preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16) customizable &= ld.type != "Concat"; @@ -1910,6 +1924,7 @@ struct Net::Impl : public detail::NetImplBase // Convert weights in FP16 for specific targets. if ((preferableTarget == DNN_TARGET_OPENCL_FP16 || preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_HDDL || preferableTarget == DNN_TARGET_FPGA) && !fused) { #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1) @@ -2104,7 +2119,7 @@ struct Net::Impl : public detail::NetImplBase bool customizable = ld.id != 0 && supportsCPUFallback; // TODO: there is a bug in Myriad plugin with custom layers shape infer. - if (preferableTarget == DNN_TARGET_MYRIAD) + if (preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL) { for (int i = 0; customizable && i < ld.inputBlobs.size(); ++i) { @@ -2114,6 +2129,7 @@ struct Net::Impl : public detail::NetImplBase // TODO: fix these workarounds if (preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_HDDL || preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16) customizable &= ld.type != "Concat"; @@ -2654,15 +2670,17 @@ struct Net::Impl : public detail::NetImplBase // OpenCL: fuse convolution layer followed by eltwise + relu // CUDA: fuse convolution layer followed by eltwise (and optional activation) - if ((IS_DNN_OPENCL_TARGET(preferableTarget) || IS_DNN_CUDA_TARGET(preferableTarget)) && - ld.layerInstance->type == "Convolution" ) + while (nextData && + (IS_DNN_OPENCL_TARGET(preferableTarget) || IS_DNN_CUDA_TARGET(preferableTarget)) && + ld.layerInstance->type == "Convolution" + ) // semantic of 'if' { - Ptr nextEltwiseLayer; - if( nextData ) - nextEltwiseLayer = nextData->layerInstance.dynamicCast(); + Ptr nextEltwiseLayer = nextData->layerInstance.dynamicCast(); + if (nextEltwiseLayer.empty()) + break; + #ifdef HAVE_CUDA // CUDA backend supports fusion with eltwise sum (without variable channels) - // `nextEltwiseLayer` is reset if eltwise layer doesn't have a compatible configuration for fusion if (IS_DNN_CUDA_TARGET(preferableTarget) && !nextEltwiseLayer.empty()) { // we create a temporary backend node for eltwise layer to obtain the eltwise configuration @@ -2672,10 +2690,43 @@ struct Net::Impl : public detail::NetImplBase // CUDA backend uses EltwiseOp when all operands have the same number of channels; otherwise, ShortcutOp is used. // Hence, a successful cast to EltwiseOp implies that the number of channels is same in all operand tensors. if (eltwiseNode.empty() || eltwiseNode->op != cuda4dnn::EltwiseOpType::SUM || !eltwiseNode->coeffs.empty()) - nextEltwiseLayer = Ptr(); + break; } #endif - if (!nextEltwiseLayer.empty() && nextData && nextData->inputBlobsId.size() == 2) + + if (IS_DNN_OPENCL_TARGET(preferableTarget) && pinsToKeep.count(lpNext) != 0) + break; + if (nextData->inputBlobsId.size() != 2) + break; + + if (IS_DNN_OPENCL_TARGET(preferableTarget)) + { + if (!nextData->params.has("operation") || toLowerCase(nextData->params.get("operation")) == "sum") + { + if (nextData->params.has("coeff")) + { + DictValue paramCoeff = nextData->params.get("coeff"); + int n = paramCoeff.size(); + bool isCoeffOneOne = (n == 2); + for (int i = 0; isCoeffOneOne && i < n; i++) + { + float c = paramCoeff.get(i); + isCoeffOneOne &= (c == 1.0f); + } + if (!isCoeffOneOne) + { + CV_LOG_DEBUG(NULL, "DNN/OpenCL: fusion of 'Sum' without coeffs (or {1.0, 1.0}) is supported only"); + break; + } + } + } + else + { + CV_LOG_DEBUG(NULL, "DNN/OpenCL: fusion with eltwise operation is not supported: " << nextData->params.get("operation")); + break; + } + } + { LayerData *eltwiseData = nextData; @@ -2732,11 +2783,13 @@ struct Net::Impl : public detail::NetImplBase // we need to check them separately; hence, the fuse variables bool fuse_eltwise = false, fuse_activation = false; + Ptr activ_power; if (IS_DNN_OPENCL_TARGET(preferableTarget) && !nextFusabeleActivLayer.empty() && nextData && (!nextData->type.compare("ReLU") || !nextData->type.compare("ChannelsPReLU") || - !nextData->type.compare("Power")) && + (!nextData->type.compare("Power") && (activ_power = nextFusabeleActivLayer.dynamicCast()) && activ_power->scale == 1.0f) + ) && currLayer->setActivation(nextFusabeleActivLayer)) { fuse_eltwise = true; @@ -2868,6 +2921,8 @@ struct Net::Impl : public detail::NetImplBase } } } + + break; } } @@ -3107,11 +3162,11 @@ struct Net::Impl : public detail::NetImplBase Ptr layer = ld.layerInstance; - TickMeter tm; - tm.start(); - if( !ld.skip ) { + TickMeter tm; + tm.start(); + std::map >::iterator it = ld.backendNodes.find(preferableBackend); if (preferableBackend == DNN_BACKEND_OPENCV || it == ld.backendNodes.end() || it->second.empty()) { @@ -3320,12 +3375,15 @@ struct Net::Impl : public detail::NetImplBase CV_Error(Error::StsNotImplemented, "Unknown backend identifier"); } } + + tm.stop(); + int64 t = tm.getTimeTicks(); + layersTimings[ld.id] = (t > 0) ? t : t + 1; // zero for skipped layers only } else - tm.reset(); - - tm.stop(); - layersTimings[ld.id] = tm.getTimeTicks(); + { + layersTimings[ld.id] = 0; + } ld.flag = 1; } @@ -3485,6 +3543,46 @@ struct Net::Impl : public detail::NetImplBase shapes = inOutShapes[layerId]; } + void updateLayersShapes() + { + CV_Assert(!layers[0].outputBlobs.empty()); + ShapesVec inputShapes; + for(int i = 0; i < layers[0].outputBlobs.size(); i++) + { + Mat& inp = layers[0].outputBlobs[i]; + CV_Assert(inp.total()); + if (preferableBackend == DNN_BACKEND_OPENCV && + preferableTarget == DNN_TARGET_OPENCL_FP16) + { + layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); + } + inputShapes.push_back(shape(inp)); + } + LayersShapesMap layersShapes; + layersShapes[0].in = inputShapes; + for (MapIdToLayerData::iterator it = layers.begin(); + it != layers.end(); it++) + { + int layerId = it->first; + std::vector& inputLayerIds = it->second.inputBlobsId; + if (layersShapes[layerId].in.empty()) + { + for(int i = 0; i < inputLayerIds.size(); i++) + { + int inputLayerId = inputLayerIds[i].lid; + LayersShapesMap::iterator inputIt = layersShapes.find(inputLayerId); + if(inputIt == layersShapes.end() || inputIt->second.out.empty()) + { + getLayerShapesRecursively(inputLayerId, layersShapes); + } + const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid]; + layersShapes[layerId].in.push_back(shape); + } + it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in); + } + } + } + LayerPin getLatestLayerPin(const std::vector& pins) { return *std::max_element(pins.begin(), pins.end()); @@ -3898,6 +3996,8 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) int id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); + if (params.get("has_dynamic_shapes", false)) + impl->hasDynamicShapes = true; return id; } @@ -4229,8 +4329,13 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons bool oldShape = prevShape == blobShape; blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); - if (!oldShape) + if (!oldShape) { ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid]; + if (impl->hasDynamicShapes) + { + impl->updateLayersShapes(); + } + } if (!ld.outputBlobsWrappers[pin.oid].empty()) { @@ -4501,6 +4606,7 @@ string Net::Impl::dump() case DNN_TARGET_OPENCL: out << "OCL"; colorId = 1; break; case DNN_TARGET_OPENCL_FP16: out << "OCL_FP16"; colorId = 2; break; case DNN_TARGET_MYRIAD: out << "MYRIAD"; colorId = 3; break; + case DNN_TARGET_HDDL: out << "HDDL"; colorId = 8; break; case DNN_TARGET_VULKAN: out << "VULKAN"; colorId = 7; break; case DNN_TARGET_FPGA: out << "FPGA"; colorId = 4; break; case DNN_TARGET_CUDA: out << "CUDA"; colorId = 5; break; @@ -5179,6 +5285,10 @@ bool Layer::getMemoryShapes(const std::vector &inputs, return false; } +bool Layer::updateMemoryShapes(const std::vector &inputs) +{ + return true; +} ////////////////////////////////////////////////////////////////////////// static Mutex& getLayerFactoryMutex() diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index 84b984ac97..c646c1fe3a 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -556,6 +556,9 @@ void InfEngineNgraphNet::init(Target targetId) case DNN_TARGET_MYRIAD: device_name = "MYRIAD"; break; + case DNN_TARGET_HDDL: + device_name = "HDDL"; + break; case DNN_TARGET_FPGA: device_name = "FPGA"; break; @@ -683,7 +686,7 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net) #endif } std::map config; - if (device_name == "MYRIAD") { + if (device_name == "MYRIAD" || device_name == "HDDL") { #if INF_ENGINE_VER_MAJOR_GT(INF_ENGINE_RELEASE_2020_4) config.emplace("MYRIAD_DETECT_NETWORK_BATCH", CONFIG_VALUE(NO)); #else diff --git a/modules/dnn/src/layers/blank_layer.cpp b/modules/dnn/src/layers/blank_layer.cpp index 5acdc3fa1e..5f93b45886 100644 --- a/modules/dnn/src/layers/blank_layer.cpp +++ b/modules/dnn/src/layers/blank_layer.cpp @@ -125,7 +125,7 @@ public: InferenceEngine::Builder::Layer ieLayer(name); ieLayer.setName(name); - if (preferableTarget == DNN_TARGET_MYRIAD) + if (preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL) { ieLayer.setType("Copy"); } diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 023c8b40d8..02495f45ea 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -48,6 +48,8 @@ #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include + #include "opencv2/core/hal/hal.hpp" #include "opencv2/core/hal/intrin.hpp" #include @@ -119,17 +121,22 @@ public: MatSize weightShape = blobs.empty() ? inputs[1].size : blobs[0].size; CV_Assert(inputs[0].dims == outputs[0].dims); + if (weightShape.dims() == 3) + { + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + } CV_Assert(weightShape.dims() == kernel_size.size() + 2); for (int i = 0; i < kernel_size.size(); i++) { CV_Assert(weightShape[i + 2] == kernel_size[i]); } const Mat &input = inputs[0]; - CV_Assert((input.dims == 4 || input.dims == 5) && (input.type() == CV_32F || input.type() == CV_16S)); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && (input.type() == CV_32F || input.type() == CV_16S)); for (size_t i = 0; i < outputs.size(); i++) { CV_Assert(inputs[i].type() == input.type()); - CV_Assert((inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]); for (int j = 0; j < inputs[i].dims; j++) { CV_Assert(inputs[i].size[j] == input.size[j]); } @@ -300,36 +307,41 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { + size_t ksize = kernel_size.size(); +#ifdef HAVE_CUDA if (backendId == DNN_BACKEND_CUDA) { /* only convolution 2d and 3d supported */ - if(kernel_size.size() == 2 || kernel_size.size() == 3) + if (ksize == 2 || ksize == 3) return true; return false; } - +#endif #ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - if (kernel_size.size() == 3) + if (ksize == 1) + return false; + if (ksize == 3) return preferableTarget == DNN_TARGET_CPU; - if ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || preferableTarget != DNN_TARGET_MYRIAD) && blobs.empty()) + bool isMyriad = preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL; + if ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || !isMyriad) && blobs.empty()) return false; - return (preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height); + return (!isMyriad || dilation.width == dilation.height); } - else #endif - { - if (kernel_size.size() == 3) - return (preferableTarget == DNN_TARGET_CPU && backendId == DNN_BACKEND_OPENCV); - else if (kernel_size.size() == 2) - return backendId == DNN_BACKEND_OPENCV || - (backendId == DNN_BACKEND_HALIDE && !blobs.empty()) || - (backendId == DNN_BACKEND_VKCOM && haveVulkan()); - else - return false; - } + if (backendId == DNN_BACKEND_OPENCV) + return ksize >= 1 && ksize <= 3; +#ifdef HAVE_HALIDE + if (backendId == DNN_BACKEND_HALIDE) + return ksize == 2 && !blobs.empty(); +#endif +#ifdef HAVE_VULKAN + if (backendId == DNN_BACKEND_VKCOM) + return ksize == 2; +#endif + return false; } bool getMemoryShapes(const std::vector &inputs, @@ -381,18 +393,27 @@ public: inputs_arr.getMatVector(inputs); // prepare weightsMat where each row is aligned and has enough zero padding on the right to // use vectorized (i.e. with intrinsics) loops without tail processing - Mat wm = blobs.empty() ? inputs[1].reshape(1, numOutput) : blobs[0].reshape(1, numOutput); - if( wm.step1() % VEC_ALIGN != 0 ) + if (!blobs.empty()) { - int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); - Mat wm_buffer = Mat(numOutput, newcols, wm.type()); - Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); - wm_padding.setTo(Scalar::all(0.)); - Mat wm_aligned = wm_buffer.colRange(0, wm.cols); - wm.copyTo(wm_aligned); - wm = wm_aligned; + Mat wm = blobs[0].reshape(1, numOutput); + if( wm.step1() % VEC_ALIGN != 0 ) + { + int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); + Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); + wm_padding.setTo(Scalar::all(0.)); + Mat wm_aligned = wm_buffer.colRange(0, wm.cols); + wm.copyTo(wm_aligned); + wm = wm_aligned; + } + weightsMat = wm; } - weightsMat = wm; + else + { + // initialized in .forward() + weightsMat.release(); + } + weightsMultipliers.assign(numOutput, 1.0); Mat biasMat = hasBias() ? blobs[1].reshape(1, numOutput) : Mat(); @@ -436,6 +457,14 @@ public: Ptr activ_power = activ.dynamicCast(); if (!activ_power.empty()) { + if (activ_power->scale != 1.0f) // not supported well by implementation, #17964 + { + // FIXIT no way to check number of blobs (like, eltwise input) + CV_LOG_DEBUG(NULL, "DNN/OpenCL: can't configure Power activation (scale != 1.0f)"); + activ.release(); + newActiv = false; + return false; + } if (activ_power->scale != 1.f || activ_power->shift != 0.f) { const int outCh = blobs[0].size[0]; @@ -897,8 +926,11 @@ public: { size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies()); - CV_Assert_N( - (input.dims == 4 || input.dims == 5) && (input.dims == output.dims), + bool isConv1D = input.dims == 3; + bool isConv2D = input.dims == 4; + bool isConv3D = input.dims == 5; + CV_CheckEQ(static_cast(kernel_size.size()), input.dims - 2, ""); + CV_Assert_N(input.dims == output.dims, input.size[0] == output.size[0], weights.rows == output.size[1], weights.cols == (input.size[1]/ngroups)*karea, @@ -908,12 +940,15 @@ public: input.isContinuous(), output.isContinuous(), biasvec.size() == (size_t)output.size[1]+2); + CV_Check(weights.step1(), weights.step1() % VEC_ALIGN == 0, ""); + CV_CheckType(weights.type(), CV_32FC1, ""); ParallelConv p; p.input_ = &input; p.weights_ = &weights; p.output_ = &output; - for( int i = 0; i < 4; i++ ) p.outShape[i] = output.size[i]; + int max_ind = isConv1D? 3: 4; + for( int i = 0; i < max_ind; i++ ) p.outShape[i] = output.size[i]; p.outShape[1] /= ngroups; p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations; @@ -925,20 +960,19 @@ public: int inpCnAll = input.size[1]; int depth = (input.dims == 5) ? input.size[2] : 1; int width = input.size[input.dims - 1]; - int height = input.size[input.dims - 2]; + int height = isConv1D? 1 : input.size[input.dims - 2]; int inpCn = inpCnAll / ngroups; - bool isConv2D = kernel_size.size() == 2; - - p.is1x1_ = isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 && - pads_begin[0] == 0 && pads_begin[1] == 0; + p.is1x1_ = (isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 && + pads_begin[0] == 0 && pads_begin[1] == 0) || + (isConv1D && pads_begin[0] == 0 && kernel_size[0] == 1); p.useAVX = checkHardwareSupport(CPU_AVX) && isConv2D; p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; - int kernel_d = !isConv2D? kernel_size[0] : 1; - int kernel_h = kernel_size[kernel_size.size() - 2]; + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; int kernel_w = kernel_size.back(); int blk_size_cn0 = cvCeil(800./(kernel_w*kernel_h)); @@ -948,14 +982,20 @@ public: ncn = std::min(ncn, inpCn); p.blk_size_cn = ncn; - int dil_d = !isConv2D? dilations[0] : 1; - int dil_h = dilations[dilations.size() - 2]; + int dil_d = isConv3D? dilations[0] : 1; + int dil_h = isConv1D? 1 : dilations[dilations.size() - 2]; int dil_w = dilations.back(); p.ofstab_.resize(karea * ncn); int* ofstab = &p.ofstab_[0]; - if (isConv2D) + if (isConv1D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[k*kernel_w + k_c] = k*width + k_c*dil_w; + } + else if (isConv2D) { for( int k = 0; k < ncn; k++ ) for( int k_r = 0; k_r < kernel_h; k_r++ ) @@ -984,34 +1024,36 @@ public: { const int valign = ConvolutionLayerImpl::VEC_ALIGN; int ngroups = ngroups_, batchSize = input_->size[0]*ngroups; + bool isConv1D = input_->dims == 3; bool isConv2D = input_->dims == 4; + bool isConv3D = input_->dims == 5; int outW = output_->size[output_->dims - 1]; - int outH = output_->size[output_->dims - 2]; + int outH = isConv1D? 1 : output_->size[output_->dims - 2]; int outCn = output_->size[1]/ngroups; - int depth = !isConv2D? input_->size[2] : 1; - int height = input_->size[input_->dims - 2]; + int depth = isConv3D? input_->size[2] : 1; + int height = isConv1D? 1 : input_->size[input_->dims - 2]; int width = input_->size[input_->dims - 1]; int inpCn = input_->size[1]/ngroups; const int nstripes = nstripes_; - int kernel_d = !isConv2D? kernel_size[0] : 1; - int kernel_h = kernel_size[kernel_size.size() - 2]; + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; int kernel_w = kernel_size.back(); int karea = kernel_w*kernel_h*kernel_d; - int pad_d = !isConv2D? pads_begin[0] : 0; - int pad_t = pads_begin[pads_begin.size() - 2]; + int pad_d = isConv3D? pads_begin[0] : 0; + int pad_t = isConv1D? 0 : pads_begin[pads_begin.size() - 2]; int pad_l = pads_begin.back(); - int stride_d = !isConv2D? strides[0] : 0; - int stride_h = strides[strides.size() - 2]; + int stride_d = isConv3D? strides[0] : 0; + int stride_h = isConv1D? 0 : strides[strides.size() - 2]; int stride_w = strides.back(); - int dilation_d = !isConv2D? dilations[0] : 1; - int dilation_h = dilations[dilations.size() - 2]; + int dilation_d = isConv3D? dilations[0] : 1; + int dilation_h = isConv1D? 1 : dilations[dilations.size() - 2]; int dilation_w = dilations.back(); int i, j, k, d; @@ -1251,7 +1293,71 @@ public: // do im2row for a part of input tensor float* rowbuf = rowbuf0; - if (isConv2D) + if (isConv1D) + { + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_j = out_j * stride_w - pad_l; + const float* imgptr = data_inp0 + cn0*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + float v0 = imgptr[k1]; + float v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int i0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int i1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, 0, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + int imgofs = k*width + i*dilation_w; + rowbuf[k*kernel_w + i] = imgptr[imgofs]; + } + } + } + } + } + } + } + else if (isConv2D) { if( is1x1 && stride_w == 1 && stride_h == 1 ) { @@ -1484,9 +1590,12 @@ public: vs12 = v_setzero_f32(), vs13 = v_setzero_f32(); for( k = 0; k < vsz; k += 4, rptr += 4 ) { - v_float32x4 w0 = v_load_aligned(wptr0 + k), w1 = v_load_aligned(wptr1 + k); - v_float32x4 r0 = v_load_aligned(rptr), r1 = v_load_aligned(rptr + vsz_a), - r2 = v_load_aligned(rptr + vsz_a*2), r3 = v_load_aligned(rptr + vsz_a*3); + v_float32x4 w0 = v_load_aligned(wptr0 + k); + v_float32x4 w1 = v_load_aligned(wptr1 + k); + v_float32x4 r0 = v_load_aligned(rptr); + v_float32x4 r1 = v_load_aligned(rptr + vsz_a); + v_float32x4 r2 = v_load_aligned(rptr + vsz_a*2); + v_float32x4 r3 = v_load_aligned(rptr + vsz_a*3); vs00 += w0*r0; vs01 += w0*r1; @@ -1556,6 +1665,12 @@ public: #ifdef HAVE_OPENCL bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) { + if (kernel_size.size() != 2) + { + // no OpenCL optimizations, see .supportedBacked() + return false; + } + std::vector inputs; std::vector outputs; @@ -1739,26 +1854,35 @@ public: if (blobs.empty()) { Mat wm = inputs[1].reshape(1, outCn); - if( wm.step1() % VEC_ALIGN != 0 ) + if (wm.data != weightsMat.data) { - wm.copyTo(weightsMat); + int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); + Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); + wm_padding.setTo(Scalar::all(0.)); + weightsMat = wm_buffer.colRange(0, wm.cols); + + wm.copyTo((const Mat&)weightsMat); if (inputs.size() > 2) { Mat biasMat = inputs[2].reshape(1, outCn); biasMat.col(0).copyTo(biasvec); - biasvec.resize(outCn + 2); - } - else - { - biasvec.resize(outCn + 2, 0); } + biasvec.resize(outCn + 2, 0); } } - - /*printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", - name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], - kernel.width, kernel.height, pad.width, pad.height, - stride.width, stride.height, dilation.width, dilation.height);*/ + /*if (inputs[0].dims > 3) { + printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + } + else { + printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + }*/ int inpGroupCn = blobs.empty() ? inputs[1].size[1] : blobs[0].size[1]; CV_Assert_N(inputs.size() >= (size_t)1, inputs[0].size[1] % inpGroupCn == 0, outputs.size() == 1, inputs[0].data != outputs[0].data); diff --git a/modules/dnn/src/layers/mvn_layer.cpp b/modules/dnn/src/layers/mvn_layer.cpp index 4a096ce19c..db986bc897 100644 --- a/modules/dnn/src/layers/mvn_layer.cpp +++ b/modules/dnn/src/layers/mvn_layer.cpp @@ -126,7 +126,10 @@ public: { #ifdef HAVE_DNN_IE_NN_BUILDER_2019 if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - return !zeroDev && (preferableTarget != DNN_TARGET_MYRIAD || eps <= 1e-7f); + { + bool isMyriad = preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL; + return !zeroDev && (!isMyriad || eps <= 1e-7f); + } #endif #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) diff --git a/modules/dnn/src/layers/normalize_bbox_layer.cpp b/modules/dnn/src/layers/normalize_bbox_layer.cpp index 3d33511d17..a979fdedb6 100644 --- a/modules/dnn/src/layers/normalize_bbox_layer.cpp +++ b/modules/dnn/src/layers/normalize_bbox_layer.cpp @@ -75,7 +75,8 @@ public: if (pnorm != 2) return false; - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && preferableTarget == DNN_TARGET_MYRIAD) + bool isMyriad = preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL; + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && isMyriad) return !acrossSpatial; return startAxis == 1; diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index c83cf026de..b286133419 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -103,9 +103,12 @@ public: { #ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + bool isMyriad = preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL; return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1) && - (preferableTarget != DNN_TARGET_MYRIAD || + (!isMyriad || (dstRanges.size() == 4 && paddings[0].first == 0 && paddings[0].second == 0)); + } #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 0e07e5352e..621315a572 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -95,8 +95,9 @@ public: computeMaxIdx = true; globalPooling = false; isGlobalPooling = std::vector(3, false); - stride = Size(1, 1); - pad_t = pad_l = pad_b = pad_r = 0; + + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; if (params.has("pool") || params.has("kernel_size") || params.has("kernel_w") || params.has("kernel_h")) @@ -115,16 +116,6 @@ public: getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode); globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2]; - if (kernel_size.size() == 2) { - kernel = Size(kernel_size[1], kernel_size[0]); - stride = Size(strides[1], strides[0]); - pad = Size(pads_begin[1], pads_begin[0]); - - pad_t = pads_begin[0]; - pad_l = pads_begin[1]; - pad_b = pads_end[0]; - pad_r = pads_end[1]; - } } else if (params.has("pooled_w") || params.has("pooled_h")) { @@ -172,17 +163,20 @@ public: finalKernel.push_back(isGlobalPooling[idx] ? inp[i] : kernel_size[idx]); } kernel_size = finalKernel; - kernel = Size(kernel_size[1], kernel_size[0]); } getConvPoolPaddings(inp, kernel_size, strides, padMode, pads_begin, pads_end); - if (pads_begin.size() == 2) { - pad_t = pads_begin[0]; - pad_l = pads_begin[1]; - pad_b = pads_end[0]; - pad_r = pads_end[1]; + + if (inputs[0].dims == 3) + { + //Pool1D + kernel_size.erase(kernel_size.begin() + 1); + strides.erase(strides.begin() + 1); + pads_begin.erase(pads_begin.begin() + 1); + pads_end.erase(pads_end.begin() + 1); } + #ifdef HAVE_OPENCL poolOp.release(); #endif @@ -202,9 +196,11 @@ public: return false; if (kernel_size.size() == 3) return preferableTarget == DNN_TARGET_CPU; - if (preferableTarget == DNN_TARGET_MYRIAD) { + if (kernel_size.size() == 1) + return false; + if (preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL) { #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1) - if (type == MAX && (pad_l == 1 && pad_t == 1) && stride == Size(2, 2) ) { + if (type == MAX && (pads_begin[1] == 1 && pads_begin[0] == 1) && (strides[0] == 2 && strides[1] == 2)) { return !isMyriadX(); } #endif @@ -216,21 +212,30 @@ public: #endif if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - return !computeMaxIdx && type != STOCHASTIC; + return !computeMaxIdx && type != STOCHASTIC && kernel_size.size() > 1; } - if (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_VKCOM) + else if (backendId == DNN_BACKEND_OPENCV) { if (kernel_size.size() == 3) - return (backendId == DNN_BACKEND_OPENCV && preferableTarget == DNN_TARGET_CPU); - if (kernel_size.empty() || kernel_size.size() == 2) - return backendId == DNN_BACKEND_OPENCV || - (backendId == DNN_BACKEND_HALIDE && haveHalide() && - (type == MAX || (type == AVE && !pad_t && !pad_l && !pad_b && !pad_r))) || - (backendId == DNN_BACKEND_VKCOM && haveVulkan() && - (type == MAX || type == AVE)); + return preferableTarget == DNN_TARGET_CPU; + if (kernel_size.size() <= 2) + return true; else return false; } + else if (backendId == DNN_BACKEND_HALIDE) + { + if (kernel_size.empty() || kernel_size.size() == 2) + return haveHalide() && + (type == MAX || (type == AVE && !pads_begin[0] && !pads_begin[1] && !pads_end[0] && !pads_end[1])); + } + else if (backendId == DNN_BACKEND_VKCOM) + { + if (kernel_size.empty() || kernel_size.size() == 2) + return haveVulkan() && + (type == MAX || type == AVE); + return false; + } return false; } @@ -250,12 +255,25 @@ public: config.in_shape = shape(inputs[0]); config.out_shape = shape(outputs[0]); - config.kernel = kernel; - config.pad_l = pad_l; - config.pad_t = pad_t; - config.pad_r = pad_r; - config.pad_b = pad_b; - config.stride = stride; + if (inputs[0].dims == 3) + { + //Pool1D + config.kernel = Size(kernel_size[0], 1); + config.stride = Size(strides[0], 1); + config.pad_l = pads_begin[0]; + config.pad_t = 0; + config.pad_r = pads_end[0]; + config.pad_b = 0; + } + else + { + config.kernel = Size(kernel_size[1], kernel_size[0]); + config.stride = Size(strides[1], strides[0]); + config.pad_l = pads_begin[1]; + config.pad_t = pads_begin[0]; + config.pad_r = pads_end[1]; + config.pad_b = pads_end[0]; + } config.channels = inputs[0].size[1]; config.pool_method = type == MAX ? LIBDNN_POOLING_METHOD_MAX : (type == AVE ? LIBDNN_POOLING_METHOD_AVE : @@ -572,7 +590,6 @@ public: public: const Mat* src, *rois; Mat *dst, *mask; - Size kernel, stride; int pad_l, pad_t, pad_r, pad_b; bool avePoolPaddedArea; int nstripes; @@ -598,7 +615,7 @@ public: CV_Assert_N( src.isContinuous(), dst.isContinuous(), src.type() == CV_32F, src.type() == dst.type(), - src.dims == 4 || src.dims == 5, dst.dims == 4 || dst.dims == 5, + src.dims == 3 || src.dims == 4 || src.dims == 5, dst.dims == 3 || dst.dims == 4 || dst.dims == 5, (((poolingType == ROI || poolingType == PSROI) && dst.size[0] == rois.size[0]) || src.size[0] == dst.size[0]), poolingType == PSROI || src.size[1] == dst.size[1], @@ -606,6 +623,9 @@ public: PoolingInvoker p; + bool isPool1D = src.dims == 3; + bool isPool3D = src.dims == 5; + p.src = &src; p.rois = &rois; p.dst = &dst; @@ -616,12 +636,10 @@ public: p.pads_end = pads_end; p.mask = &mask; - p.kernel = Size(kernel_size[1], kernel_size[0]); - p.stride = Size(strides[1], strides[0]); p.pad_l = pads_begin.back(); - p.pad_t = pads_begin[pads_begin.size() - 2]; + p.pad_t = isPool1D ? 0 : pads_begin[pads_begin.size() - 2]; p.pad_r = pads_end.back(); - p.pad_b = pads_end[pads_end.size() - 2]; + p.pad_b = isPool1D ? 0 : pads_end[pads_end.size() - 2]; p.avePoolPaddedArea = avePoolPaddedArea; p.nstripes = nstripes; @@ -631,11 +649,11 @@ public: if( !computeMaxIdx ) { - int height = src.size[src.dims - 2]; + int height = isPool1D ? 1 : src.size[src.dims - 2]; int width = src.size[src.dims - 1]; - int kernel_d = (kernel_size.size() == 3) ? kernel_size[0] : 1; - int kernel_h = kernel_size[kernel_size.size() - 2]; + int kernel_d = isPool3D ? kernel_size[0] : 1; + int kernel_h = isPool1D ? 1 : kernel_size[kernel_size.size() - 2]; int kernel_w = kernel_size.back(); p.ofsbuf.resize(kernel_d * kernel_h * kernel_w); @@ -655,13 +673,15 @@ public: { int channels = dst->size[1]; + bool isPool3D = src->dims == 5; bool isPool2D = src->dims == 4; - int depth = !isPool2D? dst->size[2] : 1; - int height = dst->size[dst->dims - 2]; + bool isPool1D = src->dims == 3; + int depth = isPool3D? dst->size[2] : 1; + int height = isPool1D? 1 : dst->size[dst->dims - 2]; int width = dst->size[dst->dims - 1]; - int inp_depth = !isPool2D? src->size[2] : 1; - int inp_height = src->size[src->dims - 2]; + int inp_depth = isPool3D? src->size[2] : 1; + int inp_height = isPool1D? 1 : src->size[src->dims - 2]; int inp_width = src->size[src->dims - 1]; size_t total = dst->total(); @@ -669,12 +689,12 @@ public: size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(r.end*stripeSize, total); - int kernel_d = !isPool2D? kernel_size[0] : 1; - int kernel_h = kernel_size[kernel_size.size() - 2]; + int kernel_d = isPool3D? kernel_size[0] : 1; + int kernel_h = isPool1D? 1 : kernel_size[kernel_size.size() - 2]; int kernel_w = kernel_size.back(); - int stride_d = !isPool2D? strides[0] : 0; - int stride_h = strides[strides.size() - 2]; + int stride_d = isPool3D? strides[0] : 0; + int stride_h = isPool1D? 1 :strides[strides.size() - 2]; int stride_w = strides.back(); bool compMaxIdx = computeMaxIdx; @@ -865,7 +885,24 @@ public: } } else +#else + CV_UNUSED(isPool2D); #endif + if( isPool1D ) + { + const float* first = srcData + xstart; + const float* last = srcData + xend; + const float* max_elem = std::max_element(first, last); + if (max_elem!=last) + { + dstData[x0] = *max_elem; + if( compMaxIdx ) + { + dstMaskData[x0] = std::distance(first, max_elem); + } + } + } + else { float max_val = -FLT_MAX; if( compMaxIdx ) @@ -939,6 +976,14 @@ public: } else #endif + if( isPool1D ) + { + const float* first = srcData + xstart; + const float* last = srcData + xend; + float sum_val = std::accumulate(first, last, 0.f); + dstData[x0] = sum_val*inv_kernel_area; + } + else { float sum_val = 0.f; for (int d = dstart; d < dend; ++d) { @@ -1052,20 +1097,26 @@ public: Halide::Buffer inputBuffer = halideBuffer(inputs[0]); const int inWidth = inputBuffer.width(); const int inHeight = inputBuffer.height(); + const size_t kernelHeight = kernel_size[0]; + const size_t kernelWidth = kernel_size[1]; + const size_t strideHeight = strides[0]; + const size_t strideWidth = strides[1]; + const size_t paddingTop = pads_begin[0]; + const size_t paddingLeft = pads_begin[1]; Halide::Var x("x"), y("y"), c("c"), n("n"); Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name)); - Halide::RDom r(0, kernel.width, 0, kernel.height); + Halide::RDom r(0, kernelWidth, 0, kernelHeight); Halide::Expr kx, ky; - if(pad_l || pad_t) + if(paddingLeft || paddingTop) { - kx = clamp(x * stride.width + r.x - pad_l, 0, inWidth - 1); - ky = clamp(y * stride.height + r.y - pad_t, 0, inHeight - 1); + kx = clamp(x * strideWidth + r.x - paddingLeft, 0, inWidth - 1); + ky = clamp(y * strideHeight + r.y - paddingTop, 0, inHeight - 1); } else { - kx = min(x * stride.width + r.x, inWidth - 1); - ky = min(y * stride.height + r.y, inHeight - 1); + kx = min(x * strideWidth + r.x, inWidth - 1); + ky = min(y * strideHeight + r.y, inHeight - 1); } // Halide::argmax returns tuple (r.x, r.y, max). @@ -1073,17 +1124,17 @@ public: // Compute offset from argmax in range [0, kernel_size). Halide::Expr max_index; - if(pad_l || pad_t) + if(paddingLeft || paddingTop) { - max_index = clamp(y * stride.height + res[1] - pad_t, + max_index = clamp(y * strideHeight + res[1] - paddingTop, 0, inHeight - 1) * inWidth + - clamp(x * stride.width + res[0] - pad_l, + clamp(x * strideWidth + res[0] - paddingLeft, 0, inWidth - 1); } else { - max_index = min(y * stride.height + res[1], inHeight - 1) * inWidth + - min(x * stride.width + res[0], inWidth - 1); + max_index = min(y * strideHeight + res[1], inHeight - 1) * inWidth + + min(x * strideWidth + res[0], inWidth - 1); } top(x, y, c, n) = { res[2], Halide::cast(max_index) }; return Ptr(new HalideBackendNode(top)); @@ -1097,21 +1148,25 @@ public: Halide::Buffer inputBuffer = halideBuffer(inputs[0]); const int inW = inputBuffer.width(), inH = inputBuffer.height(); - if ((inW - kernel.width) % stride.width || (inH - kernel.height) % stride.height) + const size_t kernelHeight = kernel_size[0]; + const size_t kernelWidth = kernel_size[1]; + const size_t strideHeight = strides[0]; + const size_t strideWidth = strides[1]; + if ((inW - kernelWidth) % strideWidth || (inH - kernelHeight) % strideHeight) { CV_Error(cv::Error::StsNotImplemented, "Halide backend for average pooling with partial " "kernels is not implemented"); } - const float norm = 1.0f / (kernel.width * kernel.height); + const float norm = 1.0f / (kernelWidth * kernelHeight); Halide::Var x("x"), y("y"), c("c"), n("n"); Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name)); - Halide::RDom r(0, kernel.width, 0, kernel.height); + Halide::RDom r(0, kernelWidth, 0, kernelHeight); top(x, y, c, n) = sum( - inputBuffer(x * stride.width + r.x, - y * stride.height + r.y, c, n)) * norm; + inputBuffer(x * strideWidth + r.x, + y * strideHeight + r.y, c, n)) * norm; return Ptr(new HalideBackendNode(top)); #endif // HAVE_HALIDE return Ptr(); @@ -1173,6 +1228,7 @@ public: { CV_Assert(inputs.size() != 0); + bool isPool1D = inputs[0].size() == 3; std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); std::vector outShape(inputs[0].begin(), inputs[0].begin() + 2); @@ -1191,25 +1247,34 @@ public: outShape.push_back(pooledSize.height); outShape.push_back(pooledSize.width); } - else if (padMode.empty()) - { - for (int i = 0; i < local_kernel.size(); i++) { - float dst = (float)(inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i]; - outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst))); - } - - // If we have padding, ensure that the last pooling starts strictly - // inside the image (instead of at the padding); otherwise clip the last. - for (int i = 0; i < pads_end.size(); i++) { - if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) { - --outShape[2 + i]; - CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]); - } - } - } else { - getConvPoolOutParams(inpShape, local_kernel, strides, padMode, std::vector(local_kernel.size(), 1), outShape); + if (hasDynamicShapes && !shapesInitialized) + { + //Just copy input shapes for width and height to prevent errors on loading stage + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back(inpShape[i]); + } + else if (padMode.empty()) + { + int addedDims = isPool1D? inpShape.size() : local_kernel.size(); + for (int i = 0; i < addedDims; i++) { + float dst = (float) (inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i]; + outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst))); + } + + // If we have padding, ensure that the last pooling starts strictly + // inside the image (instead of at the padding); otherwise clip the last. + for (int i = 0; i < addedDims; i++) { + if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) { + --outShape[2 + i]; + CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]); + } + } + } else { + getConvPoolOutParams(inpShape, local_kernel, strides, padMode, + std::vector(local_kernel.size(), 1), outShape); + } } if (type == ROI) { @@ -1231,12 +1296,21 @@ public: return false; } + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + int dims = inputs[0].size(); + CV_Assert(inputs[0][dims - 1] > 0 && inputs[0][dims - 2] > 0); + shapesInitialized = true; + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { CV_UNUSED(inputs); // suppress unused variable warning long flops = 0; - size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), + bool isPool1D = inputs[0].size() == 3; + size_t karea = std::accumulate(kernel_size.begin(), isPool1D? kernel_size.begin() + 1 : kernel_size.end(), 1, std::multiplies()); for(int i = 0; i < outputs.size(); i++) { @@ -1262,6 +1336,8 @@ private: ROI, // RoI pooling, https://arxiv.org/pdf/1504.08083.pdf PSROI // Position-sensitive RoI pooling, https://arxiv.org/pdf/1605.06409.pdf }; + bool hasDynamicShapes; + bool shapesInitialized; }; Ptr PoolingLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/prior_box_layer.cpp b/modules/dnn/src/layers/prior_box_layer.cpp index dc1364a06b..f7340b1e67 100644 --- a/modules/dnn/src/layers/prior_box_layer.cpp +++ b/modules/dnn/src/layers/prior_box_layer.cpp @@ -607,7 +607,7 @@ public: auto priorBox = std::make_shared(slice_layer, slice_image, attrs); auto axis = std::make_shared(ngraph::element::i64, ngraph::Shape{1}, std::vector{0}); - auto unsqueeze = std::make_shared(priorBox, axis); + auto unsqueeze = std::make_shared(priorBox, axis); return Ptr(new InfEngineNgraphNode(unsqueeze)); } else @@ -628,7 +628,7 @@ public: auto priorBox = std::make_shared(slice_layer, slice_image, attrs); auto axis = std::make_shared(ngraph::element::i64, ngraph::Shape{1}, std::vector{0}); - auto unsqueeze = std::make_shared(priorBox, axis); + auto unsqueeze = std::make_shared(priorBox, axis); return Ptr(new InfEngineNgraphNode(unsqueeze)); } } diff --git a/modules/dnn/src/layers/proposal_layer.cpp b/modules/dnn/src/layers/proposal_layer.cpp index 990cfeda30..4658e7b41f 100644 --- a/modules/dnn/src/layers/proposal_layer.cpp +++ b/modules/dnn/src/layers/proposal_layer.cpp @@ -95,8 +95,14 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { - return backendId == DNN_BACKEND_OPENCV || - ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && preferableTarget != DNN_TARGET_MYRIAD); +#ifdef HAVE_INF_ENGINE + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + bool isMyriad = preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL; + return !isMyriad; + } +#endif + return backendId == DNN_BACKEND_OPENCV; } bool getMemoryShapes(const std::vector &inputs, diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index dbea5d55f4..4c603c1ac8 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -170,6 +170,9 @@ public: setParamsFrom(params); int axis = params.get("axis", 0); int numAxes = params.get("num_axes", -1); + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; + CV_Assert(numAxes >= -1); newShapeRange = (numAxes == -1) ? Range(axis, INT_MAX) : Range(axis, axis + numAxes); @@ -182,6 +185,25 @@ public: for (i = 0; i < dims; i++) newShapeDesc[i] = paramShape.get(i); } + if (hasDynamicShapes) + { + dynamicShapes.clear(); + inputIndices.clear(); + if (params.has("dynamic_axes")) { + CV_Assert(params.has("input_indices")); + const DictValue &dynamicAxes = params.get("dynamic_axes"); + const DictValue &dynamicInputShapes = params.get("input_indices"); + int i, dims = dynamicAxes.size(); + CV_Assert(dims == dynamicInputShapes.size()); + CV_Assert(dims > 0); + dynamicShapes.resize(dims); + inputIndices.resize(dims); + for (i = 0; i < dims; i++) { + dynamicShapes[i] = dynamicAxes.get(i); + inputIndices[i] = dynamicInputShapes.get(i); + } + } + } } virtual bool supportBackend(int backendId) CV_OVERRIDE @@ -196,13 +218,21 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { + if (inputs.size() == 1 || inputs.size() == requiredOutputs) { outputs.clear(); for (size_t i = 0; i < inputs.size(); i++) { - outputs.push_back(MatShape()); - computeShapeByReshapeMask(inputs[i], newShapeDesc, newShapeRange, outputs.back()); + if (hasDynamicShapes && !shapesInitialized) + { + outputs.push_back(newShapeDesc); + } + else + { + outputs.push_back(MatShape()); + computeShapeByReshapeMask(inputs[i], newShapeDesc, newShapeRange, outputs.back()); + } } } else @@ -213,6 +243,19 @@ public: return true; } + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + if (hasDynamicShapes) + { + for (int i = 0; i < dynamicShapes.size(); ++i) + { + newShapeDesc[dynamicShapes[i]] = inputs[0][inputIndices[i]]; + } + } + shapesInitialized = true; + return true; + } + void finalize(InputArrayOfArrays, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { std::vector outputs; @@ -310,6 +353,10 @@ public: private: std::vector outShapes; + std::vector dynamicShapes; // Which axes shapes are dynamic and require reinitialization with new input + std::vector inputIndices; // Which axes from input are needed to compute correct output shape + bool hasDynamicShapes; + bool shapesInitialized; }; Ptr ReshapeLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 6deabb5884..fa2d755b71 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -72,6 +72,8 @@ public: setParamsFrom(params); axis = params.get("axis", 1); num_split = params.get("num_split", 0); + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; if (params.has("slice_point")) { CV_Assert(!params.has("begin") && !params.has("size") && !params.has("end")); @@ -150,7 +152,8 @@ public: CV_Assert(sliceRanges[i].size() <= inpShape.size()); for (int j = 0; j < sliceRanges[i].size(); ++j) { - outputs[i][j] = clamp(sliceRanges[i][j], inpShape[j]).size(); + if (shapesInitialized || inpShape[j] > 0) + outputs[i][j] = clamp(sliceRanges[i][j], inpShape[j]).size(); } } } @@ -165,6 +168,12 @@ public: return false; } + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + shapesInitialized = true; + return true; + } + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { #ifdef HAVE_OPENCL @@ -492,7 +501,7 @@ public: std::vector axes, offsets, dims; int from, to, step; int numDims = finalSliceRanges[0].size(); - if (preferableTarget == DNN_TARGET_MYRIAD) + if (preferableTarget == DNN_TARGET_MYRIAD || preferableTarget == DNN_TARGET_HDDL) { from = axis; to = numDims; @@ -597,6 +606,8 @@ public: protected: // The actual non-negative values determined from @p sliceRanges depends on input size. std::vector > finalSliceRanges; + bool hasDynamicShapes; + bool shapesInitialized; }; class CropLayerImpl CV_FINAL : public SliceLayerImpl diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index 677228bcf2..aefeaa42b3 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -15,6 +15,9 @@ namespace dnn { struct Model::Impl { +//protected: + Net net; + Size size; Scalar mean; double scale = 1.0; @@ -23,7 +26,70 @@ struct Model::Impl Mat blob; std::vector outNames; - void predict(Net& net, const Mat& frame, OutputArrayOfArrays outs) +public: + virtual ~Impl() {} + Impl() {} + Impl(const Impl&) = delete; + Impl(Impl&&) = delete; + + virtual Net& getNetwork() const { return const_cast(net); } + + virtual void setPreferableBackend(Backend backendId) { net.setPreferableBackend(backendId); } + virtual void setPreferableTarget(Target targetId) { net.setPreferableTarget(targetId); } + + /*virtual*/ + void initNet(const Net& network) + { + net = network; + + outNames = net.getUnconnectedOutLayersNames(); + std::vector inLayerShapes; + std::vector outLayerShapes; + net.getLayerShapes(MatShape(), 0, inLayerShapes, outLayerShapes); + if (!inLayerShapes.empty() && inLayerShapes[0].size() == 4) + size = Size(inLayerShapes[0][3], inLayerShapes[0][2]); + else + size = Size(); + } + + /*virtual*/ + void setInputParams(double scale_, const Size& size_, const Scalar& mean_, + bool swapRB_, bool crop_) + { + size = size_; + mean = mean_; + scale = scale_; + crop = crop_; + swapRB = swapRB_; + } + /*virtual*/ + void setInputSize(const Size& size_) + { + size = size_; + } + /*virtual*/ + void setInputMean(const Scalar& mean_) + { + mean = mean_; + } + /*virtual*/ + void setInputScale(double scale_) + { + scale = scale_; + } + /*virtual*/ + void setInputCrop(bool crop_) + { + crop = crop_; + } + /*virtual*/ + void setInputSwapRB(bool swapRB_) + { + swapRB = swapRB_; + } + + /*virtual*/ + void processFrame(InputArray frame, OutputArrayOfArrays outs) { if (size.empty()) CV_Error(Error::StsBadSize, "Input size not specified"); @@ -34,96 +100,115 @@ struct Model::Impl // Faster-RCNN or R-FCN if (net.getLayer(0)->outputNameToIndex("im_info") != -1) { - Mat imInfo = (Mat_(1, 3) << size.height, size.width, 1.6f); + Mat imInfo(Matx13f(size.height, size.width, 1.6f)); net.setInput(imInfo, "im_info"); } net.forward(outs, outNames); } }; -Model::Model() : impl(new Impl) {} +Model::Model() + : impl(makePtr()) +{ + // nothing +} Model::Model(const String& model, const String& config) - : Net(readNet(model, config)), impl(new Impl) + : Model() { - impl->outNames = getUnconnectedOutLayersNames(); - std::vector inLayerShapes; - std::vector outLayerShapes; - getLayerShapes(MatShape(), 0, inLayerShapes, outLayerShapes); - if (!inLayerShapes.empty() && inLayerShapes[0].size() == 4) - impl->size = Size(inLayerShapes[0][3], inLayerShapes[0][2]); -}; + impl->initNet(readNet(model, config)); +} -Model::Model(const Net& network) : Net(network), impl(new Impl) +Model::Model(const Net& network) + : Model() { - impl->outNames = getUnconnectedOutLayersNames(); - std::vector inLayerShapes; - std::vector outLayerShapes; - getLayerShapes(MatShape(), 0, inLayerShapes, outLayerShapes); - if (!inLayerShapes.empty() && inLayerShapes[0].size() == 4) - impl->size = Size(inLayerShapes[0][3], inLayerShapes[0][2]); -}; + impl->initNet(network); +} -Model& Model::setInputSize(const Size& size) +Net& Model::getNetwork_() const { - impl->size = size; + CV_DbgAssert(impl); + return impl->getNetwork(); +} + +Model& Model::setPreferableBackend(Backend backendId) +{ + CV_DbgAssert(impl); + impl->setPreferableBackend(backendId); + return *this; +} +Model& Model::setPreferableTarget(Target targetId) +{ + CV_DbgAssert(impl); + impl->setPreferableTarget(targetId); return *this; } -Model& Model::setInputSize(int width, int height) +Model& Model::setInputSize(const Size& size) { - impl->size = Size(width, height); + CV_DbgAssert(impl); + impl->setInputSize(size); return *this; } Model& Model::setInputMean(const Scalar& mean) { - impl->mean = mean; + CV_DbgAssert(impl); + impl->setInputMean(mean); return *this; } Model& Model::setInputScale(double scale) { - impl->scale = scale; + CV_DbgAssert(impl); + impl->setInputScale(scale); return *this; } Model& Model::setInputCrop(bool crop) { - impl->crop = crop; + CV_DbgAssert(impl); + impl->setInputCrop(crop); return *this; } Model& Model::setInputSwapRB(bool swapRB) { - impl->swapRB = swapRB; + CV_DbgAssert(impl); + impl->setInputSwapRB(swapRB); return *this; } void Model::setInputParams(double scale, const Size& size, const Scalar& mean, bool swapRB, bool crop) { - impl->size = size; - impl->mean = mean; - impl->scale = scale; - impl->crop = crop; - impl->swapRB = swapRB; + CV_DbgAssert(impl); + impl->setInputParams(scale, size, mean, swapRB, crop); } -void Model::predict(InputArray frame, OutputArrayOfArrays outs) +void Model::predict(InputArray frame, OutputArrayOfArrays outs) const { - impl->predict(*this, frame.getMat(), outs); + CV_DbgAssert(impl); + impl->processFrame(frame, outs); } + ClassificationModel::ClassificationModel(const String& model, const String& config) - : Model(model, config) {}; + : Model(model, config) +{ + // nothing +} -ClassificationModel::ClassificationModel(const Net& network) : Model(network) {}; +ClassificationModel::ClassificationModel(const Net& network) + : Model(network) +{ + // nothing +} std::pair ClassificationModel::classify(InputArray frame) { std::vector outs; - impl->predict(*this, frame.getMat(), outs); + impl->processFrame(frame, outs); CV_Assert(outs.size() == 1); double conf; @@ -145,11 +230,11 @@ KeypointsModel::KeypointsModel(const Net& network) : Model(network) {}; std::vector KeypointsModel::estimate(InputArray frame, float thresh) { - int frameHeight = frame.getMat().size[0]; - int frameWidth = frame.getMat().size[1]; + int frameHeight = frame.rows(); + int frameWidth = frame.cols(); std::vector outs; - impl->predict(*this, frame.getMat(), outs); + impl->processFrame(frame, outs); CV_Assert(outs.size() == 1); Mat output = outs[0]; @@ -202,9 +287,8 @@ SegmentationModel::SegmentationModel(const Net& network) : Model(network) {}; void SegmentationModel::segment(InputArray frame, OutputArray mask) { - std::vector outs; - impl->predict(*this, frame.getMat(), outs); + impl->processFrame(frame, outs); CV_Assert(outs.size() == 1); Mat score = outs[0]; @@ -250,12 +334,14 @@ void disableRegionNMS(Net& net) } DetectionModel::DetectionModel(const String& model, const String& config) - : Model(model, config) { - disableRegionNMS(*this); + : Model(model, config) +{ + disableRegionNMS(getNetwork_()); // FIXIT Move to DetectionModel::Impl::initNet() } -DetectionModel::DetectionModel(const Net& network) : Model(network) { - disableRegionNMS(*this); +DetectionModel::DetectionModel(const Net& network) : Model(network) +{ + disableRegionNMS(getNetwork_()); // FIXIT Move to DetectionModel::Impl::initNet() } void DetectionModel::detect(InputArray frame, CV_OUT std::vector& classIds, @@ -263,7 +349,7 @@ void DetectionModel::detect(InputArray frame, CV_OUT std::vector& classIds, float confThreshold, float nmsThreshold) { std::vector detections; - impl->predict(*this, frame.getMat(), detections); + impl->processFrame(frame, detections); boxes.clear(); confidences.clear(); @@ -271,15 +357,15 @@ void DetectionModel::detect(InputArray frame, CV_OUT std::vector& classIds, int frameWidth = frame.cols(); int frameHeight = frame.rows(); - if (getLayer(0)->outputNameToIndex("im_info") != -1) + if (getNetwork_().getLayer(0)->outputNameToIndex("im_info") != -1) { frameWidth = impl->size.width; frameHeight = impl->size.height; } - std::vector layerNames = getLayerNames(); - int lastLayerId = getLayerId(layerNames.back()); - Ptr lastLayer = getLayer(lastLayerId); + std::vector layerNames = getNetwork_().getLayerNames(); + int lastLayerId = getNetwork_().getLayerId(layerNames.back()); + Ptr lastLayer = getNetwork_().getLayer(lastLayerId); if (lastLayer->type == "DetectionOutput") { diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_pool.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_pool.cpp index 47b40cc6c2..b366c97ac8 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_pool.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_pool.cpp @@ -51,18 +51,20 @@ template OCL4DNNPool::OCL4DNNPool(OCL4DNNPoolConfig config) { int dims = config.in_shape.size(); - int spatial_dims = 2; + int spatial_dims = config.in_shape.size()-2; channels_ = config.channels; pool_method_ = config.pool_method; avePoolPaddedArea = config.avePoolPaddedArea; computeMaxIdx = config.computeMaxIdx; use_half = config.use_half; + kernel_shape_.push_back(config.kernel.height); + kernel_shape_.push_back(config.kernel.width); + stride_.push_back(config.stride.height); + stride_.push_back(config.stride.width); for (int i = 0; i < spatial_dims; ++i) { - kernel_shape_.push_back(i == 0 ? config.kernel.height : config.kernel.width); - stride_.push_back(i == 0 ? config.stride.height : config.stride.width); im_in_shape_.push_back(config.in_shape[dims - spatial_dims + i]); im_out_shape_.push_back(config.out_shape[dims - spatial_dims + i]); } @@ -75,10 +77,10 @@ OCL4DNNPool::OCL4DNNPool(OCL4DNNPoolConfig config) pad_l_ = config.pad_l; pad_r_ = config.pad_r; pad_b_ = config.pad_b; - height_ = im_in_shape_[0]; - width_ = im_in_shape_[1]; - pooled_height_ = im_out_shape_[0]; - pooled_width_ = im_out_shape_[1]; + height_ = spatial_dims == 1? 1 : im_in_shape_[0]; + width_ = im_in_shape_.back(); + pooled_height_ = spatial_dims == 1? 1 : im_out_shape_[0]; + pooled_width_ = im_out_shape_.back(); count_ = 1; for (int i = 0; i < config.out_shape.size(); ++i) diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 46773220aa..ad3d903d68 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -260,6 +260,40 @@ public: addNodeToMatch("Cast", gather); setFusedNode("Gather", input, index); } + + virtual bool match(const Ptr& net, int nodeId, + std::vector& matchedNodesIds, + std::vector& targetNodesIds) CV_OVERRIDE + { + bool retVal = Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds); + size_t matchedNodesNum = matchedNodesIds.size(); + // Now we check if merging can be made for these Gather and Cast nodes + if (!retVal || matchedNodesNum < 2) + return retVal; + else { + int nodeToMatch = matchedNodesIds[matchedNodesNum - 1]; + const Ptr node = net->getNode(nodeToMatch); + if (node->getType() == "Cast") { + int inpNodeId = matchedNodesIds[matchedNodesNum - 2]; + const Ptr inpNode = net->getNode(inpNodeId); + if (inpNode->getType() == "Gather") { + int numNodes = net->getNumNodes(); + std::string inpNodeName = node->getInputName(0); + for (int i = 0; i < numNodes; ++i) { + const Ptr node_to_check = net->getNode(i); + int numInp = node_to_check->getNumInputs(); + for (int inp = 0; inp < numInp; ++inp) { + if (i != nodeToMatch && inpNodeName == node_to_check->getInputName(0)) { + // Another node has the same input node, so it cannot be merged. + return false; + } + } + } + } + } + } + return retVal; + } }; class ExpandSubgraph : public Subgraph @@ -513,6 +547,19 @@ Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) CV_Assert(!field.empty()); Mat(sizes, CV_64FC1, (void*)field.data()).convertTo(blob, CV_32FC1); } + else if (datatype == opencv_onnx::TensorProto_DataType_INT32) + { + if (!tensor_proto.int32_data().empty()) + { + const ::google::protobuf::RepeatedField field = tensor_proto.int32_data(); + Mat(sizes, CV_32SC1, (void*)field.data()).copyTo(blob); + } + else + { + char* val = const_cast(tensor_proto.raw_data().c_str()); + Mat(sizes, CV_32SC1, val).copyTo(blob); + } + } else if (datatype == opencv_onnx::TensorProto_DataType_INT64) { blob.create(sizes, CV_32SC1); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7421ad1a5c..859b595b7f 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -64,6 +64,7 @@ public: ONNXImporter(Net& net, const char *onnxFile) : dstNet(net) { + hasDynamicShapes = false; CV_Assert(onnxFile); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); @@ -84,6 +85,7 @@ public: ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) : dstNet(net) { + hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); struct _Buf : public std::streambuf @@ -115,6 +117,7 @@ protected: std::map constBlobs; std::map outShapes; // List of internal blobs shapes. + bool hasDynamicShapes; // Whether the model has inputs with dynamic shapes typedef std::map::iterator IterShape_t; std::map layer_id; @@ -200,12 +203,12 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot if(attribute_name == "kernel_shape") { - CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); + CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("kernel_size", parse(attribute_proto.ints())); } else if(attribute_name == "strides") { - CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); + CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("stride", parse(attribute_proto.ints())); } else if(attribute_name == "pads") @@ -229,7 +232,7 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot else { // Convolution or pooling. - CV_Assert(attribute_proto.ints_size() == 4 || attribute_proto.ints_size() == 6); + CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 4 || attribute_proto.ints_size() == 6); lp.set("pad", parse(attribute_proto.ints())); } } @@ -244,7 +247,7 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot } else if(attribute_name == "dilations") { - CV_Assert(attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); + CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("dilation", parse(attribute_proto.ints())); } else if (attribute_proto.has_i()) @@ -413,8 +416,10 @@ void ONNXImporter::populateNet() for (int j = 0; j < inpShape.size(); ++j) { inpShape[j] = tensorShape.dim(j).dim_value(); + if (!tensorShape.dim(j).dim_param().empty()) + hasDynamicShapes = true; } - if (!inpShape.empty()) + if (!inpShape.empty() && !hasDynamicShapes) { inpShape[0] = std::max(inpShape[0], 1); // It's OK to have undetermined batch size } @@ -461,6 +466,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.name = name; layerParams.type = layer_type; + layerParams.set("has_dynamic_shapes", hasDynamicShapes); if (layer_type == "MaxPool") { @@ -494,14 +500,17 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) MatShape inpShape = outShapes[node_proto.input(0)]; DictValue axes = layerParams.get("axes"); bool keepdims = layerParams.get("keepdims"); - MatShape targetShape = inpShape; + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); for (int i = 0; i < axes.size(); i++) { int axis = clamp(axes.get(i), inpShape.size()); - if (keepdims) { - targetShape[axis] = 1; - } else { - targetShape.erase(targetShape.begin() + axis); - } + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); } if (inpShape.size() == 3 && axes.size() <= 2) @@ -551,11 +560,36 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) CV_Assert(axes.size() <= inpShape.size() - 2); std::vector kernel_size(inpShape.size() - 2, 1); - for (int i = 0; i < axes.size(); i++) { - int axis = clamp(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; + if (axes.size() == 1 && (clamp(axes.get(0), inpShape.size()) <= 1)) + { + int axis = clamp(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); + + LayerParams reshapeLp; + reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); + + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); + + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; + node_proto.set_input(0, node_proto.output(0)); } + else + { + for (int i = 0; i < axes.size(); i++) { + int axis = clamp(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; + } + } + LayerParams poolLp = layerParams; poolLp.name = layerParams.name + "/avg"; CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); @@ -1276,6 +1310,20 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) { layerParams.type = "Reshape"; layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + { + if (!maskedAxes[index]) + inputIndices.push_back(index); + } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } } else layerParams.type = "Identity"; @@ -1338,6 +1386,19 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) outShape.insert(outShape.begin() + axis, 1); layerParams.type = "Reshape"; layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } } else if (layer_type == "Expand") { @@ -1625,6 +1686,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); sliceLp.set("begin", paramBegin); sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); if (inpShape.size() > 1) { @@ -1637,6 +1699,17 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = "Reshape"; layerParams.set("axis", 0); layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } node_proto.set_input(0, sliceLp.name); } else @@ -1676,32 +1749,45 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) for (int i = 1; i < node_proto.input_size(); i++) CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - Mat shapes = getBlob(node_proto, node_proto.input_size() - 1); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - int height = shapes.at(2); - int width = shapes.at(3); - if (node_proto.input_size() == 3) + if (layerParams.has("coordinate_transformation_mode")) { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape scales = shapeIt->second; - height *= scales[2]; - width *= scales[3]; - } - layerParams.set("width", width); - layerParams.set("height", height); + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - if (layerParams.get("mode") == "linear") { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) + { + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) + { + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); + } } replaceLayerParam(layerParams, "mode", "interpolation"); } @@ -1741,10 +1827,14 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) else { // scales as input - Mat scales = getBlob(node_proto, 1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } } replaceLayerParam(layerParams, "mode", "interpolation"); } diff --git a/modules/dnn/src/op_inf_engine.cpp b/modules/dnn/src/op_inf_engine.cpp index 745d86ef3c..b7cdc2ad94 100644 --- a/modules/dnn/src/op_inf_engine.cpp +++ b/modules/dnn/src/op_inf_engine.cpp @@ -367,6 +367,7 @@ void InfEngineBackendNet::init(Target targetId) #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1) // Inference Engine determines network precision by ports. InferenceEngine::Precision p = (targetId == DNN_TARGET_MYRIAD || + targetId == DNN_TARGET_HDDL || targetId == DNN_TARGET_OPENCL_FP16) ? InferenceEngine::Precision::FP16 : InferenceEngine::Precision::FP32; @@ -391,6 +392,9 @@ void InfEngineBackendNet::init(Target targetId) case DNN_TARGET_MYRIAD: device_name = "MYRIAD"; break; + case DNN_TARGET_HDDL: + device_name = "HDDL"; + break; case DNN_TARGET_FPGA: device_name = "FPGA"; break; @@ -652,20 +656,20 @@ InferenceEngine::Core& getCore(const std::string& id) #endif #if !defined(OPENCV_DNN_IE_VPU_TYPE_DEFAULT) -static bool detectMyriadX_() +static bool detectMyriadX_(std::string device) { AutoLock lock(getInitializationMutex()); #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R3) // Lightweight detection - InferenceEngine::Core& ie = getCore("MYRIAD"); + InferenceEngine::Core& ie = getCore(device); const std::vector devices = ie.GetAvailableDevices(); for (std::vector::const_iterator i = devices.begin(); i != devices.end(); ++i) { - if (i->find("MYRIAD") != std::string::npos) + if (i->find(device) != std::string::npos) { const std::string name = ie.GetMetric(*i, METRIC_KEY(FULL_DEVICE_NAME)).as(); CV_LOG_INFO(NULL, "Myriad device: " << name); - return name.find("MyriadX") != std::string::npos || name.find("Myriad X") != std::string::npos; + return name.find("MyriadX") != std::string::npos || name.find("Myriad X") != std::string::npos || name.find("HDDL") != std::string::npos; } } return false; @@ -702,13 +706,13 @@ static bool detectMyriadX_() InferenceEngine::InferenceEnginePluginPtr enginePtr; { auto& sharedPlugins = getSharedPlugins(); - auto pluginIt = sharedPlugins.find("MYRIAD"); + auto pluginIt = sharedPlugins.find(device); if (pluginIt != sharedPlugins.end()) { enginePtr = pluginIt->second; } else { auto dispatcher = InferenceEngine::PluginDispatcher({""}); - enginePtr = dispatcher.getPluginByDevice("MYRIAD"); - sharedPlugins["MYRIAD"] = enginePtr; + enginePtr = dispatcher.getPluginByDevice(device); + sharedPlugins[device] = enginePtr; } } auto plugin = InferenceEngine::InferencePlugin(enginePtr); @@ -719,9 +723,9 @@ static bool detectMyriadX_() try { #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R3) - auto netExec = getCore("MYRIAD").LoadNetwork(cnn, "MYRIAD", {{"VPU_PLATFORM", "VPU_2480"}}); + auto netExec = getCore(device).LoadNetwork(cnn, device, {{"VPU_PLATFORM", "VPU_2480"}}); #else - auto netExec = getCore("MYRIAD").LoadNetwork(cnn, "MYRIAD", {{"VPU_MYRIAD_PLATFORM", "VPU_MYRIAD_2480"}}); + auto netExec = getCore(device).LoadNetwork(cnn, device, {{"VPU_MYRIAD_PLATFORM", "VPU_MYRIAD_2480"}}); #endif #endif auto infRequest = netExec.CreateInferRequest(); @@ -1155,11 +1159,30 @@ void resetMyriadDevice() #endif // HAVE_INF_ENGINE } +void releaseHDDLPlugin() +{ +#ifdef HAVE_INF_ENGINE + AutoLock lock(getInitializationMutex()); +#if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1) + getSharedPlugins().erase("HDDL"); +#else + // Unregister both "HDDL" and "HETERO:HDDL,CPU" plugins + InferenceEngine::Core& ie = getCore("HDDL"); + try + { + ie.UnregisterPlugin("HDDL"); + ie.UnregisterPlugin("HETERO"); + } + catch (...) {} +#endif +#endif // HAVE_INF_ENGINE +} + #ifdef HAVE_INF_ENGINE bool isMyriadX() { - static bool myriadX = getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X; - return myriadX; + static bool myriadX = getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X; + return myriadX; } static std::string getInferenceEngineVPUType_() @@ -1170,10 +1193,11 @@ static std::string getInferenceEngineVPUType_() #if defined(OPENCV_DNN_IE_VPU_TYPE_DEFAULT) param_vpu_type = OPENCV_DNN_IE_VPU_TYPE_DEFAULT; #else - CV_LOG_INFO(NULL, "OpenCV-DNN: running Inference Engine VPU autodetection: Myriad2/X. In case of other accelerator types specify 'OPENCV_DNN_IE_VPU_TYPE' parameter"); + CV_LOG_INFO(NULL, "OpenCV-DNN: running Inference Engine VPU autodetection: Myriad2/X or HDDL. In case of other accelerator types specify 'OPENCV_DNN_IE_VPU_TYPE' parameter"); try { - bool isMyriadX_ = detectMyriadX_(); - if (isMyriadX_) + bool isMyriadX_ = detectMyriadX_("MYRIAD"); + bool isHDDL_ = detectMyriadX_("HDDL"); + if (isMyriadX_ || isHDDL_) { param_vpu_type = CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X; } diff --git a/modules/dnn/src/op_inf_engine.hpp b/modules/dnn/src/op_inf_engine.hpp index 7b3ab0fed0..fcd1a6927d 100644 --- a/modules/dnn/src/op_inf_engine.hpp +++ b/modules/dnn/src/op_inf_engine.hpp @@ -27,10 +27,11 @@ #define INF_ENGINE_RELEASE_2020_2 2020020000 #define INF_ENGINE_RELEASE_2020_3 2020030000 #define INF_ENGINE_RELEASE_2020_4 2020040000 +#define INF_ENGINE_RELEASE_2021_1 2021010000 #ifndef INF_ENGINE_RELEASE -#warning("IE version have not been provided via command-line. Using 2020.4 by default") -#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2020_4 +#warning("IE version have not been provided via command-line. Using 2021.1 by default") +#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_1 #endif #define INF_ENGINE_VER_MAJOR_GT(ver) (((INF_ENGINE_RELEASE) / 10000) > ((ver) / 10000)) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index b3e425aef7..67f5782a2e 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -321,6 +321,7 @@ TEST_P(DNNTestNetwork, SSD_VGG16) else if (target == DNN_TARGET_CUDA_FP16) { scoreDiff = 0.03; + iouDiff = 0.13; } processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", @@ -511,7 +512,7 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16) else if (target == DNN_TARGET_CUDA_FP16) { l1 = 0.3; - lInf = 7.2; + lInf = 7.6; } processNet("dnn/fast_neural_style_eccv16_starry_night.t7", "", inp, "", "", l1, lInf); #if defined(HAVE_INF_ENGINE) && INF_ENGINE_VER_MAJOR_GE(2019010000) diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index e1ffa762de..c0282207dd 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -563,7 +563,7 @@ TEST_P(Test_Caffe_nets, DenseNet_121) } normAssert(outs[0], ref, "", l1, lInf); if (target != DNN_TARGET_MYRIAD || getInferenceEngineVPUType() != CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - expectNoFallbacksFromIE(model); + expectNoFallbacksFromIE(model.getNetwork_()); } TEST(Test_Caffe, multiple_inputs) @@ -749,7 +749,7 @@ TEST_P(Test_Caffe_nets, RFCN) if (target == DNN_TARGET_CUDA_FP16) { scoreDiff = 0.0034; - iouDiff = 0.11; + iouDiff = 0.12; } static Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index fa86f9b1c7..cf1b558391 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -40,6 +40,7 @@ void PrintTo(const cv::dnn::Target& v, std::ostream* os) case DNN_TARGET_OPENCL: *os << "OCL"; return; case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return; case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return; + case DNN_TARGET_HDDL: *os << "HDDL"; return; case DNN_TARGET_VULKAN: *os << "VULKAN"; return; case DNN_TARGET_FPGA: *os << "FPGA"; return; case DNN_TARGET_CUDA: *os << "CUDA"; return; @@ -67,10 +68,10 @@ void normAssert( double l1 /*= 0.00001*/, double lInf /*= 0.0001*/) { double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total(); - EXPECT_LE(normL1, l1) << comment; + EXPECT_LE(normL1, l1) << comment << " |ref| = " << cvtest::norm(ref, cv::NORM_INF); double normInf = cvtest::norm(ref, test, cv::NORM_INF); - EXPECT_LE(normInf, lInf) << comment; + EXPECT_LE(normInf, lInf) << comment << " |ref| = " << cvtest::norm(ref, cv::NORM_INF); } std::vector matToBoxes(const cv::Mat& m) @@ -221,7 +222,7 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) { - if (*i == DNN_TARGET_MYRIAD && !withVPU) + if ((*i == DNN_TARGET_MYRIAD || *i == DNN_TARGET_HDDL) && !withVPU) continue; targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, *i)); } @@ -231,7 +232,7 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) { - if (*i == DNN_TARGET_MYRIAD && !withVPU) + if ((*i == DNN_TARGET_MYRIAD || *i == DNN_TARGET_HDDL) && !withVPU) continue; targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, *i)); } @@ -281,7 +282,7 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) { - if (*i == DNN_TARGET_MYRIAD && !withVPU) + if ((*i == DNN_TARGET_MYRIAD || *i == DNN_TARGET_HDDL) && !withVPU) continue; targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, *i)); } @@ -291,7 +292,7 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) { - if (*i == DNN_TARGET_MYRIAD && !withVPU) + if ((*i == DNN_TARGET_MYRIAD || *i == DNN_TARGET_HDDL) && !withVPU) continue; targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, *i)); } @@ -323,7 +324,7 @@ static bool validateVPUType_() bool have_vpu_target = false; for (std::vector::const_iterator i = available.begin(); i != available.end(); ++i) { - if (*i == DNN_TARGET_MYRIAD) + if (*i == DNN_TARGET_MYRIAD || *i == DNN_TARGET_HDDL) { have_vpu_target = true; break; diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 30180789d2..021603636e 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -656,6 +656,11 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB ); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021010000) // nGraph compilation failure + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + const double confThreshold = 0.5; // batchId, classId, confidence, left, top, right, bottom const int N0 = 2; @@ -672,6 +677,8 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) double scoreDiff = 0.01f; double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.15 : 0.01f; + if (target == DNN_TARGET_CUDA_FP16) + iouDiff = 0.02; std::string config_file = "yolov4-tiny.cfg"; std::string weights_file = "yolov4-tiny.weights"; diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index bd36c86d6c..b285e91d96 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -340,6 +340,8 @@ TEST_P(DNNTestOpenVINO, models) // Single Myriad device cannot be shared across multiple processes. if (targetId == DNN_TARGET_MYRIAD) resetMyriadDevice(); + if (targetId == DNN_TARGET_HDDL) + releaseHDDLPlugin(); EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE"; EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV"; diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index e3caae5c0c..61537e0e01 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -2228,7 +2228,7 @@ public: static testing::internal::ParamGenerator > dnnBackendsAndTargetsForFusionTests() { - return dnnBackendsAndTargets(false, false, true, false, false, false); // OCV OpenCL + OCV CPU + return dnnBackendsAndTargets(false, false, true, false, true, false); // OCV OpenCL + OCV CPU + CUDA } }; @@ -2264,10 +2264,6 @@ TEST_P(ConvolutionActivationFusion, Accuracy) Backend backendId = get<0>(get<2>(GetParam())); Target targetId = get<1>(get<2>(GetParam())); - // bug: https://github.com/opencv/opencv/issues/17964 - if (actType == "Power" && backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - Net net; int convId = net.addLayer(convParams.name, convParams.type, convParams); int activId = net.addLayerToPrev(activationParams.name, activationParams.type, activationParams); @@ -2280,11 +2276,16 @@ TEST_P(ConvolutionActivationFusion, Accuracy) expectedFusedLayers.push_back(activId); // all activations are fused else if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) { - if (actType == "ReLU" || actType == "ChannelsPReLU" || actType == "ReLU6" || actType == "TanH" || actType == "Power") + if (actType == "ReLU" || actType == "ChannelsPReLU" || actType == "ReLU6" || actType == "TanH" /*|| actType == "Power"*/) expectedFusedLayers.push_back(activId); } } - + else if (backendId == DNN_BACKEND_CUDA) + { + if (actType == "ReLU" || actType == "ReLU6" || actType == "TanH" || actType == "Swish" || + actType == "Mish" || actType == "Sigmoid" || actType == "Power") + expectedFusedLayers.push_back(activId); + } TestLayerFusion::test(input, net, backendId, targetId, expectedFusedLayers); } INSTANTIATE_TEST_CASE_P(TestLayerFusion, ConvolutionActivationFusion, Combine( @@ -2323,7 +2324,7 @@ TEST_P(ConvolutionEltwiseFusion, Accuracy) std::string eltwiseOp = get<1>(GetParam()); bool weightedEltwise = get<2>(GetParam()); if (eltwiseOp != "sum" && weightedEltwise) - throw SkipTestException("weighted eltwise not supported"); + throw SkipTestException("weighted eltwise not supported"); LayerParams eltwiseParams; TestLayerFusion::makeDefaultTestEltwiseLayer(eltwiseParams, eltwiseOp, weightedEltwise); @@ -2336,7 +2337,11 @@ TEST_P(ConvolutionEltwiseFusion, Accuracy) Backend backendId = get<0>(get<3>(GetParam())); Target targetId = get<1>(get<3>(GetParam())); - TestLayerFusion::test(input, net, backendId, targetId); + + std::vector expectedFusedLayers; + if (backendId == DNN_BACKEND_CUDA && eltwiseOp == "sum" && !weightedEltwise) + expectedFusedLayers.push_back(eltwiseId); + TestLayerFusion::test(input, net, backendId, targetId, expectedFusedLayers); } INSTANTIATE_TEST_CASE_P(TestLayerFusion, ConvolutionEltwiseFusion, Combine( /* bias */ testing::Bool(), @@ -2390,21 +2395,6 @@ TEST_P(ConvolutionEltwiseActivationFusion, Accuracy) Backend backendId = get<0>(get<4>(GetParam())); Target targetId = get<1>(get<4>(GetParam())); - // bug: https://github.com/opencv/opencv/issues/17945 - if ((eltwiseOp != "sum" || weightedEltwise) && backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - // bug: https://github.com/opencv/opencv/issues/17953 - if (eltwiseOp == "sum" && actType == "ChannelsPReLU" && bias_term == false && - backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - } - - // bug: https://github.com/opencv/opencv/issues/17964 - if (actType == "Power" && backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - Net net; int convId = net.addLayer(convParams.name, convParams.type, convParams); int eltwiseId = net.addLayer(eltwiseParams.name, eltwiseParams.type, eltwiseParams); @@ -2421,14 +2411,25 @@ TEST_P(ConvolutionEltwiseActivationFusion, Accuracy) expectedFusedLayers.push_back(activId); // activation is fused with eltwise layer else if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) { - if (actType == "ReLU" || actType == "ChannelsPReLU" || actType == "Power") + if (eltwiseOp == "sum" && !weightedEltwise && + (actType == "ReLU" || actType == "ChannelsPReLU" /*|| actType == "Power"*/) + ) { expectedFusedLayers.push_back(eltwiseId); expectedFusedLayers.push_back(activId); } } } - + else if(backendId == DNN_BACKEND_CUDA) + { + if (eltwiseOp == "sum" && !weightedEltwise) + { + expectedFusedLayers.push_back(eltwiseId); + if (actType == "ReLU" || actType == "ReLU6" || actType == "TanH" || actType == "Swish" || + actType == "Mish" || actType == "Sigmoid" || actType == "Power") + expectedFusedLayers.push_back(activId); + } + } TestLayerFusion::test(input, net, backendId, targetId, expectedFusedLayers); } INSTANTIATE_TEST_CASE_P(TestLayerFusion, ConvolutionEltwiseActivationFusion, Combine( @@ -2483,17 +2484,6 @@ TEST_P(ConvolutionActivationEltwiseFusion, Accuracy) Backend backendId = get<0>(get<4>(GetParam())); Target targetId = get<1>(get<4>(GetParam())); - // bug: https://github.com/opencv/opencv/issues/17964 - if (actType == "Power" && backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - // bug: https://github.com/opencv/opencv/issues/17953 - if (actType == "ChannelsPReLU" && bias_term == false && - backendId == DNN_BACKEND_OPENCV && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)) - { - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - } - Net net; int convId = net.addLayer(convParams.name, convParams.type, convParams); int activId = net.addLayer(activationParams.name, activationParams.type, activationParams); @@ -2510,11 +2500,20 @@ TEST_P(ConvolutionActivationEltwiseFusion, Accuracy) expectedFusedLayers.push_back(activId); // activation fused with convolution else if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) { - if (actType == "ReLU" || actType == "ChannelsPReLU" || actType == "ReLU6" || actType == "TanH" || actType == "Power") + if (actType == "ReLU" || actType == "ChannelsPReLU" || actType == "ReLU6" || actType == "TanH" /*|| actType == "Power"*/) expectedFusedLayers.push_back(activId); // activation fused with convolution } } - + else if(backendId == DNN_BACKEND_CUDA) + { + if (actType == "ReLU" || actType == "ReLU6" || actType == "TanH" || actType == "Swish" || + actType == "Mish" || actType == "Sigmoid" || actType == "Power") + { + expectedFusedLayers.push_back(activId); + if (eltwiseOp == "sum" && !weightedEltwise) + expectedFusedLayers.push_back(eltwiseId); + } + } TestLayerFusion::test(input, net, backendId, targetId, expectedFusedLayers); } INSTANTIATE_TEST_CASE_P(TestLayerFusion, ConvolutionActivationEltwiseFusion, Combine( diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 215cc1c743..7d516de73e 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -206,6 +206,8 @@ TEST_P(Test_Model, DetectionOutput) { if (backend == DNN_BACKEND_OPENCV) scoreDiff = 4e-3; + else + scoreDiff = 2e-2; iouDiff = 1.8e-1; } @@ -261,7 +263,7 @@ TEST_P(Test_Model, DetectionMobilenetSSD) } else if (target == DNN_TARGET_CUDA_FP16) { - scoreDiff = 4e-4; + scoreDiff = 0.002; iouDiff = 1e-2; } float confThreshold = FLT_MIN; @@ -363,7 +365,7 @@ TEST_P(Test_Model, Detection_normalized) scoreDiff = 5e-3; iouDiff = 0.09; } -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) { iouDiff = 0.095f; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index f9cfd290a9..a2c097da42 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -192,9 +192,14 @@ TEST_P(Test_ONNX_layers, Convolution3D) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA) - throw SkipTestException("Only CPU and CUDA is supported"); testONNXModels("conv3d"); +} + +TEST_P(Test_ONNX_layers, Convolution3D_bias) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif testONNXModels("conv3d_bias"); } @@ -216,7 +221,8 @@ TEST_P(Test_ONNX_layers, Deconvolution) testONNXModels("two_deconvolution", npy, 0, 0, false, false); testONNXModels("deconvolution_group", npy, 0, 0, false, false); testONNXModels("deconvolution_output_shape", npy, 0, 0, false, false); - testONNXModels("deconv_adjpad_2d", npy, 0, 0, false, false); + if (target != DNN_TARGET_CUDA_FP16) // bug + testONNXModels("deconv_adjpad_2d", npy, 0, 0, false, false); } TEST_P(Test_ONNX_layers, Deconvolution3D) @@ -275,9 +281,11 @@ TEST_P(Test_ONNX_layers, ReduceSum) testONNXModels("reduce_sum"); } -TEST_P(Test_ONNX_layers, ReduceMaxGlobal) +TEST_P(Test_ONNX_layers, ReduceMax) { testONNXModels("reduce_max"); + testONNXModels("reduce_max_axis_0"); + testONNXModels("reduce_max_axis_1"); } TEST_P(Test_ONNX_layers, Scale) @@ -543,7 +551,12 @@ TEST_P(Test_ONNX_layers, Broadcast) TEST_P(Test_ONNX_layers, DynamicResize) { - testONNXModels("dynamic_resize", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_9", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_10", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_11", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_scale_9", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_scale_10", npy, 0, 0, false, true, 2); + testONNXModels("dynamic_resize_scale_11", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, Div) @@ -663,6 +676,8 @@ TEST_P(Test_ONNX_layers, LinearWithConstant) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2020040000) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE); #endif + if (backend == DNN_BACKEND_CUDA) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); testONNXModels("lin_with_constant"); } @@ -673,6 +688,8 @@ TEST_P(Test_ONNX_layers, MatmulWithTwoInputs) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2020040000) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE); #endif + if (backend == DNN_BACKEND_CUDA) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); testONNXModels("matmul_with_two_inputs"); } @@ -681,6 +698,174 @@ TEST_P(Test_ONNX_layers, ResizeOpset11_Torch1_6) testONNXModels("resize_opset11_torch1.6"); } +TEST_P(Test_ONNX_layers, Conv1d) +{ + testONNXModels("conv1d"); +} + +TEST_P(Test_ONNX_layers, Conv1d_bias) +{ + testONNXModels("conv1d_bias"); +} + +TEST_P(Test_ONNX_layers, Conv1d_variable_weight) +{ + String basename = "conv1d_variable_w"; + Net net = readNetFromONNX(_tf("models/" + basename + ".onnx")); + ASSERT_FALSE(net.empty()); + + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + Mat input = blobFromNPY(_tf("data/input_" + basename + "_0.npy")); + Mat weights = blobFromNPY(_tf("data/input_" + basename + "_1.npy")); + Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy")); + + net.setInput(input, "0"); + net.setInput(weights, "1"); + + Mat out = net.forward(); + normAssert(ref, out, "", default_l1, default_lInf); +} + +TEST_P(Test_ONNX_layers, Conv1d_variable_weight_bias) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + String basename = "conv1d_variable_wb"; + Net net = readNetFromONNX(_tf("models/" + basename + ".onnx")); + ASSERT_FALSE(net.empty()); + + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + Mat input = blobFromNPY(_tf("data/input_" + basename + "_0.npy")); + Mat weights = blobFromNPY(_tf("data/input_" + basename + "_1.npy")); + Mat bias = blobFromNPY(_tf("data/input_" + basename + "_2.npy")); + Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy")); + + net.setInput(input, "0"); + net.setInput(weights, "1"); + net.setInput(bias, "bias"); + + Mat out = net.forward(); + normAssert(ref, out, "", default_l1, default_lInf); +} + +TEST_P(Test_ONNX_layers, GatherMultiOutput) +{ + if (cvtest::skipUnstableTests && backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + throw SkipTestException("Skip unstable test: https://github.com/opencv/opencv/issues/18937"); + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE); +#endif + + testONNXModels("gather_multi_output"); +} + +TEST_P(Test_ONNX_layers, DynamicAxes) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("squeeze_and_conv_dynamic_axes"); + testONNXModels("unsqueeze_and_conv_dynamic_axes"); + testONNXModels("gather_dynamic_axes"); + testONNXModels("gather_scalar_dynamic_axes"); + testONNXModels("slice_dynamic_axes"); + testONNXModels("slice_opset_11_dynamic_axes"); + testONNXModels("resize_opset11_torch1.6_dynamic_axes"); + testONNXModels("average_pooling_dynamic_axes"); + testONNXModels("maxpooling_sigmoid_dynamic_axes"); +} + +TEST_P(Test_ONNX_layers, MaxPool1d) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("maxpooling_1d"); +} + +TEST_P(Test_ONNX_layers, MaxPoolSigmoid1d) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("maxpooling_sigmoid_1d"); +} + +TEST_P(Test_ONNX_layers, MaxPool1d_Twise) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("two_maxpooling_1d"); +} + +TEST_P(Test_ONNX_layers, AvePool1d) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("average_pooling_1d"); +} + +TEST_P(Test_ONNX_layers, PoolConv1d) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("pool_conv_1d"); +} + +TEST_P(Test_ONNX_layers, ConvResizePool1d) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + } + testONNXModels("conv_resize_pool_1d"); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers @@ -1060,8 +1245,8 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) float l1 = 0.0013, lInf = 0.009; if (target == DNN_TARGET_CUDA_FP16) { - l1 = 0.008; - lInf = 0.04; + l1 = 0.01; + lInf = 0.06; } checkBackend(&input0, &ref0); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 5f52ef0bc8..e6cfbe6637 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -169,17 +169,10 @@ TEST_P(Test_TensorFlow_layers, Convolution3D) #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - if (backend == DNN_BACKEND_CUDA) - { - // ok - } - else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); // Only CPU on DLIE backend is supported - else if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // Only CPU on DLIE backend is supported - else if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); - runTensorFlowNet("conv3d"); } @@ -1263,7 +1256,7 @@ TEST_P(Test_TensorFlow_nets, EfficientDet) if (target == DNN_TARGET_CUDA_FP16) { scoreDiff = 0.002; - iouDiff = 0.004; + iouDiff = 0.005; } normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff); expectNoFallbacksFromIE(net); diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 9de74d6cae..54b7c1baa9 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -165,7 +165,8 @@ TEST_P(Test_Torch_layers, run_reshape_single_sample) } else if (target == DNN_TARGET_CUDA_FP16) { - l1 = 0.01; + l1 = 0.02; + lInf = 0.04; } runTorchNet("net_reshape_single_sample", "", false, false, true, l1, lInf); } @@ -409,6 +410,10 @@ TEST_P(Test_Torch_nets, ENet_accuracy) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); throw SkipTestException(""); } +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); #endif if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) { diff --git a/modules/features2d/src/evaluation.cpp b/modules/features2d/src/evaluation.cpp index 2c1a446a57..ca7ab14500 100644 --- a/modules/features2d/src/evaluation.cpp +++ b/modules/features2d/src/evaluation.cpp @@ -314,7 +314,7 @@ struct SIdx UsedFinder(const SIdx& _used) : used(_used) {} const SIdx& used; bool operator()(const SIdx& v) const { return (v.i1 == used.i1 || v.i2 == used.i2); } - UsedFinder& operator=(const UsedFinder&); + UsedFinder& operator=(const UsedFinder&) = delete; }; }; diff --git a/modules/features2d/src/keypoint.cpp b/modules/features2d/src/keypoint.cpp index 219634e5b4..e14c9da94c 100644 --- a/modules/features2d/src/keypoint.cpp +++ b/modules/features2d/src/keypoint.cpp @@ -44,9 +44,9 @@ namespace cv { -struct KeypointResponseGreaterThanThreshold +struct KeypointResponseGreaterThanOrEqualToThreshold { - KeypointResponseGreaterThanThreshold(float _value) : + KeypointResponseGreaterThanOrEqualToThreshold(float _value) : value(_value) { } @@ -83,7 +83,7 @@ void KeyPointsFilter::retainBest(std::vector& keypoints, int n_points) //use std::partition to grab all of the keypoints with the boundary response. std::vector::const_iterator new_end = std::partition(keypoints.begin() + n_points, keypoints.end(), - KeypointResponseGreaterThanThreshold(ambiguous_response)); + KeypointResponseGreaterThanOrEqualToThreshold(ambiguous_response)); //resize the keypoints, given this new end point. nth_element and partition reordered the points inplace keypoints.resize(new_end - keypoints.begin()); } @@ -151,7 +151,7 @@ public: private: const Mat mask; - MaskPredicate& operator=(const MaskPredicate&); + MaskPredicate& operator=(const MaskPredicate&) = delete; }; void KeyPointsFilter::runByPixelsMask( std::vector& keypoints, const Mat& mask ) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 0278d9326a..0067cfa389 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -38,6 +38,10 @@ if(MSVC) endif() endif() +if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") # don't add Clang here: issue should be investigated and fixed (workaround for Apple only) + ocv_warnings_disable(CMAKE_CXX_FLAGS -Wrange-loop-analysis) # https://github.com/opencv/opencv/issues/18928 +endif() + file(GLOB gapi_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/*.hpp" @@ -49,6 +53,7 @@ file(GLOB gapi_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/ocl/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/own/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/render/*.hpp" + "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/s11n/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/streaming/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp" "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/util/*.hpp" @@ -56,6 +61,7 @@ file(GLOB gapi_ext_hdrs set(gapi_srcs # Front-end part + src/api/grunarg.cpp src/api/gorigin.cpp src/api/gmat.cpp src/api/garray.cpp @@ -73,10 +79,10 @@ set(gapi_srcs src/api/kernels_imgproc.cpp src/api/kernels_video.cpp src/api/kernels_nnparsers.cpp + src/api/kernels_streaming.cpp src/api/render.cpp src/api/render_ocv.cpp src/api/ginfer.cpp - src/api/ft_render.cpp src/api/media.cpp src/api/rmat.cpp @@ -97,6 +103,7 @@ set(gapi_srcs src/compiler/passes/pattern_matching.cpp src/compiler/passes/perform_substitution.cpp src/compiler/passes/streaming.cpp + src/compiler/passes/intrin.cpp # Executor src/executor/gexecutor.cpp @@ -129,21 +136,28 @@ set(gapi_srcs src/backends/ie/giebackend.cpp src/backends/ie/giebackend/giewrapper.cpp - # Render Backend. - src/backends/render/grenderocvbackend.cpp - src/backends/render/grenderocv.cpp + # ONNX backend + src/backends/onnx/gonnxbackend.cpp - #PlaidML Backend + # Render backend + src/backends/render/grenderocv.cpp + src/backends/render/ft_render.cpp + + # PlaidML Backend src/backends/plaidml/gplaidmlcore.cpp src/backends/plaidml/gplaidmlbackend.cpp - # Compound + # Common backend code + src/backends/common/gmetabackend.cpp src/backends/common/gcompoundbackend.cpp src/backends/common/gcompoundkernel.cpp # Serialization API and routines src/api/s11n.cpp src/backends/common/serialization.cpp + + # Python bridge + src/backends/ie/bindings_ie.cpp ) ocv_add_dispatched_file(backends/fluid/gfluidimgproc_func SSE4_1 AVX2) @@ -200,10 +214,20 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() + if(WIN32) # Required for htonl/ntohl on Windows ocv_target_link_libraries(${the_module} PRIVATE wsock32 ws2_32) endif() +if(HAVE_ONNX) + ocv_target_link_libraries(${the_module} PRIVATE ${ONNX_LIBRARY}) + ocv_target_compile_definitions(${the_module} PRIVATE HAVE_ONNX=1) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE HAVE_ONNX=1) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ${ONNX_LIBRARY}) + endif() +endif() + ocv_add_perf_tests() ocv_add_samples() diff --git a/modules/gapi/cmake/standalone.cmake b/modules/gapi/cmake/standalone.cmake index ca54697524..5cc57d8269 100644 --- a/modules/gapi/cmake/standalone.cmake +++ b/modules/gapi/cmake/standalone.cmake @@ -15,6 +15,8 @@ file(GLOB FLUID_includes "${FLUID_ROOT}/include/opencv2/*.hpp" "${FLUID_ROOT}/include/opencv2/gapi/own/*.hpp" "${FLUID_ROOT}/include/opencv2/gapi/fluid/*.hpp") file(GLOB FLUID_sources "${FLUID_ROOT}/src/api/g*.cpp" + "${FLUID_ROOT}/src/api/rmat.cpp" + "${FLUID_ROOT}/src/api/media.cpp" "${FLUID_ROOT}/src/compiler/*.cpp" "${FLUID_ROOT}/src/compiler/passes/*.cpp" "${FLUID_ROOT}/src/executor/*.cpp" diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index c6ab3f13fd..8445746710 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -33,4 +33,8 @@ #include #include +// Include this file here to avoid cyclic dependency between +// Desync & GKernel & GComputation & GStreamingCompiled. +#include + #endif // OPENCV_GAPI_HPP diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp index c4ddaf6bd3..8825585696 100644 --- a/modules/gapi/include/opencv2/gapi/core.hpp +++ b/modules/gapi/include/opencv2/gapi/core.hpp @@ -508,19 +508,23 @@ namespace core { return in.withType(in.depth, in.chan).withSize(dsize); } }; +} // namespace core - G_TYPED_KERNEL(GSize, (GMat)>, "org.opencv.core.size") { - static GOpaqueDesc outMeta(const GMatDesc&) { - return empty_gopaque_desc(); - } - }; +namespace streaming { - G_TYPED_KERNEL(GSizeR, (GOpaque)>, "org.opencv.core.sizeR") { - static GOpaqueDesc outMeta(const GOpaqueDesc&) { - return empty_gopaque_desc(); - } - }; -} +// Operations for Streaming (declared in this header for convenience) +G_TYPED_KERNEL(GSize, (GMat)>, "org.opencv.streaming.size") { + static GOpaqueDesc outMeta(const GMatDesc&) { + return empty_gopaque_desc(); + } +}; + +G_TYPED_KERNEL(GSizeR, (GOpaque)>, "org.opencv.streaming.sizeR") { + static GOpaqueDesc outMeta(const GOpaqueDesc&) { + return empty_gopaque_desc(); + } +}; +} // namespace streaming //! @addtogroup gapi_math //! @{ @@ -1491,8 +1495,8 @@ Output image must be of the same size and depth as input one. CV_32FC1, or CV_32FC2. @param map2 The second map of y values having the type CV_16UC1, CV_32FC1, or none (empty map if map1 is (x,y) points), respectively. -@param interpolation Interpolation method (see cv::InterpolationFlags). The method INTER_AREA is -not supported by this function. +@param interpolation Interpolation method (see cv::InterpolationFlags). The methods #INTER_AREA +and #INTER_LINEAR_EXACT are not supported by this function. @param borderMode Pixel extrapolation method (see cv::BorderTypes). When borderMode=BORDER_TRANSPARENT, it means that the pixels in the destination image that corresponds to the "outliers" in the source image are not modified by the function. @@ -1753,9 +1757,10 @@ GAPI_EXPORTS GMat warpAffine(const GMat& src, const Mat& M, const Size& dsize, i int borderMode = cv::BORDER_CONSTANT, const Scalar& borderValue = Scalar()); //! @} gapi_transform +namespace streaming { /** @brief Gets dimensions from Mat. -@note Function textual ID is "org.opencv.core.size" +@note Function textual ID is "org.opencv.streaming.size" @param src Input tensor @return Size (tensor dimensions). @@ -1765,12 +1770,13 @@ GAPI_EXPORTS GOpaque size(const GMat& src); /** @overload Gets dimensions from rectangle. -@note Function textual ID is "org.opencv.core.sizeR" +@note Function textual ID is "org.opencv.streaming.sizeR" @param r Input rectangle. @return Size (rectangle dimensions). */ GAPI_EXPORTS GOpaque size(const GOpaque& r); +} //namespace streaming } //namespace gapi } //namespace cv diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index ef67930909..5dd70bd2e8 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -271,6 +271,11 @@ template<> struct get_out >: public get_out>/GArray> conversion should be done more gracefully in the system +template struct get_out> >: public get_out> > +{ +}; + template struct get_out> { static U& get(GCPUContext &ctx, int idx) @@ -443,7 +448,7 @@ struct OCVStCallHelper, std::tuple> : template class GCPUKernelImpl: public cv::detail::KernelTag { - using CallHelper = detail::OCVCallHelper; + using CallHelper = cv::detail::OCVCallHelper; public: using API = K; @@ -497,7 +502,7 @@ private: template gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(Callable& c) { - using P = detail::OCVCallHelper; + using P = cv::detail::OCVCallHelper; return GOCVFunctor{ K::id() , &K::getOutMeta , std::bind(&P::callFunctor, std::placeholders::_1, std::ref(c)) @@ -507,7 +512,7 @@ gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(Callable& c) template gapi::cpu::GOCVFunctor gapi::cpu::ocv_kernel(const Callable& c) { - using P = detail::OCVCallHelper; + using P = cv::detail::OCVCallHelper; return GOCVFunctor{ K::id() , &K::getOutMeta , std::bind(&P::callFunctor, std::placeholders::_1, c) diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 67ce0d990c..0838573b56 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -9,12 +9,14 @@ #define OPENCV_GAPI_GARG_HPP #include +#include #include #include #include #include +#include #include #include @@ -93,7 +95,7 @@ using GArgs = std::vector; // FIXME: Express as M::type // FIXME: Move to a separate file! -using GRunArg = util::variant< +using GRunArgBase = util::variant< #if !defined(GAPI_STANDALONE) cv::UMat, #endif // !defined(GAPI_STANDALONE) @@ -105,6 +107,61 @@ using GRunArg = util::variant< cv::detail::OpaqueRef, cv::MediaFrame >; + +namespace detail { +template +struct in_variant; + +template +struct in_variant > + : std::integral_constant::value > { +}; +} // namespace detail + +struct GAPI_EXPORTS GRunArg: public GRunArgBase +{ + // Metadata information here + using Meta = std::unordered_map; + Meta meta; + + // Mimic the old GRunArg semantics here, old of the times when + // GRunArg was an alias to variant<> + GRunArg(); + GRunArg(const cv::GRunArg &arg); + GRunArg(cv::GRunArg &&arg); + + GRunArg& operator= (const GRunArg &arg); + GRunArg& operator= (GRunArg &&arg); + + template + GRunArg(const T &t, + const Meta &m = Meta{}, + typename std::enable_if< detail::in_variant::value, int>::type = 0) + : GRunArgBase(t) + , meta(m) + { + } + template + GRunArg(T &&t, + const Meta &m = Meta{}, + typename std::enable_if< detail::in_variant::value, int>::type = 0) + : GRunArgBase(std::move(t)) + , meta(m) + { + } + template auto operator= (const T &t) + -> typename std::enable_if< detail::in_variant::value, cv::GRunArg>::type& + { + GRunArgBase::operator=(t); + return *this; + } + template auto operator= (T&& t) + -> typename std::enable_if< detail::in_variant::value, cv::GRunArg>::type& + { + GRunArgBase::operator=(std::move(t)); + return *this; + } +}; using GRunArgs = std::vector; // TODO: Think about the addition operator @@ -129,11 +186,13 @@ namespace gapi namespace wip { /** - * @brief This aggregate type represents all types which G-API can handle (via variant). + * @brief This aggregate type represents all types which G-API can + * handle (via variant). * - * It only exists to overcome C++ language limitations (where a `using`-defined class can't be forward-declared). + * It only exists to overcome C++ language limitations (where a + * `using`-defined class can't be forward-declared). */ -struct Data: public GRunArg +struct GAPI_EXPORTS Data: public GRunArg { using GRunArg::GRunArg; template diff --git a/modules/gapi/include/opencv2/gapi/garray.hpp b/modules/gapi/include/opencv2/gapi/garray.hpp index 9118f4de98..5d4b3c59e0 100644 --- a/modules/gapi/include/opencv2/gapi/garray.hpp +++ b/modules/gapi/include/opencv2/gapi/garray.hpp @@ -284,6 +284,14 @@ namespace detail return static_cast&>(*m_ref).rref(); } + // Check if was created for/from std::vector + template bool holds() const + { + if (!m_ref) return false; + using U = typename std::decay::type; + return dynamic_cast*>(m_ref.get()) != nullptr; + } + void mov(VectorRef &v) { m_ref->mov(*v.m_ref); @@ -341,15 +349,18 @@ public: explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor : m_ref(ref) { putDetails(); } // (used by GCall, not for users) - detail::GArrayU strip() const { return m_ref; } + /// @private + detail::GArrayU strip() const { + return m_ref; + } + /// @private + static void VCtor(detail::VectorRef& vref) { + vref.reset(); + } private: - static void VCTor(detail::VectorRef& vref) { - vref.reset(); - vref.storeKind(); - } void putDetails() { - m_ref.setConstructFcn(&VCTor); + m_ref.setConstructFcn(&VCtor); m_ref.specifyType(); // FIXME: to unify those 2 to avoid excessive dynamic_cast m_ref.storeKind(); // } @@ -357,6 +368,8 @@ private: detail::GArrayU m_ref; }; +using GArrayP2f = GArray; + /** @} */ } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/gcall.hpp b/modules/gapi/include/opencv2/gapi/gcall.hpp index ed5ba5fde8..511eca1408 100644 --- a/modules/gapi/include/opencv2/gapi/gcall.hpp +++ b/modules/gapi/include/opencv2/gapi/gcall.hpp @@ -56,11 +56,16 @@ public: Priv& priv(); const Priv& priv() const; -protected: - std::shared_ptr m_priv; + // GKernel and params can be modified, it's needed for infer, + // because information about output shapes doesn't exist in compile time + GKernel& kernel(); + cv::util::any& params(); void setArgs(std::vector &&args); +protected: + std::shared_ptr m_priv; + // Public versions return a typed array or opaque, those are implementation details detail::GArrayU yieldArray(int output = 0); detail::GOpaqueU yieldOpaque(int output = 0); diff --git a/modules/gapi/include/opencv2/gapi/gcommon.hpp b/modules/gapi/include/opencv2/gapi/gcommon.hpp index e008fe4bf1..a474140baa 100644 --- a/modules/gapi/include/opencv2/gapi/gcommon.hpp +++ b/modules/gapi/include/opencv2/gapi/gcommon.hpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace cv { @@ -48,6 +49,7 @@ namespace detail CV_UINT64, // uint64_t user G-API data CV_STRING, // std::string user G-API data CV_POINT, // cv::Point user G-API data + CV_POINT2F, // cv::Point2f user G-API data CV_SIZE, // cv::Size user G-API data CV_RECT, // cv::Rect user G-API data CV_SCALAR, // cv::Scalar user G-API data @@ -67,15 +69,16 @@ namespace detail template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_SIZE; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_SCALAR; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_POINT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_POINT2F; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_MAT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_RECT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_MAT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_DRAW_PRIM; }; - using GOpaqueTraitsArrayTypes = std::tuple; // GOpaque is not supporting cv::Mat and cv::Scalar since there are GScalar and GMat types - using GOpaqueTraitsOpaqueTypes = std::tuple; } // namespace detail @@ -94,6 +97,15 @@ enum class GShape: int GFRAME, }; +namespace gapi { +namespace s11n { +namespace detail { +template struct wrap_serialize; +} // namespace detail +} // namespace s11n +} // namespace gapi + + struct GCompileArg; namespace detail { @@ -139,7 +151,7 @@ namespace detail { * passed in (a variadic template parameter pack) into a vector of * cv::GCompileArg objects. */ -struct GAPI_EXPORTS_W_SIMPLE GCompileArg +struct GCompileArg { public: // NB: Required for pythnon bindings @@ -151,6 +163,9 @@ public: template::value, int>::type = 0> explicit GCompileArg(T &&t) : tag(detail::CompileArgTag::type>::tag()) + , serializeF(cv::gapi::s11n::detail::has_S11N_spec::value ? + &cv::gapi::s11n::detail::wrap_serialize::serialize : + nullptr) , arg(t) { } @@ -165,7 +180,16 @@ public: return util::any_cast(arg); } + void serialize(cv::gapi::s11n::IOStream& os) const + { + if (serializeF) + { + serializeF(os, *this); + } + } + private: + std::function serializeF; util::any arg; }; @@ -198,6 +222,19 @@ inline cv::util::optional getCompileArg(const cv::GCompileArgs &args) } return cv::util::optional(); } + +namespace s11n { +namespace detail { +template struct wrap_serialize +{ + static void serialize(IOStream& os, const GCompileArg& arg) + { + using DT = typename std::decay::type; + S11N
::serialize(os, arg.get
()); + } +}; +} // namespace detail +} // namespace s11n } // namespace gapi /** diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp index 1172c0f5d6..8732ada0d6 100644 --- a/modules/gapi/include/opencv2/gapi/gcomputation.hpp +++ b/modules/gapi/include/opencv2/gapi/gcomputation.hpp @@ -436,7 +436,7 @@ public: * * @sa @ref gapi_compile_args */ - GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); + GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); /** * @brief Compile the computation for streaming mode. @@ -457,7 +457,7 @@ public: * * @sa @ref gapi_compile_args */ - GStreamingCompiled compileStreaming(GCompileArgs &&args = {}); + GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {}); // 2. Direct metadata version /** diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index b04cedecad..0ec7dd07c0 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -26,8 +26,16 @@ namespace cv { -using GShapes = std::vector; -using GKinds = std::vector; +struct GTypeInfo +{ + GShape shape; + cv::detail::OpaqueKind kind; +}; + +using GShapes = std::vector; +using GKinds = std::vector; +using GCtors = std::vector; +using GTypesInfo = std::vector; // GKernel describes kernel API to the system // FIXME: add attributes of a kernel, (e.g. number and types @@ -41,6 +49,7 @@ struct GAPI_EXPORTS GKernel M outMeta; // generic adaptor to API::outMeta(...) GShapes outShapes; // types (shapes) kernel's outputs GKinds inKinds; // kinds of kernel's inputs (fixme: below) + GCtors outCtors; // captured constructors for template output types }; // TODO: It's questionable if inKinds should really be here. Instead, // this information could come from meta. @@ -60,30 +69,27 @@ namespace detail // yield() is used in graph construction time as a generic method to obtain // lazy "return value" of G-API operations // - namespace + template struct Yield; + template<> struct Yield { - template struct Yield; - template<> struct Yield - { - static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); } - }; - template<> struct Yield - { - static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); } - }; - template<> struct Yield - { - static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); } - }; - template struct Yield > - { - static inline cv::GArray yield(cv::GCall &call, int i) { return call.yieldArray(i); } - }; - template struct Yield > - { - static inline cv::GOpaque yield(cv::GCall &call, int i) { return call.yieldOpaque(i); } - }; - } // anonymous namespace + static inline cv::GMat yield(cv::GCall &call, int i) { return call.yield(i); } + }; + template<> struct Yield + { + static inline cv::GMatP yield(cv::GCall &call, int i) { return call.yieldP(i); } + }; + template<> struct Yield + { + static inline cv::GScalar yield(cv::GCall &call, int i) { return call.yieldScalar(i); } + }; + template struct Yield > + { + static inline cv::GArray yield(cv::GCall &call, int i) { return call.yieldArray(i); } + }; + template struct Yield > + { + static inline cv::GOpaque yield(cv::GCall &call, int i) { return call.yieldOpaque(i); } + }; //////////////////////////////////////////////////////////////////////////// // Helper classes which brings outputMeta() marshalling to kernel @@ -215,7 +221,8 @@ public: , K::tag() , &K::getOutMeta , {detail::GTypeTraits::shape...} - , {detail::GTypeTraits::op_kind...}}); + , {detail::GTypeTraits::op_kind...} + , {detail::GObtainCtor::get()...}}); call.pass(args...); // TODO: std::forward() here? return yield(call, typename detail::MkSeq::type()); } @@ -240,7 +247,8 @@ public: , K::tag() , &K::getOutMeta , {detail::GTypeTraits::shape} - , {detail::GTypeTraits::op_kind...}}); + , {detail::GTypeTraits::op_kind...} + , {detail::GObtainCtor::get()}}); call.pass(args...); return detail::Yield::yield(call, 0); } @@ -459,11 +467,6 @@ namespace gapi { std::vector m_transformations; protected: - /// @private - // Check if package contains ANY implementation of a kernel API - // by API textual id. - bool includesAPI(const std::string &id) const; - /// @private // Remove ALL implementations of the given API (identified by ID) void removeAPI(const std::string &id); @@ -566,6 +569,9 @@ namespace gapi { return includesAPI(KAPI::id()); } + /// @private + bool includesAPI(const std::string &id) const; + // FIXME: The below comment is wrong, and who needs this function? /** * @brief Find a kernel (by its API) diff --git a/modules/gapi/include/opencv2/gapi/gopaque.hpp b/modules/gapi/include/opencv2/gapi/gopaque.hpp index 3d1394473b..6117971768 100644 --- a/modules/gapi/include/opencv2/gapi/gopaque.hpp +++ b/modules/gapi/include/opencv2/gapi/gopaque.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -119,6 +120,7 @@ namespace detail virtual void mov(BasicOpaqueRef &ref) = 0; virtual const void* ptr() const = 0; + virtual void set(const cv::util::any &a) = 0; }; template class OpaqueRefT final: public BasicOpaqueRef @@ -212,6 +214,10 @@ namespace detail } virtual const void* ptr() const override { return &rref(); } + + virtual void set(const cv::util::any &a) override { + wref() = util::any_cast(a); + } }; // This class strips type information from OpaqueRefT<> and makes it usable @@ -285,6 +291,13 @@ namespace detail // May be used to uniquely identify this object internally const void *ptr() const { return m_ref->ptr(); } + + // Introduced for in-graph meta handling + OpaqueRef& operator= (const cv::util::any &a) + { + m_ref->set(a); + return *this; + } }; } // namespace detail @@ -295,25 +308,27 @@ namespace detail template class GOpaque { public: - GOpaque() { putDetails(); } // Empty constructor - explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor - : m_ref(ref) { putDetails(); } // (used by GCall, not for users) - - detail::GOpaqueU strip() const { return m_ref; } - -private: // Host type (or Flat type) - the type this GOpaque is actually // specified to. using HT = typename detail::flatten_g>::type; - static void CTor(detail::OpaqueRef& ref) { - ref.reset(); - ref.storeKind(); + GOpaque() { putDetails(); } // Empty constructor + explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor + : m_ref(ref) { putDetails(); } // (used by GCall, not for users) + + /// @private + detail::GOpaqueU strip() const { + return m_ref; } + /// @private + static void Ctor(detail::OpaqueRef& ref) { + ref.reset(); + } +private: void putDetails() { - m_ref.setConstructFcn(&CTor); - m_ref.specifyType(); // FIXME: to unify those 2 to avoid excessive dynamic_cast - m_ref.storeKind(); // + m_ref.setConstructFcn(&Ctor); + m_ref.specifyType(); + m_ref.storeKind(); } detail::GOpaqueU m_ref; diff --git a/modules/gapi/include/opencv2/gapi/gproto.hpp b/modules/gapi/include/opencv2/gapi/gproto.hpp index fbcccb38ea..f91fcdb2c8 100644 --- a/modules/gapi/include/opencv2/gapi/gproto.hpp +++ b/modules/gapi/include/opencv2/gapi/gproto.hpp @@ -135,7 +135,7 @@ GRunArg value_of(const GOrigin &origin); // Transform run-time computation arguments into a collection of metadata // extracted from that arguments GMetaArg GAPI_EXPORTS descr_of(const GRunArg &arg ); -GMetaArgs GAPI_EXPORTS descr_of(const GRunArgs &args); +GMetaArgs GAPI_EXPORTS_W descr_of(const GRunArgs &args); // Transform run-time operation result argument into metadata extracted from that argument // Used to compare the metadata, which generated at compile time with the metadata result operation in run time diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 7079042069..e09cf8d0f7 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -8,15 +8,99 @@ #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP #define OPENCV_GAPI_GSTREAMING_COMPILED_HPP +#include #include #include #include +#include #include #include namespace cv { +template using optional = cv::util::optional; + +namespace detail { +template struct wref_spec { + using type = T; +}; +template struct wref_spec > { + using type = T; +}; + +template +struct OptRef { + struct OptHolder { + virtual void mov(RefHolder &h) = 0; + virtual void reset() = 0; + virtual ~OptHolder() = default; + using Ptr = std::shared_ptr; + }; + template struct Holder final: OptHolder { + std::reference_wrapper > m_opt_ref; + + explicit Holder(cv::optional& opt) : m_opt_ref(std::ref(opt)) { + } + virtual void mov(RefHolder &h) override { + using U = typename wref_spec::type; + m_opt_ref.get() = cv::util::make_optional(std::move(h.template wref())); + } + virtual void reset() override { + m_opt_ref.get().reset(); + } + }; + template + explicit OptRef(cv::optional& t) : m_opt{new Holder(t)} {} + void mov(RefHolder &h) { m_opt->mov(h); } + void reset() { m_opt->reset();} +private: + typename OptHolder::Ptr m_opt; +}; +using OptionalVectorRef = OptRef; +using OptionalOpaqueRef = OptRef; +} // namespace detail + +// TODO: Keep it in sync with GRunArgP (derive the type automatically?) +using GOptRunArgP = util::variant< + optional*, + optional*, + optional*, + cv::detail::OptionalVectorRef, + cv::detail::OptionalOpaqueRef +>; +using GOptRunArgsP = std::vector; + +namespace detail { + +template inline GOptRunArgP wrap_opt_arg(optional& arg) { + // By default, T goes to an OpaqueRef. All other types are specialized + return GOptRunArgP{OptionalOpaqueRef(arg)}; +} + +template inline GOptRunArgP wrap_opt_arg(optional >& arg) { + return GOptRunArgP{OptionalVectorRef(arg)}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &m) { + return GOptRunArgP{&m}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &s) { + return GOptRunArgP{&s}; +} + +} // namespace detail + +// Now cv::gout() may produce an empty vector (see "dynamic graphs"), so +// there may be a conflict between these two. State here that Opt version +// _must_ have at least one input for this overload +template +inline GOptRunArgsP gout(optional&arg, optional&... args) +{ + return GOptRunArgsP{ detail::wrap_opt_arg(arg), detail::wrap_opt_arg(args)... }; +} + /** * \addtogroup gapi_main_classes * @{ @@ -49,11 +133,11 @@ namespace cv { * * @sa GCompiled */ -class GAPI_EXPORTS GStreamingCompiled +class GAPI_EXPORTS_W_SIMPLE GStreamingCompiled { public: class GAPI_EXPORTS Priv; - GStreamingCompiled(); + GAPI_WRAP GStreamingCompiled(); // FIXME: More overloads? /** @@ -96,7 +180,7 @@ public: * @param ins vector of inputs to process. * @sa gin */ - void setSource(GRunArgs &&ins); + GAPI_WRAP void setSource(GRunArgs &&ins); /** * @brief Specify an input video stream for a single-input @@ -109,7 +193,23 @@ public: * @param s a shared pointer to IStreamSource representing the * input video stream. */ - void setSource(const gapi::wip::IStreamSource::Ptr& s); + GAPI_WRAP void setSource(const gapi::wip::IStreamSource::Ptr& s); + + /** + * @brief Constructs and specifies an input video stream for a + * single-input computation pipeline with the given parameters. + * + * Throws if pipeline is already running. Use stop() and then + * setSource() to run the graph on a new video stream. + * + * @overload + * @param args arguments used to contruct and initialize a stream + * source. + */ + template + void setSource(Args&&... args) { + setSource(cv::gapi::wip::make_src(std::forward(args)...)); + } /** * @brief Start the pipeline execution. @@ -126,7 +226,7 @@ public: * start()/stop()/setSource() may be called on the same object in * multiple threads in your application. */ - void start(); + GAPI_WRAP void start(); /** * @brief Get the next processed frame from the pipeline. @@ -150,6 +250,47 @@ public: */ bool pull(cv::GRunArgsP &&outs); + // NB: Used from python + GAPI_WRAP std::tuple pull(); + + /** + * @brief Get some next available data from the pipeline. + * + * This method takes a vector of cv::optional object. An object is + * assigned to some value if this value is available (ready) at + * the time of the call, and resets the object to empty() if it is + * not. + * + * This is a blocking method which guarantees that some data has + * been written to the output vector on return. + * + * Using this method only makes sense if the graph has + * desynchronized parts (see cv::gapi::desync). If there is no + * desynchronized parts in the graph, the behavior of this + * method is identical to the regular pull() (all data objects are + * produced synchronously in the output vector). + * + * Use gout() to create an output parameter vector. + * + * Output vectors must have the same number of elements as defined + * in the cv::GComputation protocol (at the moment of its + * construction). Shapes of elements also must conform to protocol + * (e.g. cv::optional needs to be passed where cv::GMat + * has been declared as output, and so on). Run-time exception is + * generated on type mismatch. + * + * This method writes new data into objects passed via output + * vector. If there is no data ready yet, this method blocks. Use + * try_pull() if you need a non-blocking version. + * + * @param outs vector of output parameters to obtain. + * @return true if next result has been obtained, + * false marks end of the stream. + * + * @sa cv::gapi::desync + */ + bool pull(cv::GOptRunArgsP &&outs); + /** * @brief Try to get the next processed frame from the pipeline. * @@ -172,7 +313,7 @@ public: * * Throws if the pipeline is not running. */ - void stop(); + GAPI_WRAP void stop(); /** * @brief Test if the pipeline is running. @@ -184,7 +325,7 @@ public: * * @return true if the current stream is not over yet. */ - bool running() const; + GAPI_WRAP bool running() const; /// @private Priv& priv(); diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index c9800b2b16..2e8dcb1aec 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -191,6 +191,29 @@ namespace detail template using wrap_gapi_helper = WrapValue::type>; template using wrap_host_helper = WrapValue >::type>; + +// Union type for various user-defined type constructors (GArray, +// GOpaque, etc) +// +// TODO: Replace construct-only API with a more generic one (probably +// with bits of introspection) +// +// Not required for non-user-defined types (GMat, GScalar, etc) +using HostCtor = util::variant + < util::monostate + , detail::ConstructVec + , detail::ConstructOpaque + >; + +template struct GObtainCtor { + static HostCtor get() { return HostCtor{}; } +}; +template struct GObtainCtor > { + static HostCtor get() { return HostCtor{ConstructVec{&GArray::VCtor}}; }; +}; +template struct GObtainCtor > { + static HostCtor get() { return HostCtor{ConstructOpaque{&GOpaque::Ctor}}; }; +}; } // namespace detail } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/imgproc.hpp b/modules/gapi/include/opencv2/gapi/imgproc.hpp index b4905e932b..7435ec1e1d 100644 --- a/modules/gapi/include/opencv2/gapi/imgproc.hpp +++ b/modules/gapi/include/opencv2/gapi/imgproc.hpp @@ -21,14 +21,45 @@ @{ @defgroup gapi_filters Graph API: Image filters @defgroup gapi_colorconvert Graph API: Converting image from one color space to another + @defgroup gapi_feature Graph API: Image Feature Detection + @defgroup gapi_shape Graph API: Image Structural Analysis and Shape Descriptors @} */ +namespace { +void validateFindingContoursMeta(const int depth, const int chan, const int mode) +{ + GAPI_Assert(chan == 1); + switch (mode) + { + case cv::RETR_CCOMP: + GAPI_Assert(depth == CV_8U || depth == CV_32S); + break; + case cv::RETR_FLOODFILL: + GAPI_Assert(depth == CV_32S); + break; + default: + GAPI_Assert(depth == CV_8U); + break; + } +} + +// Checks if the passed mat is a set of n-dimentional points of the given depth +bool isPointsVector(const int chan, const cv::Size &size, const int depth, + const int n, const int ddepth = -1) +{ + return (ddepth == depth || ddepth < 0) && + ((chan == n && (size.height == 1 || size.width == 1)) || + (chan == 1 && size.width == n)); +} +} // anonymous namespace + namespace cv { namespace gapi { namespace imgproc { using GMat2 = std::tuple; using GMat3 = std::tuple; // FIXME: how to avoid this? + using GFindContoursOutput = std::tuple>,GArray>; G_TYPED_KERNEL(GFilter2D, ,"org.opencv.imgproc.filters.filter2D") { static GMatDesc outMeta(GMatDesc in, int ddepth, Mat, Point, Scalar, int, Scalar) { @@ -78,6 +109,14 @@ namespace imgproc { } }; + G_TYPED_KERNEL(GMorphologyEx, , + "org.opencv.imgproc.filters.morphologyEx") { + static GMatDesc outMeta(const GMatDesc &in, MorphTypes, Mat, Point, int, + BorderTypes, Scalar) { + return in; + } + }; + G_TYPED_KERNEL(GSobel, , "org.opencv.imgproc.filters.sobel") { static GMatDesc outMeta(GMatDesc in, int ddepth, int, int, int, double, double, int, Scalar) { return in.withDepth(ddepth); @@ -110,7 +149,7 @@ namespace imgproc { } }; - G_TYPED_KERNEL(GCanny, , "org.opencv.imgproc.canny"){ + G_TYPED_KERNEL(GCanny, , "org.opencv.imgproc.feature.canny"){ static GMatDesc outMeta(GMatDesc in, double, double, int, bool) { return in.withType(CV_8U, 1); } @@ -118,12 +157,153 @@ namespace imgproc { G_TYPED_KERNEL(GGoodFeatures, (GMat,int,double,double,Mat,int,bool,double)>, - "org.opencv.imgproc.goodFeaturesToTrack") { + "org.opencv.imgproc.feature.goodFeaturesToTrack") { static GArrayDesc outMeta(GMatDesc, int, double, double, const Mat&, int, bool, double) { return empty_array_desc(); } }; + using RetrMode = RetrievalModes; + using ContMethod = ContourApproximationModes; + G_TYPED_KERNEL(GFindContours, >(GMat,RetrMode,ContMethod,GOpaque)>, + "org.opencv.imgproc.shape.findContours") + { + static GArrayDesc outMeta(GMatDesc in, RetrMode mode, ContMethod, GOpaqueDesc) + { + validateFindingContoursMeta(in.depth, in.chan, mode); + return empty_array_desc(); + } + }; + + // FIXME oc: make default value offset = Point() + G_TYPED_KERNEL(GFindContoursNoOffset, >(GMat,RetrMode,ContMethod)>, + "org.opencv.imgproc.shape.findContoursNoOffset") + { + static GArrayDesc outMeta(GMatDesc in, RetrMode mode, ContMethod) + { + validateFindingContoursMeta(in.depth, in.chan, mode); + return empty_array_desc(); + } + }; + + G_TYPED_KERNEL(GFindContoursH,)>, + "org.opencv.imgproc.shape.findContoursH") + { + static std::tuple + outMeta(GMatDesc in, RetrMode mode, ContMethod, GOpaqueDesc) + { + validateFindingContoursMeta(in.depth, in.chan, mode); + return std::make_tuple(empty_array_desc(), empty_array_desc()); + } + }; + + // FIXME oc: make default value offset = Point() + G_TYPED_KERNEL(GFindContoursHNoOffset,, + "org.opencv.imgproc.shape.findContoursHNoOffset") + { + static std::tuple + outMeta(GMatDesc in, RetrMode mode, ContMethod) + { + validateFindingContoursMeta(in.depth, in.chan, mode); + return std::make_tuple(empty_array_desc(), empty_array_desc()); + } + }; + + G_TYPED_KERNEL(GBoundingRectMat, (GMat)>, + "org.opencv.imgproc.shape.boundingRectMat") { + static GOpaqueDesc outMeta(GMatDesc in) { + GAPI_Assert((in.depth == CV_8U && in.chan == 1) || + (isPointsVector(in.chan, in.size, in.depth, 2, CV_32S) || + isPointsVector(in.chan, in.size, in.depth, 2, CV_32F))); + + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GBoundingRectVector32S, (GArray)>, + "org.opencv.imgproc.shape.boundingRectVector32S") { + static GOpaqueDesc outMeta(GArrayDesc) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GBoundingRectVector32F, (GArray)>, + "org.opencv.imgproc.shape.boundingRectVector32F") { + static GOpaqueDesc outMeta(GArrayDesc) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine2DMat, (GMat,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine2DMat") { + static GOpaqueDesc outMeta(GMatDesc in,DistanceTypes,double,double,double) { + GAPI_Assert(isPointsVector(in.chan, in.size, in.depth, 2, -1)); + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine2DVector32S, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine2DVector32S") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine2DVector32F, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine2DVector32F") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine2DVector64F, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine2DVector64F") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine3DMat, (GMat,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine3DMat") { + static GOpaqueDesc outMeta(GMatDesc in,int,double,double,double) { + GAPI_Assert(isPointsVector(in.chan, in.size, in.depth, 3, -1)); + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine3DVector32S, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine3DVector32S") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine3DVector32F, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine3DVector32F") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GFitLine3DVector64F, + (GArray,DistanceTypes,double,double,double)>, + "org.opencv.imgproc.shape.fitLine3DVector64F") { + static GOpaqueDesc outMeta(GArrayDesc,DistanceTypes,double,double,double) { + return empty_gopaque_desc(); + } + }; + + G_TYPED_KERNEL(GBGR2RGB, , "org.opencv.imgproc.colorconvert.bgr2rgb") { + static GMatDesc outMeta(GMatDesc in) { + return in; // type still remains CV_8UC3; + } + }; + G_TYPED_KERNEL(GRGB2YUV, , "org.opencv.imgproc.colorconvert.rgb2yuv") { static GMatDesc outMeta(GMatDesc in) { return in; // type still remains CV_8UC3; @@ -136,6 +316,42 @@ namespace imgproc { } }; + G_TYPED_KERNEL(GBGR2I420, , "org.opencv.imgproc.colorconvert.bgr2i420") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 3); + GAPI_Assert(in.size.height % 2 == 0); + return in.withType(in.depth, 1).withSize(Size(in.size.width, in.size.height * 3 / 2)); + } + }; + + G_TYPED_KERNEL(GRGB2I420, , "org.opencv.imgproc.colorconvert.rgb2i420") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 3); + GAPI_Assert(in.size.height % 2 == 0); + return in.withType(in.depth, 1).withSize(Size(in.size.width, in.size.height * 3 / 2)); + } + }; + + G_TYPED_KERNEL(GI4202BGR, , "org.opencv.imgproc.colorconvert.i4202bgr") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 1); + GAPI_Assert(in.size.height % 3 == 0); + return in.withType(in.depth, 3).withSize(Size(in.size.width, in.size.height * 2 / 3)); + } + }; + + G_TYPED_KERNEL(GI4202RGB, , "org.opencv.imgproc.colorconvert.i4202rgb") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 1); + GAPI_Assert(in.size.height % 3 == 0); + return in.withType(in.depth, 3).withSize(Size(in.size.width, in.size.height * 2 / 3)); + } + }; + G_TYPED_KERNEL(GNV12toRGB, , "org.opencv.imgproc.colorconvert.nv12torgb") { static GMatDesc outMeta(GMatDesc in_y, GMatDesc in_uv) { GAPI_Assert(in_y.chan == 1); @@ -230,7 +446,7 @@ namespace imgproc { } }; - G_TYPED_KERNEL(GNV12toRGBp, , "org.opencv.colorconvert.imgproc.nv12torgbp") { + G_TYPED_KERNEL(GNV12toRGBp, , "org.opencv.imgproc.colorconvert.nv12torgbp") { static GMatDesc outMeta(GMatDesc inY, GMatDesc inUV) { GAPI_Assert(inY.depth == CV_8U); GAPI_Assert(inUV.depth == CV_8U); @@ -244,7 +460,7 @@ namespace imgproc { } }; - G_TYPED_KERNEL(GNV12toGray, , "org.opencv.colorconvert.imgproc.nv12togray") { + G_TYPED_KERNEL(GNV12toGray, , "org.opencv.imgproc.colorconvert.nv12togray") { static GMatDesc outMeta(GMatDesc inY, GMatDesc inUV) { GAPI_Assert(inY.depth == CV_8U); GAPI_Assert(inUV.depth == CV_8U); @@ -259,7 +475,7 @@ namespace imgproc { } }; - G_TYPED_KERNEL(GNV12toBGRp, , "org.opencv.colorconvert.imgproc.nv12tobgrp") { + G_TYPED_KERNEL(GNV12toBGRp, , "org.opencv.imgproc.colorconvert.nv12tobgrp") { static GMatDesc outMeta(GMatDesc inY, GMatDesc inUV) { GAPI_Assert(inY.depth == CV_8U); GAPI_Assert(inUV.depth == CV_8U); @@ -455,7 +671,7 @@ The median filter uses cv::BORDER_REPLICATE internally to cope with border pixel @param ksize aperture linear size; it must be odd and greater than 1, for example: 3, 5, 7 ... @sa boxFilter, gaussianBlur */ -GAPI_EXPORTS GMat medianBlur(const GMat& src, int ksize); +GAPI_EXPORTS_W GMat medianBlur(const GMat& src, int ksize); /** @brief Erodes an image by using a specific structuring element. @@ -479,7 +695,7 @@ anchor is at the element center. @param iterations number of times erosion is applied. @param borderType pixel extrapolation method, see cv::BorderTypes @param borderValue border value in case of a constant border -@sa dilate +@sa dilate, morphologyEx */ GAPI_EXPORTS GMat erode(const GMat& src, const Mat& kernel, const Point& anchor = Point(-1,-1), int iterations = 1, int borderType = BORDER_CONSTANT, @@ -554,6 +770,37 @@ GAPI_EXPORTS GMat dilate3x3(const GMat& src, int iterations = 1, int borderType = BORDER_CONSTANT, const Scalar& borderValue = morphologyDefaultBorderValue()); +/** @brief Performs advanced morphological transformations. + +The function can perform advanced morphological transformations using an erosion and dilation as +basic operations. + +Any of the operations can be done in-place. In case of multi-channel images, each channel is +processed independently. + +@note Function textual ID is "org.opencv.imgproc.filters.morphologyEx" + +@param src Input image. +@param op Type of a morphological operation, see #MorphTypes +@param kernel Structuring element. It can be created using #getStructuringElement. +@param anchor Anchor position within the element. Both negative values mean that the anchor is at +the kernel center. +@param iterations Number of times erosion and dilation are applied. +@param borderType Pixel extrapolation method, see #BorderTypes. #BORDER_WRAP is not supported. +@param borderValue Border value in case of a constant border. The default value has a special +meaning. +@sa dilate, erode, getStructuringElement +@note The number of iterations is the number of times erosion or dilatation operation will be +applied. For instance, an opening operation (#MORPH_OPEN) with two iterations is equivalent to +apply successively: erode -> erode -> dilate -> dilate +(and not erode -> dilate -> erode -> dilate). + */ +GAPI_EXPORTS GMat morphologyEx(const GMat &src, const MorphTypes op, const Mat &kernel, + const Point &anchor = Point(-1,-1), + const int iterations = 1, + const BorderTypes borderType = BORDER_CONSTANT, + const Scalar &borderValue = morphologyDefaultBorderValue()); + /** @brief Calculates the first, second, third, or mixed image derivatives using an extended Sobel operator. In all cases except one, the \f$\texttt{ksize} \times \texttt{ksize}\f$ separable kernel is used to @@ -719,6 +966,10 @@ proportional to sigmaSpace. GAPI_EXPORTS GMat bilateralFilter(const GMat& src, int d, double sigmaColor, double sigmaSpace, int borderType = BORDER_DEFAULT); +//! @} gapi_filters + +//! @addtogroup gapi_feature +//! @{ /** @brief Finds edges in an image using the Canny algorithm. The function finds edges in the input image and marks them in the output map edges using the @@ -726,7 +977,7 @@ Canny algorithm. The smallest value between threshold1 and threshold2 is used fo largest value is used to find initial segments of strong edges. See -@note Function textual ID is "org.opencv.imgproc.filters.canny" +@note Function textual ID is "org.opencv.imgproc.feature.canny" @param image 8-bit input image. @param threshold1 first threshold for the hysteresis procedure. @@ -761,7 +1012,7 @@ The function can be used to initialize a point-based tracker of an object. A \> B, the vector of returned corners with qualityLevel=A will be the prefix of the output vector with qualityLevel=B . -@note Function textual ID is "org.opencv.imgproc.goodFeaturesToTrack" +@note Function textual ID is "org.opencv.imgproc.feature.goodFeaturesToTrack" @param image Input 8-bit or floating-point 32-bit, single-channel image. @param maxCorners Maximum number of corners to return. If there are more corners than are found, @@ -784,7 +1035,7 @@ or #cornerMinEigenVal. @return vector of detected corners. */ -GAPI_EXPORTS GArray goodFeaturesToTrack(const GMat &image, +GAPI_EXPORTS_W GArray goodFeaturesToTrack(const GMat &image, int maxCorners, double qualityLevel, double minDistance, @@ -795,6 +1046,8 @@ GAPI_EXPORTS GArray goodFeaturesToTrack(const GMat &image, /** @brief Equalizes the histogram of a grayscale image. +//! @} gapi_feature + The function equalizes the histogram of the input image using the following algorithm: - Calculate the histogram \f$H\f$ for src . @@ -812,10 +1065,281 @@ The algorithm normalizes the brightness and increases the contrast of the image. */ GAPI_EXPORTS GMat equalizeHist(const GMat& src); -//! @} gapi_filters +//! @addtogroup gapi_shape +//! @{ +/** @brief Finds contours in a binary image. + +The function retrieves contours from the binary image using the algorithm @cite Suzuki85 . +The contours are a useful tool for shape analysis and object detection and recognition. +See squares.cpp in the OpenCV sample directory. + +@note Function textual ID is "org.opencv.imgproc.shape.findContours" + +@param src Input gray-scale image @ref CV_8UC1. Non-zero pixels are treated as 1's. Zero +pixels remain 0's, so the image is treated as binary . You can use #compare, #inRange, #threshold , +#adaptiveThreshold, #Canny, and others to create a binary image out of a grayscale or color one. +If mode equals to #RETR_CCOMP, the input can also be a 32-bit integer +image of labels ( @ref CV_32SC1 ). If #RETR_FLOODFILL then @ref CV_32SC1 is supported only. +@param mode Contour retrieval mode, see #RetrievalModes +@param method Contour approximation method, see #ContourApproximationModes +@param offset Optional offset by which every contour point is shifted. This is useful if the +contours are extracted from the image ROI and then they should be analyzed in the whole image +context. + +@return GArray of detected contours. Each contour is stored as a GArray of points. + */ +GAPI_EXPORTS GArray> +findContours(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method, + const GOpaque &offset); + +// FIXME oc: make default value offset = Point() +/** @overload +@note Function textual ID is "org.opencv.imgproc.shape.findContoursNoOffset" + */ +GAPI_EXPORTS GArray> +findContours(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method); + +/** @brief Finds contours and their hierarchy in a binary image. + +The function retrieves contours from the binary image using the algorithm @cite Suzuki85 +and calculates their hierarchy. +The contours are a useful tool for shape analysis and object detection and recognition. +See squares.cpp in the OpenCV sample directory. + +@note Function textual ID is "org.opencv.imgproc.shape.findContoursH" + +@param src Input gray-scale image @ref CV_8UC1. Non-zero pixels are treated as 1's. Zero +pixels remain 0's, so the image is treated as binary . You can use #compare, #inRange, #threshold , +#adaptiveThreshold, #Canny, and others to create a binary image out of a grayscale or color one. +If mode equals to #RETR_CCOMP, the input can also be a 32-bit integer +image of labels ( @ref CV_32SC1 ). If #RETR_FLOODFILL -- @ref CV_32SC1 supports only. +@param mode Contour retrieval mode, see #RetrievalModes +@param method Contour approximation method, see #ContourApproximationModes +@param offset Optional offset by which every contour point is shifted. This is useful if the +contours are extracted from the image ROI and then they should be analyzed in the whole image +context. + +@return GArray of detected contours. Each contour is stored as a GArray of points. +@return Optional output GArray of cv::Vec4i, containing information about the image topology. +It has as many elements as the number of contours. For each i-th contour contours[i], the elements +hierarchy[i][0] , hierarchy[i][1] , hierarchy[i][2] , and hierarchy[i][3] are set to 0-based +indices in contours of the next and previous contours at the same hierarchical level, the first +child contour and the parent contour, respectively. If for the contour i there are no next, +previous, parent, or nested contours, the corresponding elements of hierarchy[i] will be negative. + */ +GAPI_EXPORTS std::tuple>,GArray> +findContoursH(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method, + const GOpaque &offset); + +// FIXME oc: make default value offset = Point() +/** @overload +@note Function textual ID is "org.opencv.imgproc.shape.findContoursHNoOffset" + */ +GAPI_EXPORTS std::tuple>,GArray> +findContoursH(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method); + +/** @brief Calculates the up-right bounding rectangle of a point set or non-zero pixels +of gray-scale image. + +The function calculates and returns the minimal up-right bounding rectangle for the specified +point set or non-zero pixels of gray-scale image. + +@note Function textual ID is "org.opencv.imgproc.shape.boundingRectMat" + +@param src Input gray-scale image @ref CV_8UC1; or input set of @ref CV_32S or @ref CV_32F +2D points stored in Mat. + +@note In case of a 2D points' set given, Mat should be 2-dimensional, have a single row or column +if there are 2 channels, or have 2 columns if there is a single channel. Mat should have either +@ref CV_32S or @ref CV_32F depth + */ +GAPI_EXPORTS GOpaque boundingRect(const GMat& src); + +/** @overload + +Calculates the up-right bounding rectangle of a point set. + +@note Function textual ID is "org.opencv.imgproc.shape.boundingRectVector32S" + +@param src Input 2D point set, stored in std::vector. + */ +GAPI_EXPORTS GOpaque boundingRect(const GArray& src); + +/** @overload + +Calculates the up-right bounding rectangle of a point set. + +@note Function textual ID is "org.opencv.imgproc.shape.boundingRectVector32F" + +@param src Input 2D point set, stored in std::vector. + */ +GAPI_EXPORTS GOpaque boundingRect(const GArray& src); + +/** @brief Fits a line to a 2D point set. + +The function fits a line to a 2D point set by minimizing \f$\sum_i \rho(r_i)\f$ where +\f$r_i\f$ is a distance between the \f$i^{th}\f$ point, the line and \f$\rho(r)\f$ is a distance +function, one of the following: +- DIST_L2 +\f[\rho (r) = r^2/2 \quad \text{(the simplest and the fastest least-squares method)}\f] +- DIST_L1 +\f[\rho (r) = r\f] +- DIST_L12 +\f[\rho (r) = 2 \cdot ( \sqrt{1 + \frac{r^2}{2}} - 1)\f] +- DIST_FAIR +\f[\rho \left (r \right ) = C^2 \cdot \left ( \frac{r}{C} - \log{\left(1 + \frac{r}{C}\right)} \right ) \quad \text{where} \quad C=1.3998\f] +- DIST_WELSCH +\f[\rho \left (r \right ) = \frac{C^2}{2} \cdot \left ( 1 - \exp{\left(-\left(\frac{r}{C}\right)^2\right)} \right ) \quad \text{where} \quad C=2.9846\f] +- DIST_HUBER +\f[\rho (r) = \fork{r^2/2}{if \(r < C\)}{C \cdot (r-C/2)}{otherwise} \quad \text{where} \quad C=1.345\f] + +The algorithm is based on the M-estimator ( ) technique +that iteratively fits the line using the weighted least-squares algorithm. After each iteration the +weights \f$w_i\f$ are adjusted to be inversely proportional to \f$\rho(r_i)\f$ . + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine2DMat" + +@param src Input set of 2D points stored in one of possible containers: Mat, +std::vector, std::vector, std::vector. + +@note In case of an N-dimentional points' set given, Mat should be 2-dimensional, have a single row +or column if there are N channels, or have N columns if there is a single channel. + +@param distType Distance used by the M-estimator, see #DistanceTypes. @ref DIST_USER +and @ref DIST_C are not suppored. +@param param Numerical parameter ( C ) for some types of distances. If it is 0, an optimal value +is chosen. +@param reps Sufficient accuracy for the radius (distance between the coordinate origin and the +line). 1.0 would be a good default value for reps. If it is 0, a default value is chosen. +@param aeps Sufficient accuracy for the angle. 0.01 would be a good default value for aeps. +If it is 0, a default value is chosen. + +@return Output line parameters: a vector of 4 elements (like Vec4f) - (vx, vy, x0, y0), +where (vx, vy) is a normalized vector collinear to the line and (x0, y0) is a point on the line. + */ +GAPI_EXPORTS GOpaque fitLine2D(const GMat& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine2DVector32S" + + */ +GAPI_EXPORTS GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine2DVector32F" + + */ +GAPI_EXPORTS GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine2DVector64F" + + */ +GAPI_EXPORTS GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @brief Fits a line to a 3D point set. + +The function fits a line to a 3D point set by minimizing \f$\sum_i \rho(r_i)\f$ where +\f$r_i\f$ is a distance between the \f$i^{th}\f$ point, the line and \f$\rho(r)\f$ is a distance +function, one of the following: +- DIST_L2 +\f[\rho (r) = r^2/2 \quad \text{(the simplest and the fastest least-squares method)}\f] +- DIST_L1 +\f[\rho (r) = r\f] +- DIST_L12 +\f[\rho (r) = 2 \cdot ( \sqrt{1 + \frac{r^2}{2}} - 1)\f] +- DIST_FAIR +\f[\rho \left (r \right ) = C^2 \cdot \left ( \frac{r}{C} - \log{\left(1 + \frac{r}{C}\right)} \right ) \quad \text{where} \quad C=1.3998\f] +- DIST_WELSCH +\f[\rho \left (r \right ) = \frac{C^2}{2} \cdot \left ( 1 - \exp{\left(-\left(\frac{r}{C}\right)^2\right)} \right ) \quad \text{where} \quad C=2.9846\f] +- DIST_HUBER +\f[\rho (r) = \fork{r^2/2}{if \(r < C\)}{C \cdot (r-C/2)}{otherwise} \quad \text{where} \quad C=1.345\f] + +The algorithm is based on the M-estimator ( ) technique +that iteratively fits the line using the weighted least-squares algorithm. After each iteration the +weights \f$w_i\f$ are adjusted to be inversely proportional to \f$\rho(r_i)\f$ . + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine3DMat" + +@param src Input set of 3D points stored in one of possible containers: Mat, +std::vector, std::vector, std::vector. + +@note In case of an N-dimentional points' set given, Mat should be 2-dimensional, have a single row +or column if there are N channels, or have N columns if there is a single channel. + +@param distType Distance used by the M-estimator, see #DistanceTypes. @ref DIST_USER +and @ref DIST_C are not suppored. +@param param Numerical parameter ( C ) for some types of distances. If it is 0, an optimal value +is chosen. +@param reps Sufficient accuracy for the radius (distance between the coordinate origin and the +line). 1.0 would be a good default value for reps. If it is 0, a default value is chosen. +@param aeps Sufficient accuracy for the angle. 0.01 would be a good default value for aeps. +If it is 0, a default value is chosen. + +@return Output line parameters: a vector of 6 elements (like Vec6f) - (vx, vy, vz, x0, y0, z0), +where (vx, vy, vz) is a normalized vector collinear to the line and (x0, y0, z0) is a point on +the line. + */ +GAPI_EXPORTS GOpaque fitLine3D(const GMat& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine3DVector32S" + + */ +GAPI_EXPORTS GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine3DVector32F" + + */ +GAPI_EXPORTS GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +/** @overload + +@note Function textual ID is "org.opencv.imgproc.shape.fitLine3DVector64F" + + */ +GAPI_EXPORTS GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param = 0., const double reps = 0., + const double aeps = 0.); + +//! @} gapi_shape //! @addtogroup gapi_colorconvert //! @{ +/** @brief Converts an image from BGR color space to RGB color space. + +The function converts an input image from BGR color space to RGB. +The conventional ranges for B, G, and R channel values are 0 to 255. + +Output image is 8-bit unsigned 3-channel image @ref CV_8UC3. + +@note Function textual ID is "org.opencv.imgproc.colorconvert.bgr2rgb" + +@param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3. +@sa RGB2BGR +*/ +GAPI_EXPORTS GMat BGR2RGB(const GMat& src); + /** @brief Converts an image from RGB color space to gray-scaled. The conventional ranges for R, G, and B channel values are 0 to 255. Resulting gray color value computed as @@ -826,7 +1350,7 @@ Resulting gray color value computed as @param src input image: 8-bit unsigned 3-channel image @ref CV_8UC1. @sa RGB2YUV */ -GAPI_EXPORTS GMat RGB2Gray(const GMat& src); +GAPI_EXPORTS_W GMat RGB2Gray(const GMat& src); /** @overload Resulting gray color value computed as @@ -871,6 +1395,70 @@ Output image must be 8-bit unsigned 3-channel image @ref CV_8UC3. */ GAPI_EXPORTS GMat RGB2YUV(const GMat& src); +/** @brief Converts an image from BGR color space to I420 color space. + +The function converts an input image from BGR color space to I420. +The conventional ranges for R, G, and B channel values are 0 to 255. + +Output image must be 8-bit unsigned 1-channel image. @ref CV_8UC1. +Width of I420 output image must be the same as width of input image. +Height of I420 output image must be equal 3/2 from height of input image. + +@note Function textual ID is "org.opencv.imgproc.colorconvert.bgr2i420" + +@param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3. +@sa I4202BGR +*/ +GAPI_EXPORTS GMat BGR2I420(const GMat& src); + +/** @brief Converts an image from RGB color space to I420 color space. + +The function converts an input image from RGB color space to I420. +The conventional ranges for R, G, and B channel values are 0 to 255. + +Output image must be 8-bit unsigned 1-channel image. @ref CV_8UC1. +Width of I420 output image must be the same as width of input image. +Height of I420 output image must be equal 3/2 from height of input image. + +@note Function textual ID is "org.opencv.imgproc.colorconvert.rgb2i420" + +@param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3. +@sa I4202RGB +*/ +GAPI_EXPORTS GMat RGB2I420(const GMat& src); + +/** @brief Converts an image from I420 color space to BGR color space. + +The function converts an input image from I420 color space to BGR. +The conventional ranges for B, G, and R channel values are 0 to 255. + +Output image must be 8-bit unsigned 3-channel image. @ref CV_8UC3. +Width of BGR output image must be the same as width of input image. +Height of BGR output image must be equal 2/3 from height of input image. + +@note Function textual ID is "org.opencv.imgproc.colorconvert.i4202bgr" + +@param src input image: 8-bit unsigned 1-channel image @ref CV_8UC1. +@sa BGR2I420 +*/ +GAPI_EXPORTS GMat I4202BGR(const GMat& src); + +/** @brief Converts an image from I420 color space to BGR color space. + +The function converts an input image from I420 color space to BGR. +The conventional ranges for B, G, and R channel values are 0 to 255. + +Output image must be 8-bit unsigned 3-channel image. @ref CV_8UC3. +Width of RGB output image must be the same as width of input image. +Height of RGB output image must be equal 2/3 from height of input image. + +@note Function textual ID is "org.opencv.imgproc.colorconvert.i4202rgb" + +@param src input image: 8-bit unsigned 1-channel image @ref CV_8UC1. +@sa RGB2I420 +*/ +GAPI_EXPORTS GMat I4202RGB(const GMat& src); + /** @brief Converts an image from BGR color space to LUV color space. The function converts an input image from BGR color space to LUV. diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index 50086dd848..b850775a62 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019 Intel Corporation +// Copyright (C) 2019-2020 Intel Corporation #ifndef OPENCV_GAPI_INFER_HPP @@ -77,6 +77,9 @@ public: using ResultL = std::tuple< cv::GArray... >; using APIList = std::function, Args...)>; + + // FIXME: Args... must be limited to a single GMat + using APIRoi = std::function, Args...)>; }; // Single-return-value network definition (specialized base class) @@ -92,6 +95,9 @@ public: using ResultL = cv::GArray; using APIList = std::function, Args...)>; + + // FIXME: Args... must be limited to a single GMat + using APIRoi = std::function, Args...)>; }; // APIList2 is also template to allow different calling options @@ -114,22 +120,75 @@ struct InferAPIList2 { // a particular backend, not by a network itself. struct GInferBase { static constexpr const char * id() { - return "org.opencv.dnn.infer"; // Universal stub + return "org.opencv.dnn.infer"; // Universal stub } static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) { - return GMetaArgs{}; // One more universal stub + return GMetaArgs{}; // One more universal stub } }; +// Struct stores network input/output names. +// Used by infer +struct InOutInfo +{ + std::vector in_names; + std::vector out_names; +}; + +/** + * @{ + * @brief G-API object used to collect network inputs + */ +class GAPI_EXPORTS_W_SIMPLE GInferInputs +{ +using Map = std::unordered_map; +public: + GAPI_WRAP GInferInputs(); + GAPI_WRAP void setInput(const std::string& name, const cv::GMat& value); + + cv::GMat& operator[](const std::string& name); + const Map& getBlobs() const; + +private: + std::shared_ptr in_blobs; +}; +/** @} */ + +/** + * @{ + * @brief G-API object used to collect network outputs + */ +struct GAPI_EXPORTS_W_SIMPLE GInferOutputs +{ +public: + GAPI_WRAP GInferOutputs() = default; + GInferOutputs(std::shared_ptr call); + GAPI_WRAP cv::GMat at(const std::string& name); + +private: + struct Priv; + std::shared_ptr m_priv; +}; +/** @} */ +// Base "InferROI" kernel. +// All notes from "Infer" kernel apply here as well. +struct GInferROIBase { + static constexpr const char * id() { + return "org.opencv.dnn.infer-roi"; // Universal stub + } + static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) { + return GMetaArgs{}; // One more universal stub + } +}; // Base "Infer list" kernel. // All notes from "Infer" kernel apply here as well. struct GInferListBase { static constexpr const char * id() { - return "org.opencv.dnn.infer-roi"; // Universal stub + return "org.opencv.dnn.infer-roi-list-1"; // Universal stub } static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) { - return GMetaArgs{}; // One more universal stub + return GMetaArgs{}; // One more universal stub } }; @@ -137,10 +196,10 @@ struct GInferListBase { // All notes from "Infer" kernel apply here as well. struct GInferList2Base { static constexpr const char * id() { - return "org.opencv.dnn.infer-roi-list"; // Universal stub + return "org.opencv.dnn.infer-roi-list-2"; // Universal stub } static GMetaArgs getOutMeta(const GMetaArgs &, const GArgs &) { - return GMetaArgs{}; // One more universal stub + return GMetaArgs{}; // One more universal stub } }; @@ -157,6 +216,19 @@ struct GInfer final static constexpr const char* tag() { return Net::tag(); } }; +// A specific roi-inference kernel. API (::on()) is fixed here and +// verified against Net. +template +struct GInferROI final + : public GInferROIBase + , public detail::KernelTypeMedium< GInferROI + , typename Net::APIRoi > { + using GInferROIBase::getOutMeta; // FIXME: name lookup conflict workaround? + + static constexpr const char* tag() { return Net::tag(); } +}; + + // A generic roi-list inference kernel. API (::on()) is derived from // the Net template parameter (see more in infer<> overload). template @@ -195,6 +267,23 @@ struct GInferList2 final namespace cv { namespace gapi { +/** @brief Calculates response for the specified network (template + * parameter) for the specified region in the source image. + * Currently expects a single-input network only. + * + * @tparam A network type defined with G_API_NET() macro. + * @param in input image where to take ROI from. + * @param roi an object describing the region of interest + * in the source image. May be calculated in the same graph dynamically. + * @return an object of return type as defined in G_API_NET(). + * If a network has multiple return values (defined with a tuple), a tuple of + * objects of appropriate type is returned. + * @sa G_API_NET() + */ +template +typename Net::Result infer(cv::GOpaque roi, cv::GMat in) { + return GInferROI::on(roi, in); +} /** @brief Calculates responses for the specified network (template * parameter) for every region in the source image. @@ -254,6 +343,51 @@ typename Net::Result infer(Args&&... args) { return GInfer::on(std::forward(args)...); } +/** + * @brief Special network type + */ +struct Generic { }; + +/** + * @brief Calculates response for generic network + * + * @param tag a network tag + * @param inputs networks's inputs + * @return a GInferOutputs + */ +template GInferOutputs +infer(const std::string& tag, const GInferInputs& inputs) +{ + std::vector input_args; + std::vector input_names; + + const auto& blobs = inputs.getBlobs(); + for (auto&& p : blobs) + { + input_names.push_back(p.first); + input_args.emplace_back(p.second); + } + + GKinds kinds(blobs.size(), cv::detail::OpaqueKind::CV_MAT); + auto call = std::make_shared(GKernel{ + GInferBase::id(), + tag, + GInferBase::getOutMeta, + {}, // outShape will be filled later + std::move(kinds), + {}, // outCtors will be filled later + }); + + call->setArgs(std::move(input_args)); + call->params() = InOutInfo{input_names, {}}; + + return GInferOutputs{std::move(call)}; +} + +GAPI_EXPORTS_W inline GInferOutputs infer(const String& name, const GInferInputs& inputs) +{ + return infer(name, inputs); +} } // namespace gapi } // namespace cv @@ -283,8 +417,8 @@ struct GAPI_EXPORTS GNetParam { * * @sa cv::gapi::networks */ -struct GAPI_EXPORTS GNetPackage { - GNetPackage() : GNetPackage({}) {} +struct GAPI_EXPORTS_W_SIMPLE GNetPackage { + GAPI_WRAP GNetPackage() : GNetPackage({}) {} explicit GNetPackage(std::initializer_list &&ii); std::vector backends() const; std::vector networks; diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp new file mode 100644 index 0000000000..fdd4128b1a --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -0,0 +1,56 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_INFER_BINDINGS_IE_HPP +#define OPENCV_GAPI_INFER_BINDINGS_IE_HPP + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#include // GKernelPackage +#include // Params + +#include + +namespace cv { +namespace gapi { +namespace ie { + +// NB: Used by python wrapper +// This class can be marked as SIMPLE, because it's implemented as pimpl +class GAPI_EXPORTS_W_SIMPLE PyParams { +public: + PyParams() = default; + + PyParams(const std::string &tag, + const std::string &model, + const std::string &weights, + const std::string &device); + + PyParams(const std::string &tag, + const std::string &model, + const std::string &device); + + GBackend backend() const; + std::string tag() const; + cv::util::any params() const; + +private: + std::shared_ptr> m_priv; +}; + +GAPI_EXPORTS_W PyParams params(const std::string &tag, + const std::string &model, + const std::string &weights, + const std::string &device); + +GAPI_EXPORTS_W PyParams params(const std::string &tag, + const std::string &model, + const std::string &device); +} // namespace ie +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_INFER_BINDINGS_IE_HPP diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index c6d7f272a8..53e31fbb09 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -11,12 +11,14 @@ #include #include #include // tuple, tuple_size +#include #include #include #include // GAPI_EXPORTS #include // GKernelPackage +#include // Generic namespace cv { namespace gapi { @@ -41,6 +43,8 @@ enum class TraitAs: int IMAGE //!< G-API traits an associated cv::Mat as an image so creates an "image" blob (NCHW/NHWC, etc) }; +using IEConfig = std::map; + namespace detail { struct ParamDesc { std::string model_path; @@ -58,6 +62,11 @@ namespace detail { // (e.g. topology's partial execution) std::size_t num_in; // How many inputs are defined in the operation std::size_t num_out; // How many outputs are defined in the operation + + enum class Kind { Load, Import }; + Kind kind; + bool is_generic; + IEConfig config; }; } // namespace detail @@ -80,7 +89,19 @@ public: : desc{ model, weights, device, {}, {}, {} , std::tuple_size::value // num_in , std::tuple_size::value // num_out - } { + , detail::ParamDesc::Kind::Load + , false + , {}} { + }; + + Params(const std::string &model, + const std::string &device) + : desc{ model, {}, device, {}, {}, {} + , std::tuple_size::value // num_in + , std::tuple_size::value // num_out + , detail::ParamDesc::Kind::Import + , false + , {}} { }; Params& cfgInputLayers(const typename PortCfg::In &ll) { @@ -106,18 +127,65 @@ public: return *this; } + Params& pluginConfig(IEConfig&& cfg) { + desc.config = std::move(cfg); + return *this; + } + + Params& pluginConfig(const IEConfig& cfg) { + desc.config = cfg; + return *this; + } + // BEGIN(G-API's network parametrization API) - GBackend backend() const { return cv::gapi::ie::backend(); } - std::string tag() const { return Net::tag(); } - cv::util::any params() const { return { desc }; } + GBackend backend() const { return cv::gapi::ie::backend(); } + std::string tag() const { return Net::tag(); } + cv::util::any params() const { return { desc }; } // END(G-API's network parametrization API) protected: detail::ParamDesc desc; }; +template<> +class Params { +public: + Params(const std::string &tag, + const std::string &model, + const std::string &weights, + const std::string &device) + : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}}, m_tag(tag) { + }; + + Params(const std::string &tag, + const std::string &model, + const std::string &device) + : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}}, m_tag(tag) { + }; + + Params& pluginConfig(IEConfig&& cfg) { + desc.config = std::move(cfg); + return *this; + } + + Params& pluginConfig(const IEConfig& cfg) { + desc.config = cfg; + return *this; + } + + // BEGIN(G-API's network parametrization API) + GBackend backend() const { return cv::gapi::ie::backend(); } + std::string tag() const { return m_tag; } + cv::util::any params() const { return { desc }; } + // END(G-API's network parametrization API) + +protected: + detail::ParamDesc desc; + std::string m_tag; +}; + } // namespace ie } // namespace gapi } // namespace cv -#endif // OPENCV_GAPI_INFER_HPP +#endif // OPENCV_GAPI_INFER_IE_HPP diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp new file mode 100644 index 0000000000..d61ceb3dca --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -0,0 +1,138 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_INFER_ONNX_HPP +#define OPENCV_GAPI_INFER_ONNX_HPP + +#include +#include +#include +#include // tuple, tuple_size + +#include +#include + +#include // GAPI_EXPORTS +#include // GKernelPackage + +namespace cv { +namespace gapi { +namespace onnx { + +GAPI_EXPORTS cv::gapi::GBackend backend(); + +enum class TraitAs: int { + TENSOR, //!< G-API traits an associated cv::Mat as a raw tensor + // and passes dimensions as-is + IMAGE //!< G-API traits an associated cv::Mat as an image so + // creates an "image" blob (NCHW/NHWC, etc) +}; + +using PostProc = std::function &, + std::unordered_map &)>; + + +namespace detail { +struct ParamDesc { + std::string model_path; + + // NB: nun_* may differ from topology's real input/output port numbers + // (e.g. topology's partial execution) + std::size_t num_in; // How many inputs are defined in the operation + std::size_t num_out; // How many outputs are defined in the operation + + // NB: Here order follows the `Net` API + std::vector input_names; + std::vector output_names; + + using ConstInput = std::pair; + std::unordered_map const_inputs; + + std::vector mean; + std::vector stdev; + + std::vector out_metas; + PostProc custom_post_proc; + + std::vector normalize; +}; +} // namespace detail + +template +struct PortCfg { + using In = std::array + < std::string + , std::tuple_size::value >; + using Out = std::array + < std::string + , std::tuple_size::value >; + using NormCoefs = std::array + < cv::Scalar + , std::tuple_size::value >; + using Normalize = std::array + < bool + , std::tuple_size::value >; +}; + +template class Params { +public: + Params(const std::string &model) { + desc.model_path = model; + desc.num_in = std::tuple_size::value; + desc.num_out = std::tuple_size::value; + }; + + // BEGIN(G-API's network parametrization API) + GBackend backend() const { return cv::gapi::onnx::backend(); } + std::string tag() const { return Net::tag(); } + cv::util::any params() const { return { desc }; } + // END(G-API's network parametrization API) + + Params& cfgInputLayers(const typename PortCfg::In &ll) { + desc.input_names.assign(ll.begin(), ll.end()); + return *this; + } + + Params& cfgOutputLayers(const typename PortCfg::Out &ll) { + desc.output_names.assign(ll.begin(), ll.end()); + return *this; + } + + Params& constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint = TraitAs::TENSOR) { + desc.const_inputs[layer_name] = {data, hint}; + return *this; + } + + Params& cfgMeanStd(const typename PortCfg::NormCoefs &m, + const typename PortCfg::NormCoefs &s) { + desc.mean.assign(m.begin(), m.end()); + desc.stdev.assign(s.begin(), s.end()); + return *this; + } + + Params& cfgPostProc(const std::vector &outs, + const PostProc &pp) { + desc.out_metas = outs; + desc.custom_post_proc = pp; + return *this; + } + + Params& cfgNormalize(const typename PortCfg::Normalize &n) { + desc.normalize.assign(n.begin(), n.end()); + return *this; + } + +protected: + detail::ParamDesc desc; +}; + +} // namespace onnx +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_INFER_HPP diff --git a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp index c3488f5799..15742c6e55 100644 --- a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp @@ -122,4 +122,16 @@ GAPI_EXPORTS std::tuple, GArray> parseYolo(const GMat& in, } // namespace gapi } // namespace cv +// Reimport parseSSD & parseYolo under their initial namespace +namespace cv { +namespace gapi { +namespace streaming { + +using cv::gapi::parseSSD; +using cv::gapi::parseYolo; + +} // namespace streaming +} // namespace gapi +} // namespace cv + #endif // OPENCV_GAPI_PARSERS_HPP diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index a7fe258757..f27cb80913 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -51,6 +51,7 @@ public: View(Ptrs&& ptrs, Strides&& strs, Callback &&cb = [](){}); View(const View&) = delete; View(View&&) = default; + View& operator = (const View&) = delete; ~View(); Ptrs ptr; diff --git a/modules/gapi/include/opencv2/gapi/opencv_includes.hpp b/modules/gapi/include/opencv2/gapi/opencv_includes.hpp index 5f25fe4af7..08b2d6ed02 100644 --- a/modules/gapi/include/opencv2/gapi/opencv_includes.hpp +++ b/modules/gapi/include/opencv2/gapi/opencv_includes.hpp @@ -21,11 +21,12 @@ # include // replacement of cv's structures: namespace cv { - using Rect = gapi::own::Rect; - using Size = gapi::own::Size; - using Point = gapi::own::Point; - using Scalar = gapi::own::Scalar; - using Mat = gapi::own::Mat; + using Rect = gapi::own::Rect; + using Size = gapi::own::Size; + using Point = gapi::own::Point; + using Point2f = gapi::own::Point2f; + using Scalar = gapi::own::Scalar; + using Mat = gapi::own::Mat; } // namespace cv #endif // !defined(GAPI_STANDALONE) diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index 20445ee0fd..c77a62ca53 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -28,6 +28,16 @@ public: int y = 0; }; +class Point2f +{ +public: + Point2f() = default; + Point2f(float _x, float _y) : x(_x), y(_y) {}; + + float x = 0.f; + float y = 0.f; +}; + class Rect { public: diff --git a/modules/gapi/include/opencv2/gapi/render/render_types.hpp b/modules/gapi/include/opencv2/gapi/render/render_types.hpp index 08b14d1ddd..ca403be361 100644 --- a/modules/gapi/include/opencv2/gapi/render/render_types.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render_types.hpp @@ -252,7 +252,7 @@ struct Mosaic { } - Mosaic() = default; + Mosaic() : cellSz(0), decim(0) {} /*@{*/ cv::Rect mos; //!< Coordinates of the mosaic diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index 626e67e9ee..f50bd08b65 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -10,6 +10,16 @@ #include #include +// Forward declaration +namespace cv { +namespace gapi { +namespace s11n { + struct IOStream; + struct IIStream; +} // namespace s11n +} // namespace gapi +} // namespace cv + namespace cv { // "Remote Mat", a general class which provides an abstraction layer over the data @@ -44,11 +54,11 @@ public: { public: using DestroyCallback = std::function; + using stepsT = std::vector; View() = default; - View(const GMatDesc& desc, uchar* data, size_t step = 0u, DestroyCallback&& cb = nullptr) - : m_desc(desc), m_data(data), m_step(step == 0u ? elemSize()*cols() : step), m_cb(std::move(cb)) - {} + View(const GMatDesc& desc, uchar* data, const stepsT& steps = {}, DestroyCallback&& cb = nullptr); + View(const GMatDesc& desc, uchar* data, size_t step, DestroyCallback&& cb = nullptr); View(const View&) = delete; View& operator=(const View&) = delete; @@ -60,23 +70,30 @@ public: const std::vector& dims() const { return m_desc.dims; } int cols() const { return m_desc.size.width; } int rows() const { return m_desc.size.height; } - int type() const { return CV_MAKE_TYPE(depth(), chan()); } + int type() const; int depth() const { return m_desc.depth; } int chan() const { return m_desc.chan; } size_t elemSize() const { return CV_ELEM_SIZE(type()); } - template T* ptr(int y = 0, int x = 0) { - return reinterpret_cast(m_data + m_step*y + x*CV_ELEM_SIZE(type())); + template T* ptr(int y = 0) { + return reinterpret_cast(m_data + step()*y); } - template const T* ptr(int y = 0, int x = 0) const { - return reinterpret_cast(m_data + m_step*y + x*CV_ELEM_SIZE(type())); + template const T* ptr(int y = 0) const { + return reinterpret_cast(m_data + step()*y); } - size_t step() const { return m_step; } + template T* ptr(int y, int x) { + return reinterpret_cast(m_data + step()*y + step(1)*x); + } + template const T* ptr(int y, int x) const { + return reinterpret_cast(m_data + step()*y + step(1)*x); + } + size_t step(size_t i = 0) const { GAPI_DbgAssert(i; @@ -113,6 +136,10 @@ public: return dynamic_cast(m_adapter.get()); } + void serialize(cv::gapi::s11n::IOStream& os) const { + m_adapter->serialize(os); + } + private: AdapterP m_adapter = nullptr; }; diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 0b61304c5c..0e2c4c239b 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -10,21 +10,25 @@ #include #include #include +#include #include +#include namespace cv { namespace gapi { namespace detail { GAPI_EXPORTS cv::GComputation getGraph(const std::vector &p); -} // namespace detail -namespace detail { GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &p); -} // namespace detail -namespace detail { GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &p); + + template + cv::GCompileArgs getCompileArgs(const std::vector &p); + + template + cv::GRunArgs getRunArgsWithRMats(const std::vector &p); } // namespace detail GAPI_EXPORTS std::vector serialize(const cv::GComputation &c); @@ -35,6 +39,7 @@ T deserialize(const std::vector &p); //} //ananymous namespace +GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs&); GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs&); GAPI_EXPORTS std::vector serialize(const cv::GRunArgs&); @@ -53,6 +58,17 @@ cv::GRunArgs deserialize(const std::vector &p) { return detail::getRunArgs(p); } +template inline +typename std::enable_if::value, GCompileArgs>:: +type deserialize(const std::vector &p) { + return detail::getCompileArgs(p); +} + +template inline +typename std::enable_if::value, GRunArgs>:: +type deserialize(const std::vector &p) { + return detail::getRunArgsWithRMats(p); +} } // namespace gapi } // namespace cv @@ -91,6 +107,10 @@ struct GAPI_EXPORTS IIStream { virtual IIStream& operator>> (std::string &) = 0; }; +namespace detail { +GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &p); +} // namespace detail + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// // S11N operators @@ -101,6 +121,9 @@ struct GAPI_EXPORTS IIStream { GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::Point &pt); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Point &pt); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::Point2f &pt); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Point2f &pt); + GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::Size &sz); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Size &sz); @@ -113,6 +136,27 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Scalar &s); GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::Mat &m); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Mat &m); +// FIXME: for GRunArgs serailization +#if !defined(GAPI_STANDALONE) +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &); +#endif // !defined(GAPI_STANDALONE) + +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::RMat &r); + +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &); + +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &); + +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &); + +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &); + // Generic STL types //////////////////////////////////////////////////////////////// template IOStream& operator<< (IOStream& os, const std::map &m) { @@ -175,16 +219,146 @@ IIStream& operator>> (IIStream& is, std::vector &ts) { return is; } +// Generic: variant serialization namespace detail { - // Will be used along with default types if possible in specific cases (compile args, etc) - // Note: actual implementation is defined by user - template - struct GAPI_EXPORTS S11N { - static void serialize(IOStream &, const T &) {} - static T deserialize(IIStream &) { T t; return t; } - }; +template +IOStream& put_v(IOStream&, const V&, std::size_t) { + GAPI_Assert(false && "variant>>: requested index is invalid"); +}; +template +IOStream& put_v(IOStream& os, const V& v, std::size_t x) { + return (x == 0u) + ? os << cv::util::get(v) + : put_v(os, v, x-1); +} +template +IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) { + GAPI_Assert(false && "variant<<: requested index is invalid"); +} +template +IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { + if (i == gi) { + X x{}; + is >> x; + v = V{std::move(x)}; + return is; + } else return get_v(is, v, i+1, gi); +} } // namespace detail + +template +IOStream& operator<< (IOStream& os, const cv::util::variant &v) { + os << static_cast(v.index()); + return detail::put_v, Ts...>(os, v, v.index()); +} +template +IIStream& operator>> (IIStream& is, cv::util::variant &v) { + int idx = -1; + is >> idx; + GAPI_Assert(idx >= 0 && idx < (int)sizeof...(Ts)); + return detail::get_v, Ts...>(is, v, 0u, idx); +} + +// FIXME: consider a better solution +template +void getRunArgByIdx (IIStream& is, cv::util::variant &v, uint32_t idx) { + is = detail::get_v, Ts...>(is, v, 0u, idx); +} } // namespace s11n + +namespace detail +{ +template struct try_deserialize_comparg; + +template<> struct try_deserialize_comparg> { +static cv::util::optional exec(const std::string&, cv::gapi::s11n::IIStream&) { + return { }; + } +}; + +template +struct try_deserialize_comparg> { +static cv::util::optional exec(const std::string& tag, cv::gapi::s11n::IIStream& is) { + if (tag == cv::detail::CompileArgTag::tag()) { + static_assert(cv::gapi::s11n::detail::has_S11N_spec::value, + "cv::gapi::deserialize expects Types to have S11N " + "specializations with deserialization callbacks!"); + return cv::util::optional( + GCompileArg { cv::gapi::s11n::detail::S11N::deserialize(is) }); + } + return try_deserialize_comparg>::exec(tag, is); +} +}; + +template struct deserialize_runarg; + +template +struct deserialize_runarg { +static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { + if (idx == GRunArg::index_of()) { + auto ptr = std::make_shared(); + ptr->deserialize(is); + return GRunArg { RMat(std::move(ptr)) }; + } else { // non-RMat arg - use default deserialization + GRunArg arg; + getRunArgByIdx(is, arg, idx); + return arg; + } +} +}; + +template +inline cv::util::optional tryDeserializeCompArg(const std::string& tag, + const std::vector& sArg) { + std::unique_ptr pArgIs = cv::gapi::s11n::detail::getInStream(sArg); + return try_deserialize_comparg>::exec(tag, *pArgIs); +} + +template +cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { + cv::GCompileArgs args; + + std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(sArgs); + cv::gapi::s11n::IIStream& is = *pIs; + + uint32_t sz = 0; + is >> sz; + for (uint32_t i = 0; i < sz; ++i) { + std::string tag; + is >> tag; + + std::vector sArg; + is >> sArg; + + cv::util::optional dArg = + cv::gapi::detail::tryDeserializeCompArg(tag, sArg); + + if (dArg.has_value()) + { + args.push_back(dArg.value()); + } + } + + return args; +} + +template +cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { + std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(p); + cv::gapi::s11n::IIStream& is = *pIs; + cv::GRunArgs args; + + uint32_t sz = 0; + is >> sz; + for (uint32_t i = 0; i < sz; ++i) { + uint32_t idx = 0; + is >> idx; + args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); + } + + return args; +} +} // namespace detail } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n/base.hpp b/modules/gapi/include/opencv2/gapi/s11n/base.hpp new file mode 100644 index 0000000000..d9335ee9f7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/s11n/base.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_S11N_BASE_HPP +#define OPENCV_GAPI_S11N_BASE_HPP + +#include +#include + +namespace cv { +namespace gapi { +namespace s11n { +struct IOStream; +struct IIStream; + +namespace detail { + +struct NotImplemented { +}; + +// The default S11N for custom types is NotImplemented +// Don't! sublass from NotImplemented if you actually implement S11N. +template +struct S11N: public NotImplemented { + static void serialize(IOStream &, const T &) { + GAPI_Assert(false && "No serialization routine is provided!"); + } + static T deserialize(IIStream &) { + GAPI_Assert(false && "No deserialization routine is provided!"); + } +}; + +template struct has_S11N_spec { + static constexpr bool value = !std::is_base_of::type>>::value; +}; + +} // namespace detail +} // namespace s11n +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_S11N_BASE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/cap.hpp b/modules/gapi/include/opencv2/gapi/streaming/cap.hpp index faa555063a..aad6af618c 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/cap.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/cap.hpp @@ -21,9 +21,11 @@ * Note for developers: please don't put videoio dependency in G-API * because of this file. */ +#include #include #include +#include namespace cv { namespace gapi { @@ -55,6 +57,7 @@ protected: cv::VideoCapture cap; cv::Mat first; bool first_pulled = false; + int64_t counter = 0; void prep() { @@ -80,19 +83,26 @@ protected: GAPI_Assert(!first.empty()); first_pulled = true; data = first; // no need to clone here since it was cloned already - return true; } - - if (!cap.isOpened()) return false; - - cv::Mat frame; - if (!cap.read(frame)) + else { - // end-of-stream happened - return false; + if (!cap.isOpened()) return false; + + cv::Mat frame; + if (!cap.read(frame)) + { + // end-of-stream happened + return false; + } + // Same reason to clone as in prep() + data = frame.clone(); } - // Same reason to clone as in prep() - data = frame.clone(); + // Tag data with seq_id/ts + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + data.meta[cv::gapi::streaming::meta_tag::timestamp] = int64_t{dur.count()}; + data.meta[cv::gapi::streaming::meta_tag::seq_id] = int64_t{counter++}; return true; } @@ -103,6 +113,12 @@ protected: } }; +// NB: Overload for using from python +GAPI_EXPORTS_W cv::Ptr inline make_capture_src(const std::string& path) +{ + return make_src(path); +} + } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp new file mode 100644 index 0000000000..86de279fe9 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -0,0 +1,84 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP +#define OPENCV_GAPI_GSTREAMING_DESYNC_HPP + +#include + +#include +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace streaming { + +namespace detail { +struct GDesync { + static const char *id() { + return "org.opencv.streaming.desync"; + } + + // An universal yield for desync. + // Yields output objects according to the input Types... + // Reuses gkernel machinery. + // FIXME: This function can be generic and declared in gkernel.hpp + // (it is there already, but a part of GKernelType[M] + template + static std::tuple yield(cv::GCall &call, cv::detail::Seq) { + return std::make_tuple(cv::detail::Yield::yield(call, IIs)...); + } +}; + +template +G desync(const G &g) { + cv::GKernel k{ + GDesync::id() // kernel id + , "" // kernel tag + , [](const GMetaArgs &a, const GArgs &) {return a;} // outMeta callback + , {cv::detail::GTypeTraits::shape} // output Shape + , {cv::detail::GTypeTraits::op_kind} // input data kinds + , {cv::detail::GObtainCtor::get()} // output template ctors + }; + cv::GCall call(std::move(k)); + call.pass(g); + return std::get<0>(GDesync::yield(call, cv::detail::MkSeq<1>::type())); +} +} // namespace detail + +/** + * @brief Starts a desynchronized branch in the graph. + * + * This operation takes a single G-API data object and returns a + * graph-level "duplicate" of this object. + * + * Operations which use this data object can be desynchronized + * from the rest of the graph. + * + * This operation has no effect when a GComputation is compiled with + * regular cv::GComputation::compile(), since cv::GCompiled objects + * always produce their full output vectors. + * + * This operation only makes sense when a GComputation is compiled in + * straming mode with cv::GComputation::compileStreaming(). If this + * operation is used and there are desynchronized outputs, the user + * should use a special version of cv::GStreamingCompiled::pull() + * which produces an array of cv::util::optional<> objects. + * + * @note This feature is highly experimental now and is currently + * limited to a single GMat argument only. + */ +GAPI_EXPORTS GMat desync(const GMat &g); + +} // namespace streaming +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMING_DESYNC_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/meta.hpp b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp new file mode 100644 index 0000000000..cbcfc3aa37 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp @@ -0,0 +1,79 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#ifndef OPENCV_GAPI_GSTREAMING_META_HPP +#define OPENCV_GAPI_GSTREAMING_META_HPP + +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace streaming { + +// FIXME: the name is debatable +namespace meta_tag { +static constexpr const char * timestamp = "org.opencv.gapi.meta.timestamp"; +static constexpr const char * seq_id = "org.opencv.gapi.meta.seq_id"; +} // namespace meta_tag + +namespace detail { +struct GMeta { + static const char *id() { + return "org.opencv.streaming.meta"; + } + // A universal yield for meta(), same as in GDesync + template + static std::tuple yield(cv::GCall &call, cv::detail::Seq) { + return std::make_tuple(cv::detail::Yield::yield(call, IIs)...); + } + // Also a universal outMeta stub here + static GMetaArgs getOutMeta(const GMetaArgs &args, const GArgs &) { + return args; + } +}; +} // namespace detail + +template +cv::GOpaque meta(G g, const std::string &tag) { + using O = cv::GOpaque; + cv::GKernel k{ + detail::GMeta::id() // kernel id + , tag // kernel tag. Use meta tag here + , &detail::GMeta::getOutMeta // outMeta callback + , {cv::detail::GTypeTraits::shape} // output Shape + , {cv::detail::GTypeTraits::op_kind} // input data kinds + , {cv::detail::GObtainCtor::get()} // output template ctors + }; + cv::GCall call(std::move(k)); + call.pass(g); + return std::get<0>(detail::GMeta::yield(call, cv::detail::MkSeq<1>::type())); +} + +template +cv::GOpaque timestamp(G g) { + return meta(g, meta_tag::timestamp); +} + +template +cv::GOpaque seq_id(G g) { + return meta(g, meta_tag::seq_id); +} + +template +cv::GOpaque seqNo(G g) { + // Old name, compatibility only + return seq_id(g); +} + +} // namespace streaming +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_GSTREAMING_META_HPP diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 702e8c4032..e25328e64f 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -3,7 +3,16 @@ #ifdef HAVE_OPENCV_GAPI +// NB: Python wrapper replaces :: with _ for classes using gapi_GKernelPackage = cv::gapi::GKernelPackage; +using gapi_GNetPackage = cv::gapi::GNetPackage; +using gapi_ie_PyParams = cv::gapi::ie::PyParams; +using gapi_wip_IStreamSource_Ptr = cv::Ptr; + +// FIXME: Python wrapper generate code without namespace std, +// so it cause error: "string wasn't declared" +// WA: Create using +using std::string; template<> bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) @@ -38,8 +47,20 @@ static PyObject* from_grunarg(const GRunArg& v) const auto& s = util::get(v); return pyopencv_from(s); } - + case GRunArg::index_of(): + { + const auto& vref = util::get(v); + switch (vref.getKind()) + { + case cv::detail::OpaqueKind::CV_POINT2F: + return pyopencv_from(vref.rref()); + default: + PyErr_SetString(PyExc_TypeError, "Unsupported kind for GArray"); + return NULL; + } + } default: + PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); return NULL; } GAPI_Assert(false); @@ -56,7 +77,6 @@ PyObject* pyopencv_from(const GRunArgs& value) PyObject* item = from_grunarg(value[0]); if(!item) { - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); return NULL; } return item; @@ -78,6 +98,18 @@ PyObject* pyopencv_from(const GRunArgs& value) return list; } +template<> +bool pyopencv_to(PyObject* obj, GMetaArgs& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + +template<> +PyObject* pyopencv_from(const GMetaArgs& value) +{ + return pyopencv_from_generic_vec(value); +} + template static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) { @@ -96,9 +128,13 @@ static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) { args.emplace_back(reinterpret_cast(item)->v); } + else if (PyObject_TypeCheck(item, reinterpret_cast(pyopencv_GArrayP2f_TypePtr))) + { + args.emplace_back(reinterpret_cast(item)->v.strip()); + } else { - PyErr_SetString(PyExc_TypeError, "cv.GIn() supports only cv.GMat and cv.GScalar"); + PyErr_SetString(PyExc_TypeError, "Unsupported type for cv.GIn()/cv.GOut()"); return NULL; } } @@ -151,6 +187,19 @@ static PyObject* pyopencv_cv_gin(PyObject* , PyObject* py_args, PyObject* kw) return NULL; } } + else if (PyObject_TypeCheck(item, + reinterpret_cast(pyopencv_gapi_wip_IStreamSource_TypePtr))) + { + cv::gapi::wip::IStreamSource::Ptr source = + reinterpret_cast(item)->v; + args.emplace_back(source); + } + else + { + PyErr_SetString(PyExc_TypeError, "cv.gin can works only with cv::Mat," + "cv::Scalar, cv::gapi::wip::IStreamSource::Ptr"); + return NULL; + } } return pyopencv_from_generic_vec(args); diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index dab083def7..792314512c 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,13 +3,30 @@ namespace cv { - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg); + struct GAPI_EXPORTS_W_SIMPLE GCompileArg { }; + GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg); + GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GNetPackage pkg); + + // NB: This classes doesn't exist in *.so + // HACK: Mark them as a class to force python wrapper generate code for this entities class GAPI_EXPORTS_W_SIMPLE GProtoArg { }; class GAPI_EXPORTS_W_SIMPLE GProtoInputArgs { }; class GAPI_EXPORTS_W_SIMPLE GProtoOutputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GRunArg { }; + class GAPI_EXPORTS_W_SIMPLE GRunArg { }; + class GAPI_EXPORTS_W_SIMPLE GMetaArg { }; + + class GAPI_EXPORTS_W_SIMPLE GArrayP2f { }; using GProtoInputArgs = GIOProtoArgs; using GProtoOutputArgs = GIOProtoArgs; + + namespace gapi + { + GAPI_EXPORTS_W gapi::GNetPackage networks(const cv::gapi::ie::PyParams& params); + namespace wip + { + class GAPI_EXPORTS_W IStreamSource { }; + } // namespace wip + } // namespace gapi } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_core.py b/modules/gapi/misc/python/test/test_gapi_core.py index cd85d9cadb..267037a78d 100644 --- a/modules/gapi/misc/python/test/test_gapi_core.py +++ b/modules/gapi/misc/python/test/test_gapi_core.py @@ -2,26 +2,27 @@ import numpy as np import cv2 as cv +import os from tests_common import NewOpenCVTests # Plaidml is an optional backend pkgs = [ - cv.gapi.core.ocl.kernels(), - cv.gapi.core.cpu.kernels(), - cv.gapi.core.fluid.kernels() - # cv.gapi.core.plaidml.kernels() - ] + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] class gapi_core_test(NewOpenCVTests): def test_add(self): # TODO: Extend to use any type and size here - sz = (1280, 720) - in1 = np.random.randint(0, 100, sz) - in2 = np.random.randint(0, 100, sz) + sz = (720, 1280) + in1 = np.full(sz, 100) + in2 = np.full(sz, 50) # OpenCV expected = cv.add(in1, in2) @@ -32,17 +33,18 @@ class gapi_core_test(NewOpenCVTests): g_out = cv.gapi.add(g_in1, g_in2) comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - self.assertEqual(expected.dtype, actual.dtype) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') def test_add_uint8(self): - sz = (1280, 720) - in1 = np.random.randint(0, 100, sz).astype(np.uint8) - in2 = np.random.randint(0, 100, sz).astype(np.uint8) + sz = (720, 1280) + in1 = np.full(sz, 100, dtype=np.uint8) + in2 = np.full(sz, 50 , dtype=np.uint8) # OpenCV expected = cv.add(in1, in2) @@ -53,16 +55,17 @@ class gapi_core_test(NewOpenCVTests): g_out = cv.gapi.add(g_in1, g_in2) comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - self.assertEqual(expected.dtype, actual.dtype) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') def test_mean(self): - sz = (1280, 720, 3) - in_mat = np.random.randint(0, 100, sz) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) # OpenCV expected = cv.mean(in_mat) @@ -72,15 +75,16 @@ class gapi_core_test(NewOpenCVTests): g_out = cv.gapi.mean(g_in) comp = cv.GComputation(g_in, g_out) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') def test_split3(self): - sz = (1280, 720, 3) - in_mat = np.random.randint(0, 100, sz) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) # OpenCV expected = cv.split(in_mat) @@ -90,19 +94,19 @@ class gapi_core_test(NewOpenCVTests): b, g, r = cv.gapi.split3(g_in) comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) # Comparison for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) - self.assertEqual(e.dtype, a.dtype) + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') def test_threshold(self): - sz = (1280, 720) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) - rand_int = np.random.randint(0, 50) - maxv = (rand_int, rand_int) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + maxv = (30, 30) # OpenCV expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) @@ -113,12 +117,15 @@ class gapi_core_test(NewOpenCVTests): mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.compile_args(pkg)) # Comparison - self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF)) - self.assertEqual(expected_mat.dtype, actual_mat.dtype) - self.assertEqual(expected_thresh, actual_thresh[0]) + self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_mat.dtype, actual_mat.dtype, + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_thresh, actual_thresh[0], + 'Failed on ' + pkg_name + ' backend') if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_imgproc.py b/modules/gapi/misc/python/test/test_gapi_imgproc.py new file mode 100644 index 0000000000..dd1e397081 --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_imgproc.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os + +from tests_common import NewOpenCVTests + + +# Plaidml is an optional backend +pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] + + +class gapi_imgproc_test(NewOpenCVTests): + + def test_good_features_to_track(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None + + # OpenCV + expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + # Comparison + self.assertEqual(0.0, cv.norm(expected.flatten(), actual.flatten(), cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + + + def test_rgb2gray(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.imread(img_path) + + # OpenCV + expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.RGB2Gray(g_in) + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_infer.py b/modules/gapi/misc/python/test/test_gapi_infer.py new file mode 100644 index 0000000000..a6fabf7253 --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_infer.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os + +from tests_common import NewOpenCVTests + + +class test_gapi_infer(NewOpenCVTests): + + def test_getAvailableTargets(self): + targets = cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_OPENCV) + self.assertTrue(cv.dnn.DNN_TARGET_CPU in targets) + + + def test_age_gender_infer(self): + + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (62,62)) + + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + blob = cv.dnn.blobFromImage(img) + + net.setInput(blob) + dnn_age, dnn_gender = net.forward(net.getUnconnectedOutLayersNames()) + + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + nets = cv.gapi.networks(pp) + args = cv.compile_args(nets) + gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) + + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py index 8000496f79..53304fcb26 100644 --- a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py +++ b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py @@ -2,25 +2,26 @@ import numpy as np import cv2 as cv +import os from tests_common import NewOpenCVTests # Plaidml is an optional backend pkgs = [ - cv.gapi.core.ocl.kernels(), - cv.gapi.core.cpu.kernels(), - cv.gapi.core.fluid.kernels() - # cv.gapi.core.plaidml.kernels() - ] + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] class gapi_sample_pipelines(NewOpenCVTests): # NB: This test check multiple outputs for operation def test_mean_over_r(self): - sz = (100, 100, 3) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) # # OpenCV _, _, r_ch = cv.split(in_mat) @@ -32,10 +33,11 @@ class gapi_sample_pipelines(NewOpenCVTests): g_out = cv.gapi.mean(r) comp = cv.GComputation(g_in, g_out) - for pkg in pkgs: + for pkg_name, pkg in pkgs: actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py new file mode 100644 index 0000000000..ae7ef5d338 --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os + +from tests_common import NewOpenCVTests + +class test_gapi_streaming(NewOpenCVTests): + + def test_image_input(self): + sz = (1280, 720) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + + # OpenCV + expected = cv.medianBlur(in_mat, 3) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, 3) + c = cv.GComputation(g_in, g_out) + ccomp = c.compileStreaming(cv.descr_of(cv.gin(in_mat))) + ccomp.setSource(cv.gin(in_mat)) + ccomp.start() + + _, actual = ccomp.pull() + + # Assert + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + + def test_video_input(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, ksize) + c = cv.GComputation(g_in, g_out) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, expected = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; + + + def test_video_split3(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.split(frame) + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; + + + def test_video_add(self): + sz = (576, 768, 3) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + out = cv.gapi.add(g_in1, g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source, in_mat)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.add(frame, in_mat) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; + + + def test_video_good_features_to_track(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_gray = cv.gapi.RGB2Gray(g_in) + g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + # OpenCV + frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) + expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + for e, a in zip(expected, actual): + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + self.assertEqual(0.0, cv.norm(e.flatten(), a.flatten(), cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp index 91d08bba06..ac90181184 100644 --- a/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp +++ b/modules/gapi/perf/common/gapi_core_perf_tests_inl.hpp @@ -2124,7 +2124,7 @@ PERF_TEST_P_(SizePerfTest, TestPerformance) // G-API code ////////////////////////////////////////////////////////////// cv::GMat in; - auto out = cv::gapi::size(in); + auto out = cv::gapi::streaming::size(in); cv::GComputation c(cv::GIn(in), cv::GOut(out)); cv::Size out_sz; @@ -2156,7 +2156,7 @@ PERF_TEST_P_(SizeRPerfTest, TestPerformance) // G-API code ////////////////////////////////////////////////////////////// cv::GOpaque op_rect; - auto out = cv::gapi::size(op_rect); + auto out = cv::gapi::streaming::size(op_rect); cv::GComputation c(cv::GIn(op_rect), cv::GOut(out)); cv::Size out_sz; diff --git a/modules/gapi/samples/infer_ie_onnx_hybrid.cpp b/modules/gapi/samples/infer_ie_onnx_hybrid.cpp new file mode 100644 index 0000000000..b8612a25ca --- /dev/null +++ b/modules/gapi/samples/infer_ie_onnx_hybrid.cpp @@ -0,0 +1,195 @@ +#include +#include + +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" + +#include "opencv2/gapi.hpp" +#include "opencv2/gapi/core.hpp" +#include "opencv2/gapi/imgproc.hpp" +#include "opencv2/gapi/infer.hpp" +#include "opencv2/gapi/infer/ie.hpp" +#include "opencv2/gapi/infer/onnx.hpp" +#include "opencv2/gapi/cpu/gcpukernel.hpp" +#include "opencv2/gapi/streaming/cap.hpp" + +namespace { +const std::string keys = + "{ h help | | print this help message }" + "{ input | | Path to an input video file }" + "{ fdm | | IE face detection model IR }" + "{ fdw | | IE face detection model weights }" + "{ fdd | | IE face detection device }" + "{ emom | | ONNX emotions recognition model }" + "{ output | | (Optional) Path to an output video file }" + ; +} // namespace + +namespace custom { +G_API_NET(Faces, , "face-detector"); +G_API_NET(Emotions, , "emotions-recognition"); + +G_API_OP(PostProc, (cv::GMat, cv::GMat)>, "custom.fd_postproc") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GMatDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVPostProc, PostProc) { + static void run(const cv::Mat &in_ssd_result, + const cv::Mat &in_frame, + std::vector &out_faces) { + const int MAX_PROPOSALS = 200; + const int OBJECT_SIZE = 7; + const cv::Size upscale = in_frame.size(); + const cv::Rect surface({0,0}, upscale); + + out_faces.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; // batch id + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + + if (image_id < 0.f) { // indicates end of detections + break; + } + if (confidence < 0.5f) { + continue; + } + + cv::Rect rc; + rc.x = static_cast(rc_left * upscale.width); + rc.y = static_cast(rc_top * upscale.height); + rc.width = static_cast(rc_right * upscale.width) - rc.x; + rc.height = static_cast(rc_bottom * upscale.height) - rc.y; + out_faces.push_back(rc & surface); + } + } +}; +//! [Postproc] + +} // namespace custom + +namespace labels { +// Labels as defined in +// https://github.com/onnx/models/tree/master/vision/body_analysis/emotion_ferplus +// +const std::string emotions[] = { + "neutral", "happiness", "surprise", "sadness", "anger", "disgust", "fear", "contempt" +}; +namespace { +template +std::vector softmax(Iter begin, Iter end) { + std::vector prob(end - begin, 0.f); + std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); }); + float sum = std::accumulate(prob.begin(), prob.end(), 0.0f); + for (int i = 0; i < static_cast(prob.size()); i++) + prob[i] /= sum; + return prob; +} + +void DrawResults(cv::Mat &frame, + const std::vector &faces, + const std::vector &out_emotions) { + CV_Assert(faces.size() == out_emotions.size()); + + for (auto it = faces.begin(); it != faces.end(); ++it) { + const auto idx = std::distance(faces.begin(), it); + const auto &rc = *it; + + const float *emotions_data = out_emotions[idx].ptr(); + auto sm = softmax(emotions_data, emotions_data + 8); + const auto emo_id = std::max_element(sm.begin(), sm.end()) - sm.begin(); + + const int ATTRIB_OFFSET = 15; + cv::rectangle(frame, rc, {0, 255, 0}, 4); + cv::putText(frame, emotions[emo_id], + cv::Point(rc.x, rc.y - ATTRIB_OFFSET), + cv::FONT_HERSHEY_COMPLEX_SMALL, + 1, + cv::Scalar(0, 0, 255)); + + std::cout << emotions[emo_id] << " at " << rc << std::endl; + } +} +} // anonymous namespace +} // namespace labels + +int main(int argc, char *argv[]) +{ + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + const std::string input = cmd.get("input"); + const std::string output = cmd.get("output"); + + // OpenVINO FD parameters here + auto det_net = cv::gapi::ie::Params { + cmd.get("fdm"), // read cmd args: path to topology IR + cmd.get("fdw"), // read cmd args: path to weights + cmd.get("fdd"), // read cmd args: device specifier + }; + + // ONNX Emotions parameters here + auto emo_net = cv::gapi::onnx::Params { + cmd.get("emom"), // read cmd args: path to the ONNX model + }.cfgNormalize({false}); // model accepts 0..255 range in FP32 + + auto kernels = cv::gapi::kernels(); + auto networks = cv::gapi::networks(det_net, emo_net); + + cv::GMat in; + cv::GMat bgr = cv::gapi::copy(in); + cv::GMat frame = cv::gapi::streaming::desync(bgr); + cv::GMat detections = cv::gapi::infer(frame); + cv::GArray faces = custom::PostProc::on(detections, frame); + cv::GArray emotions = cv::gapi::infer(faces, frame); + auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(bgr, faces, emotions)) + .compileStreaming(cv::compile_args(kernels, networks)); + + auto in_src = cv::gapi::wip::make_src(input); + pipeline.setSource(cv::gin(in_src)); + pipeline.start(); + + cv::util::optional out_frame; + cv::util::optional> out_faces; + cv::util::optional> out_emotions; + + cv::Mat last_mat; + std::vector last_faces; + std::vector last_emotions; + + cv::VideoWriter writer; + + while (pipeline.pull(cv::gout(out_frame, out_faces, out_emotions))) { + if (out_faces && out_emotions) { + last_faces = *out_faces; + last_emotions = *out_emotions; + } + if (out_frame) { + last_mat = *out_frame; + labels::DrawResults(last_mat, last_faces, last_emotions); + + if (!output.empty()) { + if (!writer.isOpened()) { + const auto sz = cv::Size{last_mat.cols, last_mat.rows}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + writer << last_mat; + } + } + if (!last_mat.empty()) { + cv::imshow("Out", last_mat); + cv::waitKey(1); + } + } + return 0; +} diff --git a/modules/gapi/samples/infer_single_roi.cpp b/modules/gapi/samples/infer_single_roi.cpp new file mode 100644 index 0000000000..6054a3f4a6 --- /dev/null +++ b/modules/gapi/samples/infer_single_roi.cpp @@ -0,0 +1,264 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input video file }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }" + "{ r roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }"; + +namespace { + +std::string weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} + +cv::util::optional parse_roi(const std::string &rc) { + cv::Rect rv; + char delim[3]; + + std::stringstream is(rc); + is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height; + if (is.bad()) { + return cv::util::optional(); // empty value + } + const auto is_delim = [](char c) { + return c == ','; + }; + if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) { + return cv::util::optional(); // empty value + + } + if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) { + return cv::util::optional(); // empty value + } + return cv::util::make_optional(std::move(rv)); +} + +} // namespace + +namespace custom { + +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(GetSize, , "sample.custom.get-size") { + static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVGetSize, GetSize) { + static void run(const cv::Mat &in, cv::Size &out) { + out = {in.cols, in.rows}; + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Mat &in_mat, cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_mat.cols/2, in_mat.rows/2}; + auto sqside = std::min(in_mat.cols, in_mat.rows); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +int main(int argc, char *argv[]) +{ + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // Prepare parameters first + const std::string input = cmd.get("input"); + const auto opt_roi = parse_roi(cmd.get("roi")); + + const auto face_model_path = cmd.get("facem"); + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier + }; + auto kernels = cv::gapi::kernels + < custom::OCVGetSize + , custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + + // Now build the graph. The graph structure may vary + // pased on the input parameters + cv::GStreamingCompiled pipeline; + auto inputs = cv::gin(cv::gapi::wip::make_src(input)); + + if (opt_roi.has_value()) { + // Use the value provided by user + std::cout << "Will run inference for static region " + << opt_roi.value() + << " only" + << std::endl; + cv::GMat in; + cv::GOpaque in_roi; + auto blob = cv::gapi::infer(in_roi, in); + auto rcs = custom::ParseSSD::on(blob, in_roi, custom::GetSize::on(in)); + auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, in_roi)); + pipeline = cv::GComputation(cv::GIn(in, in_roi), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + + // Since the ROI to detect is manual, make it part of the input vector + inputs.push_back(cv::gin(opt_roi.value())[0]); + } else { + // Automatically detect ROI to infer. Make it output parameter + std::cout << "ROI is not set or invalid. Locating it automatically" + << std::endl; + cv::GMat in; + cv::GOpaque roi = custom::LocateROI::on(in); + auto blob = cv::gapi::infer(roi, in); + auto rcs = custom::ParseSSD::on(blob, roi, custom::GetSize::on(in)); + auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs, roi)); + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + } + + // The execution part + pipeline.setSource(std::move(inputs)); + pipeline.start(); + + cv::Mat out; + while (pipeline.pull(cv::gout(out))) { + cv::imshow("Out", out); + cv::waitKey(1); + } + return 0; +} diff --git a/modules/gapi/samples/infer_ssd_onnx.cpp b/modules/gapi/samples/infer_ssd_onnx.cpp new file mode 100644 index 0000000000..fc26ca1e36 --- /dev/null +++ b/modules/gapi/samples/infer_ssd_onnx.cpp @@ -0,0 +1,213 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace custom { + +G_API_NET(ObjDetector, , "object-detector"); + +using GDetections = cv::GArray; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(GetSize, , "sample.custom.get-size") { + static cv::GOpaqueDesc outMeta(const cv::GMatDesc &) { + return cv::empty_gopaque_desc(); + } +}; +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVGetSize, GetSize) { + static void run(const cv::Mat &in, cv::Size &out) { + out = {in.cols, in.rows}; + } +}; +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + cv::Rect rc; + rc.x = static_cast(rc_left * in_parent_size.width); + rc.y = static_cast(rc_top * in_parent_size.height); + rc.width = static_cast(rc_right * in_parent_size.width) - rc.x; + rc.height = static_cast(rc_bottom * in_parent_size.height) - rc.y; + out_objects.emplace_back(rc & surface); + } + } +}; +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_obj_rcs, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + for (auto &&rc : in_obj_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + + std::cout << "Detections:"; + for (auto &&rc : in_obj_rcs) std::cout << ' ' << rc; + std::cout << std::endl; + } +}; + +} // namespace custom + +namespace { +void remap_ssd_ports(const std::unordered_map &onnx, + std::unordered_map &gapi) { + // Assemble ONNX-processed outputs back to a single 1x1x200x7 blob + // to preserve compatibility with OpenVINO-based SSD pipeline + const cv::Mat &num_detections = onnx.at("num_detections:0"); + const cv::Mat &detection_boxes = onnx.at("detection_boxes:0"); + const cv::Mat &detection_scores = onnx.at("detection_scores:0"); + const cv::Mat &detection_classes = onnx.at("detection_classes:0"); + + GAPI_Assert(num_detections.depth() == CV_32F); + GAPI_Assert(detection_boxes.depth() == CV_32F); + GAPI_Assert(detection_scores.depth() == CV_32F); + GAPI_Assert(detection_classes.depth() == CV_32F); + + cv::Mat &ssd_output = gapi.at("detection_output"); + + const int num_objects = static_cast(num_detections.ptr()[0]); + const float *in_boxes = detection_boxes.ptr(); + const float *in_scores = detection_scores.ptr(); + const float *in_classes = detection_classes.ptr(); + float *ptr = ssd_output.ptr(); + + for (int i = 0; i < num_objects; i++) { + ptr[0] = 0.f; // "image_id" + ptr[1] = in_classes[i]; // "label" + ptr[2] = in_scores[i]; // "confidence" + ptr[3] = in_boxes[4*i + 1]; // left + ptr[4] = in_boxes[4*i + 0]; // top + ptr[5] = in_boxes[4*i + 3]; // right + ptr[6] = in_boxes[4*i + 2]; // bottom + + ptr += 7; + in_boxes += 4; + } + if (num_objects < ssd_output.size[2]-1) { + // put a -1 mark at the end of output blob if there is space left + ptr[0] = -1.f; + } +} +} // anonymous namespace + + +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input video file }" + "{ output | | (Optional) path to output video file }" + "{ detm | | Path to an ONNX SSD object detection model (.onnx) }" + ; + +int main(int argc, char *argv[]) +{ + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // Prepare parameters first + const std::string input = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto obj_model_path = cmd.get("detm"); + + auto obj_net = cv::gapi::onnx::Params{obj_model_path} + .cfgOutputLayers({"detection_output"}) + .cfgPostProc({cv::GMatDesc{CV_32F, {1,1,200,7}}}, remap_ssd_ports); + auto kernels = cv::gapi::kernels< custom::OCVGetSize + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(obj_net); + + // Now build the graph + cv::GMat in; + auto blob = cv::gapi::infer(in); + auto rcs = custom::ParseSSD::on(blob, custom::GetSize::on(in)); + auto out = cv::gapi::wip::draw::render3ch(in, custom::BBoxes::on(rcs)); + cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + + auto inputs = cv::gin(cv::gapi::wip::make_src(input)); + + // The execution part + pipeline.setSource(std::move(inputs)); + pipeline.start(); + + cv::VideoWriter writer; + + cv::Mat outMat; + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + if (!writer.isOpened()) { + const auto sz = cv::Size{outMat.cols, outMat.rows}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + writer << outMat; + } + } + return 0; +} diff --git a/modules/gapi/samples/text_detection.cpp b/modules/gapi/samples/text_detection.cpp new file mode 100644 index 0000000000..da1bab6ca9 --- /dev/null +++ b/modules/gapi/samples/text_detection.cpp @@ -0,0 +1,698 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +const std::string about = + "This is an OpenCV-based version of OMZ Text Detection example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input video file }" + "{ tdm | text-detection-0004.xml | Path to OpenVINO text detection model (.xml), versions 0003 and 0004 work }" + "{ tdd | CPU | Target device for the text detector (e.g. CPU, GPU, VPU, ...) }" + "{ trm | text-recognition-0012.xml | Path to OpenVINO text recognition model (.xml) }" + "{ trd | CPU | Target device for the text recognition (e.g. CPU, GPU, VPU, ...) }" + "{ bw | 0 | CTC beam search decoder bandwidth, if 0, a CTC greedy decoder is used}" + "{ sset | 0123456789abcdefghijklmnopqrstuvwxyz | Symbol set to use with text recognition decoder. Shouldn't contain symbol #. }" + "{ thr | 0.2 | Text recognition confidence threshold}" + ; + +namespace { +std::string weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + const auto ext = model_path.substr(sz - EXT_LEN); + CV_Assert(cv::toLowerCase(ext) == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} + +////////////////////////////////////////////////////////////////////// +// Taken from OMZ samples as-is +template +void softmax_and_choose(Iter begin, Iter end, int *argmax, float *prob) { + auto max_element = std::max_element(begin, end); + *argmax = static_cast(std::distance(begin, max_element)); + float max_val = *max_element; + double sum = 0; + for (auto i = begin; i != end; i++) { + sum += std::exp((*i) - max_val); + } + if (std::fabs(sum) < std::numeric_limits::epsilon()) { + throw std::logic_error("sum can't be equal to zero"); + } + *prob = 1.0f / static_cast(sum); +} + +template +std::vector softmax(Iter begin, Iter end) { + std::vector prob(end - begin, 0.f); + std::transform(begin, end, prob.begin(), [](float x) { return std::exp(x); }); + float sum = std::accumulate(prob.begin(), prob.end(), 0.0f); + for (int i = 0; i < static_cast(prob.size()); i++) + prob[i] /= sum; + return prob; +} + +struct BeamElement { + std::vector sentence; //!< The sequence of chars that will be a result of the beam element + + float prob_blank; //!< The probability that the last char in CTC sequence + //!< for the beam element is the special blank char + + float prob_not_blank; //!< The probability that the last char in CTC sequence + //!< for the beam element is NOT the special blank char + + float prob() const { //!< The probability of the beam element. + return prob_blank + prob_not_blank; + } +}; + +std::string CTCGreedyDecoder(const float *data, + const std::size_t sz, + const std::string &alphabet, + const char pad_symbol, + double *conf) { + std::string res = ""; + bool prev_pad = false; + *conf = 1; + + const auto num_classes = alphabet.length(); + for (auto it = data; it != (data+sz); it += num_classes) { + int argmax = 0; + float prob = 0.f; + + softmax_and_choose(it, it + num_classes, &argmax, &prob); + (*conf) *= prob; + + auto symbol = alphabet[argmax]; + if (symbol != pad_symbol) { + if (res.empty() || prev_pad || (!res.empty() && symbol != res.back())) { + prev_pad = false; + res += symbol; + } + } else { + prev_pad = true; + } + } + return res; +} + +std::string CTCBeamSearchDecoder(const float *data, + const std::size_t sz, + const std::string &alphabet, + double *conf, + int bandwidth) { + const auto num_classes = alphabet.length(); + + std::vector curr; + std::vector last; + + last.push_back(BeamElement{std::vector(), 1.f, 0.f}); + + for (auto it = data; it != (data+sz); it += num_classes) { + curr.clear(); + + std::vector prob = softmax(it, it + num_classes); + + for(const auto& candidate: last) { + float prob_not_blank = 0.f; + const std::vector& candidate_sentence = candidate.sentence; + if (!candidate_sentence.empty()) { + int n = candidate_sentence.back(); + prob_not_blank = candidate.prob_not_blank * prob[n]; + } + float prob_blank = candidate.prob() * prob[num_classes - 1]; + + auto check_res = std::find_if(curr.begin(), + curr.end(), + [&candidate_sentence](const BeamElement& n) { + return n.sentence == candidate_sentence; + }); + if (check_res == std::end(curr)) { + curr.push_back(BeamElement{candidate.sentence, prob_blank, prob_not_blank}); + } else { + check_res->prob_not_blank += prob_not_blank; + if (check_res->prob_blank != 0.f) { + throw std::logic_error("Probability that the last char in CTC-sequence " + "is the special blank char must be zero here"); + } + check_res->prob_blank = prob_blank; + } + + for (int i = 0; i < static_cast(num_classes) - 1; i++) { + auto extend = candidate_sentence; + extend.push_back(i); + + if (candidate_sentence.size() > 0 && candidate.sentence.back() == i) { + prob_not_blank = prob[i] * candidate.prob_blank; + } else { + prob_not_blank = prob[i] * candidate.prob(); + } + + auto check_res2 = std::find_if(curr.begin(), + curr.end(), + [&extend](const BeamElement &n) { + return n.sentence == extend; + }); + if (check_res2 == std::end(curr)) { + curr.push_back(BeamElement{extend, 0.f, prob_not_blank}); + } else { + check_res2->prob_not_blank += prob_not_blank; + } + } + } + + sort(curr.begin(), curr.end(), [](const BeamElement &a, const BeamElement &b) -> bool { + return a.prob() > b.prob(); + }); + + last.clear(); + int num_to_copy = std::min(bandwidth, static_cast(curr.size())); + for (int b = 0; b < num_to_copy; b++) { + last.push_back(curr[b]); + } + } + + *conf = last[0].prob(); + std::string res=""; + for (const auto& idx: last[0].sentence) { + res += alphabet[idx]; + } + + return res; +} + +////////////////////////////////////////////////////////////////////// +} // anonymous namespace + +namespace custom { +namespace { + +////////////////////////////////////////////////////////////////////// +// Define networks for this sample +using GMat2 = std::tuple; +G_API_NET(TextDetection, + , + "sample.custom.text_detect"); + +G_API_NET(TextRecognition, + , + "sample.custom.text_recogn"); + +// Define custom operations +using GSize = cv::GOpaque; +using GRRects = cv::GArray; +G_API_OP(PostProcess, + , + "sample.custom.text.post_proc") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, + const cv::GMatDesc &, + const cv::GOpaqueDesc &, + float, + float) { + return cv::empty_array_desc(); + } +}; + +using GMats = cv::GArray; +G_API_OP(CropLabels, + , + "sample.custom.text.crop") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, + const cv::GArrayDesc &, + const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +////////////////////////////////////////////////////////////////////// +// Implement custom operations +GAPI_OCV_KERNEL(OCVPostProcess, PostProcess) { + static void run(const cv::Mat &link, + const cv::Mat &segm, + const cv::Size &img_size, + const float link_threshold, + const float segm_threshold, + std::vector &out) { + // NOTE: Taken from the OMZ text detection sample almost as-is + const int kMinArea = 300; + const int kMinHeight = 10; + + const float *link_data_pointer = link.ptr(); + std::vector link_data(link_data_pointer, link_data_pointer + link.total()); + link_data = transpose4d(link_data, dimsToShape(link.size), {0, 2, 3, 1}); + softmax(link_data); + link_data = sliceAndGetSecondChannel(link_data); + std::vector new_link_data_shape = { + link.size[0], + link.size[2], + link.size[3], + link.size[1]/2, + }; + + const float *cls_data_pointer = segm.ptr(); + std::vector cls_data(cls_data_pointer, cls_data_pointer + segm.total()); + cls_data = transpose4d(cls_data, dimsToShape(segm.size), {0, 2, 3, 1}); + softmax(cls_data); + cls_data = sliceAndGetSecondChannel(cls_data); + std::vector new_cls_data_shape = { + segm.size[0], + segm.size[2], + segm.size[3], + segm.size[1]/2, + }; + + out = maskToBoxes(decodeImageByJoin(cls_data, new_cls_data_shape, + link_data, new_link_data_shape, + segm_threshold, link_threshold), + static_cast(kMinArea), + static_cast(kMinHeight), + img_size); + } + + static std::vector dimsToShape(const cv::MatSize &sz) { + const int n_dims = sz.dims(); + std::vector result; + result.reserve(n_dims); + + // cv::MatSize is not iterable... + for (int i = 0; i < n_dims; i++) { + result.emplace_back(static_cast(sz[i])); + } + return result; + } + + static void softmax(std::vector &rdata) { + // NOTE: Taken from the OMZ text detection sample almost as-is + const size_t last_dim = 2; + for (size_t i = 0 ; i < rdata.size(); i+=last_dim) { + float m = std::max(rdata[i], rdata[i+1]); + rdata[i] = std::exp(rdata[i] - m); + rdata[i + 1] = std::exp(rdata[i + 1] - m); + float s = rdata[i] + rdata[i + 1]; + rdata[i] /= s; + rdata[i + 1] /= s; + } + } + + static std::vector transpose4d(const std::vector &data, + const std::vector &shape, + const std::vector &axes) { + // NOTE: Taken from the OMZ text detection sample almost as-is + if (shape.size() != axes.size()) + throw std::runtime_error("Shape and axes must have the same dimension."); + + for (size_t a : axes) { + if (a >= shape.size()) + throw std::runtime_error("Axis must be less than dimension of shape."); + } + size_t total_size = shape[0]*shape[1]*shape[2]*shape[3]; + std::vector steps { + shape[axes[1]]*shape[axes[2]]*shape[axes[3]], + shape[axes[2]]*shape[axes[3]], + shape[axes[3]], + 1 + }; + + size_t source_data_idx = 0; + std::vector new_data(total_size, 0); + std::vector ids(shape.size()); + for (ids[0] = 0; ids[0] < shape[0]; ids[0]++) { + for (ids[1] = 0; ids[1] < shape[1]; ids[1]++) { + for (ids[2] = 0; ids[2] < shape[2]; ids[2]++) { + for (ids[3]= 0; ids[3] < shape[3]; ids[3]++) { + size_t new_data_idx = ids[axes[0]]*steps[0] + ids[axes[1]]*steps[1] + + ids[axes[2]]*steps[2] + ids[axes[3]]*steps[3]; + new_data[new_data_idx] = data[source_data_idx++]; + } + } + } + } + return new_data; + } + + static std::vector sliceAndGetSecondChannel(const std::vector &data) { + // NOTE: Taken from the OMZ text detection sample almost as-is + std::vector new_data(data.size() / 2, 0); + for (size_t i = 0; i < data.size() / 2; i++) { + new_data[i] = data[2 * i + 1]; + } + return new_data; + } + + static void join(const int p1, + const int p2, + std::unordered_map &group_mask) { + // NOTE: Taken from the OMZ text detection sample almost as-is + const int root1 = findRoot(p1, group_mask); + const int root2 = findRoot(p2, group_mask); + if (root1 != root2) { + group_mask[root1] = root2; + } + } + + static cv::Mat decodeImageByJoin(const std::vector &cls_data, + const std::vector &cls_data_shape, + const std::vector &link_data, + const std::vector &link_data_shape, + float cls_conf_threshold, + float link_conf_threshold) { + // NOTE: Taken from the OMZ text detection sample almost as-is + const int h = cls_data_shape[1]; + const int w = cls_data_shape[2]; + + std::vector pixel_mask(h * w, 0); + std::unordered_map group_mask; + std::vector points; + for (int i = 0; i < static_cast(pixel_mask.size()); i++) { + pixel_mask[i] = cls_data[i] >= cls_conf_threshold; + if (pixel_mask[i]) { + points.emplace_back(i % w, i / w); + group_mask[i] = -1; + } + } + std::vector link_mask(link_data.size(), 0); + for (size_t i = 0; i < link_mask.size(); i++) { + link_mask[i] = link_data[i] >= link_conf_threshold; + } + size_t neighbours = size_t(link_data_shape[3]); + for (const auto &point : points) { + size_t neighbour = 0; + for (int ny = point.y - 1; ny <= point.y + 1; ny++) { + for (int nx = point.x - 1; nx <= point.x + 1; nx++) { + if (nx == point.x && ny == point.y) + continue; + if (nx >= 0 && nx < w && ny >= 0 && ny < h) { + uchar pixel_value = pixel_mask[size_t(ny) * size_t(w) + size_t(nx)]; + uchar link_value = link_mask[(size_t(point.y) * size_t(w) + size_t(point.x)) + *neighbours + neighbour]; + if (pixel_value && link_value) { + join(point.x + point.y * w, nx + ny * w, group_mask); + } + } + neighbour++; + } + } + } + return get_all(points, w, h, group_mask); + } + + static cv::Mat get_all(const std::vector &points, + const int w, + const int h, + std::unordered_map &group_mask) { + // NOTE: Taken from the OMZ text detection sample almost as-is + std::unordered_map root_map; + cv::Mat mask(h, w, CV_32S, cv::Scalar(0)); + for (const auto &point : points) { + int point_root = findRoot(point.x + point.y * w, group_mask); + if (root_map.find(point_root) == root_map.end()) { + root_map.emplace(point_root, static_cast(root_map.size() + 1)); + } + mask.at(point.x + point.y * w) = root_map[point_root]; + } + return mask; + } + + static int findRoot(const int point, + std::unordered_map &group_mask) { + // NOTE: Taken from the OMZ text detection sample almost as-is + int root = point; + bool update_parent = false; + while (group_mask.at(root) != -1) { + root = group_mask.at(root); + update_parent = true; + } + if (update_parent) { + group_mask[point] = root; + } + return root; + } + + static std::vector maskToBoxes(const cv::Mat &mask, + const float min_area, + const float min_height, + const cv::Size &image_size) { + // NOTE: Taken from the OMZ text detection sample almost as-is + std::vector bboxes; + double min_val = 0.; + double max_val = 0.; + cv::minMaxLoc(mask, &min_val, &max_val); + int max_bbox_idx = static_cast(max_val); + cv::Mat resized_mask; + cv::resize(mask, resized_mask, image_size, 0, 0, cv::INTER_NEAREST); + + for (int i = 1; i <= max_bbox_idx; i++) { + cv::Mat bbox_mask = resized_mask == i; + std::vector> contours; + + cv::findContours(bbox_mask, contours, cv::RETR_CCOMP, cv::CHAIN_APPROX_SIMPLE); + if (contours.empty()) + continue; + cv::RotatedRect r = cv::minAreaRect(contours[0]); + if (std::min(r.size.width, r.size.height) < min_height) + continue; + if (r.size.area() < min_area) + continue; + bboxes.emplace_back(r); + } + return bboxes; + } +}; // GAPI_OCV_KERNEL(PostProcess) + +GAPI_OCV_KERNEL(OCVCropLabels, CropLabels) { + static void run(const cv::Mat &image, + const std::vector &detections, + const cv::Size &outSize, + std::vector &out) { + out.clear(); + out.reserve(detections.size()); + cv::Mat crop(outSize, CV_8UC3, cv::Scalar(0)); + cv::Mat gray(outSize, CV_8UC1, cv::Scalar(0)); + std::vector blob_shape = {1,1,outSize.height,outSize.width}; + + for (auto &&rr : detections) { + std::vector points(4); + rr.points(points.data()); + + const auto top_left_point_idx = topLeftPointIdx(points); + cv::Point2f point0 = points[static_cast(top_left_point_idx)]; + cv::Point2f point1 = points[(top_left_point_idx + 1) % 4]; + cv::Point2f point2 = points[(top_left_point_idx + 2) % 4]; + + std::vector from{point0, point1, point2}; + std::vector to{ + cv::Point2f(0.0f, 0.0f), + cv::Point2f(static_cast(outSize.width-1), 0.0f), + cv::Point2f(static_cast(outSize.width-1), + static_cast(outSize.height-1)) + }; + cv::Mat M = cv::getAffineTransform(from, to); + cv::warpAffine(image, crop, M, outSize); + cv::cvtColor(crop, gray, cv::COLOR_BGR2GRAY); + + cv::Mat blob; + gray.convertTo(blob, CV_32F); + out.push_back(blob.reshape(1, blob_shape)); // pass as 1,1,H,W instead of H,W + } + } + + static int topLeftPointIdx(const std::vector &points) { + // NOTE: Taken from the OMZ text detection sample almost as-is + cv::Point2f most_left(std::numeric_limits::max(), + std::numeric_limits::max()); + cv::Point2f almost_most_left(std::numeric_limits::max(), + std::numeric_limits::max()); + int most_left_idx = -1; + int almost_most_left_idx = -1; + + for (size_t i = 0; i < points.size() ; i++) { + if (most_left.x > points[i].x) { + if (most_left.x < std::numeric_limits::max()) { + almost_most_left = most_left; + almost_most_left_idx = most_left_idx; + } + most_left = points[i]; + most_left_idx = static_cast(i); + } + if (almost_most_left.x > points[i].x && points[i] != most_left) { + almost_most_left = points[i]; + almost_most_left_idx = static_cast(i); + } + } + + if (almost_most_left.y < most_left.y) { + most_left = almost_most_left; + most_left_idx = almost_most_left_idx; + } + return most_left_idx; + } + +}; // GAPI_OCV_KERNEL(CropLabels) + +} // anonymous namespace +} // namespace custom + +namespace vis { +namespace { + +void drawRotatedRect(cv::Mat &m, const cv::RotatedRect &rc) { + std::vector tmp_points(5); + rc.points(tmp_points.data()); + tmp_points[4] = tmp_points[0]; + auto prev = tmp_points.begin(), it = prev+1; + for (; it != tmp_points.end(); ++it) { + cv::line(m, *prev, *it, cv::Scalar(50, 205, 50), 2); + prev = it; + } +} + +void drawText(cv::Mat &m, const cv::RotatedRect &rc, const std::string &str) { + const int fface = cv::FONT_HERSHEY_SIMPLEX; + const double scale = 0.7; + const int thick = 1; + int base = 0; + const auto text_size = cv::getTextSize(str, fface, scale, thick, &base); + + std::vector tmp_points(4); + rc.points(tmp_points.data()); + const auto tl_point_idx = custom::OCVCropLabels::topLeftPointIdx(tmp_points); + cv::Point text_pos = tmp_points[tl_point_idx]; + text_pos.x = std::max(0, text_pos.x); + text_pos.y = std::max(text_size.height, text_pos.y); + + cv::rectangle(m, + text_pos + cv::Point{0, base}, + text_pos + cv::Point{text_size.width, -text_size.height}, + CV_RGB(50, 205, 50), + cv::FILLED); + const auto white = CV_RGB(255, 255, 255); + cv::putText(m, str, text_pos, fface, scale, white, thick, 8); +} + +} // anonymous namespace +} // namespace vis + +int main(int argc, char *argv[]) +{ + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + const auto input_file_name = cmd.get("input"); + const auto tdet_model_path = cmd.get("tdm"); + const auto trec_model_path = cmd.get("trm"); + const auto tdet_target_dev = cmd.get("tdd"); + const auto trec_target_dev = cmd.get("trd"); + const auto ctc_beam_dec_bw = cmd.get("bw"); + const auto dec_conf_thresh = cmd.get("thr"); + + const auto pad_symbol = '#'; + const auto symbol_set = cmd.get("sset") + pad_symbol; + + cv::GMat in; + cv::GOpaque in_rec_sz; + cv::GMat link, segm; + std::tie(link, segm) = cv::gapi::infer(in); + cv::GOpaque size = cv::gapi::streaming::size(in); + cv::GArray rrs = custom::PostProcess::on(link, segm, size, 0.8f, 0.8f); + cv::GArray labels = custom::CropLabels::on(in, rrs, in_rec_sz); + cv::GArray text = cv::gapi::infer2(in, labels); + + cv::GComputation graph(cv::GIn(in, in_rec_sz), + cv::GOut(cv::gapi::copy(in), rrs, text)); + + // Text detection network + auto tdet_net = cv::gapi::ie::Params { + tdet_model_path, // path to topology IR + weights_path(tdet_model_path), // path to weights + tdet_target_dev, // device specifier + }.cfgOutputLayers({"model/link_logits_/add", "model/segm_logits/add"}); + + auto trec_net = cv::gapi::ie::Params { + trec_model_path, // path to topology IR + weights_path(trec_model_path), // path to weights + trec_target_dev, // device specifier + }; + auto networks = cv::gapi::networks(tdet_net, trec_net); + + auto kernels = cv::gapi::kernels< custom::OCVPostProcess + , custom::OCVCropLabels + >(); + auto pipeline = graph.compileStreaming(cv::compile_args(kernels, networks)); + + std::cout << "Reading " << input_file_name << std::endl; + + // Input stream + auto in_src = cv::gapi::wip::make_src(input_file_name); + + // Text recognition input size (also an input parameter to the graph) + auto in_rsz = cv::Size{ 120, 32 }; + + // Set the pipeline source & start the pipeline + pipeline.setSource(cv::gin(in_src, in_rsz)); + pipeline.start(); + + // Declare the output data & run the processing loop + cv::TickMeter tm; + cv::Mat image; + std::vector out_rcs; + std::vector out_text; + + tm.start(); + int frames = 0; + while (pipeline.pull(cv::gout(image, out_rcs, out_text))) { + frames++; + + CV_Assert(out_rcs.size() == out_text.size()); + const auto num_labels = out_rcs.size(); + + std::vector tmp_points(4); + for (std::size_t l = 0; l < num_labels; l++) { + // Decode the recognized text in the rectangle + const auto &blob = out_text[l]; + const float *data = blob.ptr(); + const auto sz = blob.total(); + double conf = 1.0; + const std::string res = ctc_beam_dec_bw == 0 + ? CTCGreedyDecoder(data, sz, symbol_set, pad_symbol, &conf) + : CTCBeamSearchDecoder(data, sz, symbol_set, &conf, ctc_beam_dec_bw); + + // Draw a bounding box for this rotated rectangle + const auto &rc = out_rcs[l]; + vis::drawRotatedRect(image, rc); + + // Draw text, if decoded + if (conf >= dec_conf_thresh) { + vis::drawText(image, rc, res); + } + } + tm.stop(); + cv::imshow("Out", image); + cv::waitKey(1); + tm.start(); + } + tm.stop(); + std::cout << "Processed " << frames << " frames" + << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; + return 0; +} diff --git a/modules/gapi/src/api/gbackend.cpp b/modules/gapi/src/api/gbackend.cpp index 600e5cc84d..fd4a5eb38b 100644 --- a/modules/gapi/src/api/gbackend.cpp +++ b/modules/gapi/src/api/gbackend.cpp @@ -67,6 +67,21 @@ cv::gapi::GKernelPackage cv::gapi::GBackend::Priv::auxiliaryKernels() const return {}; } +bool cv::gapi::GBackend::Priv::controlsMerge() const +{ + return false; +} + +bool cv::gapi::GBackend::Priv::allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const +{ + GAPI_Assert(controlsMerge()); + return true; +} + + // GBackend public implementation ////////////////////////////////////////////// cv::gapi::GBackend::GBackend() { @@ -128,6 +143,14 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle if (handleRMat == HandleRMat::SKIP) return; GAPI_Assert(arg.index() == GRunArg::index_of()); bindRMat(mag, rc, util::get(arg), RMat::Access::R); + + // FIXME: Here meta may^WWILL be copied multiple times! + // Replace it is reference-counted object? + mag.meta()[rc.id] = arg.meta; + mag.meta()[rc.id] = arg.meta; +#if !defined(GAPI_STANDALONE) + mag.meta()[rc.id] = arg.meta; +#endif break; } @@ -139,19 +162,23 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle case GRunArg::index_of() : mag_scalar = util::get(arg); break; default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } + mag.meta()[rc.id] = arg.meta; break; } case GShape::GARRAY: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; case GShape::GOPAQUE: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; case GShape::GFRAME: - mag.template slot()[rc.id] = util::get(arg); + mag.slot()[rc.id] = util::get(arg); + mag.meta()[rc.id] = arg.meta; break; default: @@ -235,13 +262,23 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref) // Wrap associated CPU object (either host or an internal one) switch (ref.shape) { - case GShape::GMAT: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GSCALAR: return GRunArg(mag.template slot().at(ref.id)); + case GShape::GMAT: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GSCALAR: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); // Note: .at() is intentional for GArray and GOpaque as objects MUST be already there // (and constructed by either bindIn/Out or resetInternal) - case GShape::GARRAY: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GOPAQUE: return GRunArg(mag.template slot().at(ref.id)); - case GShape::GFRAME: return GRunArg(mag.template slot().at(ref.id)); + case GShape::GARRAY: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GOPAQUE: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); + case GShape::GFRAME: + return GRunArg(mag.slot().at(ref.id), + mag.meta().at(ref.id)); default: util::throw_error(std::logic_error("Unsupported GShape type")); break; diff --git a/modules/gapi/src/api/gbackend_priv.hpp b/modules/gapi/src/api/gbackend_priv.hpp index 13f39acc86..45237514a5 100644 --- a/modules/gapi/src/api/gbackend_priv.hpp +++ b/modules/gapi/src/api/gbackend_priv.hpp @@ -19,7 +19,7 @@ #include "opencv2/gapi/gkernel.hpp" #include "compiler/gmodel.hpp" - +#include "compiler/gislandmodel.hpp" namespace cv { @@ -68,6 +68,22 @@ public: virtual cv::gapi::GKernelPackage auxiliaryKernels() const; + // Ask backend if it has a custom control over island fusion process + // This method is quite redundant but there's nothing better fits + // the current fusion process. By default, [existing] backends don't + // control the merge. + // FIXME: Refactor to a single entity? + virtual bool controlsMerge() const; + + // Ask backend if it is ok to merge these two islands connected + // via a data slot. By default, [existing] backends allow to merge everything. + // FIXME: Refactor to a single entity? + // FIXME: Strip down the type details form graph? (make it ade::Graph?) + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &g, + const ade::NodeHandle &a_nh, + const ade::NodeHandle &slot_nh, + const ade::NodeHandle &b_nh) const; + virtual ~Priv() = default; }; diff --git a/modules/gapi/src/api/gcall.cpp b/modules/gapi/src/api/gcall.cpp index 6f5f65bbfd..6a2121bd36 100644 --- a/modules/gapi/src/api/gcall.cpp +++ b/modules/gapi/src/api/gcall.cpp @@ -78,3 +78,13 @@ const cv::GCall::Priv& cv::GCall::priv() const { return *m_priv; } + +cv::GKernel& cv::GCall::kernel() +{ + return m_priv->m_k; +} + +cv::util::any& cv::GCall::params() +{ + return m_priv->m_params; +} diff --git a/modules/gapi/src/api/gcall_priv.hpp b/modules/gapi/src/api/gcall_priv.hpp index edc2c225dc..b142432c78 100644 --- a/modules/gapi/src/api/gcall_priv.hpp +++ b/modules/gapi/src/api/gcall_priv.hpp @@ -42,10 +42,11 @@ class GCall::Priv { public: std::vector m_args; - const GKernel m_k; + GKernel m_k; // TODO: Rename to "constructionNode" or smt to reflect its lifetime GNode m_node; + cv::util::any m_params; explicit Priv(const GKernel &k); }; diff --git a/modules/gapi/src/api/gcomputation.cpp b/modules/gapi/src/api/gcomputation.cpp index 9ff0273b40..5668cddc93 100644 --- a/modules/gapi/src/api/gcomputation.cpp +++ b/modules/gapi/src/api/gcomputation.cpp @@ -9,6 +9,7 @@ #include // remove_if #include // isspace (non-locale version) #include +#include // util::indexed #include "logger.hpp" // GAPI_LOG @@ -21,6 +22,7 @@ #include "compiler/gmodelbuilder.hpp" #include "compiler/gcompiler.hpp" +#include "compiler/gcompiled_priv.hpp" // cv::GComputation private implementation ///////////////////////////////////// // @@ -174,28 +176,42 @@ cv::GRunArgs cv::GComputation::apply(GRunArgs &&ins, GCompileArgs &&args) { recompile(descr_of(ins), std::move(args)); - const auto& out_metas = m_priv->m_lastCompiled.outMetas(); + const auto& out_info = m_priv->m_lastCompiled.priv().outInfo(); + GRunArgs run_args; GRunArgsP outs; - run_args.reserve(out_metas.size()); - outs.reserve(out_metas.size()); + run_args.reserve(out_info.size()); + outs.reserve(out_info.size()); - for (auto&& meta : out_metas) + for (auto&& info : out_info) { - switch (meta.index()) + switch (info.shape) { - case cv::GMetaArg::index_of(): + case cv::GShape::GMAT: { run_args.emplace_back(cv::Mat{}); outs.emplace_back(&cv::util::get(run_args.back())); break; } - case cv::GMetaArg::index_of(): + case cv::GShape::GSCALAR: { run_args.emplace_back(cv::Scalar{}); outs.emplace_back(&cv::util::get(run_args.back())); break; } + case cv::GShape::GARRAY: + { + switch (info.kind) + { + case cv::detail::OpaqueKind::CV_POINT2F: + run_args.emplace_back(cv::detail::VectorRef{std::vector{}}); + outs.emplace_back(cv::util::get(run_args.back())); + break; + default: + util::throw_error(std::logic_error("Unsupported kind for GArray")); + } + break; + } default: util::throw_error(std::logic_error("Only cv::GMat and cv::GScalar are supported for python output")); } diff --git a/modules/gapi/src/api/ginfer.cpp b/modules/gapi/src/api/ginfer.cpp index 98eeef5ab6..156f8938c4 100644 --- a/modules/gapi/src/api/ginfer.cpp +++ b/modules/gapi/src/api/ginfer.cpp @@ -25,3 +25,59 @@ std::vector cv::gapi::GNetPackage::backends() const { for (const auto &nn : networks) unique_set.insert(nn.backend); return std::vector(unique_set.begin(), unique_set.end()); } + +// FIXME: Inference API is currently only available in full mode +#if !defined(GAPI_STANDALONE) + +cv::GInferInputs::GInferInputs() + : in_blobs(std::make_shared()) +{ +} + +cv::GMat& cv::GInferInputs::operator[](const std::string& name) { + return (*in_blobs)[name]; +} + +const cv::GInferInputs::Map& cv::GInferInputs::getBlobs() const { + return *in_blobs; +} + +void cv::GInferInputs::setInput(const std::string& name, const cv::GMat& value) { + in_blobs->emplace(name, value); +} + +struct cv::GInferOutputs::Priv +{ + Priv(std::shared_ptr); + + std::shared_ptr call; + InOutInfo* info = nullptr; + std::unordered_map out_blobs; +}; + +cv::GInferOutputs::Priv::Priv(std::shared_ptr c) + : call(std::move(c)), info(cv::util::any_cast(&call->params())) +{ +} + +cv::GInferOutputs::GInferOutputs(std::shared_ptr call) + : m_priv(std::make_shared(std::move(call))) +{ +} + +cv::GMat cv::GInferOutputs::at(const std::string& name) +{ + auto it = m_priv->out_blobs.find(name); + if (it == m_priv->out_blobs.end()) { + // FIXME: Avoid modifying GKernel + // Expect output to be always GMat + m_priv->call->kernel().outShapes.push_back(cv::GShape::GMAT); + // ...so _empty_ constructor is passed here. + m_priv->call->kernel().outCtors.emplace_back(cv::util::monostate{}); + int out_idx = static_cast(m_priv->out_blobs.size()); + it = m_priv->out_blobs.emplace(name, m_priv->call->yield(out_idx)).first; + m_priv->info->out_names.push_back(name); + } + return it->second; +} +#endif // GAPI_STANDALONE diff --git a/modules/gapi/src/api/gmat.cpp b/modules/gapi/src/api/gmat.cpp index d9f135222b..08bb170a86 100644 --- a/modules/gapi/src/api/gmat.cpp +++ b/modules/gapi/src/api/gmat.cpp @@ -144,7 +144,7 @@ bool GMatDesc::canDescribe(const cv::Mat& mat) const bool GMatDesc::canDescribe(const cv::RMat& mat) const { - return *this == mat.desc(); + return canDescribeHelper(*this, mat); } }// namespace cv diff --git a/modules/gapi/src/api/grunarg.cpp b/modules/gapi/src/api/grunarg.cpp new file mode 100644 index 0000000000..30ae2adbc0 --- /dev/null +++ b/modules/gapi/src/api/grunarg.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" +#include + +cv::GRunArg::GRunArg() { +} + +cv::GRunArg::GRunArg(const cv::GRunArg &arg) + : cv::GRunArgBase(static_cast(arg)) + , meta(arg.meta) { +} + +cv::GRunArg::GRunArg(cv::GRunArg &&arg) + : cv::GRunArgBase(std::move(static_cast(arg))) + , meta(std::move(arg.meta)) { +} + +cv::GRunArg& cv::GRunArg::operator= (const cv::GRunArg &arg) { + cv::GRunArgBase::operator=(static_cast(arg)); + meta = arg.meta; + return *this; +} + +cv::GRunArg& cv::GRunArg::operator= (cv::GRunArg &&arg) { + cv::GRunArgBase::operator=(std::move(static_cast(arg))); + meta = std::move(arg.meta); + return *this; +} diff --git a/modules/gapi/src/api/kernels_core.cpp b/modules/gapi/src/api/kernels_core.cpp index 55c43594af..82aceb1f26 100644 --- a/modules/gapi/src/api/kernels_core.cpp +++ b/modules/gapi/src/api/kernels_core.cpp @@ -388,14 +388,14 @@ GMat warpAffine(const GMat& src, const Mat& M, const Size& dsize, int flags, return core::GWarpAffine::on(src, M, dsize, flags, borderMode, borderValue); } -GOpaque size(const GMat& src) +GOpaque streaming::size(const GMat& src) { - return core::GSize::on(src); + return streaming::GSize::on(src); } -GOpaque size(const GOpaque& r) +GOpaque streaming::size(const GOpaque& r) { - return core::GSizeR::on(r); + return streaming::GSizeR::on(r); } } //namespace gapi diff --git a/modules/gapi/src/api/kernels_imgproc.cpp b/modules/gapi/src/api/kernels_imgproc.cpp index 108eefcb81..41085a7ebf 100644 --- a/modules/gapi/src/api/kernels_imgproc.cpp +++ b/modules/gapi/src/api/kernels_imgproc.cpp @@ -73,6 +73,13 @@ GMat dilate3x3(const GMat& src, int iterations, return dilate(src, cv::Mat(), cv::Point(-1,-1), iterations, borderType, borderValue); } +GMat morphologyEx(const GMat &src, const MorphTypes op, const Mat &kernel, const Point &anchor, + const int iterations, const BorderTypes borderType, const Scalar &borderValue) +{ + return imgproc::GMorphologyEx::on(src, op, kernel, anchor, iterations, + borderType, borderValue); +} + GMat Sobel(const GMat& src, int ddepth, int dx, int dy, int ksize, double scale, double delta, int borderType, const Scalar& bordVal) @@ -115,6 +122,101 @@ cv::GArray goodFeaturesToTrack(const GMat& image, int maxCorners, d useHarrisDetector, k); } +GArray> +findContours(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method, + const GOpaque &offset) +{ + return imgproc::GFindContours::on(src, mode, method, offset); +} + +GArray> +findContours(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method) +{ + return imgproc::GFindContoursNoOffset::on(src, mode, method); +} + + +std::tuple>,GArray> +findContoursH(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method, + const GOpaque &offset) +{ + return imgproc::GFindContoursH::on(src, mode, method, offset); +} + +std::tuple>,GArray> +findContoursH(const GMat &src, const RetrievalModes mode, const ContourApproximationModes method) +{ + return imgproc::GFindContoursHNoOffset::on(src, mode, method); +} + +GOpaque boundingRect(const GMat& src) +{ + return imgproc::GBoundingRectMat::on(src); +} + +GOpaque boundingRect(const GArray& src) +{ + return imgproc::GBoundingRectVector32S::on(src); +} + +GOpaque boundingRect(const GArray& src) +{ + return imgproc::GBoundingRectVector32F::on(src); +} + +GOpaque fitLine2D(const GMat& src, const DistanceTypes distType, const double param, + const double reps, const double aeps) +{ + return imgproc::GFitLine2DMat::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine2DVector32S::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine2DVector32F::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine2D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine2DVector64F::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine3D(const GMat& src, const DistanceTypes distType, const double param, + const double reps, const double aeps) +{ + return imgproc::GFitLine3DMat::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine3DVector32S::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine3DVector32F::on(src, distType, param, reps, aeps); +} + +GOpaque fitLine3D(const GArray& src, const DistanceTypes distType, + const double param, const double reps, const double aeps) +{ + return imgproc::GFitLine3DVector64F::on(src, distType, param, reps, aeps); +} + +GMat BGR2RGB(const GMat& src) +{ + return imgproc::GBGR2RGB::on(src); +} + GMat RGB2Gray(const GMat& src) { return imgproc::GRGB2Gray::on(src); @@ -160,6 +262,26 @@ GMat YUV2RGB(const GMat& src) return imgproc::GYUV2RGB::on(src); } +GMat BGR2I420(const GMat& src) +{ + return imgproc::GBGR2I420::on(src); +} + +GMat RGB2I420(const GMat& src) +{ + return imgproc::GRGB2I420::on(src); +} + +GMat I4202BGR(const GMat& src) +{ + return imgproc::GI4202BGR::on(src); +} + +GMat I4202RGB(const GMat& src) +{ + return imgproc::GI4202RGB::on(src); +} + GMat NV12toRGB(const GMat& src_y, const GMat& src_uv) { return imgproc::GNV12toRGB::on(src_y, src_uv); diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp new file mode 100644 index 0000000000..af7bd19dd1 --- /dev/null +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -0,0 +1,74 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" + +#include +#include + +cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { + // FIXME: this is a limited implementation of desync + // The real implementation must be generic (template) and + // reside in desync.hpp (and it is detail::desync<>()) + + // FIXME: Put a copy here to solve the below problem + // FIXME: Because of the copy, the desync functionality is limited + // to GMat only (we don't have generic copy kernel for other + // object types) + return cv::gapi::copy(detail::desync(g)); + + // FIXME + // + // If consumed by multiple different islands (OCV and Fluid by + // example, an object needs to be desynchronized individually + // for every path. + // + // This is a limitation of the current implementation. It works + // this way: every "desync" link from the main path to a new + // desync path gets its "DesyncQueue" object which stores only the + // last value written before of the desync object (DO) it consumes + // (the container of type "last written value" or LWV. + // + // LWV + // [Sync path] -> desync() - - > DO -> [ISL0 @ Desync path #1] + // + // At the same time, generally, every island in the streaming + // graph gets its individual input as a queue (so normally, a + // writer pushes the same output MULTIPLE TIMES if it has mutliple + // readers): + // + // LWV + // [Sync path] -> desync() - - > DO1 -> [ISL0 @ Desync path #1] + // : LWV + // ' - - > DO2 -> [ISL1 @ Desync path #1] + // + // For users, it may seem legit to use desync here only once, and + // it MUST BE legit once the problem is fixed. + // But the problem with the current implementation is that islands + // on the same desync path get different desync queues and in fact + // stay desynchronized between each other. One shouldn't consider + // this as a single desync path anymore. + // If these two ISLs are then merged e.g. with add(a,b), the + // results will be inconsistent, given that the latency of ISL0 + // and ISL1 may be different. This is not the same frame anymore + // coming as `a` and `b` to add(a,b) because of it. + // + // To make things clear, we forbid this now and ask to call + // desync one more time to allow that. It is bad since the graph + // structure and island layout depends on kernel packages used, + // not on the sole GComputation structure. This needs to be fixed! + // Here's the working configuration: + // + // LWV + // [Sync path] -> desync() - - > DO1 -> [ISL0 @ Desync path #1] + // : LWV + // '-> desync() - - > DO2 -> [ISL1 @ Desync path #2] <-(!) + // + // Put an operation right after desync() is a quick workaround to + // this synchronization problem. There will be one "last_written_value" + // connected to a desynchronized data object, and this sole last_written_value + // object will feed both branches of the streaming executable. +} diff --git a/modules/gapi/src/api/render_ocv.cpp b/modules/gapi/src/api/render_ocv.cpp index a298a958bd..5ab2e1dd07 100644 --- a/modules/gapi/src/api/render_ocv.cpp +++ b/modules/gapi/src/api/render_ocv.cpp @@ -2,7 +2,7 @@ #include // Kernel API's #include "api/render_ocv.hpp" -#include "api/ft_render.hpp" +#include "backends/render/ft_render.hpp" namespace cv { @@ -146,12 +146,8 @@ struct EmptyConverter template void drawPrimitivesOCV(cv::Mat& in, const cv::gapi::wip::draw::Prims& prims, - cv::gapi::wip::draw::FTTextRender* ftpr) + std::shared_ptr& ftpr) { -#ifndef HAVE_FREETYPE - cv::util::suppress_unused_warning(ftpr); -#endif - using namespace cv::gapi::wip::draw; ColorConverter converter; @@ -177,7 +173,6 @@ void drawPrimitivesOCV(cv::Mat& in, case Prim::index_of(): { -#ifdef HAVE_FREETYPE const auto& ftp = cv::util::get(p); const auto color = converter.cvtColor(ftp.color); @@ -196,9 +191,6 @@ void drawPrimitivesOCV(cv::Mat& in, cv::Point tl(ftp.org.x, ftp.org.y - mask.size().height + baseline); blendTextMask(in, mask, tl, color); -#else - cv::util::throw_error(std::runtime_error("FreeType not found !")); -#endif break; } @@ -251,16 +243,16 @@ void drawPrimitivesOCV(cv::Mat& in, } } -void drawPrimitivesOCVBGR(cv::Mat &in, - const cv::gapi::wip::draw::Prims &prims, - cv::gapi::wip::draw::FTTextRender* ftpr) +void drawPrimitivesOCVBGR(cv::Mat &in, + const cv::gapi::wip::draw::Prims &prims, + std::shared_ptr &ftpr) { drawPrimitivesOCV(in, prims, ftpr); } -void drawPrimitivesOCVYUV(cv::Mat &in, - const cv::gapi::wip::draw::Prims &prims, - cv::gapi::wip::draw::FTTextRender* ftpr) +void drawPrimitivesOCVYUV(cv::Mat &in, + const cv::gapi::wip::draw::Prims &prims, + std::shared_ptr &ftpr) { drawPrimitivesOCV(in, prims, ftpr); } diff --git a/modules/gapi/src/api/render_ocv.hpp b/modules/gapi/src/api/render_ocv.hpp index 91194dcdc1..a9a98f93fb 100644 --- a/modules/gapi/src/api/render_ocv.hpp +++ b/modules/gapi/src/api/render_ocv.hpp @@ -1,6 +1,6 @@ #include #include "render_priv.hpp" -#include "ft_render.hpp" +#include "backends/render/ft_render.hpp" #ifndef OPENCV_RENDER_OCV_HPP #define OPENCV_RENDER_OCV_HPP @@ -15,8 +15,8 @@ namespace draw { // FIXME only for tests -void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc); -void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc); +void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, std::shared_ptr& mc); +void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, std::shared_ptr& mc); } // namespace draw } // namespace wip diff --git a/modules/gapi/src/api/rmat.cpp b/modules/gapi/src/api/rmat.cpp index 9c2da2ebc7..12ba4e5e0e 100644 --- a/modules/gapi/src/api/rmat.cpp +++ b/modules/gapi/src/api/rmat.cpp @@ -8,16 +8,68 @@ using View = cv::RMat::View; +namespace { +cv::GMatDesc checkDesc(const cv::GMatDesc& desc) { + if (!desc.dims.empty() && desc.chan != -1) { + cv::util::throw_error( + std::logic_error("Multidimesional RMat::Views with chan different from -1 are not supported!")); + } + return desc; +} + +int typeFromDesc(const cv::GMatDesc& desc) { + // In multidimensional case GMatDesc::chan is -1, + // change it to 1 when calling CV_MAKE_TYPE + return CV_MAKE_TYPE(desc.depth, desc.chan == -1 ? 1 : desc.chan); +} + +static View::stepsT defaultSteps(const cv::GMatDesc& desc) { + const auto& dims = desc.dims.empty() + ? std::vector{desc.size.height, desc.size.width} + : desc.dims; + View::stepsT steps(dims.size(), 0u); + auto type = typeFromDesc(desc); + steps.back() = CV_ELEM_SIZE(type); + for (int i = static_cast(dims.size())-2; i >= 0; i--) { + steps[i] = steps[i+1]*dims[i]; + } + return steps; +} +} // anonymous namespace + +View::View(const cv::GMatDesc& desc, uchar* data, size_t step, DestroyCallback&& cb) + : m_desc(checkDesc(desc)) + , m_data(data) + , m_steps([this, step](){ + GAPI_Assert(m_desc.dims.empty()); + auto steps = defaultSteps(m_desc); + if (step != 0u) { + steps[0] = step; + } + return steps; + }()) + , m_cb(std::move(cb)) { +} + +View::View(const cv::GMatDesc& desc, uchar* data, const stepsT &steps, DestroyCallback&& cb) + : m_desc(checkDesc(desc)) + , m_data(data) + , m_steps(steps == stepsT{} ? defaultSteps(m_desc): steps) + , m_cb(std::move(cb)) { +} + +int View::type() const { return typeFromDesc(m_desc); } + // There is an issue with default generated operator=(View&&) on Mac: -// it doesn't nullify m_cb of a moved object +// it doesn't nullify m_cb of the moved object View& View::operator=(View&& v) { - m_desc = v.m_desc; - m_data = v.m_data; - m_step = v.m_step; - m_cb = v.m_cb; - v.m_desc = {}; - v.m_data = nullptr; - v.m_step = 0u; - v.m_cb = nullptr; + m_desc = v.m_desc; + m_data = v.m_data; + m_steps = v.m_steps; + m_cb = v.m_cb; + v.m_desc = {}; + v.m_data = nullptr; + v.m_steps = {0u}; + v.m_cb = nullptr; return *this; } diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index 54a0850394..b6acf28ea4 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -44,6 +44,13 @@ std::vector cv::gapi::serialize(const cv::GRunArgs& ra) return os.data(); } +std::vector cv::gapi::serialize(const cv::GCompileArgs& ca) +{ + cv::gapi::s11n::ByteMemoryOutStream os; + serialize(os, ca); + return os.data(); +} + // FIXME: This function should move from S11N to GRunArg-related entities. // it has nothing to do with the S11N as it is cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) @@ -72,6 +79,9 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) case T::index_of() : outputs.emplace_back(cv::util::get(res_obj)); break; + case cv::GRunArg::index_of() : + outputs.emplace_back((cv::RMat*)(&(cv::util::get(res_obj)))); + break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. break; @@ -105,6 +115,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out) case T::index_of() : return cv::GRunArg(*cv::util::get(out)); + case T::index_of() : + return cv::GRunArg(*cv::util::get(out)); + default: // ...maybe our types were extended GAPI_Assert(false && "This value type is UNKNOWN!"); diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp index f747a0dd1c..576168db53 100644 --- a/modules/gapi/src/backends/common/gbackend.hpp +++ b/modules/gapi/src/backends/common/gbackend.hpp @@ -23,11 +23,26 @@ namespace cv { namespace gimpl { inline cv::Mat asMat(RMat::View& v) { +#if !defined(GAPI_STANDALONE) + return v.dims().empty() ? cv::Mat(v.rows(), v.cols(), v.type(), v.ptr(), v.step()) + : cv::Mat(v.dims(), v.type(), v.ptr(), v.steps().data()); +#else + // FIXME: add a check that steps are default return v.dims().empty() ? cv::Mat(v.rows(), v.cols(), v.type(), v.ptr(), v.step()) : cv::Mat(v.dims(), v.type(), v.ptr()); + +#endif } inline RMat::View asView(const Mat& m, RMat::View::DestroyCallback&& cb = nullptr) { +#if !defined(GAPI_STANDALONE) + RMat::View::stepsT steps(m.dims); + for (int i = 0; i < m.dims; i++) { + steps[i] = m.step[i]; + } + return RMat::View(cv::descr_of(m), m.data, steps, std::move(cb)); +#else return RMat::View(cv::descr_of(m), m.data, m.step, std::move(cb)); +#endif } class RMatAdapter : public RMat::Adapter { @@ -47,6 +62,8 @@ namespace magazine { template struct Class { template using MapT = std::unordered_map; + using MapM = std::unordered_map; + template MapT& slot() { return std::get::value>(slots); @@ -55,8 +72,17 @@ namespace magazine { { return std::get::value>(slots); } + template MapM& meta() + { + return metas[ade::util::type_list_index::value]; + } + template const MapM& meta() const + { + return metas[ade::util::type_list_index::value]; + } private: std::tuple...> slots; + std::array metas; }; } // namespace magazine @@ -133,7 +159,7 @@ inline cv::util::optional getCompileArg(const cv::GCompileArgs &args) return cv::gapi::getCompileArg(args); } -void createMat(const cv::GMatDesc& desc, cv::Mat& mat); +void GAPI_EXPORTS createMat(const cv::GMatDesc& desc, cv::Mat& mat); }} // cv::gimpl diff --git a/modules/gapi/src/backends/common/gmetabackend.cpp b/modules/gapi/src/backends/common/gmetabackend.cpp new file mode 100644 index 0000000000..5364152b65 --- /dev/null +++ b/modules/gapi/src/backends/common/gmetabackend.cpp @@ -0,0 +1,105 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" + +#include // compile args +#include // any +#include // GMeta + +#include "compiler/gobjref.hpp" // RcDesc +#include "compiler/gmodel.hpp" // GModel, Op +#include "backends/common/gbackend.hpp" +#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! + +#include "backends/common/gmetabackend.hpp" + +namespace { + +class GraphMetaExecutable final: public cv::gimpl::GIslandExecutable { + std::string m_meta_tag; + +public: + GraphMetaExecutable(const ade::Graph& g, + const std::vector& nodes); + bool canReshape() const override; + void reshape(ade::Graph&, const cv::GCompileArgs&) override; + + void run(std::vector &&input_objs, + std::vector &&output_objs) override; +}; + +bool GraphMetaExecutable::canReshape() const { + return true; +} +void GraphMetaExecutable::reshape(ade::Graph&, const cv::GCompileArgs&) { + // do nothing here +} + +GraphMetaExecutable::GraphMetaExecutable(const ade::Graph& g, + const std::vector& nodes) { + // There may be only one node in the graph + GAPI_Assert(nodes.size() == 1u); + + cv::gimpl::GModel::ConstGraph cg(g); + const auto &op = cg.metadata(nodes[0]).get(); + GAPI_Assert(op.k.name == cv::gapi::streaming::detail::GMeta::id()); + m_meta_tag = op.k.tag; +} + +void GraphMetaExecutable::run(std::vector &&input_objs, + std::vector &&output_objs) { + GAPI_Assert(input_objs.size() == 1u); + GAPI_Assert(output_objs.size() == 1u); + + const cv::GRunArg in_arg = input_objs[0].second; + cv::GRunArgP out_arg = output_objs[0].second; + + auto it = in_arg.meta.find(m_meta_tag); + if (it == in_arg.meta.end()) { + cv::util::throw_error + (std::logic_error("Run-time meta " + + m_meta_tag + + " is not found in object " + + std::to_string(static_cast(input_objs[0].first.shape)) + + "/" + + std::to_string(input_objs[0].first.id))); + } + cv::util::get(out_arg) = it->second; +} + +class GraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { + virtual void unpackKernel(ade::Graph &, + const ade::NodeHandle &, + const cv::GKernelImpl &) override { + // Do nothing here + } + + virtual EPtr compile(const ade::Graph& graph, + const cv::GCompileArgs&, + const std::vector& nodes, + const std::vector&, + const std::vector&) const override { + return EPtr{new GraphMetaExecutable(graph, nodes)}; + } +}; + +cv::gapi::GBackend graph_meta_backend() { + static cv::gapi::GBackend this_backend(std::make_shared()); + return this_backend; +} + +struct InGraphMetaKernel final: public cv::detail::KernelTag { + using API = cv::gapi::streaming::detail::GMeta; + static cv::gapi::GBackend backend() { return graph_meta_backend(); } + static int kernel() { return 42; } +}; + +} // anonymous namespace + +cv::gapi::GKernelPackage cv::gimpl::meta::kernels() { + return cv::gapi::kernels(); +} diff --git a/modules/gapi/src/backends/common/gmetabackend.hpp b/modules/gapi/src/backends/common/gmetabackend.hpp new file mode 100644 index 0000000000..56f61d0e3d --- /dev/null +++ b/modules/gapi/src/backends/common/gmetabackend.hpp @@ -0,0 +1,16 @@ +#ifndef OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP +#define OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP + +#include + +namespace cv { +namespace gimpl { +namespace meta { + +cv::gapi::GKernelPackage kernels(); + +} // namespace meta +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index c0b3281449..8c2313b292 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -94,13 +94,14 @@ void linkNodes(ade::Graph& g) { } void relinkProto(ade::Graph& g) { + using namespace cv::gimpl; // identify which node handles map to the protocol // input/output object in the reconstructed graph - using S = std::set; // FIXME: use ... - using M = std::map; // FIXME: unordered! + using S = std::set; // FIXME: use ... + using M = std::map; // FIXME: unordered! - cv::gimpl::GModel::Graph gm(g); - auto &proto = gm.metadata().get(); + GModel::Graph gm(g); + auto &proto = gm.metadata().get(); const S set_in(proto.inputs.begin(), proto.inputs.end()); const S set_out(proto.outputs.begin(), proto.outputs.end()); @@ -108,9 +109,9 @@ void relinkProto(ade::Graph& g) { // Associate the protocol node handles with their resource identifiers for (auto &&nh : gm.nodes()) { - if (gm.metadata(nh).get().t == cv::gimpl::NodeType::DATA) { - const auto &d = gm.metadata(nh).get(); - const auto rc = cv::gimpl::RcDesc{d.rc, d.shape, d.ctor}; + if (gm.metadata(nh).get().t == NodeType::DATA) { + const auto &d = gm.metadata(nh).get(); + const auto rc = RcDesc{d.rc, d.shape, d.ctor}; if (set_in.count(rc) > 0) { GAPI_DbgAssert(set_out.count(rc) == 0); map_in[rc] = nh; @@ -128,6 +129,12 @@ void relinkProto(ade::Graph& g) { proto.out_nhs.clear(); for (auto &rc : proto.inputs) { proto.in_nhs .push_back(map_in .at(rc)); } for (auto &rc : proto.outputs) { proto.out_nhs.push_back(map_out.at(rc)); } + + // If a subgraph is being serialized it's possible that + // some of its in/out nodes are INTERNAL in the full graph. + // Set their storage apporpriately + for (auto &nh : proto.in_nhs) { gm.metadata(nh).get().storage = Data::Storage::INPUT; } + for (auto &nh : proto.out_nhs) { gm.metadata(nh).get().storage = Data::Storage::OUTPUT; } } } // anonymous namespace @@ -145,6 +152,13 @@ IIStream& operator>> (IIStream& is, cv::Point& pt) { return is >> pt.x >> pt.y; } +IOStream& operator<< (IOStream& os, const cv::Point2f &pt) { + return os << pt.x << pt.y; +} +IIStream& operator>> (IIStream& is, cv::Point2f& pt) { + return is >> pt.x >> pt.y; +} + IOStream& operator<< (IOStream& os, const cv::Size &sz) { return os << sz.width << sz.height; } @@ -165,12 +179,12 @@ IOStream& operator<< (IOStream& os, const cv::Scalar &s) { IIStream& operator>> (IIStream& is, cv::Scalar& s) { return is >> s.val[0] >> s.val[1] >> s.val[2] >> s.val[3]; } -IOStream& operator<< (IOStream& os, const cv::RMat&) { - util::throw_error(std::logic_error("Serialization of RMat is not supported")); +IOStream& operator<< (IOStream& os, const cv::RMat& mat) { + mat.serialize(os); return os; } IIStream& operator>> (IIStream& is, cv::RMat&) { - util::throw_error(std::logic_error("Serialization of RMat is not supported")); + util::throw_error(std::logic_error("operator>> for RMat should never be called")); return is; } @@ -329,6 +343,18 @@ IIStream& operator>> (IIStream& is, cv::gapi::wip::draw::Line &l) { // G-API types ///////////////////////////////////////////////////////////////// +IOStream& operator<< (IOStream& os, const cv::GCompileArg& arg) +{ + ByteMemoryOutStream tmpS; + arg.serialize(tmpS); + std::vector data = tmpS.data(); + + os << arg.tag; + os << data; + + return os; +} + // Stubs (empty types) IOStream& operator<< (IOStream& os, cv::util::monostate ) {return os;} @@ -497,17 +523,17 @@ IOStream& operator<< (IOStream& os, const cv::GArg &arg) { GAPI_Assert(arg.kind == cv::detail::ArgKind::OPAQUE_VAL); GAPI_Assert(arg.opaque_kind != cv::detail::OpaqueKind::CV_UNKNOWN); switch (arg.opaque_kind) { - case cv::detail::OpaqueKind::CV_BOOL: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_INT: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_UINT64: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_DOUBLE: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_FLOAT: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_STRING: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_POINT: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_SIZE: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_RECT: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_SCALAR: os << arg.get(); break; - case cv::detail::OpaqueKind::CV_MAT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_BOOL: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_INT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_UINT64: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_DOUBLE: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_FLOAT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_STRING: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_POINT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_SIZE: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_RECT: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_SCALAR: os << arg.get(); break; + case cv::detail::OpaqueKind::CV_MAT: os << arg.get(); break; default: GAPI_Assert(false && "GArg: Unsupported (unknown?) opaque value type"); } } @@ -531,17 +557,18 @@ IIStream& operator>> (IIStream& is, cv::GArg &arg) { switch (arg.opaque_kind) { #define HANDLE_CASE(E,T) case cv::detail::OpaqueKind::CV_##E: \ { T t{}; is >> t; arg = (cv::GArg(t)); } break - HANDLE_CASE(BOOL , bool); - HANDLE_CASE(INT , int); - HANDLE_CASE(UINT64 , uint64_t); - HANDLE_CASE(DOUBLE , double); - HANDLE_CASE(FLOAT , float); - HANDLE_CASE(STRING , std::string); - HANDLE_CASE(POINT , cv::Point); - HANDLE_CASE(SIZE , cv::Size); - HANDLE_CASE(RECT , cv::Rect); - HANDLE_CASE(SCALAR , cv::Scalar); - HANDLE_CASE(MAT , cv::Mat); + HANDLE_CASE(BOOL , bool); + HANDLE_CASE(INT , int); + HANDLE_CASE(UINT64 , uint64_t); + HANDLE_CASE(DOUBLE , double); + HANDLE_CASE(FLOAT , float); + HANDLE_CASE(STRING , std::string); + HANDLE_CASE(POINT , cv::Point); + HANDLE_CASE(POINT2F , cv::Point2f); + HANDLE_CASE(SIZE , cv::Size); + HANDLE_CASE(RECT , cv::Rect); + HANDLE_CASE(SCALAR , cv::Scalar); + HANDLE_CASE(MAT , cv::Mat); #undef HANDLE_CASE default: GAPI_Assert(false && "GArg: Unsupported (unknown?) opaque value type"); } @@ -865,6 +892,14 @@ IIStream& ByteMemoryInStream::operator>> (std::string& str) { return *this; } +GAPI_EXPORTS std::unique_ptr detail::getInStream(const std::vector &p) { + return std::unique_ptr(new ByteMemoryInStream(p)); +} + +GAPI_EXPORTS void serialize(IOStream& os, const cv::GCompileArgs &ca) { + os << ca; +} + GAPI_EXPORTS void serialize(IOStream& os, const cv::GMetaArgs &ma) { os << ma; } @@ -882,7 +917,6 @@ GAPI_EXPORTS GRunArgs run_args_deserialize(IIStream& is) { return s; } - } // namespace s11n } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/common/serialization.hpp b/modules/gapi/src/backends/common/serialization.hpp index 4c60e71d87..a3134d84d2 100644 --- a/modules/gapi/src/backends/common/serialization.hpp +++ b/modules/gapi/src/backends/common/serialization.hpp @@ -40,6 +40,8 @@ struct GSerialized { // G-API types ///////////////////////////////////////////////////////////////// +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GCompileArg& arg); + GAPI_EXPORTS IOStream& operator<< (IOStream& os, cv::util::monostate ); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::util::monostate &); @@ -86,26 +88,6 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::GArrayDesc &); GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GFrameDesc &); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::GFrameDesc &); -#if !defined(GAPI_STANDALONE) -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &); -#endif // !defined(GAPI_STANDALONE) - -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::RMat &r); - -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &); - -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &); - -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &); - -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &); - GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::RcDesc &rc); @@ -176,46 +158,6 @@ GAPI_EXPORTS void serialize( IOStream& os GAPI_EXPORTS GSerialized deserialize(IIStream& is); GAPI_EXPORTS void reconstruct(const GSerialized &s, ade::Graph &g); -// Generic: variant serialization ////////////////////////////////////////////// -namespace detail { // FIXME: breaks old code -template -IOStream& put_v(IOStream&, const V&, std::size_t) { - GAPI_Assert(false && "variant>>: requested index is invalid"); -}; -template -IOStream& put_v(IOStream& os, const V& v, std::size_t x) { - return (x == 0u) - ? os << cv::util::get(v) - : put_v(os, v, x-1); -} -template -IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) { - GAPI_Assert(false && "variant<<: requested index is invalid"); -} -template -IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { - if (i == gi) { - X x{}; - is >> x; - v = std::move(x); - return is; - } else return get_v(is, v, i+1, gi); -} -} // namespace detail FIXME: breaks old code - -template -IOStream& operator<< (IOStream& os, const cv::util::variant &v) { - os << (uint32_t)v.index(); - return detail::put_v, Ts...>(os, v, v.index()); -} -template -IIStream& operator>> (IIStream& is, cv::util::variant &v) { - int idx = -1; - is >> idx; - GAPI_Assert(idx >= 0 && idx < (int)sizeof...(Ts)); - return detail::get_v, Ts...>(is, v, 0u, idx); -} - // FIXME: Basic Stream implementaions ////////////////////////////////////////// // Basic in-memory stream implementations. @@ -268,6 +210,11 @@ public: virtual IIStream& operator>> (std::string &) override; }; +namespace detail { +GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &p); +} // namespace detail + +GAPI_EXPORTS void serialize(IOStream& os, const cv::GCompileArgs &ca); GAPI_EXPORTS void serialize(IOStream& os, const cv::GMetaArgs &ma); GAPI_EXPORTS void serialize(IOStream& os, const cv::GRunArgs &ra); GAPI_EXPORTS GMetaArgs meta_args_deserialize(IIStream& is); diff --git a/modules/gapi/src/backends/cpu/gcpucore.cpp b/modules/gapi/src/backends/cpu/gcpucore.cpp index f2b8f7077d..fc460149c6 100644 --- a/modules/gapi/src/backends/cpu/gcpucore.cpp +++ b/modules/gapi/src/backends/cpu/gcpucore.cpp @@ -625,7 +625,7 @@ GAPI_OCV_KERNEL(GCPUParseYolo, cv::gapi::nn::parsers::GParseYolo) } }; -GAPI_OCV_KERNEL(GCPUSize, cv::gapi::core::GSize) +GAPI_OCV_KERNEL(GCPUSize, cv::gapi::streaming::GSize) { static void run(const cv::Mat& in, cv::Size& out) { @@ -634,7 +634,7 @@ GAPI_OCV_KERNEL(GCPUSize, cv::gapi::core::GSize) } }; -GAPI_OCV_KERNEL(GCPUSizeR, cv::gapi::core::GSizeR) +GAPI_OCV_KERNEL(GCPUSizeR, cv::gapi::streaming::GSizeR) { static void run(const cv::Rect& in, cv::Size& out) { diff --git a/modules/gapi/src/backends/cpu/gcpuimgproc.cpp b/modules/gapi/src/backends/cpu/gcpuimgproc.cpp index 8104565f03..6cbf0d32f0 100644 --- a/modules/gapi/src/backends/cpu/gcpuimgproc.cpp +++ b/modules/gapi/src/backends/cpu/gcpuimgproc.cpp @@ -145,6 +145,16 @@ GAPI_OCV_KERNEL(GCPUDilate, cv::gapi::imgproc::GDilate) } }; +GAPI_OCV_KERNEL(GCPUMorphologyEx, cv::gapi::imgproc::GMorphologyEx) +{ + static void run(const cv::Mat &in, const cv::MorphTypes op, const cv::Mat &kernel, + const cv::Point &anchor, const int iterations, + const cv::BorderTypes borderType, const cv::Scalar &borderValue, cv::Mat &out) + { + cv::morphologyEx(in, out, op, kernel, anchor, iterations, borderType, borderValue); + } +}; + GAPI_OCV_KERNEL(GCPUSobel, cv::gapi::imgproc::GSobel) { static void run(const cv::Mat& in, int ddepth, int dx, int dy, int ksize, double scale, double delta, int borderType, @@ -211,6 +221,182 @@ GAPI_OCV_KERNEL(GCPUGoodFeatures, cv::gapi::imgproc::GGoodFeatures) } }; +GAPI_OCV_KERNEL(GCPUFindContours, cv::gapi::imgproc::GFindContours) +{ + static void run(const cv::Mat& image, const cv::RetrievalModes mode, + const cv::ContourApproximationModes method, const cv::Point& offset, + std::vector> &outConts) + { + cv::findContours(image, outConts, mode, method, offset); + } +}; + +GAPI_OCV_KERNEL(GCPUFindContoursNoOffset, cv::gapi::imgproc::GFindContoursNoOffset) +{ + static void run(const cv::Mat& image, const cv::RetrievalModes mode, + const cv::ContourApproximationModes method, + std::vector> &outConts) + { + cv::findContours(image, outConts, mode, method); + } +}; + +GAPI_OCV_KERNEL(GCPUFindContoursH, cv::gapi::imgproc::GFindContoursH) +{ + static void run(const cv::Mat& image, const cv::RetrievalModes mode, + const cv::ContourApproximationModes method, const cv::Point& offset, + std::vector> &outConts, std::vector &outHier) + { + cv::findContours(image, outConts, outHier, mode, method, offset); + } +}; + +GAPI_OCV_KERNEL(GCPUFindContoursHNoOffset, cv::gapi::imgproc::GFindContoursHNoOffset) +{ + static void run(const cv::Mat& image, const cv::RetrievalModes mode, + const cv::ContourApproximationModes method, + std::vector> &outConts, std::vector &outHier) + { + cv::findContours(image, outConts, outHier, mode, method); + } +}; + +GAPI_OCV_KERNEL(GCPUBoundingRectMat, cv::gapi::imgproc::GBoundingRectMat) +{ + static void run(const cv::Mat& in, cv::Rect& out) + { + out = cv::boundingRect(in); + } +}; + +GAPI_OCV_KERNEL(GCPUBoundingRectVector32S, cv::gapi::imgproc::GBoundingRectVector32S) +{ + static void run(const std::vector& in, cv::Rect& out) + { + out = cv::boundingRect(in); + } +}; + +GAPI_OCV_KERNEL(GCPUBoundingRectVector32F, cv::gapi::imgproc::GBoundingRectVector32F) +{ + static void run(const std::vector& in, cv::Rect& out) + { + out = cv::boundingRect(in); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine2DMat, cv::gapi::imgproc::GFitLine2DMat) +{ + static void run(const cv::Mat& in, const cv::DistanceTypes distType, const double param, + const double reps, const double aeps, cv::Vec4f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine2DVector32S, cv::gapi::imgproc::GFitLine2DVector32S) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec4f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine2DVector32F, cv::gapi::imgproc::GFitLine2DVector32F) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec4f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine2DVector64F, cv::gapi::imgproc::GFitLine2DVector64F) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec4f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine3DMat, cv::gapi::imgproc::GFitLine3DMat) +{ + static void run(const cv::Mat& in, const cv::DistanceTypes distType, const double param, + const double reps, const double aeps, cv::Vec6f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine3DVector32S, cv::gapi::imgproc::GFitLine3DVector32S) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec6f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine3DVector32F, cv::gapi::imgproc::GFitLine3DVector32F) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec6f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUFitLine3DVector64F, cv::gapi::imgproc::GFitLine3DVector64F) +{ + static void run(const std::vector& in, const cv::DistanceTypes distType, + const double param, const double reps, const double aeps, cv::Vec6f& out) + { + cv::fitLine(in, out, distType, param, reps, aeps); + } +}; + +GAPI_OCV_KERNEL(GCPUBGR2RGB, cv::gapi::imgproc::GBGR2RGB) +{ + static void run(const cv::Mat& in, cv::Mat &out) + { + cv::cvtColor(in, out, cv::COLOR_BGR2RGB); + } +}; + +GAPI_OCV_KERNEL(GCPUBGR2I420, cv::gapi::imgproc::GBGR2I420) +{ + static void run(const cv::Mat& in, cv::Mat &out) + { + cv::cvtColor(in, out, cv::COLOR_BGR2YUV_I420); + } +}; + +GAPI_OCV_KERNEL(GCPURGB2I420, cv::gapi::imgproc::GRGB2I420) +{ + static void run(const cv::Mat& in, cv::Mat &out) + { + cv::cvtColor(in, out, cv::COLOR_RGB2YUV_I420); + } +}; + +GAPI_OCV_KERNEL(GCPUI4202BGR, cv::gapi::imgproc::GI4202BGR) +{ + static void run(const cv::Mat& in, cv::Mat &out) + { + cv::cvtColor(in, out, cv::COLOR_YUV2BGR_I420); + } +}; + +GAPI_OCV_KERNEL(GCPUI4202RGB, cv::gapi::imgproc::GI4202RGB) +{ + static void run(const cv::Mat& in, cv::Mat &out) + { + cv::cvtColor(in, out, cv::COLOR_YUV2RGB_I420); + } +}; + GAPI_OCV_KERNEL(GCPURGB2YUV, cv::gapi::imgproc::GRGB2YUV) { static void run(const cv::Mat& in, cv::Mat &out) @@ -438,6 +624,7 @@ cv::gapi::GKernelPackage cv::gapi::imgproc::cpu::kernels() , GCPUMedianBlur , GCPUErode , GCPUDilate + , GCPUMorphologyEx , GCPUSobel , GCPUSobelXY , GCPULaplacian @@ -445,8 +632,28 @@ cv::gapi::GKernelPackage cv::gapi::imgproc::cpu::kernels() , GCPUCanny , GCPUGoodFeatures , GCPUEqualizeHist + , GCPUFindContours + , GCPUFindContoursNoOffset + , GCPUFindContoursH + , GCPUFindContoursHNoOffset + , GCPUBGR2RGB , GCPURGB2YUV + , GCPUBoundingRectMat + , GCPUBoundingRectVector32S + , GCPUBoundingRectVector32F + , GCPUFitLine2DMat + , GCPUFitLine2DVector32S + , GCPUFitLine2DVector32F + , GCPUFitLine2DVector64F + , GCPUFitLine3DMat + , GCPUFitLine3DVector32S + , GCPUFitLine3DVector32F + , GCPUFitLine3DVector64F , GCPUYUV2RGB + , GCPUBGR2I420 + , GCPURGB2I420 + , GCPUI4202BGR + , GCPUI4202RGB , GCPUNV12toRGB , GCPUNV12toBGR , GCPURGB2Lab diff --git a/modules/gapi/src/backends/cpu/gnnparsers.cpp b/modules/gapi/src/backends/cpu/gnnparsers.cpp index 234382d530..a5e4bf5f85 100644 --- a/modules/gapi/src/backends/cpu/gnnparsers.cpp +++ b/modules/gapi/src/backends/cpu/gnnparsers.cpp @@ -246,6 +246,28 @@ void parseSSD(const cv::Mat& in_ssd_result, } } +static void checkYoloDims(const MatSize& dims) { + const auto d = dims.dims(); + // Accept 1x13x13xN and 13x13xN + GAPI_Assert(d >= 2); + if (d >= 3) { + if (dims[d-2] == 13) { + GAPI_Assert(dims[d-1]%5 == 0); + GAPI_Assert(dims[d-2] == 13); + GAPI_Assert(dims[d-3] == 13); + for (int i = 0; i < d-3; i++) { + GAPI_Assert(dims[i] == 1); + } + return; + } + } + // Accept 1x1x1xN, 1x1xN, 1xN + GAPI_Assert(dims[d-1]%(5*13*13) == 0); + for (int i = 0; i < d-1; i++) { + GAPI_Assert(dims[i] == 1); + } +} + void parseYolo(const cv::Mat& in_yolo_result, const cv::Size& in_size, const float confidence_threshold, @@ -255,12 +277,12 @@ void parseYolo(const cv::Mat& in_yolo_result, std::vector& out_labels) { const auto& dims = in_yolo_result.size; - GAPI_Assert(dims.dims() == 4); - GAPI_Assert(dims[0] == 1); - GAPI_Assert(dims[1] == 13); - GAPI_Assert(dims[2] == 13); - GAPI_Assert(dims[3] % 5 == 0); // 5 boxes - const auto num_classes = dims[3] / 5 - 5; + checkYoloDims(dims); + int acc = 1; + for (int i = 0; i < dims.dims(); i++) { + acc *= dims[i]; + } + const auto num_classes = acc/(5*13*13)-5; GAPI_Assert(num_classes > 0); GAPI_Assert(0 < nms_threshold && nms_threshold <= 1); out_boxes.clear(); diff --git a/modules/gapi/src/backends/fluid/gfluidbackend.cpp b/modules/gapi/src/backends/fluid/gfluidbackend.cpp index 9b95dff036..030bb10198 100644 --- a/modules/gapi/src/backends/fluid/gfluidbackend.cpp +++ b/modules/gapi/src/backends/fluid/gfluidbackend.cpp @@ -952,7 +952,7 @@ namespace using namespace cv::gimpl; GModel::Graph g(graph); GFluidModel fg(graph); - for (const auto node : g.nodes()) + for (const auto& node : g.nodes()) { if (g.metadata(node).get().t == NodeType::DATA) { @@ -1440,7 +1440,7 @@ void GFluidBackendImpl::addMetaSensitiveBackendPasses(ade::ExecutionEngineSetupC { // Add FluidData to all data nodes inside island, // set internal = true if node is not a slot in terms of higher-level GIslandModel - for (const auto node : isl->contents()) + for (const auto& node : isl->contents()) { if (g.metadata(node).get().t == NodeType::DATA && !fg.metadata(node).contains()) diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index a6f8d56e4c..edc91f0179 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -151,6 +151,348 @@ GAPI_FLUID_KERNEL(GFluidAddW, cv::gapi::core::GAddW, false) enum Arithm { ARITHM_ABSDIFF, ARITHM_ADD, ARITHM_SUBTRACT, ARITHM_MULTIPLY, ARITHM_DIVIDE }; +#if CV_SIMD +CV_ALWAYS_INLINE void absdiff_store(short out[], const v_int16& a, const v_int16& b, int x) +{ + vx_store(&out[x], v_absdiffs(a, b)); +} + +CV_ALWAYS_INLINE void absdiff_store(ushort out[], const v_uint16& a, const v_uint16& b, int x) +{ + vx_store(&out[x], v_absdiff(a, b)); +} + +CV_ALWAYS_INLINE void absdiff_store(uchar out[], const v_uint8& a, const v_uint8& b, int x) +{ + vx_store(&out[x], v_absdiff(a, b)); +} + +CV_ALWAYS_INLINE void absdiff_store(float out[], const v_float32& a, const v_float32& b, int x) +{ + vx_store(&out[x], v_absdiff(a, b)); +} + +template +CV_ALWAYS_INLINE int absdiff_impl(const T in1[], const T in2[], T out[], int length) +{ + constexpr int nlanes = static_cast(VT::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + VT a = vx_load(&in1[x]); + VT b = vx_load(&in2[x]); + absdiff_store(out, a, b, x); + } + + if (x < length && (in1 != out) && (in2 != out)) + { + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; +} + +template +CV_ALWAYS_INLINE int absdiff_simd(const T in1[], const T in2[], T out[], int length) +{ + if (std::is_same::value) + { + return absdiff_impl(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return absdiff_impl(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return absdiff_impl(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return absdiff_impl(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + + return 0; +} + +template +CV_ALWAYS_INLINE int add_simd_sametype(const T in1[], const T in2[], T out[], int length) +{ + constexpr int nlanes = static_cast(VT::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + VT a = vx_load(&in1[x]); + VT b = vx_load(&in2[x]); + vx_store(&out[x], a + b); + } + + if (x < length && (in1 != out) && (in2 != out)) + { + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; +} + +template +CV_ALWAYS_INLINE int add_simd(const SRC in1[], const SRC in2[], DST out[], int length) +{ + if (std::is_same::value && !std::is_same::value) + return 0; + + if (std::is_same::value) + { + if (std::is_same::value) + { + return add_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return add_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return add_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + } + else if (std::is_same::value && std::is_same::value) + { + constexpr int nlanes = static_cast(v_uint8::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_int16 a1 = vx_load(reinterpret_cast(&in1[x])); + v_int16 a2 = vx_load(reinterpret_cast(&in1[x + nlanes / 2])); + v_int16 b1 = vx_load(reinterpret_cast(&in2[x])); + v_int16 b2 = vx_load(reinterpret_cast(&in2[x + nlanes / 2])); + + vx_store(reinterpret_cast(&out[x]), v_pack_u(a1 + b1, a2 + b2)); + } + + if (x < length) + { + CV_DbgAssert((reinterpret_cast(in1) != reinterpret_cast(out)) && + (reinterpret_cast(in2) != reinterpret_cast(out))); + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; + } + else if (std::is_same::value && std::is_same::value) + { + constexpr int nlanes = static_cast(v_uint8::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_float32 a1 = vx_load(reinterpret_cast(&in1[x])); + v_float32 a2 = vx_load(reinterpret_cast(&in1[x + nlanes / 4])); + v_float32 a3 = vx_load(reinterpret_cast(&in1[x + 2 * nlanes / 4])); + v_float32 a4 = vx_load(reinterpret_cast(&in1[x + 3 * nlanes / 4])); + + v_float32 b1 = vx_load(reinterpret_cast(&in2[x])); + v_float32 b2 = vx_load(reinterpret_cast(&in2[x + nlanes / 4])); + v_float32 b3 = vx_load(reinterpret_cast(&in2[x + 2 * nlanes / 4])); + v_float32 b4 = vx_load(reinterpret_cast(&in2[x + 3 * nlanes / 4])); + + vx_store(reinterpret_cast(&out[x]), v_pack_u(v_pack(v_round(a1 + b1), v_round(a2 + b2)), + v_pack(v_round(a3 + b3), v_round(a4 + b4)))); + } + + if (x < length) + { + CV_DbgAssert((reinterpret_cast(in1) != reinterpret_cast(out)) && + (reinterpret_cast(in2) != reinterpret_cast(out))); + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; + } + + return 0; +} + +template +CV_ALWAYS_INLINE int sub_simd_sametype(const T in1[], const T in2[], T out[], int length) +{ + constexpr int nlanes = static_cast(VT::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + VT a = vx_load(&in1[x]); + VT b = vx_load(&in2[x]); + vx_store(&out[x], a - b); + } + + if (x < length && (in1 != out) && (in2 != out)) + { + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; +} + +template +CV_ALWAYS_INLINE int sub_simd(const SRC in1[], const SRC in2[], DST out[], int length) +{ + if (std::is_same::value && !std::is_same::value) + return 0; + + if (std::is_same::value) + { + if (std::is_same::value) + { + return sub_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return sub_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + else if (std::is_same::value) + { + return sub_simd_sametype(reinterpret_cast(in1), + reinterpret_cast(in2), + reinterpret_cast(out), length); + } + } + else if (std::is_same::value && std::is_same::value) + { + constexpr int nlanes = static_cast(v_uint8::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_int16 a1 = vx_load(reinterpret_cast(&in1[x])); + v_int16 a2 = vx_load(reinterpret_cast(&in1[x + nlanes / 2])); + v_int16 b1 = vx_load(reinterpret_cast(&in2[x])); + v_int16 b2 = vx_load(reinterpret_cast(&in2[x + nlanes / 2])); + + vx_store(reinterpret_cast(&out[x]), v_pack_u(a1 - b1, a2 - b2)); + } + + if (x < length) + { + CV_DbgAssert((reinterpret_cast(in1) != reinterpret_cast(out)) && + (reinterpret_cast(in2) != reinterpret_cast(out))); + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; + } + else if (std::is_same::value && std::is_same::value) + { + constexpr int nlanes = static_cast(v_uint8::nlanes); + + if (length < nlanes) + return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_float32 a1 = vx_load(reinterpret_cast(&in1[x])); + v_float32 a2 = vx_load(reinterpret_cast(&in1[x + nlanes / 4])); + v_float32 a3 = vx_load(reinterpret_cast(&in1[x + 2 * nlanes / 4])); + v_float32 a4 = vx_load(reinterpret_cast(&in1[x + 3 * nlanes / 4])); + + v_float32 b1 = vx_load(reinterpret_cast(&in2[x])); + v_float32 b2 = vx_load(reinterpret_cast(&in2[x + nlanes / 4])); + v_float32 b3 = vx_load(reinterpret_cast(&in2[x + 2 * nlanes / 4])); + v_float32 b4 = vx_load(reinterpret_cast(&in2[x + 3 * nlanes / 4])); + + vx_store(reinterpret_cast(&out[x]), v_pack_u(v_pack(v_round(a1 - b1), v_round(a2 - b2)), + v_pack(v_round(a3 - b3), v_round(a4 - b4)))); + } + + if (x < length) + { + CV_DbgAssert((reinterpret_cast(in1) != reinterpret_cast(out)) && + (reinterpret_cast(in2) != reinterpret_cast(out))); + x = length - nlanes; + continue; // process one more time (unaligned tail) + } + break; + } + + return x; + } + + return 0; +} +#endif + template static void run_arithm(Buffer &dst, const View &src1, const View &src2, Arithm arithm, double scale=1) @@ -168,29 +510,37 @@ static void run_arithm(Buffer &dst, const View &src1, const View &src2, Arithm a // NB: assume in/out types are not 64-bits float _scale = static_cast( scale ); + int x = 0; + switch (arithm) { - case ARITHM_ABSDIFF: - for (int l=0; l < length; l++) - out[l] = absdiff(in1[l], in2[l]); - break; - case ARITHM_ADD: - for (int l=0; l < length; l++) - out[l] = add(in1[l], in2[l]); - break; - case ARITHM_SUBTRACT: - for (int l=0; l < length; l++) - out[l] = sub(in1[l], in2[l]); - break; - case ARITHM_MULTIPLY: - for (int l=0; l < length; l++) - out[l] = mul(in1[l], in2[l], _scale); - break; - case ARITHM_DIVIDE: - for (int l=0; l < length; l++) - out[l] = div(in1[l], in2[l], _scale); - break; - default: CV_Error(cv::Error::StsBadArg, "unsupported arithmetic operation"); + case ARITHM_ADD: + { +#if CV_SIMD + x = add_simd(in1, in2, out, length); +#endif + for (; x < length; ++x) + out[x] = add(in1[x], in2[x]); + break; + } + case ARITHM_SUBTRACT: + { +#if CV_SIMD + x = sub_simd(in1, in2, out, length); +#endif + for (; x < length; ++x) + out[x] = sub(in1[x], in2[x]); + break; + } + case ARITHM_MULTIPLY: + for (; x < length; ++x) + out[x] = mul(in1[x], in2[x], _scale); + break; + case ARITHM_DIVIDE: + for (; x < length; ++x) + out[x] = div(in1[x], in2[x], _scale); + break; + default: CV_Error(cv::Error::StsBadArg, "unsupported arithmetic operation"); } } @@ -270,6 +620,29 @@ GAPI_FLUID_KERNEL(GFluidDiv, cv::gapi::core::GDiv, false) } }; +template +static void run_absdiff(Buffer &dst, const View &src1, const View &src2) +{ + static_assert(std::is_same::value, "wrong types"); + static_assert(std::is_same::value, "wrong types"); + + const auto *in1 = src1.InLine(0); + const auto *in2 = src2.InLine(0); + auto *out = dst.OutLine(); + + int width = dst.length(); + int chan = dst.meta().chan; + int length = width * chan; + + int x = 0; + +#if CV_SIMD + x = absdiff_simd(in1, in2, out, length); +#endif + for (; x < length; ++x) + out[x] = absdiff(in1[x], in2[x]); +} + GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false) { static const int Window = 1; @@ -277,10 +650,10 @@ GAPI_FLUID_KERNEL(GFluidAbsDiff, cv::gapi::core::GAbsDiff, false) static void run(const View &src1, const View &src2, Buffer &dst) { // DST SRC1 SRC2 OP __VA_ARGS__ - BINARY_(uchar , uchar , uchar , run_arithm, dst, src1, src2, ARITHM_ABSDIFF); - BINARY_(ushort, ushort, ushort, run_arithm, dst, src1, src2, ARITHM_ABSDIFF); - BINARY_( short, short, short, run_arithm, dst, src1, src2, ARITHM_ABSDIFF); - BINARY_( float, float, float, run_arithm, dst, src1, src2, ARITHM_ABSDIFF); + BINARY_(uchar , uchar , uchar , run_absdiff, dst, src1, src2); + BINARY_(ushort, ushort, ushort, run_absdiff, dst, src1, src2); + BINARY_( short, short, short, run_absdiff, dst, src1, src2); + BINARY_( float, float, float, run_absdiff, dst, src1, src2); CV_Error(cv::Error::StsBadArg, "unsupported combination of types"); } diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp new file mode 100644 index 0000000000..35191d7bcb --- /dev/null +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -0,0 +1,39 @@ +#include + +cv::gapi::ie::PyParams::PyParams(const std::string &tag, + const std::string &model, + const std::string &weights, + const std::string &device) + : m_priv(std::make_shared>(tag, model, weights, device)) { +} + +cv::gapi::ie::PyParams::PyParams(const std::string &tag, + const std::string &model, + const std::string &device) + : m_priv(std::make_shared>(tag, model, device)) { +} + +cv::gapi::GBackend cv::gapi::ie::PyParams::backend() const { + return m_priv->backend(); +} + +std::string cv::gapi::ie::PyParams::tag() const { + return m_priv->tag(); +} + +cv::util::any cv::gapi::ie::PyParams::params() const { + return m_priv->params(); +} + +cv::gapi::ie::PyParams cv::gapi::ie::params(const std::string &tag, + const std::string &model, + const std::string &weights, + const std::string &device) { + return {tag, model, weights, device}; +} + +cv::gapi::ie::PyParams cv::gapi::ie::params(const std::string &tag, + const std::string &model, + const std::string &device) { + return {tag, model, device}; +} diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 1565d03aec..85c0236ff1 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -175,11 +175,27 @@ struct IEUnit { IE::InputsDataMap inputs; IE::OutputsDataMap outputs; + IE::ExecutableNetwork this_network; + cv::gimpl::ie::wrap::Plugin this_plugin; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { - net = cv::gimpl::ie::wrap::readNetwork(params); - inputs = net.getInputsInfo(); - outputs = net.getOutputsInfo(); + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + net = cv::gimpl::ie::wrap::readNetwork(params); + inputs = net.getInputsInfo(); + outputs = net.getOutputsInfo(); + } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { + this_plugin = cv::gimpl::ie::wrap::getPlugin(params); + this_plugin.SetConfig(params.config); + this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params); + // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, + // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap + inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); + outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo()); + } else { + cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind")); + } + // The practice shows that not all inputs and not all outputs // are mandatory to specify in IE model. // So what we're concerned here about is: @@ -205,10 +221,16 @@ struct IEUnit { // This method is [supposed to be] called at Island compilation stage cv::gimpl::ie::IECompiled compile() const { - auto plugin = cv::gimpl::ie::wrap::getPlugin(params); - auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); - auto this_request = this_network.CreateInferRequest(); + IEUnit* non_const_this = const_cast(this); + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // FIXME: In case importNetwork for fill inputs/outputs need to obtain ExecutableNetwork, but + // for loadNetwork they can be obtained by using readNetwork + non_const_this->this_plugin = cv::gimpl::ie::wrap::getPlugin(params); + non_const_this->this_plugin.SetConfig(params.config); + non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, net, params); + } + auto this_request = non_const_this->this_network.CreateInferRequest(); // Bind const data to infer request for (auto &&p : params.const_inputs) { // FIXME: SetBlob is known to be inefficient, @@ -217,7 +239,16 @@ struct IEUnit { // Still, constant data is to set only once. this_request.SetBlob(p.first, wrapIE(p.second.first, p.second.second)); } - return {plugin, this_network, this_request}; + // Bind const data to infer request + for (auto &&p : params.const_inputs) { + // FIXME: SetBlob is known to be inefficient, + // it is worth to make a customizable "initializer" and pass the + // cv::Mat-wrapped blob there to support IE's optimal "GetBlob idiom" + // Still, constant data is to set only once. + this_request.SetBlob(p.first, wrapIE(p.second.first, p.second.second)); + } + + return {this_plugin, this_network, this_request}; } }; @@ -490,6 +521,65 @@ struct Infer: public cv::detail::KernelTag { } }; +struct InferROI: public cv::detail::KernelTag { + using API = cv::GInferROIBase; + static cv::gapi::GBackend backend() { return cv::gapi::ie::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + cv::GMetaArgs result; + + GConstGIEModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + + // Initialize input information + // FIXME: So far it is pretty limited + GAPI_Assert(1u == uu.params.input_names.size()); + GAPI_Assert(2u == in_metas.size()); + + // 0th is ROI, 1st is in0put image + auto &&ii = uu.inputs.at(uu.params.input_names.at(0)); + const auto &meta = util::get(in_metas.at(1)); + ii->setPrecision(toIE(meta.depth)); + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + + // FIXME: It would be nice here to have an exact number of network's + // input/output parameters. Probably GCall should store it here for us. + // It doesn't, as far as I know.. + for (const auto &out_name : uu.params.output_names) { + // NOTE: our output_names vector follows the API order + // of this operation's outputs + const IE::DataPtr& ie_out = uu.outputs.at(out_name); + const IE::SizeVector dims = ie_out->getTensorDesc().getDims(); + + cv::GMatDesc outm(toCV(ie_out->getPrecision()), + toCV(ie_out->getTensorDesc().getDims())); + result.emplace_back(outm); + } + return result; + } + + static void run(IECompiled &iec, const IEUnit &uu, IECallContext &ctx) { + // non-generic version for now, per the InferROI's definition + GAPI_Assert(uu.params.num_in == 1); + const auto& this_roi = ctx.inArg(0).rref(); + const auto this_mat = ctx.inMat(1); + IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(this_roi)); + iec.this_request.SetBlob(*uu.params.input_names.begin(), roi_blob); + iec.this_request.Infer(); + for (auto i : ade::util::iota(uu.params.num_out)) { + cv::Mat& out_mat = ctx.outMatR(i); + IE::Blob::Ptr out_blob = iec.this_request.GetBlob(uu.params.output_names[i]); + copyFromIE(out_blob, out_mat); + } + } +}; + + struct InferList: public cv::detail::KernelTag { using API = cv::GInferListBase; static cv::gapi::GBackend backend() { return cv::gapi::ie::backend(); } @@ -721,9 +811,23 @@ namespace { // FIXME: Introduce a DNNBackend interface which'd specify // the framework for this??? GIEModel gm(gr); - const auto &np = gm.metadata(nh).get(); - const auto &pp = cv::util::any_cast(np.opaque); + auto &np = gm.metadata(nh).get(); + auto &pp = cv::util::any_cast(np.opaque); const auto &ki = cv::util::any_cast(ii.opaque); + + GModel::Graph model(gr); + auto& op = model.metadata(nh).get(); + + // NB: In case generic infer, info about in/out names is stored in operation (op.params) + if (pp.is_generic) + { + auto& info = cv::util::any_cast(op.params); + pp.input_names = info.in_names; + pp.output_names = info.out_names; + pp.num_in = info.in_names.size(); + pp.num_out = info.out_names.size(); + } + gm.metadata(nh).set(IEUnit{pp}); gm.metadata(nh).set(IECallable{ki.run}); gm.metadata(nh).set(CustomMetaFunction{ki.customMetaFunc}); @@ -737,6 +841,7 @@ namespace { virtual cv::gapi::GKernelPackage auxiliaryKernels() const override { return cv::gapi::kernels< cv::gimpl::ie::Infer + , cv::gimpl::ie::InferROI , cv::gimpl::ie::InferList , cv::gimpl::ie::InferList2 >(); diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index 444d9553e7..ba0632d4f0 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -22,6 +22,24 @@ namespace IE = InferenceEngine; namespace giewrap = cv::gimpl::ie::wrap; using GIEParam = cv::gapi::ie::detail::ParamDesc; +IE::InputsDataMap giewrap::toInputsDataMap (const IE::ConstInputsDataMap& inputs) { + IE::InputsDataMap transformed; + auto convert = [](const std::pair& p) { + return std::make_pair(p.first, std::const_pointer_cast(p.second)); + }; + std::transform(inputs.begin(), inputs.end(), std::inserter(transformed, transformed.end()), convert); + return transformed; +} + +IE::OutputsDataMap giewrap::toOutputsDataMap (const IE::ConstOutputsDataMap& outputs) { + IE::OutputsDataMap transformed; + auto convert = [](const std::pair& p) { + return std::make_pair(p.first, std::const_pointer_cast(p.second)); + }; + std::transform(outputs.begin(), outputs.end(), std::inserter(transformed, transformed.end()), convert); + return transformed; +} + #if INF_ENGINE_RELEASE < 2020000000 // < 2020.1 // Load extensions (taken from DNN module) std::vector giewrap::getExtensions(const GIEParam& params) { diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 7871942d26..3927c802b7 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -28,7 +28,11 @@ namespace wrap { GAPI_EXPORTS std::vector getExtensions(const GIEParam& params); GAPI_EXPORTS IE::CNNNetwork readNetwork(const GIEParam& params); +IE::InputsDataMap toInputsDataMap (const IE::ConstInputsDataMap& inputs); +IE::OutputsDataMap toOutputsDataMap(const IE::ConstOutputsDataMap& outputs); + #if INF_ENGINE_RELEASE < 2019020000 // < 2019.R2 +using Plugin = IE::InferencePlugin; GAPI_EXPORTS IE::InferencePlugin getPlugin(const GIEParam& params); GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::InferencePlugin& plugin, const IE::CNNNetwork& net, @@ -36,7 +40,12 @@ GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::InferencePlugin& return plugin.LoadNetwork(net, {}); // FIXME: 2nd parameter to be // configurable via the API } +GAPI_EXPORTS inline IE::ExecutableNetwork importNetwork( IE::CNNNetwork& plugin, + const GIEParam& param) { + return plugin.ImportNetwork(param.model_path, param.device_id, {}); +} #else // >= 2019.R2 +using Plugin = IE::Core; GAPI_EXPORTS IE::Core getCore(); GAPI_EXPORTS IE::Core getPlugin(const GIEParam& params); GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::Core& core, @@ -44,6 +53,10 @@ GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::Core& core const GIEParam& params) { return core.LoadNetwork(net, params.device_id); } +GAPI_EXPORTS inline IE::ExecutableNetwork importNetwork( IE::Core& core, + const GIEParam& param) { + return core.ImportNetwork(param.model_path, param.device_id, {}); +} #endif // INF_ENGINE_RELEASE < 2019020000 }}}} diff --git a/modules/gapi/src/backends/ocl/goclbackend.cpp b/modules/gapi/src/backends/ocl/goclbackend.cpp index 34dba01afe..847b802fd2 100644 --- a/modules/gapi/src/backends/ocl/goclbackend.cpp +++ b/modules/gapi/src/backends/ocl/goclbackend.cpp @@ -272,4 +272,8 @@ void cv::gimpl::GOCLExecutable::run(std::vector &&input_objs, GAPI_Assert((out_arg_data == (mag_mat.getMat(ACCESS_RW).data)) && " data for output parameters was reallocated ?"); } } + + // In/Out args clean-up is mandatory now with RMat + for (auto &it : input_objs) magazine::unbind(m_res, it.first); + for (auto &it : output_objs) magazine::unbind(m_res, it.first); } diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp new file mode 100644 index 0000000000..7ab386ecab --- /dev/null +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -0,0 +1,963 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "precomp.hpp" +#include "backends/onnx/gonnxbackend.hpp" + +#ifdef HAVE_ONNX + +#include // any_of +#include +#include +#include + +#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! + +namespace cv { +namespace gimpl { +namespace onnx { + +enum TensorPosition : int { + INPUT, + OUTPUT +}; + +struct TensorInfo { + TensorInfo() = default; + explicit TensorInfo(const Ort::TensorTypeAndShapeInfo& info) + : dims(info.GetShape()) + , type(info.GetElementType()) + , is_dynamic(std::find(dims.begin(), dims.end(), -1) != dims.end()) { + if (!is_dynamic) { + size = std::accumulate(dims.begin(), + dims.end(), + static_cast(1), + std::multiplies()); + } + // Heuristic: check if the tensor is grayscale input + if (dims.size() == 4u + && dims[0] == 1 + && dims[1] == 1 + && dims[2] > 1 + && dims[3] > 1) { + is_grayscale = true; + } + } + + std::string name; + std::vector dims; + ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + int64_t size = -1; + + bool normalize = true; + + bool is_dynamic = false; + bool is_grayscale = false; + + struct MeanStdev { + cv::Scalar mean; + cv::Scalar stdev; + }; + cv::util::optional mstd; +}; + +class ONNXCompiled { + // ONNX Resources + // NOTE: Env must live with the session, otherwise segfaults. + Ort::Env this_env{nullptr}; + Ort::Session this_session{nullptr}; + Ort::MemoryInfo this_memory_info{nullptr}; + + std::vector in_tensor_info; + std::vector out_tensor_info; + bool is_dynamic = false; + + // G-API description + gapi::onnx::detail::ParamDesc params; + + // Input/output tensor information + std::vector getTensorInfo(TensorPosition pos); + + // Run-time data structures + std::vector in_data; + std::vector out_data; + + void Run(const std::vector& ins, + const std::vector& outs); + +public: + explicit ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp); + + // Extract the information about output layer #i + cv::GMatDesc outMeta(int i) const; + + // Assign input/output info + std::size_t numInputs() const { return params.num_in; } + std::size_t numOutputs() const { return params.num_out; } + void setInput(int i, const cv::Mat &m); + void setOutput(int i, cv::Mat &m); + cv::Mat allocOutput(int i) const; + + // Run with the assigned inputs/outputs + void run(); +}; + +} // namespace onnx +} // namespace gimpl +} // namespace cv + +namespace { + +inline std::vector getCharNames(const std::vector& names) { + std::vector out_vec; + for (const auto& el : names) { + out_vec.push_back(el.data()); + } + return out_vec; +} + +inline int getIdxByName(const std::vector& info, const std::string& name) { + // FIXME: Cache the ordering + const auto it = std::find_if(info.begin(), info.end(), [&](const cv::gimpl::onnx::TensorInfo &i) { + return i.name == name; + }); + GAPI_Assert(it != info.end()); + return std::distance(info.begin(), it); +} + +inline int toCV(ONNXTensorElementDataType prec) { + switch (prec) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return CV_8U; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return CV_32F; + default: GAPI_Assert(false && "Unsupported data type"); + } + return -1; +} + +inline std::vector toCV(const std::vector &vsz) { + std::vector result; + result.reserve(vsz.size()); + for (auto sz : vsz) { + result.push_back(ade::util::checked_cast(sz)); + } + return result; +} + +inline cv::Mat toCV(Ort::Value &v) { + auto info = v.GetTensorTypeAndShapeInfo(); + return cv::Mat(toCV(info.GetShape()), + toCV(info.GetElementType()), + reinterpret_cast(v.GetTensorMutableData())); +} + +inline std::vector toORT(const cv::MatSize &sz) { + return cv::to_own(sz); +} + +inline void preprocess(const cv::Mat& src, + const cv::gimpl::onnx::TensorInfo& ti, + cv::Mat& dst) { + GAPI_Assert(src.depth() == CV_32F || src.depth() == CV_8U); + + if (src.depth() == CV_32F) { + // Just pass the tensor as-is. + // No layout or dimension transformations done here! + // TODO: This needs to be aligned across all NN backends. + GAPI_Assert(toCV(ti.type) == CV_32F && "Only 32F model input is supported for 32F data"); + const auto tensor_dims = toORT(src.size); + if (tensor_dims.size() == ti.dims.size()) { + for (size_t i = 0; i < ti.dims.size(); ++i) { + GAPI_Assert((ti.dims[i] == -1 || ti.dims[i] == tensor_dims[i]) && + "32F tensor dimensions should match with all non-dynamic NN input dimensions"); + } + } else { + GAPI_Assert(false && "32F tensor size should match with NN input"); + } + + dst = src; + } else { + // 8U input: full preprocessing path + GAPI_Assert(src.depth() == CV_8U && "Only 8U data type is supported for preproc"); + GAPI_Assert(ti.dims.size() == 4u && "Only NCHW/NHWC layouts are supported for preproc"); + + const auto ddepth = toCV(ti.type); + GAPI_Assert((ddepth == CV_8U || ddepth == CV_32F) + && "Only 8U and 32F model input is supported for 8U data"); + + // Assess the expected input layout + const bool is_hwc = [&](int ch) { + if (ti.is_grayscale) return false; // 1,1,h,w + else if (ti.dims[3] == ch) return true; // _,_,_,c + else if (ti.dims[1] == ch) return false; // _,c,_,_ + else cv::util::throw_error(std::logic_error("Couldn't identify input tensor layout")); + } (src.channels()); + + int new_c = src.channels(); + cv::Mat csc; + if (ti.is_grayscale && new_c == 3) { + cv::cvtColor(src, csc, cv::COLOR_BGR2GRAY); + new_c = 1; + } else { + csc = src; + } + + // NHWC vs NCHW + int new_h = -1, new_w = -1; + if (ti.is_dynamic) { + // reuse h & w from the input image + new_h = src.rows; + new_w = src.cols; + } else { + // take h & w from the ONNX tensor info + new_h = ti.dims[is_hwc ? 1 : 2]; + new_w = ti.dims[is_hwc ? 2 : 3]; + } + GAPI_Assert(new_h != -1 && new_w != -1); + + cv::Mat rsz, pp; + cv::resize(csc, rsz, cv::Size(new_w, new_h)); + if (src.depth() == CV_8U && ddepth == CV_32F) { + rsz.convertTo(pp, ddepth, ti.normalize ? 1.f / 255 : 1.f); + if (ti.mstd.has_value()) { + pp -= ti.mstd->mean; + pp /= ti.mstd->stdev; + } + } else { + pp = rsz; + } + + if (!is_hwc && new_c > 1) { + // Convert to CHW + dst.create(cv::Size(new_w, new_h * new_c), ddepth); + std::vector planes(new_c); + for (int ch = 0; ch < new_c; ++ch) { + planes[ch] = dst.rowRange(ch * new_h, (ch + 1) * new_h); + } + cv::split(pp, planes); + } else { + // Keep HWC + dst = pp; + } + + // Ensure dst is a tensor shape (not a 2D image) + if (ti.is_dynamic) { + // Reshape to input dimensions + const std::vector out_dims = is_hwc + ? std::vector{1, new_h, new_w, new_c} + : std::vector{1, new_c, new_h, new_w}; + dst = dst.reshape(1, out_dims); + } else { + // Reshape to ONNX dimensions (no -1s there!) + dst = dst.reshape(1, toCV(ti.dims)); + } + } +} + +template +inline Ort::Value createTensor(const Ort::MemoryInfo& memory_info, + const cv::gimpl::onnx::TensorInfo& tensor_params, + const cv::Mat& data) { + (void) tensor_params; + auto ort_dims = toORT(data.size); + return Ort::Value::CreateTensor(memory_info, + const_cast(data.ptr()), + data.total(), + ort_dims.data(), + ort_dims.size()); +} + +inline Ort::Value createTensor(const Ort::MemoryInfo& memory_info, + const cv::gimpl::onnx::TensorInfo& tensor_params, + const cv::Mat& data) { + GAPI_Assert(data.isContinuous ()); + switch (tensor_params.type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + return createTensor(memory_info, tensor_params, data); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + return createTensor(memory_info, tensor_params, data); + default: + GAPI_Assert(false && "Unsupported data type"); + } + return Ort::Value{nullptr}; +} + +struct ONNXUnit { + static const char *name() { return "ONNXModelConfig"; } + + std::shared_ptr oc; + + explicit ONNXUnit(const cv::gapi::onnx::detail::ParamDesc &pp) + : oc(new cv::gimpl::onnx::ONNXCompiled(pp)) { + } +}; + +struct ONNXCallContext { + // Input parameters passed to an inference operation. + std::vector args; + + //FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call + //to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run, + //once on enter for input and output arguments, and once before return for output arguments only + //FIXME: check if the above applies to this backend (taken from CPU) + std::unordered_map results; + + // Generic accessor API + template + const T& inArg(std::size_t input) { return args.at(input).get(); } + + // Syntax sugar + const cv::Mat& inMat(std::size_t input) { + return inArg(input); + } + cv::Mat& outMatR(std::size_t output) { + return *cv::util::get(results.at(output)); + } + + template std::vector& outVecR(std::size_t output) { // FIXME: the same issue + return outVecRef(output).wref(); + } + cv::detail::VectorRef& outVecRef(std::size_t output) { + return cv::util::get(results.at(output)); + } +}; + +struct ONNXCallable { + static const char *name() { return "ONNXRequestCallable"; } + using Run = std::function; + Run run; +}; + +struct KImpl { + cv::gimpl::CustomMetaFunction::CM customMetaFunc; + ONNXCallable::Run run; +}; + +// FIXME: Is there a way to take a typed graph (our GModel), +// and create a new typed graph _ATOP_ of that (by extending with a couple of +// new types?). +// Alternatively, is there a way to compose types graphs? +// +// If not, we need to introduce that! +using GONNXModel = ade::TypedGraph + < cv::gimpl::Protocol + , cv::gimpl::Op + , cv::gimpl::NetworkParams + , cv::gimpl::CustomMetaFunction + , ONNXUnit + , ONNXCallable + >; + +// FIXME: Same issue with Typed and ConstTyped +using GConstGONNXModel = ade::ConstTypedGraph + < cv::gimpl::Protocol + , cv::gimpl::Op + , cv::gimpl::NetworkParams + , cv::gimpl::CustomMetaFunction + , ONNXUnit + , ONNXCallable + >; +} // anonymous namespace + +// GCPUExcecutable implementation ////////////////////////////////////////////// +cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g, + const std::vector &nodes) + : m_g(g), m_gm(m_g) { + // FIXME: Currently this backend is capable to run a single inference node only. + // Need to extend our island fusion with merge/not-to-merge decision making parametrization + GConstGONNXModel iem(g); + + for (auto &nh : nodes) { + switch (m_gm.metadata(nh).get().t) { + case NodeType::OP: + if (this_nh == nullptr) { + this_nh = nh; + } + else { + util::throw_error(std::logic_error("Multi-node inference is not supported!")); + } + break; + + case NodeType::DATA: { + m_dataNodes.push_back(nh); + const auto &desc = m_gm.metadata(nh).get(); + if (desc.storage == Data::Storage::CONST_VAL) { + util::throw_error(std::logic_error("No const data supported in backend!")); + } + if (desc.storage == Data::Storage::INTERNAL) { + util::throw_error(std::logic_error("No internal data supported in backend!")); + } + break; + } + default: util::throw_error(std::logic_error("Unsupported NodeType")); + } + } +} + +// FIXME: Document what it does +cv::GArg cv::gimpl::onnx::GONNXExecutable::packArg(const cv::GArg &arg) { + // No API placeholders allowed at this point + // FIXME: this check has to be done somewhere in compilation stage. + GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT + && arg.kind != cv::detail::ArgKind::GSCALAR + && arg.kind != cv::detail::ArgKind::GARRAY + && arg.kind != cv::detail::ArgKind::GOPAQUE); + + if (arg.kind != cv::detail::ArgKind::GOBJREF) { + util::throw_error(std::logic_error("Inference supports G-types ONLY!")); + } + GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF); + + // Wrap associated CPU object (either host or an internal one) + // FIXME: object can be moved out!!! GExecutor faced that. + const cv::gimpl::RcDesc &ref = arg.get(); + switch (ref.shape) + { + case GShape::GMAT: return GArg(m_res.slot()[ref.id]); + + // Note: .at() is intentional for GArray as object MUST be already there + // (and constructed by either bindIn/Out or resetInternal) + case GShape::GARRAY: return GArg(m_res.slot().at(ref.id)); + + // Note: .at() is intentional for GOpaque as object MUST be already there + // (and constructed by either bindIn/Out or resetInternal) + case GShape::GOPAQUE: return GArg(m_res.slot().at(ref.id)); + + default: + util::throw_error(std::logic_error("Unsupported GShape type")); + break; + } +} + +void cv::gimpl::onnx::GONNXExecutable::run(std::vector &&input_objs, + std::vector &&output_objs) { + // Update resources with run-time information - what this Island + // has received from user (or from another Island, or mix...) + // FIXME: Check input/output objects against GIsland protocol + + for (auto& it : input_objs) magazine::bindInArg (m_res, it.first, it.second); + for (auto& it : output_objs) magazine::bindOutArg(m_res, it.first, it.second); + + // FIXME: Running just a single node now. + // Not sure if need to support many of them, though + // FIXME: Make this island-unmergeable? + const auto &op = m_gm.metadata(this_nh).get(); + + // Initialize kernel's execution context: + // - Input parameters + ONNXCallContext context; + context.args.reserve(op.args.size()); + using namespace std::placeholders; + ade::util::transform(op.args, + std::back_inserter(context.args), + std::bind(&GONNXExecutable::packArg, this, _1)); + + // - Output parameters. + for (const auto &out_it : ade::util::indexed(op.outs)) { + // FIXME: Can the same GArg type resolution mechanism be reused here? + const auto out_port = ade::util::index(out_it); + const auto out_desc = ade::util::value(out_it); + context.results[out_port] = magazine::getObjPtr(m_res, out_desc); + } + + // And now trigger the execution + GConstGONNXModel giem(m_g); + const auto &uu = giem.metadata(this_nh).get(); + const auto &kk = giem.metadata(this_nh).get(); + kk.run(uu, context); + + for (auto &it : output_objs) magazine::writeBack(m_res, it.first, it.second); +} + +namespace cv { +namespace gimpl { +namespace onnx { + +ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) + : params(pp) { + + // Validate input parameters before allocating any resources + if (params.num_in > 1u && params.num_in != params.input_names.size()) { + cv::util::throw_error(std::logic_error("Please specify input layer names for " + + params.model_path)); + } + if (params.num_out > 1u && params.num_out != params.output_names.size()) { + cv::util::throw_error(std::logic_error("Please specify output layer names for " + + params.model_path)); + } + + // Create and initialize the ONNX session + Ort::SessionOptions session_options; + this_env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, ""); + this_session = Ort::Session(this_env, params.model_path.data(), session_options); + this_memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + + in_tensor_info = getTensorInfo(INPUT); + out_tensor_info = getTensorInfo(OUTPUT); + + const auto is_dyn = [](const TensorInfo &ti) { + return ti.is_dynamic; + }; + is_dynamic = ade::util::any_of(in_tensor_info, is_dyn) + || ade::util::any_of(out_tensor_info, is_dyn); + if (is_dynamic && !params.custom_post_proc) { + util::throw_error(std::logic_error("This network has dynamic shapes. " + "Please provide a custom post-processing function " + "(.cfgPostProc) in network parameters")); + } + + // Update parameters based on session information + if (params.num_in == 1u && params.input_names.empty()) { + params.input_names = { in_tensor_info.front().name }; + } + if (params.num_out == 1u && params.output_names.empty()) { + params.output_names = { out_tensor_info.front().name }; + } + + // Validate what is supported currently + GAPI_Assert(params.const_inputs.empty() + && "Const inputs are not currently supported"); + GAPI_Assert(std::all_of(in_tensor_info.begin(), + in_tensor_info.end(), + [](const cv::gimpl::onnx::TensorInfo &p) { + return p.type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT + || p.type == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; + }) + && "Only FP32 and U8 inputs for NN are supported"); + + // Put mean and std in appropriate tensor params + if (!params.mean.empty() || !params.stdev.empty()) { + GAPI_Assert(params.mean.size() == params.stdev.size() && + params.mean.size() == params.input_names.size()); + for (auto idx : ade::util::iota(params.num_in)) { + const auto ort_idx = getIdxByName(in_tensor_info, params.input_names[idx]); + using M = TensorInfo::MeanStdev; + in_tensor_info[ort_idx].mstd = util::make_optional(M{ params.mean[idx] + , params.stdev[idx] }); + } + } + + // Update normalize flags for input tensors + if (!params.normalize.empty()) { + for (auto idx : ade::util::iota(params.num_in)) { + const auto ort_idx = getIdxByName(in_tensor_info, params.input_names[idx]); + in_tensor_info[ort_idx].normalize = params.normalize[idx]; + } + } + + // Pre-allocate vectors (not buffers) for runtime info + in_data.resize(params.num_in); + out_data.resize(params.num_out); +} + +std::vector ONNXCompiled::getTensorInfo(TensorPosition pos) { + GAPI_Assert(pos == INPUT || pos == OUTPUT); + + const auto num_nodes = pos == INPUT + ? this_session.GetInputCount() + : this_session.GetOutputCount(); + + std::vector tensor_info; + tensor_info.reserve(num_nodes); + + Ort::AllocatorWithDefaultOptions allocator; + for (auto i : ade::util::iota(num_nodes)) { + const auto info = pos == INPUT + ? this_session.GetInputTypeInfo(i) + : this_session.GetOutputTypeInfo(i); + tensor_info.emplace_back(info.GetTensorTypeAndShapeInfo()); + + char *name_p = pos == INPUT + ? this_session.GetInputName(i, allocator) + : this_session.GetOutputName(i, allocator); + tensor_info.back().name = name_p; + allocator.Free(name_p); + } + + return tensor_info; +} + +cv::GMatDesc ONNXCompiled::outMeta(int idx) const { + if (is_dynamic) { + GAPI_Assert(!params.out_metas.empty() + && "Metadata must be specified if NN has dynamic inputs!"); + return params.out_metas.at(idx); + } + const auto ort_idx = getIdxByName(out_tensor_info, params.output_names[idx]); + return cv::GMatDesc(toCV(out_tensor_info[ort_idx].type), + toCV(out_tensor_info[ort_idx].dims)); +} + +void ONNXCompiled::setInput(int i, const cv::Mat &m) { + const auto in_idx = i; + const auto in_name = params.input_names[in_idx]; + const auto ort_idx = getIdxByName(in_tensor_info, in_name); + preprocess(m, in_tensor_info[ort_idx], in_data[in_idx]); +} + +void ONNXCompiled::setOutput(int i, cv::Mat &m) { + // FIXME: No need in double-indexing? + out_data[i] = m; +} + +cv::Mat ONNXCompiled::allocOutput(int i) const { + cv::Mat m; + m.create(toCV(out_tensor_info[i].dims), + toCV(out_tensor_info[i].type)); + return m; +} + +void ONNXCompiled::Run(const std::vector& ins, + const std::vector& outs) { + std::vector in_tensors, out_tensors; + + auto in_run_names = getCharNames(params.input_names); + + for (const auto it : ade::util::indexed(params.input_names)) { + auto i = ade::util::index(it); + auto in_name = ade::util::value(it); + const auto idx = getIdxByName(in_tensor_info, in_name); + in_tensors.emplace_back(createTensor(this_memory_info, + in_tensor_info[idx], + ins[i])); + } + + if (!is_dynamic) { + // Easy path - just run the session which is bound to G-API's + // internal data + for (auto i : ade::util::iota(params.output_names.size())) { + out_tensors.emplace_back(createTensor(this_memory_info, + out_tensor_info[i], + outs[i])); + } + auto out_run_names = getCharNames(params.output_names); + this_session.Run(Ort::RunOptions{nullptr}, + in_run_names.data(), + &in_tensors.front(), + params.input_names.size(), + out_run_names.data(), + &out_tensors.front(), + params.output_names.size()); + } else { + // Hard path - run session & user-defined post-processing + // NOTE: use another list of output names here + std::vector out_names; + for (auto &&ti : out_tensor_info) { + out_names.push_back(ti.name.c_str()); + } + + auto outputs = this_session.Run(Ort::RunOptions{nullptr}, + in_run_names.data(), + &in_tensors.front(), + params.input_names.size(), + out_names.data(), + out_names.size()); + std::unordered_map onnx_outputs; + std::unordered_map gapi_outputs; + + GAPI_Assert(outputs.size() == out_names.size()); + // Fill in ONNX tensors + for (auto &&iter : ade::util::zip(ade::util::toRange(out_tensor_info), + ade::util::toRange(outputs))) { + const auto &out_name = std::get<0>(iter).name; + auto &out_tensor = std::get<1>(iter); + onnx_outputs[out_name] = toCV(out_tensor); + } + + // Fill in G-API outputs + for (auto &&it: ade::util::indexed(params.output_names)) { + gapi_outputs[ade::util::value(it)] = outs[ade::util::index(it)]; + } + params.custom_post_proc(onnx_outputs, gapi_outputs); + } +} + +void ONNXCompiled::run() { + Run(in_data, out_data); +} + +struct Infer: public cv::detail::KernelTag { + using API = cv::GInferBase; + static cv::gapi::GBackend backend() { return cv::gapi::onnx::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + cv::GMetaArgs result; + + GConstGONNXModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + + GAPI_Assert(uu.oc->numInputs() == in_metas.size() + && "Known input layers count doesn't match input meta count"); + for (auto &&mm : in_metas) { + GAPI_Assert(util::holds_alternative(mm) + && "Non-GMat inputs are not supported"); + } + for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) { + result.emplace_back(uu.oc->outMeta(idx)); + } + return result; + } + + static void run(const ONNXUnit &uu, ONNXCallContext &ctx) { + for (auto &&idx : ade::util::iota(uu.oc->numInputs())) { + uu.oc->setInput(idx, ctx.inMat(idx)); + } + for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) { + uu.oc->setOutput(idx, ctx.outMatR(idx)); + } + uu.oc->run(); + } +}; + +struct InferROI: public cv::detail::KernelTag { + using API = cv::GInferROIBase; + static cv::gapi::GBackend backend() { return cv::gapi::onnx::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + cv::GMetaArgs result; + + GConstGONNXModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + GAPI_Assert(1u == uu.oc->numInputs()); + GAPI_Assert(2u == in_metas.size()); + + for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) { + result.emplace_back(uu.oc->outMeta(idx)); + } + return result; + } + + static void run(const ONNXUnit &uu, ONNXCallContext &ctx) { + // non-generic version for now, per the InferROI's definition + GAPI_Assert(uu.oc->numInputs() == 1u); + const auto& this_roi = ctx.inArg(0).rref(); + const auto this_mat = ctx.inMat(1); + + uu.oc->setInput(0, this_mat(this_roi)); + for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) { + uu.oc->setOutput(idx, ctx.outMatR(idx)); + } + uu.oc->run(); + } +}; + +struct InferList: public cv::detail::KernelTag { + using API = cv::GInferListBase; + static cv::gapi::GBackend backend() { return cv::gapi::onnx::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + GConstGONNXModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + + // Note our input layers list order matches the API order and so + // meta order. + GAPI_Assert(uu.oc->numInputs() == (in_metas.size() - 1u) + && "Known input layers count doesn't match input meta count"); + + for (auto i : ade::util::iota(uu.oc->numInputs())) { + const auto & mm = in_metas[i + 1]; + + GAPI_Assert(util::holds_alternative(mm) + && "Non-GMat inputs are not supported"); + } + + // roi-list version is much easier at the moment. + // All our outputs are vectors which don't have + // metadata at the moment - so just create a vector of + // "empty" array metadatas of the required size. + return cv::GMetaArgs(uu.oc->numOutputs(), + cv::GMetaArg{cv::empty_array_desc()}); + } + + static void run(const ONNXUnit &uu, ONNXCallContext &ctx) { + // non-generic version for now: + // - assumes input 0 is always ROI list + // - assumes all inputs/outputs are always Mats + GAPI_Assert(uu.oc->numInputs() == 1); // roi list is not counted in net's inputs + + const auto& in_roi_vec = ctx.inArg(0u).rref(); + const cv::Mat this_mat = ctx.inMat(1u); + + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + ctx.outVecR(i).clear(); + } + for (const auto &rc : in_roi_vec) { + uu.oc->setInput(0, this_mat(rc)); + std::vector out_mats(uu.oc->numOutputs()); + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + out_mats[i] = uu.oc->allocOutput(i); + uu.oc->setOutput(i, out_mats[i]); + } + uu.oc->run(); + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + std::vector &out_vec = ctx.outVecR(i); + out_vec.push_back(std::move(out_mats[i])); + } + } + } +}; + +struct InferList2: public cv::detail::KernelTag { + using API = cv::GInferList2Base; + static cv::gapi::GBackend backend() { return cv::gapi::onnx::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + + GConstGONNXModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + + // Note our input layers list order matches the API order and so + // meta order. + GAPI_Assert(uu.oc->numInputs() == (in_metas.size() - 1u) + && "Known input layers count doesn't match input meta count"); + + // In contrast to InferList, the InferList2 has only one + // "full-frame" image argument, and all the rest are arrays of + // ether ROI or blobs. So here we set the 0th arg image format + // to all inputs which are ROI-based (skipping the + // "blob"-based ones) + // FIXME: this is filtering not done, actually! GArrayDesc has + // no hint for type! + const auto &mm_0 = in_metas[0u]; + const auto &meta_0 = util::get(mm_0); + GAPI_Assert( !meta_0.isND() + && !meta_0.planar + && "Only images are supported as the 0th argument"); + for (auto i : ade::util::iota(uu.oc->numInputs())) { + const auto &mm = in_metas[i + 1]; + GAPI_Assert(util::holds_alternative(mm) + && "Non-array inputs are not supported"); + } + + // roi-list version is much easier at the moment. + // All our outputs are vectors which don't have + // metadata at the moment - so just create a vector of + // "empty" array metadatas of the required size. + return cv::GMetaArgs(uu.oc->numOutputs(), + cv::GMetaArg{cv::empty_array_desc()}); + } + + static void run(const ONNXUnit &uu, ONNXCallContext &ctx) { + GAPI_Assert(ctx.args.size() > 1u + && "This operation must have at least two arguments"); + + // Since we do a ROI list inference, always assume our input buffer is image + const cv::Mat mat_0 = ctx.inMat(0u); + // Take the next argument, which must be vector (of any kind). + // Use this only to obtain the ROI list size (sizes of all + // other vectors must be equal to this one) + const auto list_size = ctx.inArg(1u).size(); + + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + ctx.outVecR(i).clear(); + } + // For every ROI in the list {{{ + for (const auto &list_idx : ade::util::iota(list_size)) { + std::vector in_tensors, out_tensors; + std::vector in_mats(uu.oc->numInputs()); + // For every input of the net {{{ + for (auto in_idx : ade::util::iota(uu.oc->numInputs())) { + const auto &this_vec = ctx.inArg(in_idx+1u); + GAPI_Assert(this_vec.size() == list_size); + // Prepare input {{{ + // FIXME: Terrible run-time logic based on RTTI! + // FIXME: Will never work on non-RTTI systems! + // FIXME: Need to replace with a static type tags + // (like with serialization) instead! + if (this_vec.holds()) { + // ROI case - create an ROI blob + const auto &vec = this_vec.rref(); + uu.oc->setInput(in_idx, mat_0(vec[list_idx])); + } else if (this_vec.holds()) { + // Mat case - create a regular blob + // FIXME: NOW Assume Mats are always BLOBS (not + // images) + const auto &vec = this_vec.rref(); + uu.oc->setInput(in_idx, vec[list_idx]); + } else { + GAPI_Assert(false && "Only Rect and Mat types are supported for infer list 2!"); + } + // }}} (Preapre input) + } // }}} (For every input of the net) + + std::vector out_mats(uu.oc->numOutputs()); + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + out_mats[i] = uu.oc->allocOutput(i); + uu.oc->setOutput(i, out_mats[i]); + } + uu.oc->run(); + + for (auto i : ade::util::iota(uu.oc->numOutputs())) { + std::vector &out_vec = ctx.outVecR(i); + out_vec.push_back(std::move(out_mats[i])); + } + } // }}} (For every ROI in the list) + } +}; + +} // namespace onnx +} // namespace gapi +} // namespace cv + +namespace { + class GONNXBackendImpl final: public cv::gapi::GBackend::Priv { + virtual void unpackKernel(ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GKernelImpl &ii) override { + using namespace cv::gimpl; + // FIXME: Introduce a DNNBackend interface which'd specify + // the framework for this??? + GONNXModel gm(gr); + const auto &np = gm.metadata(nh).get(); + const auto &pp = cv::util::any_cast(np.opaque); + const auto &ki = cv::util::any_cast(ii.opaque); + gm.metadata(nh).set(ONNXUnit{pp}); + gm.metadata(nh).set(ONNXCallable{ki.run}); + gm.metadata(nh).set(CustomMetaFunction{ki.customMetaFunc}); + } + + virtual EPtr compile(const ade::Graph &graph, + const cv::GCompileArgs &, + const std::vector &nodes) const override { + return EPtr{new cv::gimpl::onnx::GONNXExecutable(graph, nodes)}; + } + + virtual cv::gapi::GKernelPackage auxiliaryKernels() const override { + return cv::gapi::kernels< cv::gimpl::onnx::Infer + , cv::gimpl::onnx::InferROI + , cv::gimpl::onnx::InferList + , cv::gimpl::onnx::InferList2 + >(); + } + }; +} + +cv::gapi::GBackend cv::gapi::onnx::backend() { + static cv::gapi::GBackend this_backend(std::make_shared()); + return this_backend; +} +#else // HAVE_ONNX + +cv::gapi::GBackend cv::gapi::onnx::backend() { + // Still provide this symbol to avoid linking issues + util::throw_error(std::runtime_error("G-API has been compiled without ONNX support")); +} +#endif // HAVE_ONNX diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.hpp b/modules/gapi/src/backends/onnx/gonnxbackend.hpp new file mode 100644 index 0000000000..a3cc897030 --- /dev/null +++ b/modules/gapi/src/backends/onnx/gonnxbackend.hpp @@ -0,0 +1,56 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_GONNXBACKEND_HPP +#define OPENCV_GAPI_GONNXBACKEND_HPP + +#include "opencv2/gapi/infer/onnx.hpp" +#ifdef HAVE_ONNX + +#include +#include // type_list_index + +#include "backends/common/gbackend.hpp" + +namespace cv { +namespace gimpl { +namespace onnx { + +class GONNXExecutable final: public GIslandExecutable +{ + const ade::Graph &m_g; + GModel::ConstGraph m_gm; + + // The only executable stuff in this graph + // (assuming it is always single-op) + ade::NodeHandle this_nh; + + // List of all resources in graph (both internal and external) + std::vector m_dataNodes; + + // Actual data of all resources in graph (both internal and external) + Mag m_res; + + // Execution helpers + GArg packArg(const GArg &arg); + +public: + GONNXExecutable(const ade::Graph &graph, + const std::vector &nodes); + + virtual inline bool canReshape() const override { return false; } + virtual inline void reshape(ade::Graph&, const GCompileArgs&) override { + GAPI_Assert(false); // Not implemented yet + } + + virtual void run(std::vector &&input_objs, + std::vector &&output_objs) override; +}; + +}}} // namespace cv::gimpl::onnx + +#endif // HAVE_ONNX +#endif // OPENCV_GAPI_GONNXBACKEND_HPP diff --git a/modules/gapi/src/api/ft_render.cpp b/modules/gapi/src/backends/render/ft_render.cpp similarity index 92% rename from modules/gapi/src/api/ft_render.cpp rename to modules/gapi/src/backends/render/ft_render.cpp index 7561dff833..fcf84713ff 100644 --- a/modules/gapi/src/api/ft_render.cpp +++ b/modules/gapi/src/backends/render/ft_render.cpp @@ -5,11 +5,11 @@ // Copyright (C) 2019 Intel Corporation #include "precomp.hpp" +#include "ft_render.hpp" #ifdef HAVE_FREETYPE -#include "api/ft_render.hpp" -#include "api/ft_render_priv.hpp" +#include "ft_render_priv.hpp" #include #include @@ -166,6 +166,11 @@ void cv::gapi::wip::draw::FTTextRender::Priv::putText(cv::Mat& mat, "Failed to load char"); FT_Bitmap *bitmap = &(m_face->glyph->bitmap); + // FIXME: Skip glyph, if size is 0 + if (bitmap->rows == 0 || bitmap->width == 0) { + continue; + } + cv::Mat glyph(bitmap->rows, bitmap->width, CV_8UC1, bitmap->buffer, bitmap->pitch); int left = m_face->glyph->bitmap_left; @@ -211,4 +216,21 @@ void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat& mat, m_priv->putText(mat, text, org, fh); } +#else + +cv::Size cv::gapi::wip::draw::FTTextRender::getTextSize(const std::wstring&, int, int*) +{ + cv::util::throw_error(std::runtime_error("Freetype not found")); +} + +void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat&, const std::wstring&, const cv::Point&, int) +{ + cv::util::throw_error(std::runtime_error("Freetype not found")); +} + +cv::gapi::wip::draw::FTTextRender::FTTextRender(const std::string&) +{ + cv::util::throw_error(std::runtime_error("Freetype not found")); +} + #endif // HAVE_FREETYPE diff --git a/modules/gapi/src/api/ft_render.hpp b/modules/gapi/src/backends/render/ft_render.hpp similarity index 91% rename from modules/gapi/src/api/ft_render.hpp rename to modules/gapi/src/backends/render/ft_render.hpp index 2556c7269c..068c0d4d3f 100644 --- a/modules/gapi/src/api/ft_render.hpp +++ b/modules/gapi/src/backends/render/ft_render.hpp @@ -23,8 +23,6 @@ namespace wip namespace draw { -#ifdef HAVE_FREETYPE - class GAPI_EXPORTS FTTextRender { public: @@ -38,12 +36,6 @@ private: std::shared_ptr m_priv; }; -#else - -class GAPI_EXPORTS FTTextRender {}; - -#endif // HAVE_FREETYPE - } // namespace draw } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/api/ft_render_priv.hpp b/modules/gapi/src/backends/render/ft_render_priv.hpp similarity index 96% rename from modules/gapi/src/api/ft_render_priv.hpp rename to modules/gapi/src/backends/render/ft_render_priv.hpp index 5a0679dd99..903f439b96 100644 --- a/modules/gapi/src/api/ft_render_priv.hpp +++ b/modules/gapi/src/backends/render/ft_render_priv.hpp @@ -10,7 +10,7 @@ #ifndef OPENCV_FT_RENDER_PRIV_HPP #define OPENCV_FT_RENDER_PRIV_HPP -#include "api/ft_render.hpp" +#include "ft_render.hpp" #include #include FT_FREETYPE_H diff --git a/modules/gapi/src/backends/render/grenderocv.cpp b/modules/gapi/src/backends/render/grenderocv.cpp index cb4fd1be3a..71be889d79 100644 --- a/modules/gapi/src/backends/render/grenderocv.cpp +++ b/modules/gapi/src/backends/render/grenderocv.cpp @@ -1,16 +1,21 @@ #include #include "api/render_ocv.hpp" -#include "backends/render/grenderocv.hpp" #include +#include -GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR) +struct RenderOCVState +{ + std::shared_ptr ftpr; +}; + +GAPI_OCV_KERNEL_ST(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR, RenderOCVState) { static void run(const cv::Mat& in, const cv::gapi::wip::draw::Prims& prims, - cv::gapi::wip::draw::FTTextRender* ftpr, - cv::Mat& out) + cv::Mat& out, + RenderOCVState& state) { // NB: If in and out cv::Mats are the same object // we can avoid copy and render on out cv::Mat @@ -19,18 +24,33 @@ GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR) in.copyTo(out); } - cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, ftpr); + cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, state.ftpr); + } + + static void setup(const cv::GMatDesc& /* in */, + const cv::GArrayDesc& /* prims */, + std::shared_ptr& state, + const cv::GCompileArgs& args) + { + using namespace cv::gapi::wip::draw; + auto opt_freetype_font = cv::gapi::getCompileArg(args); + state = std::make_shared(); + + if (opt_freetype_font.has_value()) + { + state->ftpr = std::make_shared(opt_freetype_font->path); + } } }; -GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12) +GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOCVState) { static void run(const cv::Mat& in_y, const cv::Mat& in_uv, const cv::gapi::wip::draw::Prims& prims, - cv::gapi::wip::draw::FTTextRender* ftpr, cv::Mat& out_y, - cv::Mat& out_uv) + cv::Mat& out_uv, + RenderOCVState& state) { // NB: If in and out cv::Mats are the same object // we can avoid copy and render on out cv::Mat @@ -67,7 +87,7 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12) cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR); cv::merge(std::vector{in_y, upsample_uv}, yuv); - cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, ftpr); + cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr); // YUV -> NV12 cv::Mat out_u, out_v, uv_plane; @@ -76,6 +96,22 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12) cv::merge(std::vector{chs[1], chs[2]}, uv_plane); cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR); } + + static void setup(const cv::GMatDesc& /* in_y */, + const cv::GMatDesc& /* in_uv */, + const cv::GArrayDesc& /* prims */, + std::shared_ptr& state, + const cv::GCompileArgs& args) + { + using namespace cv::gapi::wip::draw; + auto has_freetype_font = cv::gapi::getCompileArg(args); + state = std::make_shared(); + + if (has_freetype_font) + { + state->ftpr = std::make_shared(has_freetype_font->path); + } + } }; cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels() diff --git a/modules/gapi/src/backends/render/grenderocv.hpp b/modules/gapi/src/backends/render/grenderocv.hpp deleted file mode 100644 index e5091042b2..0000000000 --- a/modules/gapi/src/backends/render/grenderocv.hpp +++ /dev/null @@ -1,55 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2019 Intel Corporation - -#ifndef OPENCV_GAPI_GRENDEROCV_HPP -#define OPENCV_GAPI_GRENDEROCV_HPP - -#include -#include "api/render_priv.hpp" -#include "api/ft_render.hpp" - -namespace cv -{ -namespace gapi -{ -namespace render -{ -namespace ocv -{ - -GAPI_EXPORTS cv::gapi::GBackend backend(); - -template -struct add_type_to_tuple; - -template -struct add_type_to_tuple> -{ - using type = std::tuple; -}; - -template -class GRenderKernelImpl: public cv::detail::OCVCallHelper, - public cv::detail::KernelTag -{ - using InArgs = typename add_type_to_tuple::type; - using P = detail::OCVCallHelper; - -public: - using API = K; - - static cv::gapi::GBackend backend() { return cv::gapi::render::ocv::backend(); } - static cv::GCPUKernel kernel() { return GCPUKernel(&P::call); } -}; - -#define GAPI_RENDER_OCV_KERNEL(Name, API) struct Name: public cv::gapi::render::ocv::GRenderKernelImpl - -} // namespace ocv -} // namespace render -} // namespace gapi -} // namespace cv - -#endif // OPENCV_GAPI_GRENDEROCV_HPP diff --git a/modules/gapi/src/backends/render/grenderocvbackend.cpp b/modules/gapi/src/backends/render/grenderocvbackend.cpp deleted file mode 100644 index 413d0c3f9c..0000000000 --- a/modules/gapi/src/backends/render/grenderocvbackend.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2018-2020 Intel Corporation - -#include "precomp.hpp" - -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "compiler/gobjref.hpp" -#include "compiler/gmodel.hpp" - -#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! -#include "api/render_ocv.hpp" - -#include "backends/render/grenderocvbackend.hpp" - -#include -#include "api/ocv_mask_creator.hpp" -#include "api/ft_render.hpp" - - -using GRenderModel = ade::TypedGraph - < cv::gimpl::render::ocv::RenderUnit - >; - -// FIXME: Same issue with Typed and ConstTyped -using GConstRenderModel = ade::ConstTypedGraph - < cv::gimpl::render::ocv::RenderUnit - >; - -cv::gimpl::render::ocv::GRenderExecutable::GRenderExecutable(const ade::Graph &g, - const std::vector &nodes, - std::unique_ptr&& ftpr) - : m_g(g), m_gm(m_g), m_ftpr(std::move(ftpr)) { - GConstRenderModel gcm(m_g); - - auto is_op = [&](ade::NodeHandle nh) { - return m_gm.metadata(nh).get().t == NodeType::OP; - }; - - auto it = ade::util::find_if(nodes, is_op); - - GAPI_Assert(it != nodes.end()); - this_nh = *it; - - if (!std::none_of(std::next(it), nodes.end(), is_op)) { - util::throw_error(std::logic_error("Multi-node rendering is not supported!")); - } -} - -void cv::gimpl::render::ocv::GRenderExecutable::run(std::vector &&input_objs, - std::vector &&output_objs) { - GConstRenderModel gcm(m_g); - - for (auto& it : input_objs) magazine::bindInArg (m_res, it.first, it.second); - for (auto& it : output_objs) magazine::bindOutArg(m_res, it.first, it.second); - - const auto &op = m_gm.metadata(this_nh).get(); - - // Initialize kernel's execution context: - // - Input parameters - GCPUContext context; - context.m_args.reserve(op.args.size()); - using namespace std::placeholders; - ade::util::transform(op.args, - std::back_inserter(context.m_args), - std::bind(&GRenderExecutable::packArg, this, _1)); - - // - Output parameters. - for (const auto &out_it : ade::util::indexed(op.outs)) { - // FIXME: Can the same GArg type resolution mechanism be reused here? - const auto out_port = ade::util::index(out_it); - const auto out_desc = ade::util::value(out_it); - context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc); - } - - auto k = gcm.metadata(this_nh).get().k; - - context.m_args.emplace_back(m_ftpr.get()); - - k.m_runF(context); - - for (auto &it : output_objs) magazine::writeBack(m_res, it.first, it.second); - - // In/Out args clean-up is mandatory now with RMat - for (auto &it : input_objs) magazine::unbind(m_res, it.first); - for (auto &it : output_objs) magazine::unbind(m_res, it.first); -} - -cv::GArg cv::gimpl::render::ocv::GRenderExecutable::packArg(const cv::GArg &arg) { - // No API placeholders allowed at this point - // FIXME: this check has to be done somewhere in compilation stage. - GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT - && arg.kind != cv::detail::ArgKind::GSCALAR - && arg.kind != cv::detail::ArgKind::GARRAY); - - if (arg.kind != cv::detail::ArgKind::GOBJREF) { - util::throw_error(std::logic_error("Render supports G-types ONLY!")); - } - GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF); - - const cv::gimpl::RcDesc &ref = arg.get(); - switch (ref.shape) - { - case GShape::GMAT: return GArg(m_res.slot()[ref.id]); - case GShape::GARRAY: return GArg(m_res.slot().at(ref.id)); - default: - util::throw_error(std::logic_error("Unsupported GShape type")); - break; - } -} - -namespace { - class GRenderBackendImpl final: public cv::gapi::GBackend::Priv { - virtual void unpackKernel(ade::Graph &gr, - const ade::NodeHandle &op_node, - const cv::GKernelImpl &impl) override { - GRenderModel rm(gr); - auto render_impl = cv::util::any_cast(impl.opaque); - rm.metadata(op_node).set(cv::gimpl::render::ocv::RenderUnit{render_impl}); - } - - virtual EPtr compile(const ade::Graph &graph, - const cv::GCompileArgs& args, - const std::vector &nodes) const override { - - using namespace cv::gapi::wip::draw; - auto has_freetype_font = cv::gapi::getCompileArg(args); - std::unique_ptr ftpr; - if (has_freetype_font) - { -#ifndef HAVE_FREETYPE - throw std::runtime_error("Freetype not found"); -#else - ftpr.reset(new FTTextRender(has_freetype_font.value().path)); -#endif - } - return EPtr{new cv::gimpl::render::ocv::GRenderExecutable(graph, nodes, std::move(ftpr))}; - } - }; -} - -cv::gapi::GBackend cv::gapi::render::ocv::backend() { - static cv::gapi::GBackend this_backend(std::make_shared()); - return this_backend; -} diff --git a/modules/gapi/src/backends/render/grenderocvbackend.hpp b/modules/gapi/src/backends/render/grenderocvbackend.hpp deleted file mode 100644 index 69d388ffe6..0000000000 --- a/modules/gapi/src/backends/render/grenderocvbackend.hpp +++ /dev/null @@ -1,73 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2019 Intel Corporation - -#ifndef OPENCV_GAPI_GRENDEROCVBACKEND_HPP -#define OPENCV_GAPI_GRENDEROCVBACKEND_HPP - -#include -#include -#include - -#include "api/gorigin.hpp" -#include "backends/common/gbackend.hpp" -#include "compiler/gislandmodel.hpp" - -#include "backends/render/grenderocv.hpp" - -#include - -namespace cv -{ -namespace gimpl -{ -namespace render -{ -namespace ocv -{ - -struct RenderUnit -{ - static const char *name() { return "RenderUnit"; } - GCPUKernel k; -}; - -class GRenderExecutable final: public GIslandExecutable -{ - const ade::Graph &m_g; - GModel::ConstGraph m_gm; - std::unique_ptr m_ftpr; - - // The only executable stuff in this graph - // (assuming it is always single-op) - ade::NodeHandle this_nh; - - //// Actual data of all resources in graph (both internal and external) - Mag m_res; - - //// Execution helpers - GArg packArg(const GArg &arg); - -public: - GRenderExecutable(const ade::Graph &graph, - const std::vector &nodes, - std::unique_ptr&& ftpr); - - virtual inline bool canReshape() const override { return false; } - - virtual inline void reshape(ade::Graph&, const GCompileArgs&) override { - GAPI_Assert(false); // Not implemented yet - } - - virtual void run(std::vector &&input_objs, - std::vector &&output_objs) override; -}; - -} // namespace ocv -} // namespace render -} // namespace gimpl -} // namespace cv - -#endif // OPENCV_GAPI_GRENDEROCVBACKEND_HPP diff --git a/modules/gapi/src/compiler/gcompiled_priv.hpp b/modules/gapi/src/compiler/gcompiled_priv.hpp index f21bfc80bc..b08b1f9c59 100644 --- a/modules/gapi/src/compiler/gcompiled_priv.hpp +++ b/modules/gapi/src/compiler/gcompiled_priv.hpp @@ -38,6 +38,10 @@ class GAPI_EXPORTS GCompiled::Priv GMetaArgs m_outMetas; // inferred by compiler std::unique_ptr m_exec; + // NB: Used by python wrapper to clarify input/output types + GTypesInfo m_out_info; + GTypesInfo m_in_info; + void checkArgs(const cv::gimpl::GRuntimeArgs &args) const; public: @@ -55,6 +59,12 @@ public: const GMetaArgs& outMetas() const; const cv::gimpl::GModel::Graph& model() const; + + void setOutInfo(const GTypesInfo& info) { m_out_info = std::move(info); } + const GTypesInfo& outInfo() const { return m_out_info; } + + void setInInfo(const GTypesInfo& info) { m_in_info = std::move(info); } + const GTypesInfo& inInfo() const { return m_in_info; } }; } diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index 2f46ea873b..4d050dbabd 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -35,6 +35,7 @@ #include "executor/gexecutor.hpp" #include "executor/gstreamingexecutor.hpp" #include "backends/common/gbackend.hpp" +#include "backends/common/gmetabackend.hpp" // #if !defined(GAPI_STANDALONE) @@ -58,7 +59,8 @@ namespace for (const auto &b : pkg.backends()) { aux_pkg = combine(aux_pkg, b.priv().auxiliaryKernels()); } - return combine(pkg, aux_pkg); + // Always include built-in meta<> implementation + return combine(pkg, aux_pkg, cv::gimpl::meta::kernels()); }; auto has_use_only = cv::gapi::getCompileArg(args); @@ -238,6 +240,11 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c, // (no compound backend present here) m_e.addPass("kernels", "check_islands_content", passes::checkIslandsContent); + // Special stage for intrinsics handling + m_e.addPassStage("intrin"); + m_e.addPass("intrin", "desync", passes::intrinDesync); + m_e.addPass("intrin", "finalizeIntrin", passes::intrinFinalize); + //Input metas may be empty when a graph is compiled for streaming m_e.addPassStage("meta"); if (!m_metas.empty()) @@ -384,6 +391,9 @@ cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::generateGraph() { GModel::Graph(*g).metadata().set(OriginalInputMeta{m_metas}); } + // FIXME: remove m_args, remove GCompileArgs from backends' method signatures, + // rework backends to access GCompileArgs from graph metadata + GModel::Graph(*g).metadata().set(CompileArgs{m_args}); return g; } @@ -407,6 +417,19 @@ void cv::gimpl::GCompiler::compileIslands(ade::Graph &g, const cv::GCompileArgs GIslandModel::compileIslands(gim, g, args); } +static cv::GTypesInfo collectInfo(const cv::gimpl::GModel::ConstGraph& g, + const std::vector& nhs) { + cv::GTypesInfo info; + info.reserve(nhs.size()); + + ade::util::transform(nhs, std::back_inserter(info), [&g](const ade::NodeHandle& nh) { + const auto& data = g.metadata(nh).get(); + return cv::GTypeInfo{data.shape, data.kind}; + }); + + return info; +} + cv::GCompiled cv::gimpl::GCompiler::produceCompiled(GPtr &&pg) { // This is the final compilation step. Here: @@ -425,6 +448,8 @@ cv::GCompiled cv::gimpl::GCompiler::produceCompiled(GPtr &&pg) // an execution plan for it (backend-specific execution) // ...before call to produceCompiled(); + GModel::ConstGraph cgr(*pg); + const auto &outMetas = GModel::ConstGraph(*pg).metadata() .get().outMeta; std::unique_ptr pE(new GExecutor(std::move(pg))); @@ -433,6 +458,14 @@ cv::GCompiled cv::gimpl::GCompiler::produceCompiled(GPtr &&pg) GCompiled compiled; compiled.priv().setup(m_metas, outMetas, std::move(pE)); + + // NB: Need to store input/output GTypeInfo to allocate output arrays for python bindings + auto out_meta = collectInfo(cgr, cgr.metadata().get().out_nhs); + auto in_meta = collectInfo(cgr, cgr.metadata().get().in_nhs); + + compiled.priv().setOutInfo(std::move(out_meta)); + compiled.priv().setInInfo(std::move(in_meta)); + return compiled; } @@ -448,6 +481,16 @@ cv::GStreamingCompiled cv::gimpl::GCompiler::produceStreamingCompiled(GPtr &&pg) outMetas = GModel::ConstGraph(*pg).metadata().get().outMeta; } + + GModel::ConstGraph cgr(*pg); + + // NB: Need to store input/output GTypeInfo to allocate output arrays for python bindings + auto out_meta = collectInfo(cgr, cgr.metadata().get().out_nhs); + auto in_meta = collectInfo(cgr, cgr.metadata().get().in_nhs); + + compiled.priv().setOutInfo(std::move(out_meta)); + compiled.priv().setInInfo(std::move(in_meta)); + std::unique_ptr pE(new GStreamingExecutor(std::move(pg), m_args)); if (!m_metas.empty() && !outMetas.empty()) diff --git a/modules/gapi/src/compiler/gislandmodel.cpp b/modules/gapi/src/compiler/gislandmodel.cpp index aee0477e08..4d0feaea71 100644 --- a/modules/gapi/src/compiler/gislandmodel.cpp +++ b/modules/gapi/src/compiler/gislandmodel.cpp @@ -175,13 +175,26 @@ void GIslandModel::generateInitial(GIslandModel::Graph &g, { auto src_data_nh = in_edge->srcNode(); auto isl_slot_nh = data_to_slot.at(src_data_nh); - g.link(isl_slot_nh, nh); // no other data stored yet + auto isl_new_eh = g.link(isl_slot_nh, nh); // no other data stored yet + // Propagate some special metadata from the GModel to GIslandModel + // TODO: Make it a single place (a function) for both inputs/outputs? + // (since it is duplicated in the below code block) + if (src_g.metadata(in_edge).contains()) + { + const auto idx = src_g.metadata(in_edge).get().index; + g.metadata(isl_new_eh).set(DesyncIslEdge{idx}); + } } for (auto out_edge : src_op_nh->outEdges()) { auto dst_data_nh = out_edge->dstNode(); auto isl_slot_nh = data_to_slot.at(dst_data_nh); - g.link(nh, isl_slot_nh); + auto isl_new_eh = g.link(nh, isl_slot_nh); + if (src_g.metadata(out_edge).contains()) + { + const auto idx = src_g.metadata(out_edge).get().index; + g.metadata(isl_new_eh).set(DesyncIslEdge{idx}); + } } } // for(all_operations) } @@ -254,6 +267,9 @@ void GIslandModel::syncIslandTags(Graph &g, ade::Graph &orig_g) void GIslandModel::compileIslands(Graph &g, const ade::Graph &orig_g, const GCompileArgs &args) { GModel::ConstGraph gm(orig_g); + if (gm.metadata().contains()) { + util::throw_error(std::logic_error("FATAL: The graph has unresolved intrinsics")); + } auto original_sorted = gm.metadata().get(); for (auto nh : g.nodes()) @@ -341,26 +357,21 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO for (auto &&it: ade::util::zip(ade::util::toRange(in_desc), ade::util::toRange(in_vector))) { - // FIXME: Not every Island expects a cv::Mat instead of own::Mat on input - // This kludge should go as a result of de-ownification const cv::GRunArg& in_data_orig = std::get<1>(it); cv::GRunArg in_data; -#if !defined(GAPI_STANDALONE) switch (in_data_orig.index()) { case cv::GRunArg::index_of(): - in_data = cv::GRunArg{cv::make_rmat(cv::util::get(in_data_orig))}; - break; - case cv::GRunArg::index_of(): - in_data = cv::GRunArg{(cv::util::get(in_data_orig))}; + // FIXME: This whole construct is ugly, from + // its writing to a need in this in general + in_data = cv::GRunArg{ cv::make_rmat(cv::util::get(in_data_orig)) + , in_data_orig.meta + }; break; default: in_data = in_data_orig; break; } -#else - in_data = in_data_orig; -#endif // GAPI_STANDALONE in_objs.emplace_back(std::get<0>(it), std::move(in_data)); } for (auto &&it: ade::util::indexed(ade::util::toRange(out_desc))) @@ -369,9 +380,27 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO out.get(ade::util::checked_cast(ade::util::index(it)))); } run(std::move(in_objs), std::move(out_objs)); + + // Propagate in-graph meta down to the graph + // Note: this is not a complete implementation! Mainly this is a stub + // and the proper implementation should come later. + // + // Propagating the meta information here has its pros and cons. + // Pros: it works here uniformly for both regular and streaming cases, + // also for the majority of old-fashioned (synchronous) backends + // Cons: backends implementing the asynchronous run(IInput,IOutput) + // won't get it out of the box + cv::GRunArg::Meta stub_meta; + for (auto &&in_arg : in_vector) + { + stub_meta.insert(in_arg.meta.begin(), in_arg.meta.end()); + } + // Report output objects as "ready" to the executor, also post + // calculated in-graph meta for the objects for (auto &&it: out_objs) { - out.post(std::move(it.second)); // report output objects as "ready" to the executor + out.meta(it.second, stub_meta); + out.post(std::move(it.second)); } } diff --git a/modules/gapi/src/compiler/gislandmodel.hpp b/modules/gapi/src/compiler/gislandmodel.hpp index 6cf8f98667..e8eb73692b 100644 --- a/modules/gapi/src/compiler/gislandmodel.hpp +++ b/modules/gapi/src/compiler/gislandmodel.hpp @@ -142,6 +142,14 @@ public: // at that stage. virtual void handleNewStream() {}; // do nothing here by default + // This method is called for every IslandExecutable when + // the stream-based execution is stopped. + // All processing is guaranteed to be stopped by this moment, + // with no pending or running 'run()' processes ran in background. + // FIXME: This method is tightly bound to the GStreamingExecutor + // now. + virtual void handleStopStream() {} // do nothing here by default + virtual ~GIslandExecutable() = default; }; @@ -164,6 +172,10 @@ struct GIslandExecutable::IOutput: public GIslandExecutable::IODesc { virtual GRunArgP get(int idx) = 0; // Allocate (wrap) a new data object for output idx virtual void post(GRunArgP&&) = 0; // Release the object back to the framework (mark available) virtual void post(EndOfStream&&) = 0; // Post end-of-stream marker back to the framework + + // Assign accumulated metadata to the given output object. + // This method can only be called after get() and before post(). + virtual void meta(const GRunArgP&, const GRunArg::Meta &) = 0; }; // GIslandEmitter - a backend-specific thing which feeds data into @@ -222,8 +234,19 @@ struct IslandsCompiled static const char *name() { return "IslandsCompiled"; } }; +// This flag marks an edge in an GIslandModel as "desynchronized" +// i.e. it starts a new desynchronized subgraph +struct DesyncIslEdge +{ + static const char *name() { return "DesynchronizedIslandEdge"; } + + // Projection from GModel/DesyncEdge.index + int index; +}; + namespace GIslandModel { + using Graph = ade::TypedGraph < NodeKind , FusedIsland @@ -232,6 +255,7 @@ namespace GIslandModel , Emitter , Sink , IslandsCompiled + , DesyncIslEdge , ade::passes::TopologicalSortData >; @@ -244,6 +268,7 @@ namespace GIslandModel , Emitter , Sink , IslandsCompiled + , DesyncIslEdge , ade::passes::TopologicalSortData >; diff --git a/modules/gapi/src/compiler/gmodel.cpp b/modules/gapi/src/compiler/gmodel.cpp index 39dc1da33b..ea4eb880a4 100644 --- a/modules/gapi/src/compiler/gmodel.cpp +++ b/modules/gapi/src/compiler/gmodel.cpp @@ -23,12 +23,16 @@ namespace cv { namespace gimpl { -ade::NodeHandle GModel::mkOpNode(GModel::Graph &g, const GKernel &k, const std::vector &args, const std::string &island) +ade::NodeHandle GModel::mkOpNode(GModel::Graph &g, + const GKernel &k, + const std::vector &args, + const cv::util::any ¶ms, + const std::string &island) { ade::NodeHandle op_h = g.createNode(); g.metadata(op_h).set(NodeType{NodeType::OP}); //These extra empty {} are to please GCC (-Wmissing-field-initializers) - g.metadata(op_h).set(Op{k, args, {}, {}}); + g.metadata(op_h).set(Op{k, args, {}, {}, params}); if (!island.empty()) g.metadata(op_h).set(Island{island}); return op_h; @@ -73,7 +77,7 @@ ade::NodeHandle GModel::mkDataNode(GModel::Graph &g, const GShape shape) return data_h; } -void GModel::linkIn(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::size_t in_port) +ade::EdgeHandle GModel::linkIn(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::size_t in_port) { // Check if input is already connected for (const auto& in_e : opH->inEdges()) @@ -92,9 +96,11 @@ void GModel::linkIn(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::si // Replace an API object with a REF (G* -> GOBJREF) op.args[in_port] = cv::GArg(RcDesc{gm.rc, gm.shape, {}}); + + return eh; } -void GModel::linkOut(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::size_t out_port) +ade::EdgeHandle GModel::linkOut(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::size_t out_port) { // FIXME: check validity using kernel prototype @@ -117,6 +123,8 @@ void GModel::linkOut(Graph &g, ade::NodeHandle opH, ade::NodeHandle objH, std::s const auto min_out_size = std::max(op.outs.size(), storage_with_port); op.outs.resize(min_out_size, RcDesc{-1,GShape::GMAT,{}}); // FIXME: Invalid shape instead? op.outs[out_port] = RcDesc{gm.rc, gm.shape, {}}; + + return eh; } std::vector GModel::orderedInputs(const ConstGraph &g, ade::NodeHandle nh) @@ -206,26 +214,29 @@ ade::NodeHandle GModel::detail::dataNodeOf(const ConstLayoutGraph &g, const GOri return g.metadata().get().object_nodes.at(origin); } -void GModel::redirectReaders(Graph &g, ade::NodeHandle from, ade::NodeHandle to) +std::vector GModel::redirectReaders(Graph &g, ade::NodeHandle from, ade::NodeHandle to) { std::vector ehh(from->outEdges().begin(), from->outEdges().end()); + std::vector ohh; + ohh.reserve(ehh.size()); for (auto e : ehh) { auto dst = e->dstNode(); auto input = g.metadata(e).get(); g.erase(e); - linkIn(g, dst, to, input.port); + ohh.push_back(linkIn(g, dst, to, input.port)); } + return ohh; } -void GModel::redirectWriter(Graph &g, ade::NodeHandle from, ade::NodeHandle to) +ade::EdgeHandle GModel::redirectWriter(Graph &g, ade::NodeHandle from, ade::NodeHandle to) { GAPI_Assert(from->inEdges().size() == 1); auto e = from->inEdges().front(); auto op = e->srcNode(); auto output = g.metadata(e).get(); g.erase(e); - linkOut(g, op, to, output.port); + return linkOut(g, op, to, output.port); } GMetaArgs GModel::collectInputMeta(const GModel::ConstGraph &cg, ade::NodeHandle node) diff --git a/modules/gapi/src/compiler/gmodel.hpp b/modules/gapi/src/compiler/gmodel.hpp index 8f78ba49b7..d016766fb5 100644 --- a/modules/gapi/src/compiler/gmodel.hpp +++ b/modules/gapi/src/compiler/gmodel.hpp @@ -61,6 +61,7 @@ struct Op std::vector outs; // TODO: Introduce a new type for resource references cv::gapi::GBackend backend; + cv::util::any params; // Operation specific information }; struct Data @@ -210,6 +211,58 @@ struct CustomMetaFunction CM customOutMeta; }; +// This is a general flag indicating that this GModel has intrinsics. +// In the beginning of the compilation, it is a quick check to +// indicate there are intrinsics. +// +// In the end of the compilation, having this flag is fatal -- all +// intrinsics must be resolved. +struct HasIntrinsics +{ + static const char *name() { return "HasIntrinsicsFlag"; } +}; + +// This is a special tag for both DATA and OP nodes indicating +// which desynchronized path this node belongs to. +// This tag is set by a special complex pass intrinDesync/accept. +struct DesyncPath +{ + static const char *name() { return "DesynchronizedPath"; } + + // A zero-based index of the desynchronized path in the graph. + // Set by intrinDesync() compiler pass + int index; +}; + +// This is a special tag for graph Edges indicating that this +// particular edge starts a desynchronized path in the graph. +// At the execution stage, the data coming "through" these edges +// (virtually, of course, since our GModel edges never transfer the +// actual data, they just represent these transfers) is desynchronized +// from the rest of the pipeline, i.e. may be "lost" (stay unconsumed +// and then overwritten with some new data when streaming). +struct DesyncEdge +{ + static const char *name() { return "DesynchronizedEdge"; } + + // A zero-based index of the desynchronized path in the graph. + // Set by intrinDesync/apply() compiler pass + int index; +}; + +// This flag marks the island graph as "desynchronized" +struct Desynchronized +{ + static const char *name() { return "Desynchronized"; } +}; + +// Reference to compile args of the computation +struct CompileArgs +{ + static const char *name() { return "CompileArgs"; } + GCompileArgs args; +}; + namespace GModel { using Graph = ade::TypedGraph @@ -231,6 +284,11 @@ namespace GModel , CustomMetaFunction , Streaming , Deserialized + , HasIntrinsics + , DesyncPath + , DesyncEdge + , Desynchronized + , CompileArgs >; // FIXME: How to define it based on GModel??? @@ -253,6 +311,11 @@ namespace GModel , CustomMetaFunction , Streaming , Deserialized + , HasIntrinsics + , DesyncPath + , DesyncEdge + , Desynchronized + , CompileArgs >; // FIXME: @@ -262,7 +325,11 @@ namespace GModel // GAPI_EXPORTS for tests GAPI_EXPORTS void init (Graph& g); - GAPI_EXPORTS ade::NodeHandle mkOpNode(Graph &g, const GKernel &k, const std::vector& args, const std::string &island); + GAPI_EXPORTS ade::NodeHandle mkOpNode(Graph &g, + const GKernel &k, + const std::vector& args, + const cv::util::any& params, + const std::string &island); // Isn't used by the framework or default backends, required for external backend development GAPI_EXPORTS ade::NodeHandle mkDataNode(Graph &g, const GShape shape); @@ -273,11 +340,11 @@ namespace GModel // Clears logged messages of a node. GAPI_EXPORTS void log_clear(Graph &g, ade::NodeHandle node); - GAPI_EXPORTS void linkIn (Graph &g, ade::NodeHandle op, ade::NodeHandle obj, std::size_t in_port); - GAPI_EXPORTS void linkOut (Graph &g, ade::NodeHandle op, ade::NodeHandle obj, std::size_t out_port); + GAPI_EXPORTS ade::EdgeHandle linkIn (Graph &g, ade::NodeHandle op, ade::NodeHandle obj, std::size_t in_port); + GAPI_EXPORTS ade::EdgeHandle linkOut (Graph &g, ade::NodeHandle op, ade::NodeHandle obj, std::size_t out_port); - GAPI_EXPORTS void redirectReaders(Graph &g, ade::NodeHandle from, ade::NodeHandle to); - GAPI_EXPORTS void redirectWriter (Graph &g, ade::NodeHandle from, ade::NodeHandle to); + GAPI_EXPORTS std::vector redirectReaders(Graph &g, ade::NodeHandle from, ade::NodeHandle to); + GAPI_EXPORTS ade::EdgeHandle redirectWriter (Graph &g, ade::NodeHandle from, ade::NodeHandle to); GAPI_EXPORTS std::vector orderedInputs (const ConstGraph &g, ade::NodeHandle nh); GAPI_EXPORTS std::vector orderedOutputs(const ConstGraph &g, ade::NodeHandle nh); diff --git a/modules/gapi/src/compiler/gmodelbuilder.cpp b/modules/gapi/src/compiler/gmodelbuilder.cpp index 87e9ab55b8..5f8f3518fc 100644 --- a/modules/gapi/src/compiler/gmodelbuilder.cpp +++ b/modules/gapi/src/compiler/gmodelbuilder.cpp @@ -134,12 +134,19 @@ cv::gimpl::Unrolled cv::gimpl::unrollExpr(const GProtoArgs &ins, // Put the outputs object description of the node // so that they are not lost if they are not consumed by other operations + GAPI_Assert(call_p.m_k.outCtors.size() == call_p.m_k.outShapes.size()); for (const auto &it : ade::util::indexed(call_p.m_k.outShapes)) { std::size_t port = ade::util::index(it); GShape shape = ade::util::value(it); - GOrigin org { shape, node, port, {}, origin.kind }; + // FIXME: then use ZIP + HostCtor ctor = call_p.m_k.outCtors[port]; + + // NB: Probably this fixes all other "missing host ctor" + // problems. + // TODO: Clean-up the old workarounds if it really is. + GOrigin org {shape, node, port, std::move(ctor), origin.kind}; origins.insert(org); } @@ -286,7 +293,7 @@ ade::NodeHandle cv::gimpl::GModelBuilder::put_OpNode(const cv::GNode &node) { GAPI_Assert(node.shape() == GNode::NodeShape::CALL); const auto &call_p = node.call().priv(); - auto nh = cv::gimpl::GModel::mkOpNode(m_gm, call_p.m_k, call_p.m_args, node_p.m_island); + auto nh = cv::gimpl::GModel::mkOpNode(m_gm, call_p.m_k, call_p.m_args, call_p.m_params, node_p.m_island); m_graph_ops[&node_p] = nh; return nh; } diff --git a/modules/gapi/src/compiler/gobjref.hpp b/modules/gapi/src/compiler/gobjref.hpp index dd0939c439..bca6fa525e 100644 --- a/modules/gapi/src/compiler/gobjref.hpp +++ b/modules/gapi/src/compiler/gobjref.hpp @@ -16,15 +16,9 @@ namespace cv namespace gimpl { - // Union type for various user-defined type constructors (GArray, GOpaque, etc) - // FIXME: Replace construct-only API with a more generic one - // (probably with bits of introspection) - // Not required for non-user-defined types (GMat, GScalar, etc) - using HostCtor = util::variant - < util::monostate - , detail::ConstructVec - , detail::ConstructOpaque - >; + // HostCtor was there, but then moved to public + // Redeclare here to avoid changing tons of code + using HostCtor = cv::detail::HostCtor; using ConstVal = util::variant < util::monostate diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp index 2e9c016ceb..fa736d592e 100644 --- a/modules/gapi/src/compiler/gstreaming.cpp +++ b/modules/gapi/src/compiler/gstreaming.cpp @@ -8,6 +8,7 @@ #include "precomp.hpp" #include +#include // util::indexed #include // can_describe #include @@ -69,6 +70,11 @@ bool cv::GStreamingCompiled::Priv::pull(cv::GRunArgsP &&outs) return m_exec->pull(std::move(outs)); } +bool cv::GStreamingCompiled::Priv::pull(cv::GOptRunArgsP &&outs) +{ + return m_exec->pull(std::move(outs)); +} + bool cv::GStreamingCompiled::Priv::try_pull(cv::GRunArgsP &&outs) { return m_exec->try_pull(std::move(outs)); @@ -111,6 +117,58 @@ bool cv::GStreamingCompiled::pull(cv::GRunArgsP &&outs) return m_priv->pull(std::move(outs)); } +std::tuple cv::GStreamingCompiled::pull() +{ + // FIXME: Why it is not @ priv?? + GRunArgs run_args; + GRunArgsP outs; + const auto& out_info = m_priv->outInfo(); + run_args.reserve(out_info.size()); + outs.reserve(out_info.size()); + + for (auto&& info : out_info) + { + switch (info.shape) + { + case cv::GShape::GMAT: + { + run_args.emplace_back(cv::Mat{}); + outs.emplace_back(&cv::util::get(run_args.back())); + break; + } + case cv::GShape::GSCALAR: + { + run_args.emplace_back(cv::Scalar{}); + outs.emplace_back(&cv::util::get(run_args.back())); + break; + } + case cv::GShape::GARRAY: + { + switch (info.kind) + { + case cv::detail::OpaqueKind::CV_POINT2F: + run_args.emplace_back(cv::detail::VectorRef{std::vector{}}); + outs.emplace_back(cv::util::get(run_args.back())); + break; + default: + util::throw_error(std::logic_error("Unsupported kind for GArray")); + } + break; + } + default: + util::throw_error(std::logic_error("Only cv::GMat and cv::GScalar are supported for python output")); + } + } + + bool is_over = m_priv->pull(std::move(outs)); + return std::make_tuple(is_over, run_args); +} + +bool cv::GStreamingCompiled::pull(cv::GOptRunArgsP &&outs) +{ + return m_priv->pull(std::move(outs)); +} + bool cv::GStreamingCompiled::try_pull(cv::GRunArgsP &&outs) { return m_priv->try_pull(std::move(outs)); diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index 447bcda76e..be0869e663 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -28,6 +28,10 @@ class GAPI_EXPORTS GStreamingCompiled::Priv GMetaArgs m_outMetas; // inferred by compiler std::unique_ptr m_exec; + // NB: Used by python wrapper to clarify input/output types + GTypesInfo m_out_info; + GTypesInfo m_in_info; + public: void setup(const GMetaArgs &metaArgs, const GMetaArgs &outMetas, @@ -41,10 +45,17 @@ public: void setSource(GRunArgs &&args); void start(); bool pull(cv::GRunArgsP &&outs); + bool pull(cv::GOptRunArgsP &&outs); bool try_pull(cv::GRunArgsP &&outs); void stop(); bool running() const; + + void setOutInfo(const GTypesInfo& info) { m_out_info = std::move(info); } + const GTypesInfo& outInfo() const { return m_out_info; } + + void setInInfo(const GTypesInfo& info) { m_in_info = std::move(info); } + const GTypesInfo& inInfo() const { return m_in_info; } }; } // namespace cv diff --git a/modules/gapi/src/compiler/passes/exec.cpp b/modules/gapi/src/compiler/passes/exec.cpp index 755538bb46..f6a73489eb 100644 --- a/modules/gapi/src/compiler/passes/exec.cpp +++ b/modules/gapi/src/compiler/passes/exec.cpp @@ -20,6 +20,7 @@ #include // util::optional #include "logger.hpp" // GAPI_LOG +#include "api/gbackend_priv.hpp" // for canMerge() #include "compiler/gmodel.hpp" #include "compiler/gislandmodel.hpp" #include "compiler/passes/passes.hpp" @@ -54,11 +55,28 @@ namespace // Also check the cases backend can't handle // (e.x. GScalar connecting two fluid ops should split the graph) const GModel::ConstGraph g(src_graph); + if (g.metadata().contains()) { + // Fusion of a graph having a desynchronized path is + // definitely non-trivial + return false; + } const auto& active_backends = g.metadata().get().backends; - return active_backends.size() == 1 && - ade::util::all_of(g.nodes(), [&](ade::NodeHandle nh) { - return !g.metadata(nh).contains(); - }); + if (active_backends.size() != 1u) { + // More than 1 backend involved - non-trivial + return false; + } + const auto& has_island_tags = [&](ade::NodeHandle nh) { + return g.metadata(nh).contains(); + }; + if (ade::util::any_of(g.nodes(), has_island_tags)) { + // There are user-defined islands - non-trivial + return false; + } + if (active_backends.begin()->priv().controlsMerge()) { + // If the only backend controls Island Fusion on its own - non-trivial + return false; + } + return true; } void fuseTrivial(GIslandModel::Graph &g, const ade::Graph &src_graph) @@ -71,12 +89,12 @@ namespace all.insert(src_g.nodes().begin(), src_g.nodes().end()); - for (const auto nh : proto.in_nhs) + for (const auto& nh : proto.in_nhs) { all.erase(nh); in_ops.insert(nh->outNodes().begin(), nh->outNodes().end()); } - for (const auto nh : proto.out_nhs) + for (const auto& nh : proto.out_nhs) { all.erase(nh); out_ops.insert(nh->inNodes().begin(), nh->inNodes().end()); @@ -90,12 +108,12 @@ namespace auto ih = GIslandModel::mkIslandNode(g, std::move(isl)); - for (const auto nh : proto.in_nhs) + for (const auto& nh : proto.in_nhs) { auto slot = GIslandModel::mkSlotNode(g, nh); g.link(slot, ih); } - for (const auto nh : proto.out_nhs) + for (const auto& nh : proto.out_nhs) { auto slot = GIslandModel::mkSlotNode(g, nh); g.link(ih, slot); @@ -125,9 +143,9 @@ namespace }; bool canMerge(const GIslandModel::Graph &g, - const ade::NodeHandle a_nh, - const ade::NodeHandle /*slot_nh*/, - const ade::NodeHandle b_nh, + const ade::NodeHandle &a_nh, + const ade::NodeHandle &slot_nh, + const ade::NodeHandle &b_nh, const MergeContext &ctx = MergeContext()) { auto a_ptr = g.metadata(a_nh).get().object; @@ -142,8 +160,8 @@ namespace // Islands which cause a cycle can't be merged as well // (since the flag is set, the procedure already tried to // merge these islands in the past) - if (ade::util::contains(ctx.cycle_causers, std::make_pair(a_ptr, b_ptr))|| - ade::util::contains(ctx.cycle_causers, std::make_pair(b_ptr, a_ptr))) + if ( ade::util::contains(ctx.cycle_causers, std::make_pair(a_ptr, b_ptr)) + || ade::util::contains(ctx.cycle_causers, std::make_pair(b_ptr, a_ptr))) return false; // There may be user-defined islands. Initially user-defined @@ -163,7 +181,13 @@ namespace return false; } - // FIXME: add a backend-specified merge checker + // If available, run the backend-specified merge checker + const auto &this_backend_p = a_ptr->backend().priv(); + if ( this_backend_p.controlsMerge() + && !this_backend_p.allowsMerge(g, a_nh, slot_nh, b_nh)) + { + return false; + } return true; } @@ -205,10 +229,31 @@ namespace { using namespace std::placeholders; + // Before checking for candidates, find and ban neighbor nodes + // (input or outputs) which are connected via desynchronized + // edges. + GIsland::node_set nodes_with_desync_edges; + for (const auto& in_eh : nh->inEdges()) { + if (g.metadata(in_eh).contains()) { + nodes_with_desync_edges.insert(in_eh->srcNode()); + } + } + for (const auto& output_data_nh : nh->outNodes()) { + for (const auto &out_reader_eh : output_data_nh->outEdges()) { + if (g.metadata(out_reader_eh).contains()) { + nodes_with_desync_edges.insert(out_reader_eh->dstNode()); + } + } + } + // Find a first matching candidate GIsland for merge // among inputs - for (const auto& input_data_nh : nh->inNodes()) + for (const auto& in_eh : nh->inEdges()) { + if (ade::util::contains(nodes_with_desync_edges, in_eh->srcNode())) { + continue; // desync edges can never be fused + } + const auto& input_data_nh = in_eh->srcNode(); if (input_data_nh->inNodes().size() != 0) { // Data node must have a single producer only @@ -224,14 +269,17 @@ namespace // Ok, now try to find it among the outputs for (const auto& output_data_nh : nh->outNodes()) { - auto mergeTest = [&](ade::NodeHandle cons_nh) -> bool { - return canMerge(g, nh, output_data_nh, cons_nh, ctx); + auto mergeTest = [&](ade::EdgeHandle cons_eh) -> bool { + if (ade::util::contains(nodes_with_desync_edges, cons_eh->dstNode())) { + return false; // desync edges can never be fused + } + return canMerge(g, nh, output_data_nh, cons_eh->dstNode(), ctx); }; - auto cand_it = std::find_if(output_data_nh->outNodes().begin(), - output_data_nh->outNodes().end(), + auto cand_it = std::find_if(output_data_nh->outEdges().begin(), + output_data_nh->outEdges().end(), mergeTest); - if (cand_it != output_data_nh->outNodes().end()) - return std::make_tuple(*cand_it, + if (cand_it != output_data_nh->outEdges().end()) + return std::make_tuple((*cand_it)->dstNode(), output_data_nh, Direction::Out); } // for(outNodes) @@ -251,6 +299,7 @@ namespace ade::NodeHandle m_slot; ade::NodeHandle m_cons; + using Change = ChangeT; Change::List m_changes; struct MergeObjects @@ -423,10 +472,10 @@ namespace auto backend = m_gim.metadata(m_prod).get() .object->backend(); auto merged = std::make_shared(backend, - std::move(mo.all), - std::move(mo.in_ops), - std::move(mo.out_ops), - std::move(maybe_user_tag)); + std::move(mo.all), + std::move(mo.in_ops), + std::move(mo.out_ops), + std::move(maybe_user_tag)); // FIXME: move this debugging to some user-controllable log-level #ifdef DEBUG_MERGE merged->debug(); @@ -440,7 +489,9 @@ namespace m_prod->inEdges().end()); for (auto in_edge : input_edges) { - m_changes.enqueue(m_g, in_edge->srcNode(), new_nh); + // FIXME: Introduce a Relink primitive instead? + // (combining the both actions into one?) + m_changes.enqueue(m_g, in_edge->srcNode(), new_nh, in_edge); m_changes.enqueue(m_g, m_prod, in_edge); } @@ -450,7 +501,7 @@ namespace m_cons->outEdges().end()); for (auto out_edge : output_edges) { - m_changes.enqueue(m_g, new_nh, out_edge->dstNode()); + m_changes.enqueue(m_g, new_nh, out_edge->dstNode(), out_edge); m_changes.enqueue(m_g, m_cons, out_edge); } @@ -491,6 +542,10 @@ namespace m_changes.enqueue(m_g, non_opt_slot_nh, eh); } } + // FIXME: No metadata copied here (from where??) + // For DesyncIslEdges it still works, as these tags are + // placed to Data->Op edges and this one is an Op->Data + // edge. m_changes.enqueue(m_g, new_nh, non_opt_slot_nh); } @@ -502,7 +557,7 @@ namespace m_prod->outEdges().end()); for (auto extra_out : prod_extra_out_edges) { - m_changes.enqueue(m_g, new_nh, extra_out->dstNode()); + m_changes.enqueue(m_g, new_nh, extra_out->dstNode(), extra_out); m_changes.enqueue(m_g, m_prod, extra_out); } @@ -514,7 +569,7 @@ namespace m_cons->inEdges().end()); for (auto extra_in : cons_extra_in_edges) { - m_changes.enqueue(m_g, extra_in->srcNode(), new_nh); + m_changes.enqueue(m_g, extra_in->srcNode(), new_nh, extra_in); m_changes.enqueue(m_g, m_cons, extra_in); } @@ -557,10 +612,10 @@ namespace there_was_a_merge = false; // FIXME: move this debugging to some user-controllable log level - #ifdef DEBUG_MERGE +#ifdef DEBUG_MERGE GAPI_LOG_INFO(NULL, "Before next merge attempt " << iteration << "..."); merge_debug(g, iteration); - #endif +#endif iteration++; auto sorted = pass_helpers::topoSort(im); for (auto nh : sorted) @@ -600,9 +655,9 @@ namespace "merge(" << l_obj->name() << "," << r_obj->name() << ") was successful!"); action.commit(); - #ifdef DEBUG_MERGE +#ifdef DEBUG_MERGE GIslandModel::syncIslandTags(gim, g); - #endif +#endif there_was_a_merge = true; break; // start do{}while from the beginning } diff --git a/modules/gapi/src/compiler/passes/intrin.cpp b/modules/gapi/src/compiler/passes/intrin.cpp new file mode 100644 index 0000000000..5d2707570a --- /dev/null +++ b/modules/gapi/src/compiler/passes/intrin.cpp @@ -0,0 +1,305 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + + +#include "precomp.hpp" + +#include +#include +#include // GDesync intrinsic + +#include "compiler/gmodel.hpp" +#include "compiler/passes/passes.hpp" + +namespace desync { +namespace { + +// Drop the desynchronized node `nh` from the graph, reconnect the +// graph structure properly. This is a helper function which is used +// in both drop(g) and apply(g) passes. +// +// @return a vector of new edge handles connecting the "main" graph +// with its desynchronized part. +std::vector drop(cv::gimpl::GModel::Graph &g, + ade::NodeHandle nh) { + using namespace cv::gimpl; + + // What we need to do here: + // 1. Connect the readers of its produced data objects + // to the input data objects of desync; + // 2. Drop the data object it produces. + // 3. Drop the desync operation itself; + std::vector in_data_objs = GModel::orderedInputs(g, nh); + std::vector out_data_objs = GModel::orderedOutputs(g, nh); + std::vector new_links; + GAPI_Assert(in_data_objs.size() == out_data_objs.size()); + GAPI_DbgAssert(ade::util::all_of + (out_data_objs, + [&](const ade::NodeHandle &oh) { + return g.metadata(oh).contains(); + })); + // (1) + for (auto &&it: ade::util::zip(ade::util::toRange(in_data_objs), + ade::util::toRange(out_data_objs))) { + auto these_new_links = GModel::redirectReaders(g, + std::get<1>(it), + std::get<0>(it)); + new_links.insert(new_links.end(), + these_new_links.begin(), + these_new_links.end()); + } + // (2) + for (auto &&old_out_nh : out_data_objs) { + g.erase(old_out_nh); + } + // (3) + g.erase(nh); + + return new_links; +} + +// Tracing a desynchronizing subgraph is somewhat tricky and happens +// in both directions: downwards and upwards. +// +// The downward process is the basic one: we start with a "desync" +// OP node and go down to the graph using the "output" edges. We check +// if all nodes on this path [can] belong to this desynchronized path +// and don't overlap with others. +// +// An important contract to maintain is that the desynchronized part +// can't have any input references from the "main" graph part or any +// other desynchronized part in the graph. This contract is validated +// by checking every node's input which must belong to the same +// desynchronized part. +// +// Here is the pitfall of this check: +// +// v +// GMat_0 +// v +// +----------+ +// | desync() | <- This point originates the traceDown process +// +----------+ +// v +// GMat_0' <- This node will be tagged for this desync at +// :--------. step 0/1 +// v : <- The order how output nodes are visited is not +// +----------+ : specified, we can visit Op2() first (as there +// | Op1() | : is a direct link) bypassing visiting and tagging +// +----------+ : Op1() and GMat_1 +// v : +// GMat_1 : +// : .---' +// v v <- When we visit Op2() via the 2nd edge on this +// +----------+ graph, we check if all inputs belong to the same +// | Op2() | desynchronized graph and GMat_1 fails this check +// +----------+ (since the traceDown() process haven't visited +// it yet). +// +// Cases like this originate the traceUp() process: if we find an +// input node in our desynchronized path which doesn't belong to this +// path YET, it is not 100% a problem, and we need to trace it back +// (upwards) to see if it is really a case. + +// This recursive function checks the desync_id in the graph upwards. +// The process doesn't continue for nodes which have a valid +// desync_id already. +// The process only continues for nodes which have no desync_id +// assigned. If there's no such nodes anymore, the procedure is +// considered complete and a list of nodes to tag is returned to the +// caller. +// +// If NO inputs of this node have a valid desync_id, the desync +// invariant is broken and the function throws. +void traceUp(cv::gimpl::GModel::Graph &g, + const ade::NodeHandle &nh, + int desync_id, + std::vector &path) { + using namespace cv::gimpl; + + GAPI_Assert(!nh->inNodes().empty() + && "traceUp: a desynchronized part of the graph is not isolated?"); + + if (g.metadata(nh).contains()) { + // We may face nodes which have DesyncPath already visited during + // this recursive process (e.g. via some other output or branch in the + // subgraph) + if (g.metadata(nh).get().index != desync_id) { + GAPI_Assert(false && "Desynchronization can't be nested!"); + } + return; // This object belongs to the desync path - exit early. + } + + // Regardless of the result, put this nh to the path + path.push_back(nh); + + // Check if the input nodes are OK + std::vector nodes_to_trace; + nodes_to_trace.reserve(nh->inNodes().size()); + for (auto &&in_nh : nh->inNodes()) { + if (g.metadata(in_nh).contains()) { + // We may face nodes which have DesyncPath already visited during + // this recursive process (e.g. via some other output or branch in the + // subgraph) + GAPI_Assert(g.metadata(in_nh).get().index == desync_id + && "Desynchronization can't be nested!"); + } else { + nodes_to_trace.push_back(in_nh); + } + } + + // If there are nodes to trace, continue the recursion + for (auto &&up_nh : nodes_to_trace) { + traceUp(g, up_nh, desync_id, path); + } +} + +// This recursive function propagates the desync_id down to the graph +// starting at nh, and also checks: +// - if this desync path is isolated; +// - if this desync path is not overlapped. +// It also originates the traceUp() process at the points of +// uncertainty (as described in the comment above). +void traceDown(cv::gimpl::GModel::Graph &g, + const ade::NodeHandle &nh, + int desync_id) { + using namespace cv::gimpl; + + if (g.metadata(nh).contains()) { + // We may face nodes which have DesyncPath already visited during + // this recursive process (e.g. via some other output or branch in the + // subgraph) + GAPI_Assert(g.metadata(nh).get().index == desync_id + && "Desynchronization can't be nested!"); + } else { + g.metadata(nh).set(DesyncPath{desync_id}); + } + + // All inputs of this data object must belong to the same + // desync path. + for (auto &&in_nh : nh->inNodes()) { + // If an input object is not assigned to this desync path, + // it does not means that the object doesn't belong to + // this path. Check it. + std::vector path_up; + traceUp(g, in_nh, desync_id, path_up); + // We get here on success. Just set the proper tags for + // the identified input path. + for (auto &&up_nh : path_up) { + g.metadata(up_nh).set(DesyncPath{desync_id}); + } + } + + // Propagate the tag & check down + for (auto &&out_nh : nh->outNodes()) { + traceDown(g, out_nh, desync_id); + } +} + +// Streaming case: ensure the graph has proper isolation of the +// desynchronized parts, set proper Edge metadata hints for +// GStreamingExecutable +void apply(cv::gimpl::GModel::Graph &g) { + using namespace cv::gimpl; + + // Stage 0. Trace down the desync operations in the graph. + // Tag them with their unique (per graph) identifiers. + int total_desync = 0; + for (auto &&nh : g.nodes()) { + if (g.metadata(nh).get().t == NodeType::OP) { + const auto &op = g.metadata(nh).get(); + if (op.k.name == cv::gapi::streaming::detail::GDesync::id()) { + GAPI_Assert(!g.metadata(nh).contains() + && "Desynchronization can't be nested!"); + const int this_desync_id = total_desync++; + g.metadata(nh).set(DesyncPath{this_desync_id}); + for (auto &&out_nh: nh->outNodes()) { + traceDown(g, out_nh, this_desync_id); + } + } // if (desync) + } // if(OP) + } // for(nodes) + + // Tracing is done for all desync ops in the graph now. + // Stage 1. Drop the desync operations from the graph, but mark + // the desynchronized edges a special way. + // The desynchronized edge is the edge which connects a main + // subgraph data with a desynchronized subgraph data. + std::vector nodes(g.nodes().begin(), g.nodes().end()); + for (auto &&nh : nodes) { + if (nh == nullptr) { + // Some nodes could be dropped already during the procedure + // thanks ADE their NodeHandles updated automatically + continue; + } + if (g.metadata(nh).get().t == NodeType::OP) { + const auto &op = g.metadata(nh).get(); + if (op.k.name == cv::gapi::streaming::detail::GDesync::id()) { + auto index = g.metadata(nh).get().index; + auto new_links = drop(g, nh); + for (auto &&eh : new_links) { + g.metadata(eh).set(DesyncEdge{index}); + } + } // if (desync) + } // if (Op) + } // for(nodes) + + // Stage 2. Put a synchronized tag if there were changes applied + if (total_desync > 0) { + g.metadata().set(Desynchronized{}); + } +} + +// Probably the simplest case: desync makes no sense in the regular +// compilation process, so just drop all its occurences in the graph, +// reconnecting nodes properly. +void drop(cv::gimpl::GModel::Graph &g) { + // FIXME: LOG here that we're dropping the desync operations as + // they have no sense when compiling in the regular mode. + using namespace cv::gimpl; + std::vector nodes(g.nodes().begin(), g.nodes().end()); + for (auto &&nh : nodes) { + if (nh == nullptr) { + // Some nodes could be dropped already during the procedure + // thanks ADE their NodeHandles updated automatically + continue; + } + if (g.metadata(nh).get().t == NodeType::OP) { + const auto &op = g.metadata(nh).get(); + if (op.k.name == cv::gapi::streaming::detail::GDesync::id()) { + drop(g, nh); + } // if (desync) + } // if (Op) + } // for(nodes) +} + +} // anonymous namespace +} // namespace desync + +void cv::gimpl::passes::intrinDesync(ade::passes::PassContext &ctx) { + GModel::Graph gr(ctx.graph); + if (!gr.metadata().contains()) + return; + + gr.metadata().contains() + ? desync::apply(gr) // Streaming compilation + : desync::drop(gr); // Regular compilation +} + +// Clears the HasIntrinsics flag if all intrinsics have been handled. +void cv::gimpl::passes::intrinFinalize(ade::passes::PassContext &ctx) { + GModel::Graph gr(ctx.graph); + for (auto &&nh : gr.nodes()) { + if (gr.metadata(nh).get().t == NodeType::OP) { + const auto &op = gr.metadata(nh).get(); + if (is_intrinsic(op.k.name)) { + return; + } + } + } + // If reached here, really clear the flag + gr.metadata().erase(); +} diff --git a/modules/gapi/src/compiler/passes/kernels.cpp b/modules/gapi/src/compiler/passes/kernels.cpp index 69b339fb1e..837e21f19a 100644 --- a/modules/gapi/src/compiler/passes/kernels.cpp +++ b/modules/gapi/src/compiler/passes/kernels.cpp @@ -14,6 +14,7 @@ #include // compound::backend() #include // GKernelPackage #include // GNetPackage +#include // GDesync intrinsic #include "compiler/gmodel.hpp" #include "compiler/passes/passes.hpp" @@ -24,6 +25,20 @@ #include "logger.hpp" // GAPI_LOG #include "api/gproto_priv.hpp" // is_dynamic, rewrap +namespace +{ + // FIXME: This may be not the right design choice, but so far it works + const std::vector known_intrinsics = { + cv::gapi::streaming::detail::GDesync::id() + }; +} +bool cv::gimpl::is_intrinsic(const std::string &s) { + // FIXME: This search might be better in time once we start using string + return std::find(known_intrinsics.begin(), + known_intrinsics.end(), + s) != known_intrinsics.end(); +} + namespace { struct ImplInfo @@ -126,12 +141,18 @@ void cv::gimpl::passes::bindNetParams(ade::passes::PassContext &ctx, continue; pgr.metadata(nh).set(NetworkParams{it->params}); + op.backend = it->backend; } } } -// This pass, given the kernel package, selects a kernel implementation -// for every operation in the graph +// This pass, given the kernel package, selects a kernel +// implementation for every operation in the graph +// +// Starting OpenCV 4.3, G-API may have some special "intrinsic" +// operations. Those can be implemented by backends as regular +// kernels, but if not, they are handled by the framework itself in +// its optimization/execution passes. void cv::gimpl::passes::resolveKernels(ade::passes::PassContext &ctx, const gapi::GKernelPackage &kernels) { @@ -142,14 +163,44 @@ void cv::gimpl::passes::resolveKernels(ade::passes::PassContext &ctx, { if (gr.metadata(nh).get().t == NodeType::OP) { + // If the operation is known to be intrinsic and is NOT + // implemented in the package, just skip it - there should + // be some pass which handles it. auto &op = gr.metadata(nh).get(); - cv::gapi::GBackend selected_backend; - cv::GKernelImpl selected_impl; - std::tie(selected_backend, selected_impl) = kernels.lookup(op.k.name); + if (is_intrinsic(op.k.name) && !kernels.includesAPI(op.k.name)) { + gr.metadata().set(HasIntrinsics{}); + continue; + } + // FIXME: And this logic is terribly wrong. The right + // thing is to assign an intrinsic to a particular island + // if and only if it is: + // (a) surrounded by nodes of backend X, AND + // (b) is supported by backend X. + // Here we may have multiple backends supporting an + // intrinsic but only one of those gets selected. And + // this is exactly a situation we need multiple versions + // of the same kernel to be presented in the kernel + // package (as it was designed originally). - selected_backend.priv().unpackKernel(ctx.graph, nh, selected_impl); - op.backend = selected_backend; - active_backends.insert(selected_backend); + cv::GKernelImpl selected_impl; + + if (op.backend == cv::gapi::GBackend()) { + std::tie(op.backend, selected_impl) = kernels.lookup(op.k.name); + } else { + // FIXME: This needs to be reworked properly + // Lookup for implementation from the pre-assinged backend + cv::gapi::GBackend dummy; + std::tie(dummy, selected_impl) = op.backend.priv() + .auxiliaryKernels().lookup(op.k.name); + // FIXME: Warning here! + // This situation may happen when NN (infer) backend was assigned + // by tag in bindNetParams (see above) but at this stage the operation + // lookup resulted in another backend (and it is perfectly valid when + // we have multiple NN backends available). + } + + op.backend.priv().unpackKernel(ctx.graph, nh, selected_impl); + active_backends.insert(op.backend); if (gr.metadata().contains()) { @@ -181,6 +232,12 @@ void cv::gimpl::passes::expandKernels(ade::passes::PassContext &ctx, const gapi: if (gr.metadata(nh).get().t == NodeType::OP) { const auto& op = gr.metadata(nh).get(); + // FIXME: Essentially the same problem as in the above resolveKernels + if (is_intrinsic(op.k.name) && !kernels.includesAPI(op.k.name)) { + // Note: There's no need to set HasIntrinsics flag here + // since resolveKernels would do it later. + continue; + } cv::gapi::GBackend selected_backend; cv::GKernelImpl selected_impl; diff --git a/modules/gapi/src/compiler/passes/passes.hpp b/modules/gapi/src/compiler/passes/passes.hpp index 84142fc055..8f187f6bb7 100644 --- a/modules/gapi/src/compiler/passes/passes.hpp +++ b/modules/gapi/src/compiler/passes/passes.hpp @@ -31,7 +31,11 @@ namespace gapi { struct GNetPackage; } // namespace gapi -namespace gimpl { namespace passes { +namespace gimpl { + +bool is_intrinsic(const std::string &op_name); + +namespace passes { void dumpDot(const ade::Graph &g, std::ostream& os); void dumpDot(ade::passes::PassContext &ctx, std::ostream& os); @@ -66,6 +70,9 @@ void applyTransformations(ade::passes::PassContext &ctx, void addStreaming(ade::passes::PassContext &ctx); +void intrinDesync(ade::passes::PassContext &ctx); +void intrinFinalize(ade::passes::PassContext &ctx); + }} // namespace gimpl::passes } // namespace cv diff --git a/modules/gapi/src/compiler/transactions.hpp b/modules/gapi/src/compiler/transactions.hpp index 54af8a6e69..bdc1723e19 100644 --- a/modules/gapi/src/compiler/transactions.hpp +++ b/modules/gapi/src/compiler/transactions.hpp @@ -14,6 +14,7 @@ #include +#include "opencv2/gapi/util/util.hpp" // Seq #include "opencv2/gapi/own/assert.hpp" enum class Direction: int {Invalid, In, Out}; @@ -21,8 +22,50 @@ enum class Direction: int {Invalid, In, Out}; //////////////////////////////////////////////////////////////////////////// //// // TODO: Probably it can be moved to ADE +template +class Preserved +{ + using S = typename cv::detail::MkSeq::type; + std::tuple...> m_data; -namespace Change + template + cv::util::optional get(ade::ConstTypedGraph g, H h) { + return g.metadata(h).template contains() + ? cv::util::make_optional(g.metadata(h).template get()) + : cv::util::optional{}; + } + template + int set(ade::TypedGraph &g, H &h) { + const auto &opt = std::get(m_data); + if (opt.has_value()) + g.metadata(h).set(opt.value()); + return 0; + } + template + void copyTo_impl(ade::TypedGraph &g, H h, cv::detail::Seq) { + int unused[] = {0, set(g, h)...}; + (void) unused; + } +public: + Preserved(const ade::Graph &g, H h) { + ade::ConstTypedGraph tg(g); + m_data = std::make_tuple(get(tg, h)...); + } + void copyTo(ade::Graph &g, H h) { + ade::TypedGraph tg(g); + copyTo_impl(tg, h, S{}); + } +}; +// Do nothing if there's no metadata +template +class Preserved { +public: + Preserved(const ade::Graph &, H) {} + void copyTo(ade::Graph &, H) {} +}; + +template +struct ChangeT { struct Base { @@ -31,6 +74,8 @@ namespace Change virtual ~Base() = default; }; + template using Preserved = ::Preserved; + class NodeCreated final: public Base { ade::NodeHandle m_node; @@ -39,11 +84,7 @@ namespace Change virtual void rollback(ade::Graph &g) override { g.erase(m_node); } }; - // NB: Drops all metadata stored in the EdgeHandle, - // which is not restored even in the rollback - - // FIXME: either add a way for users to preserve meta manually - // or extend ADE to manipulate with meta such way + // FIXME: maybe extend ADE to clone/copy the whole metadata? class DropLink final: public Base { ade::NodeHandle m_node; @@ -51,13 +92,15 @@ namespace Change ade::NodeHandle m_sibling; + Preserved m_meta; + public: DropLink(ade::Graph &g, const ade::NodeHandle &node, const ade::EdgeHandle &edge) - : m_node(node), m_dir(node == edge->srcNode() - ? Direction::Out - : Direction::In) + : m_node(node) + , m_dir(node == edge->srcNode() ? Direction::Out : Direction::In) + , m_meta(g, edge) { m_sibling = (m_dir == Direction::In ? edge->srcNode() @@ -67,12 +110,17 @@ namespace Change virtual void rollback(ade::Graph &g) override { + // FIXME: Need to preserve metadata here! + // GIslandModel edges now have metadata + ade::EdgeHandle eh; switch(m_dir) { - case Direction::In: g.link(m_sibling, m_node); break; - case Direction::Out: g.link(m_node, m_sibling); break; + case Direction::In: eh = g.link(m_sibling, m_node); break; + case Direction::Out: eh = g.link(m_node, m_sibling); break; default: GAPI_Assert(false); } + GAPI_Assert(eh != nullptr); + m_meta.copyTo(g, eh); } }; @@ -82,10 +130,15 @@ namespace Change public: NewLink(ade::Graph &g, - const ade::NodeHandle &prod, - const ade::NodeHandle &cons) + const ade::NodeHandle &prod, + const ade::NodeHandle &cons, + const ade::EdgeHandle ©_from = ade::EdgeHandle()) : m_edge(g.link(prod, cons)) { + if (copy_from != nullptr) + { + Preserved(g, copy_from).copyTo(g, m_edge); + } } virtual void rollback(ade::Graph &g) override @@ -141,7 +194,7 @@ namespace Change } } }; -} // namespace Change +}; // struct Change //////////////////////////////////////////////////////////////////////////// #endif // OPENCV_GAPI_COMPILER_TRANSACTIONS_HPP diff --git a/modules/gapi/src/executor/conc_queue.hpp b/modules/gapi/src/executor/conc_queue.hpp index 5de50ef34b..9875e8245a 100644 --- a/modules/gapi/src/executor/conc_queue.hpp +++ b/modules/gapi/src/executor/conc_queue.hpp @@ -119,8 +119,7 @@ void concurrent_bounded_queue::set_capacity(std::size_t capacity) { // Clear the queue. Similar to the TBB version, this method is not // thread-safe. template -void concurrent_bounded_queue::clear() -{ +void concurrent_bounded_queue::clear() { m_data = std::queue{}; } diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index d9f5cfafe6..66f3b24771 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -12,6 +12,8 @@ #include #include + +#include "api/gproto_priv.hpp" // ptr(GRunArgP) #include "executor/gexecutor.hpp" #include "compiler/passes/passes.hpp" @@ -105,6 +107,9 @@ void bindInArgExec(Mag& mag, const RcDesc &rc, const GRunArg &arg) mag_rmat = util::get(arg); break; default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } + // FIXME: has to take extra care about meta here for this particuluar + // case, just because this function exists at all + mag.meta()[rc.id] = arg.meta; } void bindOutArgExec(Mag& mag, const RcDesc &rc, const GRunArgP &arg) @@ -131,7 +136,7 @@ cv::GRunArgP getObjPtrExec(Mag& mag, const RcDesc &rc) { return getObjPtr(mag, rc); } - return GRunArgP(&mag.template slot()[rc.id]); + return GRunArgP(&mag.slot()[rc.id]); } void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) @@ -155,6 +160,25 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?")); } } + +void assignMetaStubExec(Mag& mag, const RcDesc &rc, const cv::GRunArg::Meta &meta) { + switch (rc.shape) + { + case GShape::GARRAY: mag.meta()[rc.id] = meta; break; + case GShape::GOPAQUE: mag.meta()[rc.id] = meta; break; + case GShape::GSCALAR: mag.meta()[rc.id] = meta; break; + case GShape::GFRAME: mag.meta()[rc.id] = meta; break; + case GShape::GMAT: + mag.meta() [rc.id] = meta; + mag.meta()[rc.id] = meta; +#if !defined(GAPI_STANDALONE) + mag.meta()[rc.id] = meta; +#endif + break; + default: util::throw_error(std::logic_error("Unsupported GShape type")); break; + } +} + } // anonymous namespace }}} // namespace cv::gimpl::magazine @@ -231,11 +255,28 @@ public: class cv::gimpl::GExecutor::Output final: public cv::gimpl::GIslandExecutable::IOutput { cv::gimpl::Mag &mag; - virtual GRunArgP get(int idx) override { return magazine::getObjPtrExec(mag, desc()[idx]); } - virtual void post(GRunArgP&&) override { } // Do nothing here - virtual void post(EndOfStream&&) override {} // Do nothing here too + std::unordered_map out_idx; + + GRunArgP get(int idx) override + { + auto r = magazine::getObjPtrExec(mag, desc()[idx]); + // Remember the output port for this output object + out_idx[cv::gimpl::proto::ptr(r)] = idx; + return r; + } + void post(GRunArgP&&) override { } // Do nothing here + void post(EndOfStream&&) override {} // Do nothing here too + void meta(const GRunArgP &out, const GRunArg::Meta &m) override + { + const auto idx = out_idx.at(cv::gimpl::proto::ptr(out)); + magazine::assignMetaStubExec(mag, desc()[idx], m); + } public: - Output(cv::gimpl::Mag &m, const std::vector &rcs) : mag(m) { set(rcs); } + Output(cv::gimpl::Mag &m, const std::vector &rcs) + : mag(m) + { + set(rcs); + } }; void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args) @@ -330,7 +371,7 @@ void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args) // Run the script for (auto &op : m_ops) { - // (5) + // (5), (6) Input i{m_res, op.in_objects}; Output o{m_res, op.out_objects}; op.isl_exec->run(i, o); diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index afdebee020..70686699d0 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -6,12 +6,17 @@ #include "precomp.hpp" +#include // make_shared #include #include #include +#if !defined(GAPI_STANDALONE) +#include // GCopy -- FIXME - to be removed! +#endif // GAPI_STANDALONE + #include "api/gproto_priv.hpp" // ptr(GRunArgP) #include "compiler/passes/passes.hpp" #include "backends/common/gbackend.hpp" // createMat @@ -60,14 +65,27 @@ public: struct DataQueue { static const char *name() { return "StreamingDataQueue"; } + enum tag { DESYNC }; // Enum of 1 element: purely a syntax sugar explicit DataQueue(std::size_t capacity) { - if (capacity) { - q.set_capacity(capacity); + // Note: `ptr` is shared, while the `q` is a shared + auto ptr = std::make_shared(); + if (capacity != 0) { + ptr->set_capacity(capacity); } + q = std::move(ptr); + } + explicit DataQueue(tag t) + : q(new cv::gimpl::stream::DesyncQueue()) { + GAPI_Assert(t == DESYNC); } - cv::gimpl::stream::Q q; + // FIXME: ADE metadata requires types to be copiable + std::shared_ptr q; +}; + +struct DesyncSpecialCase { + static const char *name() { return "DesyncSpecialCase"; } }; std::vector reader_queues( ade::Graph &g, @@ -77,7 +95,7 @@ std::vector reader_queues( ade::Graph &g, std::vector result; for (auto &&out_eh : obj->outEdges()) { - result.push_back(&qgr.metadata(out_eh).get().q); + result.push_back(qgr.metadata(out_eh).get().q.get()); } return result; } @@ -90,7 +108,7 @@ std::vector input_queues( ade::Graph &g, for (auto &&in_eh : obj->inEdges()) { result.push_back(qgr.metadata(in_eh).contains() - ? &qgr.metadata(in_eh).get().q + ? qgr.metadata(in_eh).get().q.get() : nullptr); } return result; @@ -133,6 +151,77 @@ void sync_data(cv::GRunArgs &results, cv::GRunArgsP &outputs) } } +// FIXME: Is there a way to derive function from its GRunArgsP version? +template using O = cv::util::optional; +void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs) +{ + namespace own = cv::gapi::own; + + for (auto && it : ade::util::zip(ade::util::toRange(outputs), + ade::util::toRange(r.args), + ade::util::toRange(r.flags))) + { + auto &out_obj = std::get<0>(it); + auto &res_obj = std::get<1>(it); + bool available = std::get<2>(it); + + using T = cv::GOptRunArgP; +#define HANDLE_CASE(Type) \ + case T::index_of*>(): \ + if (available) { \ + *cv::util::get*>(out_obj) \ + = cv::util::make_optional(std::move(cv::util::get(res_obj))); \ + } else { \ + cv::util::get*>(out_obj)->reset(); \ + } + + // FIXME: this conversion should be unified + switch (out_obj.index()) + { + HANDLE_CASE(cv::Scalar); break; + HANDLE_CASE(cv::RMat); break; + + case T::index_of*>(): { + // Mat: special handling. + auto &mat_opt = *cv::util::get*>(out_obj); + if (available) { + auto q_map = cv::util::get(res_obj).access(cv::RMat::Access::R); + // FIXME: Copy! Maybe we could do some optimization for this case! + // e.g. don't handle RMat for last ilsand in the graph. + // It is not always possible though. + mat_opt = cv::util::make_optional(cv::gimpl::asMat(q_map).clone()); + } else { + mat_opt.reset(); + } + } break; + case T::index_of(): { + // std::vector<>: special handling + auto &vec_opt = cv::util::get(out_obj); + if (available) { + vec_opt.mov(cv::util::get(res_obj)); + } else { + vec_opt.reset(); + } + } break; + case T::index_of(): { + // std::vector<>: special handling + auto &opq_opt = cv::util::get(out_obj); + if (available) { + opq_opt.mov(cv::util::get(res_obj)); + } else { + opq_opt.reset(); + } + } break; + default: + // ...maybe because of STANDALONE mode. + GAPI_Assert(false && "This value type is not supported!"); + break; + } + } +#undef HANDLE_CASE +} + + // Pops an item from every input queue and combine it to the final // result. Blocks the current thread. Returns true if the vector has // been obtained successfully and false if a Stop message has been @@ -206,12 +295,39 @@ class QueueReader bool m_finishing = false; // Set to true once a "soft" stop is received std::vector m_cmd; + void rewindToStop(std::vector &in_queues, + const std::size_t this_id); + public: - bool getInputVector(std::vector &in_queues, - cv::GRunArgs &in_constants, - cv::GRunArgs &isl_inputs); + bool getInputVector (std::vector &in_queues, + cv::GRunArgs &in_constants, + cv::GRunArgs &isl_inputs); + + bool getResultsVector(std::vector &in_queues, + const std::vector &in_mapping, + const std::size_t out_size, + cv::GRunArgs &out_results); }; +// This method handles a stop sign got from some input +// island. Reiterate through all _remaining valid_ queues (some of +// them can be set to nullptr already -- see handling in +// getInputVector) and rewind data to every Stop sign per queue. +void QueueReader::rewindToStop(std::vector &in_queues, + const std::size_t this_id) +{ + for (auto &&qit : ade::util::indexed(in_queues)) + { + auto id2 = ade::util::index(qit); + auto &q2 = ade::util::value(qit); + if (this_id == id2) continue; + + Cmd cmd; + while (q2 && !cv::util::holds_alternative(cmd)) + q2->pop(cmd); + } +} + bool QueueReader::getInputVector(std::vector &in_queues, cv::GRunArgs &in_constants, cv::GRunArgs &isl_inputs) @@ -234,16 +350,14 @@ bool QueueReader::getInputVector(std::vector &in_queues, // value-initialized scalar) // It can also hold a constant value received with // Stop::Kind::CNST message (see above). - // FIXME: Variant move problem - isl_inputs[id] = const_cast(in_constants[id]); + isl_inputs[id] = in_constants[id]; continue; } q->pop(m_cmd[id]); if (!cv::util::holds_alternative(m_cmd[id])) { - // FIXME: Variant move problem - isl_inputs[id] = const_cast(cv::util::get(m_cmd[id])); + isl_inputs[id] = cv::util::get(m_cmd[id]); } else // A Stop sign { @@ -266,25 +380,12 @@ bool QueueReader::getInputVector(std::vector &in_queues, // NEXT time (on a next call to getInputVector()), the // "q==nullptr" check above will be triggered, but now // we need to make it manually: - isl_inputs[id] = const_cast(in_constants[id]); + isl_inputs[id] = in_constants[id]; } else { GAPI_Assert(stop.kind == Stop::Kind::HARD); - // Just got a stop sign. Reiterate through all - // _remaining valid_ queues (some of them can be - // set to nullptr already -- see above) and rewind - // data to every Stop sign per queue - for (auto &&qit : ade::util::indexed(in_queues)) - { - auto id2 = ade::util::index(qit); - auto &q2 = ade::util::value(qit); - if (id == id2) continue; - - Cmd cmd2; - while (q2 && !cv::util::holds_alternative(cmd2)) - q2->pop(cmd2); - } + rewindToStop(in_queues, id); // After queues are read to the proper indicator, // indicate end-of-stream return false; @@ -303,6 +404,60 @@ bool QueueReader::getInputVector(std::vector &in_queues, return true; // A regular case - there is data to process. } +// This is a special method to obtain a result vector +// for the entire pipeline's outputs. +// +// After introducing desync(), the pipeline output's vector +// can be produced just partially. Also, if a desynchronized +// path has multiple outputs for the pipeline, _these_ outputs +// should still come synchronized to the end user (via pull()) +// +// +// This method handles all this. +// It takes a number of input queues, which may or may not be +// equal to the number of pipeline outputs (<=). +// It also takes indexes saying which queue produces which +// output in the resulting pipeline. +// +// `out_results` is always produced with the size of full output +// vector. In the desync case, the number of in_queues will +// be less than this size and some of the items won't be produced. +// In the sync case, there will be a 1-1 mapping. +// +// In the desync case, there _will be_ multiple collector threads +// calling this method, and pushing their whole-pipeline outputs +// (_may be_ partially filled) to the same final output queue. +// The receiver part at the GStreamingExecutor level won't change +// because of that. +bool QueueReader::getResultsVector(std::vector &in_queues, + const std::vector &in_mapping, + const std::size_t out_size, + cv::GRunArgs &out_results) +{ + m_cmd.resize(out_size); + for (auto &&it : ade::util::indexed(in_queues)) + { + auto ii = ade::util::index(it); + auto oi = in_mapping[ii]; + auto &q = ade::util::value(it); + q->pop(m_cmd[oi]); + if (!cv::util::holds_alternative(m_cmd[oi])) + { + out_results[oi] = std::move(cv::util::get(m_cmd[oi])); + } + else // A Stop sign + { + // In theory, the CNST should never reach here. + // Collector thread never handles the inputs directly + // (collector's input queues are always produced by + // islands in the graph). + rewindToStop(in_queues, ii); + return false; + } // if(Stop) + } // for(in_queues) + return true; +} + // This thread is a plain dump source actor. What it do is just: // - Check input queue (the only one) for a control command @@ -509,8 +664,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput Cmd cmd; if (cv::util::holds_alternative(post_iter->data)) { - // FIXME: That ugly VARIANT problem - cmd = Cmd{const_cast(cv::util::get(post_iter->data))}; + cmd = Cmd{cv::util::get(post_iter->data)}; } else { @@ -520,8 +674,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput } for (auto &&q : m_out_queues[out_idx]) { - // FIXME: This ugly VARIANT problem - q->push(const_cast(cmd)); + q->push(cmd); } post_iter = m_postings[out_idx].erase(post_iter); } @@ -551,6 +704,15 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput } } } + void meta(const cv::GRunArgP &out, const cv::GRunArg::Meta &m) override + { + const auto it = m_postIdx.find(cv::gimpl::proto::ptr(out)); + GAPI_Assert(it != m_postIdx.end()); + + const auto out_iter = it->second.second; + cv::util::get(out_iter->data).meta = m; + } + public: explicit StreamingOutput(const cv::GMetaArgs &metas, std::vector< std::vector > &out_queues, @@ -603,22 +765,84 @@ void islandActorThread(std::vector in_rcs, // // and then put the resulting vector into one single queue. While it // looks redundant, it simplifies dramatically the way how try_pull() // is implemented - we need to check one queue instead of many. -void collectorThread(std::vector in_queues, - Q& out_queue) +// +// After desync() is added, there may be multiple collector threads +// running, every thread producing its own part of the partial +// pipeline output (optional...). All partial outputs are pushed +// to the same output queue and then picked by GStreamingExecutor +// in the end. +void collectorThread(std::vector in_queues, + std::vector in_mapping, + const std::size_t out_size, + const bool handle_stop, + Q& out_queue) { + // These flags are static now: regardless if the sync or + // desync branch is collected by this thread, all in_queue + // data should come in sync. + std::vector flags(out_size, false); + for (auto idx : in_mapping) { + flags[idx] = true; + } + QueueReader qr; while (true) { - cv::GRunArgs this_result(in_queues.size()); - cv::GRunArgs this_const(in_queues.size()); - if (!qr.getInputVector(in_queues, this_const, this_result)) + cv::GRunArgs this_result(out_size); + const bool ok = qr.getResultsVector(in_queues, in_mapping, out_size, this_result); + if (!ok) { - out_queue.push(Cmd{Stop{}}); + if (handle_stop) + { + out_queue.push(Cmd{Stop{}}); + } + // Terminate the thread anyway return; } - out_queue.push(Cmd{this_result}); + out_queue.push(Cmd{Result{std::move(this_result), flags}}); } } + +void check_DesyncObjectConsumedByMultipleIslands(const cv::gimpl::GIslandModel::Graph &gim) { + using namespace cv::gimpl; + + // Since the limitation exists only in this particular + // implementation, the check is also done only here but not at the + // graph compiler level. + // + // See comment in desync(GMat) src/api/kernels_streaming.cpp for details. + for (auto &&nh : gim.nodes()) { + if (gim.metadata(nh).get().k == NodeKind::SLOT) { + // SLOTs are read by ISLANDs, so look for the metadata + // of the outbound edges + std::unordered_map out_desync_islands; + for (auto &&out_eh : nh->outEdges()) { + if (gim.metadata(out_eh).contains()) { + // This is a desynchronized edge + // Look what Island it leads to + const auto out_desync_idx = gim.metadata(out_eh) + .get().index; + const auto out_island = gim.metadata(out_eh->dstNode()) + .get().object; + + auto it = out_desync_islands.find(out_desync_idx); + if (it != out_desync_islands.end()) { + // If there's already an edge with this desync + // id, it must point to the same island object + GAPI_Assert(it->second == out_island.get() + && "A single desync object may only be used by a single island!"); + } else { + // Store the island pointer for the further check + out_desync_islands[out_desync_idx] = out_island.get(); + } + } // if(desync) + } // for(out_eh) + // There must be only one backend in the end of the day + // (under this desync path) + } // if(SLOT) + } // for(nodes) +} + } // anonymous namespace // GStreamingExecutor expects compile arguments as input to have possibility to do @@ -630,20 +854,28 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && .get().model) , m_comp_args(comp_args) , m_gim(*m_island_graph) + , m_desync(GModel::Graph(*m_orig_graph).metadata() + .contains()) { GModel::Graph gm(*m_orig_graph); // NB: Right now GIslandModel is acyclic, and all the below code assumes that. - // NB: This naive execution code is taken from GExecutor nearly "as-is" + // NB: This naive execution code is taken from GExecutor nearly + // "as-is" + + if (m_desync) { + check_DesyncObjectConsumedByMultipleIslands(m_gim); + } const auto proto = gm.metadata().get(); m_emitters .resize(proto.in_nhs.size()); m_emitter_queues.resize(proto.in_nhs.size()); m_sinks .resize(proto.out_nhs.size()); - m_sink_queues .resize(proto.out_nhs.size()); + m_sink_queues .resize(proto.out_nhs.size(), nullptr); + m_sink_sync .resize(proto.out_nhs.size(), -1); // Very rough estimation to limit internal queue sizes. // Pipeline depth is equal to number of its (pipeline) steps. - const auto queue_capacity = std::count_if + const auto queue_capacity = 3*std::count_if (m_gim.nodes().begin(), m_gim.nodes().end(), [&](ade::NodeHandle nh) { @@ -723,15 +955,53 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && , isl_exec }); // Initialize queues for every operation's input - ade::TypedGraph qgr(*m_island_graph); + ade::TypedGraph qgr(*m_island_graph); + bool is_desync_start = false; for (auto eh : nh->inEdges()) { // ...only if the data is not compile-const if (const_ins.count(eh->srcNode()) == 0) { - qgr.metadata(eh).set(DataQueue(queue_capacity)); - m_internal_queues.insert(&qgr.metadata(eh).get().q); + if (m_gim.metadata(eh).contains()) { + qgr.metadata(eh).set(DataQueue(DataQueue::DESYNC)); + is_desync_start = true; + } else if (qgr.metadata(eh).contains()) { + // See comment below + // Limit queue size to 1 in this case + qgr.metadata(eh).set(DataQueue(1u)); + } else { + qgr.metadata(eh).set(DataQueue(queue_capacity)); + } + m_internal_queues.insert(qgr.metadata(eh).get().q.get()); } } + // WORKAROUND: + // Since now we always know desync() is followed by copy(), + // copy is always the island with DesyncIslEdge. + // Mark the node's outputs a special way so then its following + // queue sizes will be limited to 1 (to avoid copy reading more + // data in advance - as there's no other way for the underlying + // "slow" part to control it) + if (is_desync_start) { + auto isl = m_gim.metadata(nh).get().object; + // In the current implementation, such islands + // _must_ start with copy + GAPI_Assert(isl->in_ops().size() == 1u); +#if !defined(GAPI_STANDALONE) + GAPI_Assert(GModel::Graph(*m_orig_graph) + .metadata(*isl->in_ops().begin()) + .get() + .k.name == cv::gapi::core::GCopy::id()); +#endif // GAPI_STANDALONE + for (auto out_nh : nh->outNodes()) { + for (auto out_eh : out_nh->outEdges()) { + qgr.metadata(out_eh).set(DesyncSpecialCase{}); + } + } + } + // It is ok to do it here since the graph is visited in + // a topologic order and its consumers (those checking + // their input edges & initializing queues) are yet to be + // visited } break; case NodeKind::SLOT: @@ -760,7 +1030,14 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && ade::TypedGraph qgr(*m_island_graph); GAPI_Assert(nh->inEdges().size() == 1u); qgr.metadata(nh->inEdges().front()).set(DataQueue(queue_capacity)); - m_sink_queues[sink_idx] = &qgr.metadata(nh->inEdges().front()).get().q; + m_sink_queues[sink_idx] = qgr.metadata(nh->inEdges().front()).get().q.get(); + + // Assign a desync tag + const auto sink_out_nh = gm.metadata().get().out_nhs[sink_idx]; + if (gm.metadata(sink_out_nh).contains()) { + // metadata().get_or<> could make this thing better + m_sink_sync[sink_idx] = gm.metadata(sink_out_nh).get().index; + } } break; default: @@ -768,7 +1045,23 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && break; } // switch(kind) } // for(gim nodes) - m_out_queue.set_capacity(queue_capacity); + + // If there are desynchronized parts in the graph, there may be + // multiple theads polling every separate (desynchronized) + // branch in the graph individually. Prepare a mapping information + // for any such thread + for (auto &&idx : ade::util::iota(m_sink_queues.size())) { + auto path_id = m_sink_sync[idx]; + auto &info = m_collector_map[path_id]; + info.queues.push_back(m_sink_queues[idx]); + info.mapping.push_back(static_cast(idx)); + } + + // Reserve space in the final queue based on the number + // of desync parts (they can generate output individually + // per the same input frame, so the output traffic multiplies) + GAPI_Assert(m_collector_map.size() > 0u); + m_out_queue.set_capacity(queue_capacity * m_collector_map.size()); } cv::gimpl::GStreamingExecutor::~GStreamingExecutor() @@ -938,7 +1231,6 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) real_video_completion_cb); } - // Now do this for every island (in a topological order) for (auto &&op : m_ops) { @@ -974,10 +1266,27 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) out_queues); } - // Finally, start a collector thread. - m_threads.emplace_back(collectorThread, - m_sink_queues, - std::ref(m_out_queue)); + // Finally, start collector thread(s). + // If there are desynchronized parts in the graph, there may be + // multiple theads polling every separate (desynchronized) + // branch in the graph individually. + const bool has_main_path = m_sink_sync.end() != + std::find(m_sink_sync.begin(), m_sink_sync.end(), -1); + for (auto &&info : m_collector_map) { + m_threads.emplace_back(collectorThread, + info.second.queues, + info.second.mapping, + m_sink_queues.size(), + has_main_path ? info.first == -1 : true, // see below (*) + std::ref(m_out_queue)); + + // (*) - there may be a problem with desynchronized paths when those work + // faster than the main path. In this case, the desync paths get "Stop" message + // earlier and thus broadcast it down to pipeline gets stopped when there is + // some "main path" data to process. This new collectorThread's flag regulates it: + // - desync paths should never post Stop message if there is a main path. + // - if there is no main path, than any desync path can terminate the execution. + } state = State::READY; } @@ -1018,15 +1327,25 @@ void cv::gimpl::GStreamingExecutor::wait_shutdown() for (auto &q : m_internal_queues) q->clear(); m_out_queue.clear(); + for (auto &&op : m_ops) { + op.isl_exec->handleStopStream(); + } + state = State::STOPPED; } bool cv::gimpl::GStreamingExecutor::pull(cv::GRunArgsP &&outs) { + // This pull() can only be called when there's no desynchronized + // parts in the graph. + GAPI_Assert(!m_desync && + "This graph has desynchronized parts! Please use another pull()"); + if (state == State::STOPPED) return false; GAPI_Assert(state == State::RUNNING); - GAPI_Assert(m_sink_queues.size() == outs.size()); + GAPI_Assert(m_sink_queues.size() == outs.size() && + "Number of data objects in cv::gout() must match the number of graph outputs in cv::GOut()"); Cmd cmd; m_out_queue.pop(cmd); @@ -1036,12 +1355,39 @@ bool cv::gimpl::GStreamingExecutor::pull(cv::GRunArgsP &&outs) return false; } - GAPI_Assert(cv::util::holds_alternative(cmd)); - cv::GRunArgs &this_result = cv::util::get(cmd); + GAPI_Assert(cv::util::holds_alternative(cmd)); + cv::GRunArgs &this_result = cv::util::get(cmd).args; sync_data(this_result, outs); return true; } +bool cv::gimpl::GStreamingExecutor::pull(cv::GOptRunArgsP &&outs) +{ + // This pull() can only be called in both cases: if there are + // desyncrhonized parts or not. + + // FIXME: so far it is a full duplicate of standard pull except + // the sync_data version called. + if (state == State::STOPPED) + return false; + GAPI_Assert(state == State::RUNNING); + GAPI_Assert(m_sink_queues.size() == outs.size() && + "Number of data objects in cv::gout() must match the number of graph outputs in cv::GOut()"); + + Cmd cmd; + m_out_queue.pop(cmd); + if (cv::util::holds_alternative(cmd)) + { + wait_shutdown(); + return false; + } + + GAPI_Assert(cv::util::holds_alternative(cmd)); + sync_data(cv::util::get(cmd), outs); + return true; +} + + bool cv::gimpl::GStreamingExecutor::try_pull(cv::GRunArgsP &&outs) { if (state == State::STOPPED) @@ -1059,8 +1405,8 @@ bool cv::gimpl::GStreamingExecutor::try_pull(cv::GRunArgsP &&outs) return false; } - GAPI_Assert(cv::util::holds_alternative(cmd)); - cv::GRunArgs &this_result = cv::util::get(cmd); + GAPI_Assert(cv::util::holds_alternative(cmd)); + cv::GRunArgs &this_result = cv::util::get(cmd).args; sync_data(this_result, outs); return true; } diff --git a/modules/gapi/src/executor/gstreamingexecutor.hpp b/modules/gapi/src/executor/gstreamingexecutor.hpp index d10f9eddd0..b6093ac1ef 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.hpp +++ b/modules/gapi/src/executor/gstreamingexecutor.hpp @@ -14,6 +14,8 @@ #include // unique_ptr, shared_ptr #include // thread +#include +#include #if defined(HAVE_TBB) # include // FIXME: drop it from here! @@ -22,6 +24,7 @@ template using QueueClass = tbb::concurrent_bounded_queue; # include "executor/conc_queue.hpp" template using QueueClass = cv::gapi::own::concurrent_bounded_queue; #endif // TBB +#include "executor/last_value.hpp" #include @@ -40,14 +43,61 @@ struct Stop { cv::GRunArg cdata; // const data for CNST stop }; +struct Result { + cv::GRunArgs args; // Full results vector + std::vector flags; // Availability flags (in case of desync) +}; + using Cmd = cv::util::variant < cv::util::monostate , Start // Tells emitters to start working. Not broadcasted to workers. , Stop // Tells emitters to stop working. Broadcasted to workers. , cv::GRunArg // Workers data payload to process. - , cv::GRunArgs // Full results vector + , Result // Pipeline's data for gout() >; -using Q = QueueClass; + +// Interface over a queue. The underlying queue implementation may be +// different. This class is mainly introduced to bring some +// abstraction over the real queues (bounded in-order) and a +// desynchronized data slots (see required to implement +// cv::gapi::desync) + +class Q { +public: + virtual void push(const Cmd &cmd) = 0; + virtual void pop(Cmd &cmd) = 0; + virtual bool try_pop(Cmd &cmd) = 0; + virtual void clear() = 0; + virtual ~Q() = default; +}; + +// A regular queue implementation +class SyncQueue final: public Q { + QueueClass m_q; // FIXME: OWN or WRAP?? + +public: + virtual void push(const Cmd &cmd) override { m_q.push(cmd); } + virtual void pop(Cmd &cmd) override { m_q.pop(cmd); } + virtual bool try_pop(Cmd &cmd) override { return m_q.try_pop(cmd); } + virtual void clear() override { m_q.clear(); } + + void set_capacity(std::size_t c) { m_q.set_capacity(c);} +}; + +// Desynchronized "queue" implementation +// Every push overwrites value which is not yet popped +// This container can hold 0 or 1 element +// Special handling for Stop is implemented (FIXME: not really) +class DesyncQueue final: public Q { + cv::gapi::own::last_written_value m_v; + +public: + virtual void push(const Cmd &cmd) override { m_v.push(cmd); } + virtual void pop(Cmd &cmd) override { m_v.pop(cmd); } + virtual bool try_pop(Cmd &cmd) override { return m_v.try_pop(cmd); } + virtual void clear() override { m_v.clear(); } +}; + } // namespace stream // FIXME: Currently all GExecutor comments apply also @@ -87,6 +137,7 @@ protected: util::optional m_reshapable; cv::gimpl::GIslandModel::Graph m_gim; // FIXME: make const? + const bool m_desync; // FIXME: Naive executor details are here for now // but then it should be moved to another place @@ -117,11 +168,27 @@ protected: std::vector m_sinks; std::vector m_threads; - std::vector m_emitter_queues; - std::vector m_const_emitter_queues; // a view over m_emitter_queues - std::vector m_sink_queues; - std::unordered_set m_internal_queues; - stream::Q m_out_queue; + std::vector m_emitter_queues; + + // a view over m_emitter_queues + std::vector m_const_emitter_queues; + + std::vector m_sink_queues; + + // desync path tags for outputs. -1 means that output + // doesn't belong to a desync path + std::vector m_sink_sync; + + std::unordered_set m_internal_queues; + stream::SyncQueue m_out_queue; + + // Describes mapping from desync paths to collector threads + struct CollectorThreadInfo { + std::vector queues; + std::vector mapping; + }; + std::unordered_map m_collector_map; + void wait_shutdown(); @@ -132,6 +199,7 @@ public: void setSource(GRunArgs &&args); void start(); bool pull(cv::GRunArgsP &&outs); + bool pull(cv::GOptRunArgsP &&outs); bool try_pull(cv::GRunArgsP &&outs); void stop(); bool running() const; diff --git a/modules/gapi/src/executor/last_value.hpp b/modules/gapi/src/executor/last_value.hpp new file mode 100644 index 0000000000..152449a879 --- /dev/null +++ b/modules/gapi/src/executor/last_value.hpp @@ -0,0 +1,105 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_EXECUTOR_LAST_VALUE_HPP +#define OPENCV_GAPI_EXECUTOR_LAST_VALUE_HPP + +#include +#include + +#include +#include + +namespace cv { +namespace gapi { +namespace own { + +// This class implements a "Last Written Value" thing. Writer threads +// (in our case, it is just one) can write as many values there as it +// can. +// +// The reader thread gets only a value it gets at the time (or blocks +// if there was no value written since the last read). +// +// Again, the implementation is highly inefficient right now. +template +class last_written_value { + cv::util::optional m_data; + + std::mutex m_mutex; + std::condition_variable m_cond_empty; + + void unsafe_pop(T &t); + +public: + last_written_value() {} + last_written_value(const last_written_value &cc) + : m_data(cc.m_data) { + // FIXME: what to do with all that locks, etc? + } + last_written_value(last_written_value &&cc) + : m_data(std::move(cc.m_data)) { + // FIXME: what to do with all that locks, etc? + } + + // FIXME: && versions + void push(const T &t); + void pop(T &t); + bool try_pop(T &t); + + // Not thread-safe + void clear(); +}; + +// Internal: do shared pop things assuming the lock is already there +template +void last_written_value::unsafe_pop(T &t) { + GAPI_Assert(m_data.has_value()); + t = std::move(m_data.value()); + m_data.reset(); +} + +// Push an element to the queue. Blocking if there's no space left +template +void last_written_value::push(const T& t) { + std::unique_lock lock(m_mutex); + m_data = cv::util::make_optional(t); + lock.unlock(); + m_cond_empty.notify_one(); +} + +// Pop an element from the queue. Blocking if there's no items +template +void last_written_value::pop(T &t) { + std::unique_lock lock(m_mutex); + if (!m_data.has_value()) { + // if there is no data, wait + m_cond_empty.wait(lock, [&](){return m_data.has_value();}); + } + unsafe_pop(t); +} + +// Try pop an element from the queue. Returns false if queue is empty +template +bool last_written_value::try_pop(T &t) { + std::unique_lock lock(m_mutex); + if (!m_data.has_value()) { + // if there is no data, return + return false; + } + unsafe_pop(t); + return true; +} + +// Clear the value holder. This method is not thread-safe. +template +void last_written_value::clear() { + m_data.reset(); +} + +}}} // namespace cv::gapi::own + +#endif // OPENCV_GAPI_EXECUTOR_CONC_QUEUE_HPP diff --git a/modules/gapi/test/common/gapi_core_tests.hpp b/modules/gapi/test/common/gapi_core_tests.hpp index 4a0a7641f9..889e32f1c1 100644 --- a/modules/gapi/test/common/gapi_core_tests.hpp +++ b/modules/gapi/test/common/gapi_core_tests.hpp @@ -157,7 +157,7 @@ GAPI_TEST_EXT_BASE_FIXTURE(ParseSSDBLTest, ParserSSDTest, initNothing, GAPI_TEST_EXT_BASE_FIXTURE(ParseSSDTest, ParserSSDTest, initNothing, FIXTURE_API(float, bool, bool), 3, confidence_threshold, alignment_to_square, filter_out_of_bounds) GAPI_TEST_EXT_BASE_FIXTURE(ParseYoloTest, ParserYoloTest, initNothing, - FIXTURE_API(float, float, int), 3, confidence_threshold, nms_threshold, num_classes) + FIXTURE_API(float, float, int, std::pair), 4, confidence_threshold, nms_threshold, num_classes, dims_config) GAPI_TEST_FIXTURE(SizeTest, initMatrixRandU, <>, 0) GAPI_TEST_FIXTURE(SizeRTest, initNothing, <>, 0) } // opencv_test diff --git a/modules/gapi/test/common/gapi_core_tests_inl.hpp b/modules/gapi/test/common/gapi_core_tests_inl.hpp index e350a14e65..045b556369 100644 --- a/modules/gapi/test/common/gapi_core_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_core_tests_inl.hpp @@ -618,7 +618,8 @@ TEST_P(SumTest, AccuracyTest) #undef countNonZero TEST_P(CountNonZeroTest, AccuracyTest) { - int out_cnz_gapi, out_cnz_ocv; + int out_cnz_gapi = -1; + int out_cnz_ocv = -2; // G-API code ////////////////////////////////////////////////////////////// cv::GMat in; @@ -1665,7 +1666,7 @@ TEST_P(ParseSSDTest, ParseTest) TEST_P(ParseYoloTest, ParseTest) { - cv::Mat in_mat = generateYoloOutput(num_classes); + cv::Mat in_mat = generateYoloOutput(num_classes, dims_config); auto anchors = cv::gapi::nn::parsers::GParseYolo::defaultAnchors(); std::vector boxes_gapi, boxes_ref; std::vector labels_gapi, labels_ref; @@ -1690,7 +1691,7 @@ TEST_P(SizeTest, ParseTest) cv::GMat in; cv::Size out_sz; - auto out = cv::gapi::size(in); + auto out = cv::gapi::streaming::size(in); cv::GComputation c(cv::GIn(in), cv::GOut(out)); c.apply(cv::gin(in_mat1), cv::gout(out_sz), getCompileArgs()); @@ -1703,7 +1704,7 @@ TEST_P(SizeRTest, ParseTest) cv::Size out_sz; cv::GOpaque op_rect; - auto out = cv::gapi::size(op_rect); + auto out = cv::gapi::streaming::size(op_rect); cv::GComputation c(cv::GIn(op_rect), cv::GOut(out)); c.apply(cv::gin(rect), cv::gout(out_sz), getCompileArgs()); diff --git a/modules/gapi/test/common/gapi_imgproc_tests.hpp b/modules/gapi/test/common/gapi_imgproc_tests.hpp index cd074efda0..b48b7b6732 100644 --- a/modules/gapi/test/common/gapi_imgproc_tests.hpp +++ b/modules/gapi/test/common/gapi_imgproc_tests.hpp @@ -46,6 +46,8 @@ GAPI_TEST_FIXTURE(Erode3x3Test, initMatrixRandN, FIXTURE_API(CompareMats,int), 2 GAPI_TEST_FIXTURE(DilateTest, initMatrixRandN, FIXTURE_API(CompareMats,int,int), 3, cmpF, kernSize, kernType) GAPI_TEST_FIXTURE(Dilate3x3Test, initMatrixRandN, FIXTURE_API(CompareMats,int), 2, cmpF, numIters) +GAPI_TEST_FIXTURE(MorphologyExTest, initMatrixRandN, FIXTURE_API(CompareMats,MorphTypes), + 2, cmpF, op) GAPI_TEST_FIXTURE(SobelTest, initMatrixRandN, FIXTURE_API(CompareMats,int,int,int), 4, cmpF, kernSize, dx, dy) GAPI_TEST_FIXTURE(SobelXYTest, initMatrixRandN, FIXTURE_API(CompareMats,int,int,int,int), 5, @@ -64,9 +66,45 @@ GAPI_TEST_FIXTURE_SPEC_PARAMS(GoodFeaturesTest, double,int,bool), 8, cmpF, fileName, type, maxCorners, qualityLevel, minDistance, blockSize, useHarrisDetector) +GAPI_TEST_FIXTURE_SPEC_PARAMS(FindContoursNoOffsetTest, + FIXTURE_API(cv::Size,MatType2,cv::RetrievalModes, + cv::ContourApproximationModes), + 4, sz, type, mode, method) +GAPI_TEST_FIXTURE_SPEC_PARAMS(FindContoursOffsetTest, <>, 0) +GAPI_TEST_FIXTURE_SPEC_PARAMS(FindContoursHNoOffsetTest, + FIXTURE_API(cv::Size,MatType2,cv::RetrievalModes, + cv::ContourApproximationModes), + 4, sz, type, mode, method) +GAPI_TEST_FIXTURE_SPEC_PARAMS(FindContoursHOffsetTest, <>, 0) +GAPI_TEST_FIXTURE(BoundingRectMatTest, initMatrixRandU, FIXTURE_API(CompareRects), 1, cmpF) +GAPI_TEST_FIXTURE(BoundingRectMatVector32STest, initNothing, FIXTURE_API(CompareRects), 1, cmpF) +GAPI_TEST_FIXTURE(BoundingRectMatVector32FTest, initNothing, FIXTURE_API(CompareRects), 1, cmpF) +GAPI_TEST_FIXTURE(BoundingRectVector32STest, initNothing, FIXTURE_API(CompareRects), 1, cmpF) +GAPI_TEST_FIXTURE(BoundingRectVector32FTest, initNothing, FIXTURE_API(CompareRects), 1, cmpF) +GAPI_TEST_FIXTURE(BGR2RGBTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) GAPI_TEST_FIXTURE(RGB2GrayTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) +GAPI_TEST_FIXTURE(FitLine2DMatVectorTest, initMatByPointsVectorRandU, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine2DVector32STest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine2DVector32FTest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine2DVector64FTest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine3DMatVectorTest, initMatByPointsVectorRandU, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine3DVector32STest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine3DVector32FTest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) +GAPI_TEST_FIXTURE(FitLine3DVector64FTest, initNothing, + FIXTURE_API(CompareVecs,cv::DistanceTypes), 2, cmpF, distType) GAPI_TEST_FIXTURE(BGR2GrayTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) GAPI_TEST_FIXTURE(RGB2YUVTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) +GAPI_TEST_FIXTURE(BGR2I420Test, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) +GAPI_TEST_FIXTURE(RGB2I420Test, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) +GAPI_TEST_FIXTURE(I4202BGRTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) +GAPI_TEST_FIXTURE(I4202RGBTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) GAPI_TEST_FIXTURE(YUV2RGBTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) GAPI_TEST_FIXTURE(YUV2GrayTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) GAPI_TEST_FIXTURE(NV12toRGBTest, initMatrixRandN, FIXTURE_API(CompareMats), 1, cmpF) diff --git a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp index 4aadc17d5d..2a4f2e64ea 100644 --- a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp @@ -50,6 +50,27 @@ namespace rgb2yuyv(in_line_p, out_line_p, in.cols); } } + + // Draw random ellipses on given mat of given size and type + void initMatForFindingContours(cv::Mat& mat, const cv::Size& sz, const int type) + { + cv::RNG& rng = theRNG(); + mat = cv::Mat(sz, type, cv::Scalar::all(0)); + size_t numEllipses = rng.uniform(1, 10); + + for( size_t i = 0; i < numEllipses; i++ ) + { + cv::Point center; + cv::Size axes; + center.x = rng.uniform(0, sz.width); + center.y = rng.uniform(0, sz.height); + axes.width = rng.uniform(2, sz.width); + axes.height = rng.uniform(2, sz.height); + int color = rng.uniform(1, 256); + double angle = rng.uniform(0., 180.); + cv::ellipse(mat, center, axes, angle, 0., 360., color, 1, FILLED); + } + } } TEST_P(Filter2DTest, AccuracyTest) @@ -290,6 +311,29 @@ TEST_P(Dilate3x3Test, AccuracyTest) } } +TEST_P(MorphologyExTest, AccuracyTest) +{ + MorphShapes defShape = cv::MORPH_RECT; + int defKernSize = 3; + cv::Mat kernel = cv::getStructuringElement(defShape, cv::Size(defKernSize, defKernSize)); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::morphologyEx(in, op, kernel); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::morphologyEx(in_mat1, out_mat_ocv, op, kernel); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), sz); + } +} + TEST_P(SobelTest, AccuracyTest) { // G-API code ////////////////////////////////////////////////////////////// @@ -447,6 +491,472 @@ TEST_P(GoodFeaturesTest, AccuracyTest) } } +TEST_P(FindContoursNoOffsetTest, AccuracyTest) +{ + std::vector> outCtsOCV, outCtsGAPI; + + initMatForFindingContours(in_mat1, sz, type); + out_mat_gapi = cv::Mat(sz, type, cv::Scalar::all(0)); + out_mat_ocv = cv::Mat(sz, type, cv::Scalar::all(0)); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::findContours(in_mat1, outCtsOCV, mode, method); + } + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + cv::GArray> outCts; + outCts = cv::gapi::findContours(in, mode, method); + cv::GComputation c(GIn(in), GOut(outCts)); + c.apply(gin(in_mat1), gout(outCtsGAPI), getCompileArgs()); + + // Comparison ////////////////////////////////////////////////////////////// + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + cv::fillPoly(out_mat_ocv, outCtsOCV, cv::Scalar::all(1)); + cv::fillPoly(out_mat_gapi, outCtsGAPI, cv::Scalar::all(1)); + EXPECT_TRUE(AbsExact().to_compare_f()(out_mat_ocv, out_mat_gapi)); +} + +TEST_P(FindContoursOffsetTest, AccuracyTest) +{ + const cv::Size sz(1280, 720); + const MatType2 type = CV_8UC1; + const cv::RetrievalModes mode = cv::RETR_EXTERNAL; + const cv::ContourApproximationModes method = cv::CHAIN_APPROX_NONE; + const cv::Point offset(15, 15); + std::vector> outCtsOCV, outCtsGAPI; + + initMatForFindingContours(in_mat1, sz, type); + out_mat_gapi = cv::Mat(sz, type, cv::Scalar::all(0)); + out_mat_ocv = cv::Mat(sz, type, cv::Scalar::all(0)); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::findContours(in_mat1, outCtsOCV, mode, method, offset); + } + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + GOpaque gOffset; + cv::GArray> outCts; + outCts = cv::gapi::findContours(in, mode, method, gOffset); + cv::GComputation c(GIn(in, gOffset), GOut(outCts)); + c.apply(gin(in_mat1, offset), gout(outCtsGAPI), getCompileArgs()); + + // Comparison ////////////////////////////////////////////////////////////// + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + cv::fillPoly(out_mat_ocv, outCtsOCV, cv::Scalar::all(1)); + cv::fillPoly(out_mat_gapi, outCtsGAPI, cv::Scalar::all(1)); + EXPECT_TRUE(AbsExact().to_compare_f()(out_mat_ocv, out_mat_gapi)); +} + +TEST_P(FindContoursHNoOffsetTest, AccuracyTest) +{ + std::vector> outCtsOCV, outCtsGAPI; + std::vector outHierOCV, outHierGAPI; + + initMatForFindingContours(in_mat1, sz, type); + out_mat_gapi = cv::Mat(sz, type, cv::Scalar::all(0)); + out_mat_ocv = cv::Mat(sz, type, cv::Scalar::all(0)); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::findContours(in_mat1, outCtsOCV, outHierOCV, mode, method); + } + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + cv::GArray> outCts; + cv::GArray outHier; + std::tie(outCts, outHier) = cv::gapi::findContoursH(in, mode, method); + cv::GComputation c(GIn(in), GOut(outCts, outHier)); + c.apply(gin(in_mat1), gout(outCtsGAPI, outHierGAPI), getCompileArgs()); + + // Comparison ////////////////////////////////////////////////////////////// + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + cv::fillPoly(out_mat_ocv, outCtsOCV, cv::Scalar::all(1)); + cv::fillPoly(out_mat_gapi, outCtsGAPI, cv::Scalar::all(1)); + EXPECT_TRUE(AbsExact().to_compare_f()(out_mat_ocv, out_mat_gapi)); + + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + EXPECT_TRUE(AbsExactVector().to_compare_f()(outHierOCV, outHierGAPI)); +} + +TEST_P(FindContoursHOffsetTest, AccuracyTest) +{ + const cv::Size sz(1280, 720); + const MatType2 type = CV_8UC1; + const cv::RetrievalModes mode = cv::RETR_EXTERNAL; + const cv::ContourApproximationModes method = cv::CHAIN_APPROX_NONE; + const cv::Point offset(15, 15); + std::vector> outCtsOCV, outCtsGAPI; + std::vector outHierOCV, outHierGAPI; + + initMatForFindingContours(in_mat1, sz, type); + out_mat_gapi = cv::Mat(sz, type, cv::Scalar::all(0)); + out_mat_ocv = cv::Mat(sz, type, cv::Scalar::all(0)); + + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::findContours(in_mat1, outCtsOCV, outHierOCV, mode, method, offset); + } + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + GOpaque gOffset; + cv::GArray> outCts; + cv::GArray outHier; + std::tie(outCts, outHier) = cv::gapi::findContoursH(in, mode, method, gOffset); + cv::GComputation c(GIn(in, gOffset), GOut(outCts, outHier)); + c.apply(gin(in_mat1, offset), gout(outCtsGAPI, outHierGAPI), getCompileArgs()); + + // Comparison ////////////////////////////////////////////////////////////// + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + cv::fillPoly(out_mat_ocv, outCtsOCV, cv::Scalar::all(1)); + cv::fillPoly(out_mat_gapi, outCtsGAPI, cv::Scalar::all(1)); + EXPECT_TRUE(AbsExact().to_compare_f()(out_mat_ocv, out_mat_gapi)); + + EXPECT_TRUE(outCtsGAPI.size() == outCtsOCV.size()); + EXPECT_TRUE(AbsExactVector().to_compare_f()(outHierOCV, outHierGAPI)); +} + +TEST_P(BoundingRectMatTest, AccuracyTest) +{ + cv::Rect out_rect_gapi, out_rect_ocv; + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::boundingRect(in); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_mat1), cv::gout(out_rect_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_rect_ocv = cv::boundingRect(in_mat1); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_rect_gapi, out_rect_ocv)); + } +} + +TEST_P(BoundingRectMatVector32STest, AccuracyTest) +{ + cv::Rect out_rect_gapi, out_rect_ocv; + + std::vector in_vectorS(sz.width); + cv::randu(in_vectorS, cv::Scalar::all(0), cv::Scalar::all(255)); + in_mat1 = cv::Mat(in_vectorS); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::boundingRect(in); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_mat1), cv::gout(out_rect_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_rect_ocv = cv::boundingRect(in_mat1); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_rect_gapi, out_rect_ocv)); + } +} + +TEST_P(BoundingRectMatVector32FTest, AccuracyTest) +{ + cv::RNG& rng = theRNG(); + cv::Rect out_rect_gapi, out_rect_ocv; + + std::vector in_vectorF(sz.width); + const int fscale = 256; // avoid bits near ULP, generate stable test input + for (int i = 0; i < sz.width; i++) + { + cv::Point2f pt(rng.uniform(0, 255 * fscale) / static_cast(fscale), + rng.uniform(0, 255 * fscale) / static_cast(fscale)); + in_vectorF.push_back(pt); + } + in_mat1 = cv::Mat(in_vectorF); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::boundingRect(in); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_mat1), cv::gout(out_rect_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_rect_ocv = cv::boundingRect(in_mat1); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_rect_gapi, out_rect_ocv)); + } +} + + +TEST_P(BoundingRectVector32STest, AccuracyTest) +{ + cv::Rect out_rect_gapi, out_rect_ocv; + + std::vector in_vectorS(sz.width); + cv::randu(in_vectorS, cv::Scalar::all(0), cv::Scalar::all(255)); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::boundingRect(in); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vectorS), cv::gout(out_rect_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_rect_ocv = cv::boundingRect(in_vectorS); + } + + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_rect_gapi, out_rect_ocv)); + } +} + +TEST_P(BoundingRectVector32FTest, AccuracyTest) +{ + cv::RNG& rng = theRNG(); + cv::Rect out_rect_gapi, out_rect_ocv; + + std::vector in_vectorF(sz.width); + const int fscale = 256; // avoid bits near ULP, generate stable test input + for (int i = 0; i < sz.width; i++) + { + cv::Point2f pt(rng.uniform(0, 255 * fscale) / static_cast(fscale), + rng.uniform(0, 255 * fscale) / static_cast(fscale)); + in_vectorF.push_back(pt); + } + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::boundingRect(in); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vectorF), cv::gout(out_rect_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + out_rect_ocv = cv::boundingRect(in_vectorF); + } + + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_rect_gapi, out_rect_ocv)); + } +} + +TEST_P(FitLine2DMatVectorTest, AccuracyTest) +{ + cv::Vec4f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::fitLine2D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_mat1), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_mat1, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine2DVector32STest, AccuracyTest) +{ + cv::Vec4f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine2D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine2DVector32FTest, AccuracyTest) +{ + cv::Vec4f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine2D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine2DVector64FTest, AccuracyTest) +{ + cv::Vec4f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine2D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine3DMatVectorTest, AccuracyTest) +{ + cv::Vec6f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::fitLine3D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_mat1), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_mat1, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine3DVector32STest, AccuracyTest) +{ + cv::Vec6f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine3D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine3DVector32FTest, AccuracyTest) +{ + cv::Vec6f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine3D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(FitLine3DVector64FTest, AccuracyTest) +{ + cv::Vec6f out_vec_gapi, out_vec_ocv; + double paramDefault = 0., repsDefault = 0., aepsDefault = 0.; + + std::vector in_vec; + initPointsVectorRandU(sz.width, in_vec); + + // G-API code ////////////////////////////////////////////////////////////// + cv::GArray in; + auto out = cv::gapi::fitLine3D(in, distType, paramDefault, repsDefault, aepsDefault); + + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + c.apply(cv::gin(in_vec), cv::gout(out_vec_gapi), getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::fitLine(in_vec, out_vec_ocv, distType, paramDefault, repsDefault, aepsDefault); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_vec_gapi, out_vec_ocv)); + } +} + +TEST_P(BGR2RGBTest, AccuracyTest) +{ + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::BGR2RGB(in); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::cvtColor(in_mat1, out_mat_ocv, cv::COLOR_BGR2RGB); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), sz); + } +} + TEST_P(RGB2GrayTest, AccuracyTest) { // G-API code ////////////////////////////////////////////////////////////// @@ -523,6 +1033,82 @@ TEST_P(YUV2RGBTest, AccuracyTest) } } +TEST_P(BGR2I420Test, AccuracyTest) +{ + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::BGR2I420(in); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::cvtColor(in_mat1, out_mat_ocv, cv::COLOR_BGR2YUV_I420); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + } +} + +TEST_P(RGB2I420Test, AccuracyTest) +{ + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::RGB2I420(in); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::cvtColor(in_mat1, out_mat_ocv, cv::COLOR_RGB2YUV_I420); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + } +} + +TEST_P(I4202BGRTest, AccuracyTest) +{ + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::I4202BGR(in); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::cvtColor(in_mat1, out_mat_ocv, cv::COLOR_YUV2BGR_I420); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + } +} + +TEST_P(I4202RGBTest, AccuracyTest) +{ + // G-API code ////////////////////////////////////////////////////////////// + cv::GMat in; + auto out = cv::gapi::I4202RGB(in); + + cv::GComputation c(in, out); + c.apply(in_mat1, out_mat_gapi, getCompileArgs()); + // OpenCV code ///////////////////////////////////////////////////////////// + { + cv::cvtColor(in_mat1, out_mat_ocv, cv::COLOR_YUV2RGB_I420); + } + // Comparison ////////////////////////////////////////////////////////////// + { + EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); + EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + } +} + TEST_P(NV12toRGBTest, AccuracyTest) { // G-API code ////////////////////////////////////////////////////////////// diff --git a/modules/gapi/test/common/gapi_parsers_tests_common.hpp b/modules/gapi/test/common/gapi_parsers_tests_common.hpp index 127a1c5a5e..328f86b851 100644 --- a/modules/gapi/test/common/gapi_parsers_tests_common.hpp +++ b/modules/gapi/test/common/gapi_parsers_tests_common.hpp @@ -176,7 +176,7 @@ private: int randInRange(const int start, const int end) { GAPI_Assert(start <= end); - return start + std::rand() % (end - start + 1); + return theRNG().uniform(start, end); } cv::Rect generateBox(const cv::Size& in_sz) @@ -211,7 +211,7 @@ private: SSDitem it; it.image_id = static_cast(i); it.label = static_cast(randInRange(0, 9)); - it.confidence = static_cast(std::rand()) / RAND_MAX; + it.confidence = theRNG().uniform(0.f, 1.f); auto box = generateBox(in_sz); it.rc_left = normalize(box.x, in_sz.width); it.rc_right = normalize(box.x + box.width, in_sz.width); @@ -225,16 +225,30 @@ private: class ParserYoloTest { public: - cv::Mat generateYoloOutput(const int num_classes) + cv::Mat generateYoloOutput(const int num_classes, std::pair dims_config = {false, 4}) { - std::vector dims = { 1, 13, 13, (num_classes + 5) * 5 }; + bool one_dim = false; + int num_dims = 0; + std::tie(one_dim, num_dims) = dims_config; + GAPI_Assert(num_dims <= 4); + GAPI_Assert((!one_dim && num_dims >= 3) || + ( one_dim && num_dims >= 1)); + std::vector dims(num_dims, 1); + if (one_dim) { + dims.back() = (num_classes+5)*5*13*13; + } else { + dims.back() = (num_classes+5)*5; + dims[num_dims-2] = 13; + dims[num_dims-3] = 13; + } cv::Mat mat(dims, CV_32FC1); auto data = mat.ptr(); - const size_t range = dims[0] * dims[1] * dims[2] * dims[3]; + const size_t range = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); + cv::RNG& rng = theRNG(); for (size_t i = 0; i < range; ++i) { - data[i] = static_cast(std::rand()) / RAND_MAX; + data[i] = rng.uniform(0.f, 1.f); } return mat; } diff --git a/modules/gapi/test/common/gapi_tests_common.hpp b/modules/gapi/test/common/gapi_tests_common.hpp index 113f3c73c0..514fa2be38 100644 --- a/modules/gapi/test/common/gapi_tests_common.hpp +++ b/modules/gapi/test/common/gapi_tests_common.hpp @@ -74,6 +74,50 @@ namespace } #endif // WINRT } + + template inline void initPointRandU(cv::RNG &rng, cv::Point_& pt) + { + GAPI_Assert(std::is_integral::value); + pt = cv::Point_(static_cast(static_cast(rng(CHAR_MAX + 1U))), + static_cast(static_cast(rng(CHAR_MAX + 1U)))); + } + + template inline void initPointRandU(cv::RNG &rng, cv::Point3_& pt) + { + GAPI_Assert(std::is_integral::value); + pt = cv::Point3_(static_cast(static_cast(rng(CHAR_MAX + 1U))), + static_cast(static_cast(rng(CHAR_MAX + 1U))), + static_cast(static_cast(rng(CHAR_MAX + 1U)))); + } + + template inline void initFloatPointRandU(cv::RNG &rng, cv::Point_ &pt) + { + GAPI_Assert(std::is_floating_point::value); + static const int fscale = 256; // avoid bits near ULP, generate stable test input + pt = cv::Point_(rng.uniform(0, 255 * fscale) / static_cast(fscale), + rng.uniform(0, 255 * fscale) / static_cast(fscale)); + } + + template<> inline void initPointRandU(cv::RNG &rng, cv::Point2f &pt) + { initFloatPointRandU(rng, pt); } + + template<> inline void initPointRandU(cv::RNG &rng, cv::Point2d &pt) + { initFloatPointRandU(rng, pt); } + + template inline void initFloatPointRandU(cv::RNG &rng, cv::Point3_ &pt) + { + GAPI_Assert(std::is_floating_point::value); + static const int fscale = 256; // avoid bits near ULP, generate stable test input + pt = cv::Point3_(rng.uniform(0, 255 * fscale) / static_cast(fscale), + rng.uniform(0, 255 * fscale) / static_cast(fscale), + rng.uniform(0, 255 * fscale) / static_cast(fscale)); + } + + template<> inline void initPointRandU(cv::RNG &rng, cv::Point3f &pt) + { initFloatPointRandU(rng, pt); } + + template<> inline void initPointRandU(cv::RNG &rng, cv::Point3d &pt) + { initFloatPointRandU(rng, pt); } } // namespace namespace opencv_test @@ -279,6 +323,80 @@ public: } } + template + inline void initPointRandU(cv::RNG& rng, T& pt) + { ::initPointRandU(rng, pt); } + +// Disable unreachable code warning for MSVS 2015 +#if defined _MSC_VER && _MSC_VER < 1910 /*MSVS 2017*/ +#pragma warning(push) +#pragma warning(disable: 4702) +#endif + // initialize std::vector>/std::vector> + template class Pt> + void initPointsVectorRandU(const int sz_in, std::vector> &vec_) + { + cv::RNG& rng = theRNG(); + + vec_.clear(); + vec_.reserve(sz_in); + + for (int i = 0; i < sz_in; i++) + { + Pt pt; + initPointRandU(rng, pt); + vec_.emplace_back(pt); + } + } +#if defined _MSC_VER && _MSC_VER < 1910 /*MSVS 2017*/ +#pragma warning(pop) +#endif + + template + inline void initMatByPointsVectorRandU(const cv::Size &sz_in) + { + std::vector in_vector; + initPointsVectorRandU(sz_in.width, in_vector); + in_mat1 = cv::Mat(in_vector, true); + } + + // initialize Mat by a vector of Points + template