From 3c89a28a0612c3f44d9b44fbe28e4bf0ded564d2 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Sat, 18 Sep 2021 10:02:55 +0300 Subject: [PATCH] Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules * feat: OpenCV extension with pure Python modules * feat: cv2 is now a Python package instead of extension module Python package cv2 now can handle both Python and C extension modules properly without additional "subfolders" like "_extra_py_code". * feat: can call native function from its reimplementation in Python --- .../include/opencv2/core/bindings_utils.hpp | 6 ++ .../misc/python/package/utils/__init__.py | 14 ++++ modules/python/common.cmake | 47 +++++++---- modules/python/package/cv2/__init__.py | 83 +++++++++++++++---- .../package/cv2/_extra_py_code/__init__.py | 53 ------------ .../package/extra_modules/misc/__init__.py | 1 + .../package/extra_modules/misc/version.py | 5 ++ modules/python/python3/CMakeLists.txt | 14 ++++ modules/python/python_loader.cmake | 1 - modules/python/test/test_misc.py | 31 +++++++ 10 files changed, 170 insertions(+), 85 deletions(-) create mode 100644 modules/core/misc/python/package/utils/__init__.py delete mode 100644 modules/python/package/cv2/_extra_py_code/__init__.py create mode 100644 modules/python/package/extra_modules/misc/__init__.py create mode 100644 modules/python/package/extra_modules/misc/version.py diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index a3f45eb0c7..f9c73dd4ff 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,12 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +int testOverwriteNativeMethod(int argument) +{ + return argument; +} + CV_WRAP static inline String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) { diff --git a/modules/core/misc/python/package/utils/__init__.py b/modules/core/misc/python/package/utils/__init__.py new file mode 100644 index 0000000000..49cd40ba2d --- /dev/null +++ b/modules/core/misc/python/package/utils/__init__.py @@ -0,0 +1,14 @@ +from collections import namedtuple + +import cv2 + + +NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult", + ("py", "native")) + + +def testOverwriteNativeMethod(arg): + return NativeMethodPatchedResult( + arg + 1, + cv2.utils._native.testOverwriteNativeMethod(arg) + ) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index cedf071434..251b78c6cb 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,6 +1,31 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}") +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) + endif() + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endforeach() + message(STATUS "Found ${extra_modules} Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files") + endif() +endfunction() + ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) include_directories(SYSTEM @@ -224,23 +249,15 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" ) - set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - file(GLOB_RECURSE extra_py_files - RELATIVE "${__base}" - "${__base}/**/*.py" - ) - if(extra_py_files) - list(SORT extra_py_files) - foreach(f ${extra_py_files}) - get_filename_component(__dir "${f}" DIRECTORY) - configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY) - install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python) - endforeach() - else() - message(WARNING "Module ${m} has no .py files in misc/python/package") - endif() + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") endif() endforeach(m) + + if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() + endif() endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 27e7edd083..80e2612276 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -2,6 +2,7 @@ OpenCV Python binary extension loader ''' import os +import importlib import sys __all__ = [] @@ -15,17 +16,54 @@ except ImportError: print(' pip install numpy') raise - -py_code_loader = None -if sys.version_info[:2] >= (3, 0): - try: - from . import _extra_py_code as py_code_loader - except: - pass - # TODO # is_x64 = sys.maxsize > 2**32 + +def __load_extra_py_code_for_module(base, name, enable_debug_print=False): + module_name = "{}.{}".format(__name__, name) + export_module_name = "{}.{}".format(base, name) + native_module = sys.modules.pop(module_name, None) + try: + py_module = importlib.import_module(module_name) + except ImportError as err: + if enable_debug_print: + print("Can't load Python code for module:", module_name, + ". Reason:", err) + # Extension doesn't contain extra py code + return False + + if not hasattr(base, name): + setattr(sys.modules[base], name, py_module) + sys.modules[export_module_name] = py_module + # If it is C extension module it is already loaded by cv2 package + if native_module: + setattr(py_module, "_native", native_module) + for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), + native_module.__dict__.items()): + if enable_debug_print: print(' symbol: {} = {}'.format(k, v)) + setattr(py_module, k, v) + return True + + +def __collect_extra_submodules(enable_debug_print=False): + def modules_filter(module): + return all(( + # module is not internal + not module.startswith("_"), + # it is not a file + os.path.isdir(os.path.join(_extra_submodules_init_path, module)) + )) + if sys.version_info[0] < 3: + if enable_debug_print: + print("Extra submodules is loaded only for Python 3") + return [] + + __INIT_FILE_PATH = os.path.abspath(__file__) + _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH) + return filter(modules_filter, os.listdir(_extra_submodules_init_path)) + + def bootstrap(): import sys @@ -107,23 +145,36 @@ def bootstrap(): # amending of LD_LIBRARY_PATH works for sub-processes only os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '') - if DEBUG: print('OpenCV loader: replacing cv2 module') - del sys.modules['cv2'] - import cv2 + if DEBUG: print("Relink everything from native cv2 module to cv2 package") + + py_module = sys.modules.pop("cv2") + + native_module = importlib.import_module("cv2") + + sys.modules["cv2"] = py_module + setattr(py_module, "_native", native_module) + + for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__", + "__name__", "__package__"), + native_module.__dict__.items()): + if item_name not in g_vars: + g_vars[item_name] = item sys.path = save_sys_path # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502) try: - import sys del sys.OpenCV_LOADER - except: - pass + except Exception as e: + if DEBUG: + print("Exception during delete OpenCV_LOADER:", e) if DEBUG: print('OpenCV loader: binary extension... OK') - if py_code_loader: - py_code_loader.init('cv2') + for submodule in __collect_extra_submodules(DEBUG): + if __load_extra_py_code_for_module("cv2", submodule, DEBUG): + if DEBUG: print("Extra Python code for", submodule, "is loaded") if DEBUG: print('OpenCV loader: DONE') + bootstrap() diff --git a/modules/python/package/cv2/_extra_py_code/__init__.py b/modules/python/package/cv2/_extra_py_code/__init__.py deleted file mode 100644 index be84566825..0000000000 --- a/modules/python/package/cv2/_extra_py_code/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import importlib - -__all__ = ['init'] - - -DEBUG = False -if hasattr(sys, 'OpenCV_LOADER_DEBUG'): - DEBUG = True - - -def _load_py_code(base, name): - try: - m = importlib.import_module(__name__ + name) - except ImportError: - return # extension doesn't exist? - - if DEBUG: print('OpenCV loader: added python code extension for: ' + name) - - if hasattr(m, '__all__'): - export_members = { k : getattr(m, k) for k in m.__all__ } - else: - export_members = m.__dict__ - - for k, v in export_members.items(): - if k.startswith('_'): # skip internals - continue - if isinstance(v, type(sys)): # don't bring modules - continue - if DEBUG: print(' symbol: {} = {}'.format(k, v)) - setattr(sys.modules[base + name ], k, v) - - del sys.modules[__name__ + name] - - -# TODO: listdir -def init(base): - _load_py_code(base, '.cv2') # special case - prefix = base - prefix_len = len(prefix) - - modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ] - for m in modules: - m2 = m[prefix_len:] # strip prefix - if len(m2) == 0: - continue - if m2.startswith('._'): # skip internals - continue - if m2.startswith('.load_config_'): # skip helper files - continue - _load_py_code(base, m2) - - del sys.modules[__name__] diff --git a/modules/python/package/extra_modules/misc/__init__.py b/modules/python/package/extra_modules/misc/__init__.py new file mode 100644 index 0000000000..3e0559d3d4 --- /dev/null +++ b/modules/python/package/extra_modules/misc/__init__.py @@ -0,0 +1 @@ +from .version import get_ocv_version diff --git a/modules/python/package/extra_modules/misc/version.py b/modules/python/package/extra_modules/misc/version.py new file mode 100644 index 0000000000..d34705872e --- /dev/null +++ b/modules/python/package/extra_modules/misc/version.py @@ -0,0 +1,5 @@ +import cv2 + + +def get_ocv_version(): + return getattr(cv2, "__version__", "unavailable") diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index d95af21e04..0bb401f2ec 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -15,9 +15,23 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) +set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules) +set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + +if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +else() + list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +endif() + +unset(_ocv_extra_modules_path) + set(PYTHON PYTHON3) include(../common.cmake) +set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path}) + +unset(_old_ocv_python_extra_modules_path) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index 677111b061..a872ffd763 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -25,7 +25,6 @@ endif() set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" - "cv2/_extra_py_code/__init__.py" ) foreach(fname ${PYTHON_LOADER_FILES}) get_filename_component(__dir "${fname}" DIRECTORY) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 111b1427b4..76803992dc 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -4,6 +4,12 @@ from __future__ import print_function import ctypes from functools import partial from collections import namedtuple +import sys + +if sys.version_info[0] < 3: + from collections import Sequence +else: + from collections.abc import Sequence import numpy as np import cv2 as cv @@ -585,6 +591,31 @@ class Arguments(NewOpenCVTests): self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") +class CanUsePurePythonModuleFunction(NewOpenCVTests): + def test_can_get_ocv_version(self): + import sys + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + self.assertEqual(cv.misc.get_ocv_version(), cv.__version__, + "Can't get package version using Python misc module") + + def test_native_method_can_be_patched(self): + import sys + + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + res = cv.utils.testOverwriteNativeMethod(10) + self.assertTrue(isinstance(res, Sequence), + msg="Overwritten method should return sequence. " + "Got: {} of type {}".format(res, type(res))) + self.assertSequenceEqual(res, (11, 10), + msg="Failed to overwrite native method") + res = cv.utils._native.testOverwriteNativeMethod(123) + self.assertEqual(res, 123, msg="Failed to call native method implementation") + + class SamplesFindFile(NewOpenCVTests): def test_ExistedFile(self):