python: bindings loader package

Configures and loads OpenCV bindings extension including 3rdparty dependencies

Based on running Python specify:
- configure PYTHON_PATH (via "sys.path")
- configure LD_LIBRARY_PATH / PATH

CMakeLists.txt

@@ -299,8 +299,8 @@ OCV_OPTION(BUILD_ANDROID_EXAMPLES   "Build examples for Android platform"
 OCV_OPTION(BUILD_DOCS               "Create build rules for OpenCV Documentation" OFF  IF (NOT WINRT AND NOT APPLE_FRAMEWORK))
 OCV_OPTION(BUILD_EXAMPLES           "Build all examples"                          OFF )
 OCV_OPTION(BUILD_PACKAGE            "Enables 'make package_source' command"       ON  IF NOT WINRT)
-OCV_OPTION(BUILD_PERF_TESTS         "Build performance tests"                     ON  IF (NOT APPLE_FRAMEWORK) )
+OCV_OPTION(BUILD_PERF_TESTS         "Build performance tests"                     NOT INSTALL_CREATE_DISTRIB  IF (NOT APPLE_FRAMEWORK) )
-OCV_OPTION(BUILD_TESTS              "Build accuracy & regression tests"           ON  IF (NOT APPLE_FRAMEWORK) )
+OCV_OPTION(BUILD_TESTS              "Build accuracy & regression tests"           NOT INSTALL_CREATE_DISTRIB  IF (NOT APPLE_FRAMEWORK) )
 OCV_OPTION(BUILD_WITH_DEBUG_INFO    "Include debug info into release binaries ('OFF' means default settings)" OFF )
 OCV_OPTION(BUILD_WITH_STATIC_CRT    "Enables use of statically linked CRT for statically linked OpenCV" ON IF MSVC )
 OCV_OPTION(BUILD_WITH_DYNAMIC_IPP   "Enables dynamic linking of IPP (only for standalone IPP)" OFF )
@@ -470,6 +470,7 @@ else()
     endif()
   endif()
   ocv_update(OPENCV_INCLUDE_INSTALL_PATH "include")
+  ocv_update(OPENCV_PYTHON_INSTALL_PATH  "python")
 endif()
 
 ocv_update(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${OPENCV_LIB_INSTALL_PATH}")

cmake/OpenCVGenSetupVars.cmake

@@ -43,7 +43,12 @@ else()
 endif()
 file(RELATIVE_PATH OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG
     "${CMAKE_INSTALL_PREFIX}/${OPENCV_SETUPVARS_INSTALL_PATH}/" "${CMAKE_INSTALL_PREFIX}/")
-ocv_path_join(OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG "${OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG}" "python_loader")  # https://github.com/opencv/opencv/pull/12977
+if(IS_ABSOLUTE "${OPENCV_PYTHON_INSTALL_PATH}")
+  set(OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG "${OPENCV_PYTHON_INSTALL_PATH}")
+  message(WARNING "CONFIGURATION IS NOT SUPPORTED: validate setupvars script in install directory")
+else()
+  ocv_path_join(OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG "${OPENCV_PYTHON_DIR_RELATIVE_CMAKECONFIG}" "${OPENCV_PYTHON_INSTALL_PATH}")
+endif()
 configure_file("${OpenCV_SOURCE_DIR}/cmake/templates/${OPENCV_SETUPVARS_TEMPLATE}" "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/install/${OPENCV_SETUPVARS_FILENAME}" @ONLY)
 install(FILES "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/install/${OPENCV_SETUPVARS_FILENAME}"
     DESTINATION "${OPENCV_SETUPVARS_INSTALL_PATH}"

modules/python/CMakeLists.txt

@@ -3,11 +3,10 @@
 # ----------------------------------------------------------------------------
 if(DEFINED OPENCV_INITIAL_PASS)  # OpenCV build
 
-add_subdirectory(bindings)
-
 if(ANDROID OR APPLE_FRAMEWORK OR WINRT)
-  set(__disable_python2 ON)
+  ocv_module_disable_(python2)
-  set(__disable_python3 ON)
+  ocv_module_disable_(python3)
+  return()
 elseif(BUILD_opencv_world OR (WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug"))
   if(NOT DEFINED BUILD_opencv_python2)
     set(__disable_python2 ON)
@@ -17,6 +16,12 @@ elseif(BUILD_opencv_world OR (WIN32 AND CMAKE_BUILD_TYPE STREQUAL "Debug"))
   endif()
 endif()
 
+add_subdirectory(bindings)
+
+if(NOT OPENCV_SKIP_PYTHON_LOADER)
+  include("./python_loader.cmake")
+endif()
+
 if(__disable_python2)
   ocv_module_disable_(python2)
 endif()

modules/python/bindings/CMakeLists.txt

@@ -42,6 +42,7 @@ ocv_list_filterout(opencv_hdrs "modules/.*\\\\.inl\\\\.h*")
 ocv_list_filterout(opencv_hdrs "modules/.*_inl\\\\.h*")
 ocv_list_filterout(opencv_hdrs "modules/.*\\\\.details\\\\.h*")
 ocv_list_filterout(opencv_hdrs "modules/.*\\\\.private\\\\.h*")
+ocv_list_filterout(opencv_hdrs "modules/.*/private\\\\.h*")
 ocv_list_filterout(opencv_hdrs "modules/.*/detection_based_tracker\\\\.hpp") # Conditional compilation
 
 set(cv2_generated_files
@@ -100,6 +101,7 @@ ocv_cmake_script_append_var(PYTHON_CONFIG_SCRIPT
 
     CMAKE_MODULE_LINKER_FLAGS
     CMAKE_INSTALL_PREFIX
+    OPENCV_PYTHON_INSTALL_PATH
 
     OpenCV_SOURCE_DIR
 

modules/python/common.cmake

@@ -56,8 +56,10 @@ else()
   endif()
 endif()
 
+ocv_update(OPENCV_PYTHON_EXTENSION_BUILD_PATH "${LIBRARY_OUTPUT_PATH}/${MODULE_INSTALL_SUBDIR}")
+
 set_target_properties(${the_module} PROPERTIES
-                      LIBRARY_OUTPUT_DIRECTORY  "${LIBRARY_OUTPUT_PATH}/${MODULE_INSTALL_SUBDIR}"
+                      LIBRARY_OUTPUT_DIRECTORY  "${OPENCV_PYTHON_EXTENSION_BUILD_PATH}"
                       ARCHIVE_OUTPUT_NAME ${the_module}  # prevent name conflict for python2/3 outputs
                       DEFINE_SYMBOL CVAPI_EXPORTS
                       PREFIX ""
@@ -110,33 +112,67 @@ else()
   set(PYTHON_INSTALL_ARCHIVE ARCHIVE DESTINATION ${${PYTHON}_PACKAGES_PATH} COMPONENT python)
 endif()
 
-if(DEFINED OPENCV_${PYTHON}_INSTALL_PATH)
+ocv_assert(${PYTHON}_VERSION_MAJOR)
-  set(__dst "${OPENCV_${PYTHON}_INSTALL_PATH}")
+ocv_assert(${PYTHON}_VERSION_MINOR)
-elseif(NOT INSTALL_CREATE_DISTRIB AND DEFINED ${PYTHON}_PACKAGES_PATH)
+
-  set(__dst "${${PYTHON}_PACKAGES_PATH}")
+set(__python_loader_subdir "")
+if(NOT OPENCV_SKIP_PYTHON_LOADER)
+  set(__python_loader_subdir "cv2/")
 endif()
-if(NOT __dst)
+
-  if(DEFINED ${PYTHON}_VERSION_MAJOR)
+if(NOT " ${PYTHON}" STREQUAL " PYTHON" AND DEFINED OPENCV_${PYTHON}_INSTALL_PATH)
-    set(__ver "${${PYTHON}_VERSION_MAJOR}.${${PYTHON}_VERSION_MINOR}")
+  set(__python_binary_install_path "${OPENCV_${PYTHON}_INSTALL_PATH}")
-  elseif(DEFINED ${PYTHON}_VERSION_STRING)
+else()
-    set(__ver "${${PYTHON}_VERSION_STRING}")
+  ocv_assert(DEFINED OPENCV_PYTHON_INSTALL_PATH)
-  else()
+  set(__python_binary_install_path "${OPENCV_PYTHON_INSTALL_PATH}/${__python_loader_subdir}python-${${PYTHON}_VERSION_MAJOR}.${${PYTHON}_VERSION_MINOR}")
-    set(__ver "unknown")
-  endif()
-  if(INSTALL_CREATE_DISTRIB)
-    set(__dst "python/${__ver}/${OpenCV_ARCH}")
-  else()
-    set(__dst "python/${__ver}")
-  endif()
 endif()
 
 install(TARGETS ${the_module}
         ${PYTHON_INSTALL_CONFIGURATIONS}
-        RUNTIME DESTINATION "${__dst}" COMPONENT python
+        RUNTIME DESTINATION "${__python_binary_install_path}" COMPONENT python
-        LIBRARY DESTINATION "${__dst}" COMPONENT python
+        LIBRARY DESTINATION "${__python_binary_install_path}" COMPONENT python
         ${PYTHON_INSTALL_ARCHIVE}
         )
 
+if(NOT OPENCV_SKIP_PYTHON_LOADER)
+  ocv_assert(DEFINED OPENCV_PYTHON_INSTALL_PATH)
+  if(OpenCV_FOUND)
+    set(__loader_path "${OpenCV_BINARY_DIR}/python_loader")
+  else()
+    set(__loader_path "${CMAKE_BINARY_DIR}/python_loader")
+  endif()
+
+  set(__python_loader_install_tmp_path "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/install/python_loader/")
+  if(IS_ABSOLUTE "${OPENCV_PYTHON_INSTALL_PATH}")
+    set(OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/")
+    set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "'${CMAKE_INSTALL_PREFIX}'")
+  else()
+    file(RELATIVE_PATH OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/${OPENCV_PYTHON_INSTALL_PATH}/cv2" ${CMAKE_INSTALL_PREFIX})
+    set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "os.path.join(LOADER_DIR, '${OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE}')")
+  endif()
+
+  if(DEFINED ${PYTHON}_VERSION_MINOR)
+    set(__target_config "config-${${PYTHON}_VERSION_MAJOR}.${${PYTHON}_VERSION_MINOR}.py")
+  else()
+    set(__target_config "config-${${PYTHON}_VERSION_MAJOR}.py")
+  endif()
+
+  if(CMAKE_GENERATOR MATCHES "Visual Studio")
+    set(CMAKE_PYTHON_EXTENSION_PATH "'${OPENCV_PYTHON_EXTENSION_BUILD_PATH}/Release'")  # TODO: CMAKE_BUILD_TYPE is not defined
+  else()
+    set(CMAKE_PYTHON_EXTENSION_PATH "'${OPENCV_PYTHON_EXTENSION_BUILD_PATH}'")
+  endif()
+  configure_file("${PYTHON_SOURCE_DIR}/package/template/config-x.y.py.in" "${__loader_path}/cv2/${__target_config}" @ONLY)
+
+  if(IS_ABSOLUTE __python_binary_install_path)
+    set(CMAKE_PYTHON_EXTENSION_PATH "'${__python_binary_install_path}'")
+  else()
+    set(CMAKE_PYTHON_EXTENSION_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${__python_binary_install_path}')")
+  endif()
+  configure_file("${PYTHON_SOURCE_DIR}/package/template/config-x.y.py.in" "${__python_loader_install_tmp_path}/cv2/${__target_config}" @ONLY)
+  install(FILES "${__python_loader_install_tmp_path}/cv2/${__target_config}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python)
+endif()  # NOT OPENCV_SKIP_PYTHON_LOADER
+
 unset(PYTHON_SRC_DIR)
 unset(PYTHON_CVPY_PROCESS)
 unset(CVPY_SUFFIX)

modules/python/package/.gitignore

@@ -0,0 +1,4 @@
+__pycache__
+*.pyc
+*.egg-info
+*dist

modules/python/package/cv2/__init__.py

@@ -0,0 +1,89 @@
+'''
+OpenCV Python binary extension loader
+'''
+import os
+import sys
+
+try:
+    import numpy
+    import numpy.core.multiarray
+except ImportError:
+    print('OpenCV bindings requires "numpy" package.')
+    print('Install it via command:')
+    print('    pip install numpy')
+    raise
+
+# TODO
+# is_x64 = sys.maxsize > 2**32
+
+def bootstrap():
+    import sys
+    if hasattr(sys, 'OpenCV_LOADER'):
+        print(sys.path)
+        raise ImportError('ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.')
+    sys.OpenCV_LOADER = True
+
+    DEBUG = False
+    if hasattr(sys, 'OpenCV_LOADER_DEBUG'):
+        DEBUG = True
+
+    import platform
+    if DEBUG: print('OpenCV loader: os.name="{}"  platform.system()="{}"'.format(os.name, str(platform.system())))
+
+    LOADER_DIR=os.path.dirname(os.path.abspath(__file__))
+
+    PYTHON_EXTENSIONS_PATHS = []
+    BINARIES_PATHS = []
+
+    g_vars = globals()
+    l_vars = locals()
+
+    if sys.version_info[:2] < (3, 0):
+        from cv2.load_config_py2 import exec_file_wrapper
+    else:
+        from . load_config_py3 import exec_file_wrapper
+
+    def load_first_config(fnames, required=True):
+        for fname in fnames:
+            fpath = os.path.join(LOADER_DIR, fname)
+            if not os.path.exists(fpath):
+                if DEBUG: print('OpenCV loader: config not found, skip: {}'.format(fpath))
+                continue
+            if DEBUG: print('OpenCV loader: loading config: {}'.format(fpath))
+            exec_file_wrapper(fpath, g_vars, l_vars)
+            return True
+        if required:
+            raise ImportError('OpenCV loader: missing configuration file: {}. Check OpenCV installation.'.format(fnames))
+
+    load_first_config(['config.py'], True)
+    load_first_config([
+        'config-{}.{}.py'.format(sys.version_info[0], sys.version_info[1]),
+        'config-{}.py'.format(sys.version_info[0])
+    ], True)
+
+    if DEBUG: print('OpenCV loader: PYTHON_EXTENSIONS_PATHS={}'.format(str(l_vars['PYTHON_EXTENSIONS_PATHS'])))
+    if DEBUG: print('OpenCV loader: BINARIES_PATHS={}'.format(str(l_vars['BINARIES_PATHS'])))
+
+    for p in reversed(l_vars['PYTHON_EXTENSIONS_PATHS']):
+        sys.path.insert(0, p)
+
+    if os.name == 'nt':
+        os.environ['PATH'] = ';'.join(l_vars['BINARIES_PATHS']) + ';' + os.environ.get('PATH', '')
+        if DEBUG: print('OpenCV loader: PATH={}'.format(str(os.environ['PATH'])))
+    else:
+        # amending of LD_LIBRARY_PATH works for sub-processes only
+        os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '')
+
+    if DEBUG: print('OpenCV loader: replacing cv2 module')
+    del sys.modules['cv2']
+    import cv2
+
+    try:
+        import sys
+        del sys.OpenCV_LOADER
+    except:
+        pass
+
+    if DEBUG: print('OpenCV loader: DONE')
+
+bootstrap()

modules/python/package/cv2/load_config_py2.py

@@ -0,0 +1,6 @@
+# flake8: noqa
+import sys
+
+if sys.version_info[:2] < (3, 0):
+    def exec_file_wrapper(fpath, g_vars, l_vars):
+        execfile(fpath, g_vars, l_vars)

modules/python/package/cv2/load_config_py3.py

@@ -0,0 +1,9 @@
+# flake8: noqa
+import os
+import sys
+
+if sys.version_info[:2] >= (3, 0):
+    def exec_file_wrapper(fpath, g_vars, l_vars):
+        with open(fpath) as f:
+            code = compile(f.read(), os.path.basename(fpath), 'exec')
+            exec(code, g_vars, l_vars)

modules/python/package/setup.py

@@ -0,0 +1,57 @@
+import os
+import sys
+import platform
+import setuptools
+
+SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__))
+
+def main():
+    os.chdir(SCRIPT_DIR)
+
+    package_name = 'opencv'
+    package_version = os.environ.get('OPENCV_VERSION', '3.4.4')  # TODO
+
+    long_description = 'Open Source Computer Vision Library Python bindings'  # TODO
+
+    setuptools.setup(
+        name=package_name,
+        version=package_version,
+        url='https://github.com/opencv/opencv',
+        license='BSD',
+        description='OpenCV python bindings',
+        long_description=long_description,
+        long_description_content_type="text/markdown",
+        packages=setuptools.find_packages(),
+        maintainer="OpenCV Team",
+        install_requires="numpy",
+        classifiers=[
+          'Development Status :: 5 - Production/Stable',
+          'Environment :: Console',
+          'Intended Audience :: Developers',
+          'Intended Audience :: Education',
+          'Intended Audience :: Information Technology',
+          'Intended Audience :: Science/Research',
+          'License :: BSD License',
+          'Operating System :: MacOS',
+          'Operating System :: Microsoft :: Windows',
+          'Operating System :: POSIX',
+          'Operating System :: Unix',
+          'Programming Language :: Python',
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+          'Programming Language :: C++',
+          'Programming Language :: Python :: Implementation :: CPython',
+          'Topic :: Scientific/Engineering',
+          'Topic :: Scientific/Engineering :: Image Recognition',
+          'Topic :: Software Development',
+          'Topic :: Software Development :: Libraries',
+        ],
+    )
+
+if __name__ == '__main__':
+    main()

modules/python/package/template/config-x.y.py.in

@@ -0,0 +1,3 @@
+PYTHON_EXTENSIONS_PATHS = [
+    @CMAKE_PYTHON_EXTENSION_PATH@
+] + PYTHON_EXTENSIONS_PATHS

modules/python/package/template/config.py.in

@@ -0,0 +1,3 @@
+BINARIES_PATHS = [
+    @CMAKE_PYTHON_BINARIES_PATH@
+] + BINARIES_PATHS

modules/python/python_loader.cmake

@@ -0,0 +1,54 @@
+ocv_assert(NOT OPENCV_SKIP_PYTHON_LOADER)
+
+set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}")
+
+ocv_assert(DEFINED OPENCV_PYTHON_INSTALL_PATH)
+if(OpenCV_FOUND)
+  set(__loader_path "${OpenCV_BINARY_DIR}/python_loader")
+else()
+  set(__loader_path "${CMAKE_BINARY_DIR}/python_loader")
+endif()
+
+set(__python_loader_install_tmp_path "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/install/python_loader/")
+if(IS_ABSOLUTE "${OPENCV_PYTHON_INSTALL_PATH}")
+  set(OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/")
+  set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "'${CMAKE_INSTALL_PREFIX}'")
+else()
+  file(RELATIVE_PATH OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE "${CMAKE_INSTALL_PREFIX}/${OPENCV_PYTHON_INSTALL_PATH}/cv2" ${CMAKE_INSTALL_PREFIX})
+  set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "os.path.join(LOADER_DIR, '${OpenCV_PYTHON_INSTALL_PATH_RELATIVE_CONFIGCMAKE}')")
+endif()
+
+set(PYTHON_LOADER_FILES
+    "setup.py" "cv2/__init__.py"
+    "cv2/load_config_py2.py" "cv2/load_config_py3.py"
+)
+foreach(fname ${PYTHON_LOADER_FILES})
+  get_filename_component(__dir "${fname}" DIRECTORY)
+  file(COPY "${PYTHON_SOURCE_DIR}/package/${fname}" DESTINATION "${__loader_path}/${__dir}")
+  install(FILES "${PYTHON_SOURCE_DIR}/package/${fname}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/${__dir}" COMPONENT python)
+endforeach()
+
+if(NOT OpenCV_FOUND)  # Ignore "standalone" builds of Python bindings
+  if(WIN32)
+    if(CMAKE_GENERATOR MATCHES "Visual Studio")
+      list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'")  # TODO: CMAKE_BUILD_TYPE is not defined
+    else()
+      list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'")
+    endif()
+  else()
+    list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'")
+  endif()
+  string(REPLACE ";" ",\n    " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}")
+  configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY)
+
+  if(WIN32)
+    list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')")
+  else()
+    list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')")
+  endif()
+  string(REPLACE ";" ",\n    " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}")
+  configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY)
+  install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python)
+
+  message(STATUS "OpenCV Python: during development append to PYTHONPATH: ${__loader_path}")
+endif()