mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-24 03:00:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #20370 from ddacw:stub-gen-next

Browse Source 
Python typing stub generation #20370

Add stub generation to `gen2.py`, addressing #14590.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or other license that is incompatible with OpenCV
- [x] The PR is proposed to proper branch
- [x] There is reference to original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Duong Dac 2023-05-26 17:25:46 +02:00 committed by GitHub
parent cf0ba039c3
commit a9424868a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 3449 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/OpenCVDetectPython.cmake
View File

@ -297,3 +297,10 @@ elseif(PYTHON3_EXECUTABLE AND PYTHON3INTERP_FOUND)
set(PYTHON_DEFAULT_AVAILABLE "TRUE")
set(PYTHON_DEFAULT_EXECUTABLE "${PYTHON3_EXECUTABLE}")
endif()
if(PYTHON_DEFAULT_AVAILABLE)
execute_process(COMMAND ${PYTHON_DEFAULT_EXECUTABLE} --version
OUTPUT_VARIABLE PYTHON_DEFAULT_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REGEX MATCH "[0-9]+.[0-9]+.[0-9]+" PYTHON_DEFAULT_VERSION "${PYTHON_DEFAULT_VERSION}")
endif()

7
modules/python/bindings/CMakeLists.txt
View File

@ -76,11 +76,14 @@ set(cv2_generated_files
string(REPLACE ";" "\n" opencv_hdrs_ "${opencv_hdrs}")
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${opencv_hdrs_}")
file(GLOB_RECURSE typing_stubs_generation_files "${PYTHON_SOURCE_DIR}/src2/typing_stubs_generation/*.py")
add_custom_command(
OUTPUT ${cv2_generated_files}
COMMAND "${PYTHON_DEFAULT_EXECUTABLE}" "${PYTHON_SOURCE_DIR}/src2/gen2.py" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/headers.txt"
DEPENDS "${PYTHON_SOURCE_DIR}/src2/gen2.py"
"${PYTHON_SOURCE_DIR}/src2/hdr_parser.py"
"${typing_stubs_generation_files}"
"${PYTHON_SOURCE_DIR}/src2/typing_stubs_generator.py"
# not a real build dependency (file(WRITE) result): ${CMAKE_CURRENT_BINARY_DIR}/headers.txt
${opencv_hdrs}
COMMENT "Generate files for Python bindings and documentation"
@ -88,6 +91,10 @@ add_custom_command(
add_custom_target(gen_opencv_python_source DEPENDS ${cv2_generated_files})
if(TARGET copy_opencv_typing_stubs)
add_dependencies(copy_opencv_typing_stubs gen_opencv_python_source)
endif()
set(cv2_custom_hdr "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_custom_headers.h")
set(cv2_custom_hdr_str "//user-defined headers\n")
foreach(uh ${opencv_userdef_hdrs})

5
modules/python/common.cmake
View File

@ -34,6 +34,11 @@ if(TARGET gen_opencv_python_source)
add_dependencies(${the_module} gen_opencv_python_source)
endif()
if(TARGET copy_opencv_typing_stubs)
# Python 3.6+
add_dependencies(${the_module} copy_opencv_typing_stubs)
endif()
ocv_assert(${PYTHON}_VERSION_MAJOR)
ocv_assert(${PYTHON}_VERSION_MINOR)

27
modules/python/package/setup.py
View File

@ -1,9 +1,19 @@
import os
import sys
import platform
import setuptools
SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__))
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
def collect_module_typing_stub_files(root_module_path):
stub_files = []
for module_path, _, files in os.walk(root_module_path):
stub_files.extend(
map(lambda p: os.path.join(module_path, p),
filter(lambda f: f.endswith(".pyi"), files))
)
return stub_files
def main():
os.chdir(SCRIPT_DIR)
@ -13,6 +23,13 @@ def main():
long_description = 'Open Source Computer Vision Library Python bindings' # TODO
root_module_path = os.path.join(SCRIPT_DIR, "cv2")
py_typed_path = os.path.join(root_module_path, "py.typed")
if os.path.isfile(py_typed_path):
typing_stub_files = collect_module_typing_stub_files(root_module_path)
if len(typing_stub_files) > 0:
typing_stub_files.append(py_typed_path)
setuptools.setup(
name=package_name,
version=package_version,
@ -22,6 +39,9 @@ def main():
long_description=long_description,
long_description_content_type="text/markdown",
packages=setuptools.find_packages(),
package_data={
"cv2": typing_stub_files
},
maintainer="OpenCV Team",
install_requires="numpy",
classifiers=[
@ -55,5 +75,6 @@ def main():
],
)
if __name__ == '__main__':
main()

19
modules/python/python_loader.cmake
View File

@ -45,8 +45,6 @@ foreach(fname ${PYTHON_LOADER_FILES})
endif()
endforeach()
if(WIN32)
if(CMAKE_GENERATOR MATCHES "Visual Studio")
list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined
@ -126,3 +124,20 @@ if(NOT "${OPENCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "")
ocv_add_python_files_from_path(${extra_ocv_py_modules_path})
endforeach()
endif()
if(${PYTHON}_VERSION_STRING VERSION_GREATER "3.6" AND PYTHON_DEFAULT_VERSION VERSION_GREATER "3.6")
add_custom_target(copy_opencv_typing_stubs)
# Copy all generated stub files to python_loader directory only if
# generation succeeds, this behvoir can't be achieved with default
# CMake constructions, because failed generation produces a warning instead of
# halts on hard error.
add_custom_command(
TARGET copy_opencv_typing_stubs
COMMAND ${PYTHON_DEFAULT_EXECUTABLE} ${PYTHON_SOURCE_DIR}/src2/copy_typings_stubs_on_success.py
--stubs_dir ${OPENCV_PYTHON_BINDINGS_DIR}/cv2
--output_dir ${__loader_path}/cv2
)
if(DEFINED OPENCV_PYTHON_INSTALL_PATH)
install(DIRECTORY "${OPENCV_PYTHON_BINDINGS_DIR}/cv2" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}" COMPONENT python)
endif()
endif()

45
modules/python/src2/copy_typings_stubs_on_success.py Normal file
View File

@ -0,0 +1,45 @@
import argparse
import warnings
import os
import sys
if sys.version_info >= (3, 8, ):
# shutil.copytree received the `dirs_exist_ok` parameter
from functools import partial
import shutil
copy_tree = partial(shutil.copytree, dirs_exist_ok=True)
else:
from distutils.dir_util import copy_tree
def main():
args = parse_arguments()
py_typed_path = os.path.join(args.stubs_dir, 'py.typed')
if not os.path.isfile(py_typed_path):
warnings.warn(
'{} is missing, it means that typings stubs generation is either '
'failed or has been skipped. Ensure that Python 3.6+ is used for '
'build and there is no warnings during Python source code '
'generation phase.'.format(py_typed_path)
)
return
copy_tree(args.stubs_dir, args.output_dir)
def parse_arguments():
parser = argparse.ArgumentParser(
description='Copies generated typing stubs only when generation '
'succeeded. This is identified by presence of the `py.typed` file '
'inside typing stubs directory.'
)
parser.add_argument('--stubs_dir', type=str,
help='Path to directory containing generated typing '
'stubs file')
parser.add_argument('--output_dir', type=str,
help='Path to output directory')
return parser.parse_args()
if __name__ == '__main__':
main()

129
modules/python/src2/gen2.py
View File

@ -1,16 +1,34 @@
#!/usr/bin/env python
from __future__ import print_function
import hdr_parser, sys, re, os
import hdr_parser, sys, re
from string import Template
from pprint import pprint
from collections import namedtuple
from itertools import chain
from typing_stubs_generator import TypingStubsGenerator
if sys.version_info[0] >= 3:
from io import StringIO
else:
from cStringIO import StringIO
if sys.version_info >= (3, 6):
from typing_stubs_generation import SymbolName
else:
SymbolName = namedtuple('SymbolName', ('namespaces', 'classes', 'name'))
def parse_symbol_name(cls, full_symbol_name, known_namespaces):
chunks = full_symbol_name.split('.')
namespaces, name = chunks[:-1], chunks[-1]
classes = []
while len(namespaces) > 0 and '.'.join(namespaces) not in known_namespaces:
classes.insert(0, namespaces.pop())
return cls(tuple(namespaces), tuple(classes), name)
setattr(SymbolName, "parse", classmethod(parse_symbol_name))
forbidden_arg_types = ["void*"]
@ -182,6 +200,7 @@ ${variant}
}
""")
class FormatStrings:
string = 's'
unsigned_char = 'b'
@ -197,9 +216,9 @@ class FormatStrings:
double = 'd'
object = 'O'
ArgTypeInfo = namedtuple('ArgTypeInfo',
['atype', 'format_str', 'default_value',
'strict_conversion'])
['atype', 'format_str', 'default_value', 'strict_conversion'])
# strict_conversion is False by default
ArgTypeInfo.__new__.__defaults__ = (False,)
@ -256,12 +275,16 @@ class ClassProp(object):
class ClassInfo(object):
def __init__(self, name, decl=None, codegen=None):
# Scope name can be a module or other class e.g. cv::SimpleBlobDetector::Params
scope_name, self.original_name = name.rsplit(".", 1)
self.original_scope_name, self.original_name = name.rsplit(".", 1)
# In case scope refer the outer class exported with different name
if codegen:
scope_name = codegen.get_export_scope_name(scope_name)
self.scope_name = re.sub(r"^cv\.?", "", scope_name)
self.export_scope_name = codegen.get_export_scope_name(
self.original_scope_name
)
else:
self.export_scope_name = self.original_scope_name
self.export_scope_name = re.sub(r"^cv\.?", "", self.export_scope_name)
self.export_name = self.original_name
@ -312,8 +335,8 @@ class ClassInfo(object):
@property
def wname(self):
if len(self.scope_name) > 0:
return self.scope_name.replace(".", "_") + "_" + self.export_name
if len(self.export_scope_name) > 0:
return self.export_scope_name.replace(".", "_") + "_" + self.export_name
return self.export_name
@ -322,16 +345,16 @@ class ClassInfo(object):
return self.class_id
@property
def full_scope_name(self):
return "cv." + self.scope_name if len(self.scope_name) else "cv"
def full_export_scope_name(self):
return "cv." + self.export_scope_name if len(self.export_scope_name) else "cv"
@property
def full_export_name(self):
return self.full_scope_name + "." + self.export_name
return self.full_export_scope_name + "." + self.export_name
@property
def full_original_name(self):
return self.full_scope_name + "." + self.original_name
return self.original_scope_name + "." + self.original_name
@property
def has_export_alias(self):
@ -415,7 +438,7 @@ class ClassInfo(object):
baseptr,
constructor_name,
# Leading dot is required to provide correct class naming
"." + self.scope_name if len(self.scope_name) > 0 else self.scope_name
"." + self.export_scope_name if len(self.export_scope_name) > 0 else self.export_scope_name
)
@ -577,6 +600,10 @@ class FuncVariant(object):
self.args.append(ainfo)
self.init_pyproto(namespace, classname, known_classes)
def is_arg_optional(self, py_arg_index):
# type: (FuncVariant, int) -> bool
return py_arg_index >= len(self.py_arglist) - self.py_noptargs
def init_pyproto(self, namespace, classname, known_classes):
# string representation of argument list, with '[', ']' symbols denoting optional arguments, e.g.
# "src1, src2[, dst[, mask]]" for cv.add
@ -1053,6 +1080,7 @@ class PythonWrapperGenerator(object):
self.namespaces = {}
self.consts = {}
self.enums = {}
self.typing_stubs_generator = TypingStubsGenerator()
self.code_include = StringIO()
self.code_enums = StringIO()
self.code_types = StringIO()
@ -1099,26 +1127,19 @@ class PythonWrapperGenerator(object):
return original_scope_name
def split_decl_name(self, name):
chunks = name.split('.')
namespace = chunks[:-1]
classes = []
while namespace and '.'.join(namespace) not in self.parser.namespaces:
classes.insert(0, namespace.pop())
return namespace, classes, chunks[-1]
return SymbolName.parse(name, self.parser.namespaces)
def add_const(self, name, decl):
cname = name.replace('.','::')
namespace, classes, name = self.split_decl_name(name)
namespace = '.'.join(namespace)
name = '_'.join(classes+[name])
name = '_'.join(chain(classes, (name, )))
ns = self.namespaces.setdefault(namespace, Namespace())
if name in ns.consts:
print("Generator error: constant %s (cname=%s) already exists" \
% (name, cname))
sys.exit(-1)
ns.consts[name] = cname
value = decl[1]
py_name = '.'.join([namespace, name])
py_signatures = self.py_signatures.setdefault(cname, [])
@ -1126,20 +1147,31 @@ class PythonWrapperGenerator(object):
#print(cname + ' => ' + str(py_name) + ' (value=' + value + ')')
def add_enum(self, name, decl):
enumeration_name = SymbolName.parse(name, self.parser.namespaces)
is_scoped_enum = decl[0].startswith("enum class") \
or decl[0].startswith("enum struct")
wname = normalize_class_name(name)
if wname.endswith("<unnamed>"):
wname = None
else:
self.enums[wname] = name
const_decls = decl[3]
enum_entries = {}
for decl in const_decls:
name = decl[0]
self.add_const(name.replace("const ", "").strip(), decl)
enum_entries[decl[0].split(".")[-1]] = decl[1]
self.add_const(decl[0].replace("const ", "").strip(), decl)
# Extra enumerations tracking is required to generate stubs for
# all enumerations, including <unnamed> once, otherwise they
# will be forgiven
self.typing_stubs_generator.add_enum(enumeration_name, is_scoped_enum,
enum_entries)
def add_func(self, decl):
namespace, classes, barename = self.split_decl_name(decl[0])
cname = "::".join(namespace+classes+[barename])
cname = "::".join(chain(namespace, classes, (barename, )))
name = barename
classname = ''
bareclassname = ''
@ -1166,7 +1198,7 @@ class PythonWrapperGenerator(object):
return
if isconstructor:
name = "_".join(classes[:-1]+[name])
name = "_".join(chain(classes[:-1], (name, )))
if is_static:
# Add it as a method to the class
@ -1175,7 +1207,7 @@ class PythonWrapperGenerator(object):
func.add_variant(decl, self.classes, isphantom)
# Add it as global function
g_name = "_".join(classes+[name])
g_name = "_".join(chain(classes, (name, )))
w_classes = []
for i in range(0, len(classes)):
classes_i = classes[:i+1]
@ -1187,10 +1219,15 @@ class PythonWrapperGenerator(object):
w_classes.append(w_classname)
g_wname = "_".join(w_classes+[name])
func_map = self.namespaces.setdefault(namespace_str, Namespace()).funcs
# Static functions should be called using class names, not like
# module-level functions, so first step is to remove them from
# type hints.
self.typing_stubs_generator.add_ignored_function_name(g_name)
# Exports static function with internal name (backward compatibility)
func = func_map.setdefault(g_name, FuncInfo("", g_name, cname, isconstructor, namespace_str, False))
func.add_variant(decl, self.classes, isphantom)
if g_wname != g_name: # TODO OpenCV 5.0
self.typing_stubs_generator.add_ignored_function_name(g_wname)
wfunc = func_map.setdefault(g_wname, FuncInfo("", g_wname, cname, isconstructor, namespace_str, False))
wfunc.add_variant(decl, self.classes, isphantom)
else:
@ -1207,7 +1244,6 @@ class PythonWrapperGenerator(object):
if classname and isconstructor:
self.classes[classname].constructor = func
def gen_namespace(self, ns_name):
ns = self.namespaces[ns_name]
wname = normalize_class_name(ns_name)
@ -1260,6 +1296,7 @@ class PythonWrapperGenerator(object):
self.clear()
self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True, generate_gpumat_decls=True)
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = self.parser.parse(hdr)
@ -1355,23 +1392,36 @@ class PythonWrapperGenerator(object):
for decl_idx, name, classinfo in classlist1:
if classinfo.ismap:
continue
def _registerType(classinfo):
if classinfo.decl_idx in published_types:
#print(classinfo.decl_idx, classinfo.name, ' - already published')
return
# If class already registered it means that there is
# a correponding node in the AST. This check is partically
# useful for base classes.
return self.typing_stubs_generator.find_class_node(
classinfo, self.parser.namespaces
)
published_types.add(classinfo.decl_idx)
# Registering a class means creation of the AST node from the
# given class information
class_node = self.typing_stubs_generator.create_class_node(
classinfo, self.parser.namespaces
)
if classinfo.base and classinfo.base in self.classes:
base_classinfo = self.classes[classinfo.base]
#print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name)
_registerType(base_classinfo)
# print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name)
base_node = _registerType(base_classinfo)
class_node.add_base(base_node)
#print(classinfo.decl_idx, classinfo.name, ' - published!')
# print(classinfo.decl_idx, classinfo.name, ' - published!')
self.code_type_publish.write(classinfo.gen_def(self))
return class_node
_registerType(classinfo)
# step 3: generate the code for all the global functions
for ns_name, ns in sorted(self.namespaces.items()):
if ns_name.split('.')[0] != 'cv':
@ -1381,6 +1431,10 @@ class PythonWrapperGenerator(object):
continue
code = func.gen_code(self)
self.code_funcs.write(code)
# If function is not ignored - create an AST node for it
if name not in self.typing_stubs_generator.type_hints_ignored_functions:
self.typing_stubs_generator.create_function_node(func)
self.gen_namespace(ns_name)
self.code_ns_init.write('CVPY_MODULE("{}", {});\n'.format(ns_name[2:], normalize_class_name(ns_name)))
@ -1396,6 +1450,10 @@ class PythonWrapperGenerator(object):
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# All symbols are collected and AST is reconstructed, generating
# typing stubs...
self.typing_stubs_generator.generate(output_path)
# That's it. Now save all the files
self.save(output_path, "pyopencv_generated_include.h", self.code_include)
self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs)
@ -1406,6 +1464,7 @@ class PythonWrapperGenerator(object):
self.save(output_path, "pyopencv_generated_modules_content.h", self.code_ns_reg)
self.save_json(output_path, "pyopencv_signatures.json", self.py_signatures)
if __name__ == "__main__":
srcfiles = hdr_parser.opencv_hdr_list
dstdir = "/Users/vp/tmp"

34
modules/python/src2/typing_stubs_generation/__init__.py Normal file
View File

@ -0,0 +1,34 @@
from .nodes import (
NamespaceNode,
ClassNode,
ClassProperty,
EnumerationNode,
FunctionNode,
ConstantNode,
TypeNode,
OptionalTypeNode,
TupleTypeNode,
AliasTypeNode,
SequenceTypeNode,
AnyTypeNode,
AggregatedTypeNode,
)
from .types_conversion import (
replace_template_parameters_with_placeholders,
get_template_instantiation_type,
create_type_node
)
from .ast_utils import (
SymbolName,
ScopeNotFoundError,
SymbolNotFoundError,
find_scope,
find_class_node,
create_class_node,
create_function_node,
resolve_enum_scopes
)
from .generation import generate_typing_stubs

369
modules/python/src2/typing_stubs_generation/ast_utils.py Normal file
View File

@ -0,0 +1,369 @@
from typing import NamedTuple, Sequence, Tuple, Union, List, Dict
import keyword
from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode,
EnumerationNode, ClassProperty, OptionalTypeNode,
TupleTypeNode)
from .types_conversion import create_type_node
class ScopeNotFoundError(Exception):
pass
class SymbolNotFoundError(Exception):
pass
class SymbolName(NamedTuple):
namespaces: Tuple[str, ...]
classes: Tuple[str, ...]
name: str
def __str__(self) -> str:
return '(namespace="{}", classes="{}", name="{}")'.format(
'::'.join(self.namespaces),
'::'.join(self.classes),
self.name
)
def __repr__(self) -> str:
return str(self)
@classmethod
def parse(cls, full_symbol_name: str,
known_namespaces: Sequence[str],
symbol_parts_delimiter: str = '.') -> "SymbolName":
"""Performs contextual symbol name parsing into namespaces, classes
and "bare" symbol name.
Args:
full_symbol_name (str): Input string to parse symbol name from.
known_namespaces (Sequence[str]): Collection of namespace that was
met during C++ headers parsing.
symbol_parts_delimiter (str, optional): Delimiter string used to
split `full_symbol_name` string into chunks. Defaults to '.'.
Returns:
SymbolName: Parsed symbol name structure.
>>> SymbolName.parse('cv.ns.Feature', ('cv', 'cv.ns'))
(namespace="cv::ns", classes="", name="Feature")
>>> SymbolName.parse('cv.ns.Feature', ())
(namespace="", classes="cv::ns", name="Feature")
>>> SymbolName.parse('cv.ns.Feature.Params', ('cv', 'cv.ns'))
(namespace="cv::ns", classes="Feature", name="Params")
>>> SymbolName.parse('cv::ns::Feature::Params::serialize',
... known_namespaces=('cv', 'cv.ns'),
... symbol_parts_delimiter='::')
(namespace="cv::ns", classes="Feature::Params", name="serialize")
"""
chunks = full_symbol_name.split(symbol_parts_delimiter)
namespaces, name = chunks[:-1], chunks[-1]
classes: List[str] = []
while len(namespaces) > 0 and '.'.join(namespaces) not in known_namespaces:
classes.insert(0, namespaces.pop())
return SymbolName(tuple(namespaces), tuple(classes), name)
def find_scope(root: NamespaceNode, symbol_name: SymbolName,
create_missing_namespaces: bool = True) -> Union[NamespaceNode, ClassNode]:
"""Traverses down nodes hierarchy to the direct parent of the node referred
by `symbol_name`.
Args:
root (NamespaceNode): Root node of the hierarchy.
symbol_name (SymbolName): Full symbol name to find scope for.
create_missing_namespaces (bool, optional): Set to True to create missing
namespaces while traversing the hierarchy. Defaults to True.
Raises:
ScopeNotFoundError: If direct parent for the node referred by `symbol_name`
can't be found e.g. one of classes doesn't exist.
Returns:
Union[NamespaceNode, ClassNode]: Direct parent for the node referred by
`symbol_name`.
>>> root = NamespaceNode('cv')
>>> algorithm_node = root.add_class('Algorithm')
>>> find_scope(root, SymbolName(('cv', ), ('Algorithm',), 'Params')) == algorithm_node
True
>>> root = NamespaceNode('cv')
>>> scope = find_scope(root, SymbolName(('cv', 'gapi', 'detail'), (), 'function'))
>>> scope.full_export_name
'cv.gapi.detail'
>>> root = NamespaceNode('cv')
>>> scope = find_scope(root, SymbolName(('cv', 'gapi'), ('GOpaque',), 'function'))
Traceback (most recent call last):
...
ast_utils.ScopeNotFoundError: Can't find a scope for 'function', with \
'(namespace="cv::gapi", classes="GOpaque", name="function")', \
because 'GOpaque' class is not registered yet
"""
assert isinstance(root, NamespaceNode), \
'Wrong hierarchy root type: {}'.format(type(root))
assert symbol_name.namespaces[0] == root.name, \
"Trying to find scope for '{}' with root namespace different from: '{}'".format(
symbol_name, root.name
)
scope: Union[NamespaceNode, ClassNode] = root
for namespace in symbol_name.namespaces[1:]:
if namespace not in scope.namespaces: # type: ignore
if not create_missing_namespaces:
raise ScopeNotFoundError(
"Can't find a scope for '{}', with '{}', because namespace"
" '{}' is not created yet and `create_missing_namespaces`"
" flag is set to False".format(
symbol_name.name, symbol_name, namespace
)
)
scope = scope.add_namespace(namespace) # type: ignore
else:
scope = scope.namespaces[namespace] # type: ignore
for class_name in symbol_name.classes:
if class_name not in scope.classes:
raise ScopeNotFoundError(
"Can't find a scope for '{}', with '{}', because '{}' "
"class is not registered yet".format(
symbol_name.name, symbol_name, class_name
)
)
scope = scope.classes[class_name]
return scope
def find_class_node(root: NamespaceNode, full_class_name: str,
namespaces: Sequence[str]) -> ClassNode:
symbol_name = SymbolName.parse(full_class_name, namespaces)
scope = find_scope(root, symbol_name)
if symbol_name.name not in scope.classes:
raise SymbolNotFoundError("Can't find {} in its scope".format(symbol_name))
return scope.classes[symbol_name.name]
def create_function_node_in_scope(scope: Union[NamespaceNode, ClassNode],
func_info) -> FunctionNode:
def prepare_overload_arguments_and_return_type(variant):
arguments = [] # type: list[FunctionNode.Arg]
# Enumerate is requried, because `argno` in `variant.py_arglist`
# refers to position of argument in C++ function interface,
# but `variant.py_noptargs` refers to position in `py_arglist`
for i, (_, argno) in enumerate(variant.py_arglist):
arg_info = variant.args[argno]
type_node = create_type_node(arg_info.tp)
default_value = None
if len(arg_info.defval):
default_value = arg_info.defval
# If argument is optional and can be None - make its type optional
if variant.is_arg_optional(i):
# NOTE: should UMat be always mandatory for better type hints?
# otherwise overload won't be selected e.g. VideoCapture.read()
if arg_info.py_outputarg:
type_node = OptionalTypeNode(type_node)
default_value = "None"
elif arg_info.isbig() and "None" not in type_node.typename:
# but avoid duplication of the optioness
type_node = OptionalTypeNode(type_node)
arguments.append(
FunctionNode.Arg(arg_info.export_name, type_node=type_node,
default_value=default_value)
)
if func_info.isconstructor:
return arguments, None
# Function has more than 1 output argument, so its return type is a tuple
if len(variant.py_outlist) > 1:
ret_types = []
# Actual returned value of the function goes first
if variant.py_outlist[0][1] == -1:
ret_types.append(create_type_node(variant.rettype))
outlist = variant.py_outlist[1:]
else:
outlist = variant.py_outlist
for _, argno in outlist:
assert argno >= 0, \
"Logic Error! Outlist contains function return type: {}".format(
outlist
)
ret_types.append(create_type_node(variant.args[argno].tp))
return arguments, FunctionNode.RetType(
TupleTypeNode("return_type", ret_types)
)
# Function with 1 output argument in Python
if len(variant.py_outlist) == 1:
# Can be represented as a function with a non-void return type in C++
if variant.rettype:
return arguments, FunctionNode.RetType(
create_type_node(variant.rettype)
)
# or a function with void return type and output argument type
# such non-const reference
ret_type = variant.args[variant.py_outlist[0][1]].tp
return arguments, FunctionNode.RetType(
create_type_node(ret_type)
)
# Function without output types returns None in Python
return arguments, None
function_node = FunctionNode(func_info.name)
function_node.parent = scope
if func_info.isconstructor:
function_node.export_name = "__init__"
for variant in func_info.variants:
arguments, ret_type = prepare_overload_arguments_and_return_type(variant)
if isinstance(scope, ClassNode):
if func_info.is_static:
if ret_type is not None and ret_type.typename.endswith(scope.name):
function_node.is_classmethod = True
arguments.insert(0, FunctionNode.Arg("cls"))
else:
function_node.is_static = True
else:
arguments.insert(0, FunctionNode.Arg("self"))
function_node.add_overload(arguments, ret_type)
return function_node
def create_function_node(root: NamespaceNode, func_info) -> FunctionNode:
func_symbol_name = SymbolName(
func_info.namespace.split(".") if len(func_info.namespace) else (),
func_info.classname.split(".") if len(func_info.classname) else (),
func_info.name
)
return create_function_node_in_scope(find_scope(root, func_symbol_name),
func_info)
def create_class_node_in_scope(scope: Union[NamespaceNode, ClassNode],
symbol_name: SymbolName,
class_info) -> ClassNode:
properties = []
for property in class_info.props:
export_property_name = property.name
if keyword.iskeyword(export_property_name):
export_property_name += "_"
properties.append(
ClassProperty(
name=export_property_name,
type_node=create_type_node(property.tp),
is_readonly=property.readonly
)
)
class_node = scope.add_class(symbol_name.name,
properties=properties)
class_node.export_name = class_info.export_name
if class_info.constructor is not None:
create_function_node_in_scope(class_node, class_info.constructor)
for method in class_info.methods.values():
create_function_node_in_scope(class_node, method)
return class_node
def create_class_node(root: NamespaceNode, class_info,
namespaces: Sequence[str]) -> ClassNode:
symbol_name = SymbolName.parse(class_info.full_original_name, namespaces)
scope = find_scope(root, symbol_name)
return create_class_node_in_scope(scope, symbol_name, class_info)
def resolve_enum_scopes(root: NamespaceNode,
enums: Dict[SymbolName, EnumerationNode]):
"""Attaches all enumeration nodes to the appropriate classes and modules
If classes containing enumeration can't be found in the AST - they will
be created and marked as not exportable. This behavior is required to cover
cases, when enumeration is defined in base class, but only its derivatives
are used. Example:
```cpp
class CV_EXPORTS TermCriteria {
public:
enum Type { /* ... */ };
// ...
};
```
Args:
root (NamespaceNode): root of the reconstructed AST
enums (Dict[SymbolName, EnumerationNode]): Mapping between enumerations
symbol names and corresponding nodes without parents.
"""
for symbol_name, enum_node in enums.items():
if symbol_name.classes:
try:
scope = find_scope(root, symbol_name)
except ScopeNotFoundError:
# Scope can't be found if enumeration is a part of class
# that is not exported.
# Create class node, but mark it as not exported
for i, class_name in enumerate(symbol_name.classes):
scope = find_scope(root,
SymbolName(symbol_name.namespaces,
classes=symbol_name.classes[:i],
name=class_name))
if class_name in scope.classes:
continue
class_node = scope.add_class(class_name)
class_node.is_exported = False
scope = find_scope(root, symbol_name)
else:
scope = find_scope(root, symbol_name)
enum_node.parent = scope
def get_enclosing_namespace(node: ASTNode) -> NamespaceNode:
"""Traverses up nodes hierarchy to find closest enclosing namespace of the
passed node
Args:
node (ASTNode): Node to find a namespace for.
Returns:
NamespaceNode: Closest enclosing namespace of the provided node.
Raises:
AssertionError: if nodes hierarchy missing a namespace node.
>>> root = NamespaceNode('cv')
>>> feature_class = root.add_class("Feature")
>>> get_enclosing_namespace(feature_class) == root
True
>>> root = NamespaceNode('cv')
>>> feature_class = root.add_class("Feature")
>>> feature_params_class = feature_class.add_class("Params")
>>> serialize_params_func = feature_params_class.add_function("serialize")
>>> get_enclosing_namespace(serialize_params_func) == root
True
>>> root = NamespaceNode('cv')
>>> detail_ns = root.add_namespace('detail')
>>> flags_enum = detail_ns.add_enumeration('Flags')
>>> get_enclosing_namespace(flags_enum) == detail_ns
True
"""
parent_node = node.parent
while not isinstance(parent_node, NamespaceNode):
assert parent_node is not None, \
"Can't find enclosing namespace for '{}' known as: '{}'".format(
node.full_export_name, node.native_name
)
parent_node = parent_node.parent
return parent_node
if __name__ == '__main__':
import doctest
doctest.testmod()

671
modules/python/src2/typing_stubs_generation/generation.py Normal file
View File

@ -0,0 +1,671 @@
__all__ = ("generate_typing_stubs", )
from io import StringIO
from pathlib import Path
from typing import (Generator, Type, Callable, NamedTuple, Union, Set, Dict,
Collection)
import warnings
from .ast_utils import get_enclosing_namespace
from .predefined_types import PREDEFINED_TYPES
from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode,
EnumerationNode, ConstantNode)
from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode,
AggregatedTypeNode)
def generate_typing_stubs(root: NamespaceNode, output_path: Path):
"""Generates typing stubs for the AST with root `root` and outputs
created files tree to directory pointed by `output_path`.
Stubs generation consist from 4 steps:
1. Reconstruction of AST tree for header parser output.
2. "Lazy" AST nodes resolution (type nodes used as function arguments
and return types). Resolution procedure attaches every "lazy"
AST node to the corresponding node in the AST created during step 1.
3. Generation of the typing module content. Typing module doesn't exist
in library code, but is essential place to define aliases widely used
in stub files.
4. Generation of typing stubs from the reconstructed AST.
Every namespace corresponds to a Python module with the same name.
Generation procedure is recursive repetition of the following steps
for each namespace (module):
- Collect and write required imports for the module
- Write all module constants stubs
- Write all module enumerations stubs
- Write all module classes stubs, preserving correct declaration
order, when base classes go before their derivatives.
- Write all module functions stubs
- Repeat steps above for nested namespaces
Args:
root (NamespaceNode): Root namespace node of the library AST.
output_path (Path): Path to output directory.
"""
# Most of the time type nodes miss their full name (especially function
# arguments and return types), so resolution should start from the narrowest
# scope and gradually expanded.
# Example:
# ```cpp
# namespace cv {
# enum AlgorithmType {
# // ...
# };
# namespace detail {
# struct Algorithm {
# static Ptr<Algorithm> create(AlgorithmType alg_type);
# };
# } // namespace detail
# } // namespace cv
# ```
# To resolve `alg_type` argument of function `create` having `AlgorithmType`
# type from above example the following steps are done:
# 1. Try to resolve against `cv::detail::Algorithm` - fail
# 2. Try to resolve against `cv::detail` - fail
# 3. Try to resolve against `cv` - success
# The whole process should fail !only! when all possible scopes are
# checked and at least 1 node is still unresolved.
root.resolve_type_nodes()
_generate_typing_module(root, output_path)
_generate_typing_stubs(root, output_path)
def _generate_typing_stubs(root: NamespaceNode, output_path: Path):
output_path = Path(output_path) / root.export_name
output_path.mkdir(parents=True, exist_ok=True)
# Collect all imports required for module items declaration
required_imports = _collect_required_imports(root)
output_stream = StringIO()
# Write required imports at the top of file
_write_required_imports(required_imports, output_stream)
# Write constants section, because constants don't impose any dependencies
_generate_section_stub(StubSection("# Constants", ConstantNode), root,
output_stream, 0)
# NOTE: Enumerations require special handling, because all enumeration
# constants are exposed as module attributes
has_enums = _generate_section_stub(StubSection("# Enumerations", EnumerationNode),
root, output_stream, 0)
# Collect all enums from class level and export them to module level
for class_node in root.classes.values():
if _generate_enums_from_classes_tree(class_node, output_stream, indent=0):
has_enums = True
# 2 empty lines between enum and classes definitions
if has_enums:
output_stream.write("\n")
# Write the rest of module content - classes and functions
for section in STUB_SECTIONS:
_generate_section_stub(section, root, output_stream, 0)
# Dump content to the output file
(output_path / "__init__.pyi").write_text(output_stream.getvalue())
# Process nested namespaces
for ns in root.namespaces.values():
_generate_typing_stubs(ns, output_path)
class StubSection(NamedTuple):
name: str
node_type: Type[ASTNode]
STUB_SECTIONS = (
StubSection("# Constants", ConstantNode),
# StubSection("# Enumerations", EnumerationNode), # Skipped for now (special rules)
StubSection("# Classes", ClassNode),
StubSection("# Functions", FunctionNode)
)
def _generate_section_stub(section: StubSection, node: ASTNode,
output_stream: StringIO, indent: int) -> bool:
"""Generates stub for a single type of children nodes of the provided node.
Args:
section (StubSection): section identifier that carries section name and
type its nodes.
node (ASTNode): root node with children nodes used for
output_stream (StringIO): Output stream for all nodes stubs related to
the given section.
indent (int): Indent used for each line written to `output_stream`.
Returns:
bool: `True` if section has a content, `False` otherwise.
"""
if section.node_type not in node._children:
return False
children = node._children[section.node_type]
if len(children) == 0:
return False
output_stream.write(" " * indent)
output_stream.write(section.name)
output_stream.write("\n")
stub_generator = NODE_TYPE_TO_STUB_GENERATOR[section.node_type]
children = filter(lambda c: c.is_exported, children.values()) # type: ignore
if hasattr(section.node_type, "weight"):
children = sorted(children, key=lambda child: getattr(child, "weight")) # type: ignore
for child in children:
stub_generator(child, output_stream, indent) # type: ignore
output_stream.write("\n")
return True
def _generate_class_stub(class_node: ClassNode, output_stream: StringIO,
indent: int = 0) -> None:
"""Generates stub for the provided class node.
Rules:
- Read/write properties are converted to object attributes.
- Readonly properties are converted to functions decorated with `@property`.
- When return type of static functions matches class name - these functions
are treated as factory functions and annotated with `@classmethod`.
- In contrast to implicit `this` argument in C++ methods, in Python all
"normal" methods have explicit `self` as their first argument.
- Body of empty classes is replaced with `...`
Example:
```cpp
struct Object : public BaseObject {
struct InnerObject {
int param;
bool param2;
float readonlyParam();
};
Object(int param, bool param2 = false);
Object(InnerObject obj);
static Object create();
};
```
becomes
```python
class Object(BaseObject):
class InnerObject:
param: int
param2: bool
@property
def readonlyParam() -> float: ...
@typing.override
def __init__(self, param: int, param2: bool = ...) -> None: ...
@typing.override
def __init__(self, obj: "Object.InnerObject") -> None: ...
@classmethod
def create(cls) -> Object: ...
```
Args:
class_node (ClassNode): Class node to generate stub entry for.
output_stream (StringIO): Output stream for class stub.
indent (int, optional): Indent used for each line written to
`output_stream`. Defaults to 0.
"""
class_module = get_enclosing_namespace(class_node)
class_module_name = class_module.full_export_name
if len(class_node.bases) > 0:
bases = []
for base in class_node.bases:
base_module = get_enclosing_namespace(base) # type: ignore
if base_module != class_module:
bases.append(base.full_export_name)
else:
bases.append(base.export_name)
inheritance_str = "({})".format(
', '.join(bases)
)
else:
inheritance_str = ""
output_stream.write(
"{indent}class {name}{bases}:\n".format(
indent=" " * indent,
name=class_node.export_name,
bases=inheritance_str
)
)
has_content = len(class_node.properties) > 0
# Processing class properties
for property in class_node.properties:
if property.is_readonly:
template = "{indent}@property\n{indent}def {name}(self) -> {type}: ...\n"
else:
template = "{indent}{name}: {type}\n"
output_stream.write(
template.format(indent=" " * (indent + 4),
name=property.name,
type=property.relative_typename(class_module_name))
)
if len(class_node.properties) > 0:
output_stream.write("\n")
for section in STUB_SECTIONS:
if _generate_section_stub(section, class_node,
output_stream, indent + 4):
has_content = True
if not has_content:
output_stream.write(" " * (indent + 4))
output_stream.write("...\n\n")
def _generate_constant_stub(constant_node: ConstantNode,
output_stream: StringIO, indent: int = 0,
extra_export_prefix: str = "") -> None:
"""Generates stub for the provided constant node.
Args:
constant_node (ConstantNode): Constant node to generate stub entry for.
output_stream (StringIO): Output stream for constant stub.
indent (int, optional): Indent used for each line written to
`output_stream`. Defaults to 0.
extra_export_prefix (str, optional) Extra prefix added to the export
constant name. Defaults to empty string.
"""
output_stream.write(
"{indent}{prefix}{name}: {value_type}\n".format(
prefix=extra_export_prefix,
name=constant_node.export_name,
value_type=constant_node.value_type,
indent=" " * indent
)
)
def _generate_enumeration_stub(enumeration_node: EnumerationNode,
output_stream: StringIO, indent: int = 0,
extra_export_prefix: str = "") -> None:
"""Generates stub for the provided enumeration node. In contrast to the
Python `enum.Enum` class, C++ enumerations are exported as module-level
(or class-level) constants.
Example:
```cpp
enum Flags {
Flag1 = 0,
Flag2 = 1,
Flag3
};
```
becomes
```python
Flag1: int
Flag2: int
Flag3: int
Flags = int # One of [Flag1, Flag2, Flag3]
```
Unnamed enumerations don't export their names to Python:
```cpp
enum {
Flag1 = 0,
Flag2 = 1
};
```
becomes
```python
Flag1: int
Flag2: int
```
Scoped enumeration adds its name before each item name:
```cpp
enum struct ScopedEnum {
Flag1,
Flag2
};
```
becomes
```python
ScopedEnum_Flag1: int
ScopedEnum_Flag2: int
ScopedEnum = int # One of [ScopedEnum_Flag1, ScopedEnum_Flag2]
```
Args:
enumeration_node (EnumerationNode): Enumeration node to generate stub entry for.
output_stream (StringIO): Output stream for enumeration stub.
indent (int, optional): Indent used for each line written to `output_stream`.
Defaults to 0.
extra_export_prefix (str, optional) Extra prefix added to the export
enumeration name. Defaults to empty string.
"""
entries_extra_prefix = extra_export_prefix
if enumeration_node.is_scoped:
entries_extra_prefix += enumeration_node.export_name + "_"
for entry in enumeration_node.constants.values():
_generate_constant_stub(entry, output_stream, indent, entries_extra_prefix)
# Unnamed enumerations are skipped as definition
if enumeration_node.export_name.endswith("<unnamed>"):
output_stream.write("\n")
return
output_stream.write(
"{indent}{export_prefix}{name} = int # One of [{entries}]\n\n".format(
export_prefix=extra_export_prefix,
name=enumeration_node.export_name,
entries=", ".join(entry.export_name
for entry in enumeration_node.constants.values()),
indent=" " * indent
)
)
def _generate_function_stub(function_node: FunctionNode,
output_stream: StringIO, indent: int = 0) -> None:
"""Generates stub entry for the provided function node. Function node can
refer free function or class method.
Args:
function_node (FunctionNode): Function node to generate stub entry for.
output_stream (StringIO): Output stream for function stub.
indent (int, optional): Indent used for each line written to
`output_stream`. Defaults to 0.
"""
# Function is a stub without any arguments information
if not function_node.overloads:
warnings.warn(
'Function node "{}" exported as "{}" has no overloads'.format(
function_node.full_name, function_node.full_export_name
)
)
return
decorators = []
if function_node.is_classmethod:
decorators.append(" " * indent + "@classmethod")
elif function_node.is_static:
decorators.append(" " * indent + "@staticmethod")
if len(function_node.overloads) > 1:
decorators.append(" " * indent + "@typing.overload")
function_module = get_enclosing_namespace(function_node)
function_module_name = function_module.full_export_name
for overload in function_node.overloads:
# Annotate every function argument
annotated_args = []
for arg in overload.arguments:
annotated_arg = arg.name
typename = arg.relative_typename(function_module_name)
if typename is not None:
annotated_arg += ": " + typename
if arg.default_value is not None:
annotated_arg += " = ..."
annotated_args.append(annotated_arg)
# And convert return type to the actual type
if overload.return_type is not None:
ret_type = overload.return_type.relative_typename(function_module_name)
else:
ret_type = "None"
output_stream.write(
"{decorators}"
"{indent}def {name}({args}) -> {ret_type}: ...\n".format(
decorators="\n".join(decorators) +
"\n" if len(decorators) > 0 else "",
name=function_node.export_name,
args=", ".join(annotated_args),
ret_type=ret_type,
indent=" " * indent
)
)
output_stream.write("\n")
def _generate_enums_from_classes_tree(class_node: ClassNode,
output_stream: StringIO,
indent: int = 0,
class_name_prefix: str = "") -> bool:
"""Recursively generates class-level enumerations on the module level
starting from the `class_node`.
NOTE: This function is required, because all enumerations are exported as
module-level constants.
Example:
```cpp
namespace cv {
struct TermCriteria {
enum Type {
COUNT = 1,
MAX_ITER = COUNT,
EPS = 2
};
};
} // namespace cv
```
is exported to `__init__.pyi` of `cv` module as as
```python
TermCriteria_COUNT: int
TermCriteria_MAX_ITER: int
TermCriteria_EPS: int
TermCriteria_Type = int # One of [COUNT, MAX_ITER, EPS]
```
Args:
class_node (ClassNode): Class node to generate enumerations stubs for.
output_stream (StringIO): Output stream for enumerations stub.
indent (int, optional): Indent used for each line written to
`output_stream`. Defaults to 0.
class_name_prefix (str, optional): Prefix used for enumerations and
constants names. Defaults to "".
Returns:
bool: `True` if classes tree declares at least 1 enum, `False` otherwise.
"""
class_name_prefix = class_node.export_name + "_" + class_name_prefix
has_content = len(class_node.enumerations) > 0
for enum_node in class_node.enumerations.values():
_generate_enumeration_stub(enum_node, output_stream, indent,
class_name_prefix)
for cls in class_node.classes.values():
if _generate_enums_from_classes_tree(cls, output_stream, indent,
class_name_prefix):
has_content = True
return has_content
def check_overload_presence(node: Union[NamespaceNode, ClassNode]) -> bool:
"""Checks that node has at least 1 function with overload.
Args:
node (Union[NamespaceNode, ClassNode]): Node to check for overload
presence.
Returns:
bool: True if input node has at least 1 function with overload, False
otherwise.
"""
for func_node in node.functions.values():
if len(func_node.overloads):
return True
return False
def _for_each_class(node: Union[NamespaceNode, ClassNode]) \
-> Generator[ClassNode, None, None]:
for cls in node.classes.values():
yield cls
if len(cls.classes):
yield from _for_each_class(cls)
def _for_each_function(node: Union[NamespaceNode, ClassNode]) \
-> Generator[FunctionNode, None, None]:
for func in node.functions.values():
yield func
for cls in node.classes.values():
yield from _for_each_function(cls)
def _for_each_function_overload(node: Union[NamespaceNode, ClassNode]) \
-> Generator[FunctionNode.Overload, None, None]:
for func in _for_each_function(node):
for overload in func.overloads:
yield overload
def _collect_required_imports(root: NamespaceNode) -> Set[str]:
"""Collects all imports required for classes and functions typing stubs
declarations.
Args:
root (NamespaceNode): Namespace node to collect imports for
Returns:
Set[str]: Collection of unique `import smth` statements required for
classes and function declarations of `root` node.
"""
def _add_required_usage_imports(type_node: TypeNode, imports: Set[str]):
for required_import in type_node.required_usage_imports:
imports.add(required_import)
required_imports: Set[str] = set()
# Check if typing module is required due to @overload decorator usage
# Looking for module-level function with at least 1 overload
has_overload = check_overload_presence(root)
# if there is no module-level functions with overload, check its presence
# during class traversing, including their inner-classes
for cls in _for_each_class(root):
if not has_overload and check_overload_presence(cls):
has_overload = True
required_imports.add("import typing")
# Add required imports for class properties
for prop in cls.properties:
_add_required_usage_imports(prop.type_node, required_imports)
# Add required imports for class bases
for base in cls.bases:
base_namespace = get_enclosing_namespace(base) # type: ignore
if base_namespace != root:
required_imports.add(
"import " + base_namespace.full_export_name
)
if has_overload:
required_imports.add("import typing")
# Importing modules required to resolve functions arguments
for overload in _for_each_function_overload(root):
for arg in filter(lambda a: a.type_node is not None, overload.arguments):
_add_required_usage_imports(arg.type_node, required_imports) # type: ignore
if overload.return_type is not None:
_add_required_usage_imports(overload.return_type.type_node,
required_imports)
root_import = "import " + root.full_export_name
if root_import in required_imports:
required_imports.remove(root_import)
return required_imports
def _write_required_imports(required_imports: Collection[str],
output_stream: StringIO) -> None:
"""Writes all entries of `required_imports` to the `output_stream`.
Args:
required_imports (Collection[str]): Imports to write into the output
stream.
output_stream (StringIO): Output stream for import statements.
"""
for required_import in sorted(required_imports):
output_stream.write(required_import)
output_stream.write("\n")
if len(required_imports):
output_stream.write("\n\n")
def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None:
"""Generates stub file for typings module.
Actual module doesn't exist, but it is an appropriate place to define
all widely-used aliases.
Args:
root (NamespaceNode): AST root node used for type nodes resolution.
output_path (Path): Path to typing module directory, where __init__.pyi
will be written.
"""
def register_alias_links_from_aggregated_type(type_node: TypeNode) -> None:
assert isinstance(type_node, AggregatedTypeNode), \
"Provided type node '{}' is not an aggregated type".format(
type_node.ctype_name
)
for item in filter(lambda i: isinstance(i, AliasRefTypeNode), type_node):
register_alias(PREDEFINED_TYPES[item.ctype_name]) # type: ignore
def register_alias(alias_node: AliasTypeNode) -> None:
typename = alias_node.typename
# Check if alias is already registered
if typename in aliases:
return
if isinstance(alias_node.value, AggregatedTypeNode):
# Check if collection contains a link to another alias
register_alias_links_from_aggregated_type(alias_node.value)
# Strip module prefix from aliased types
aliases[typename] = alias_node.value.full_typename.replace(
root.export_name + ".typing.", ""
)
if alias_node.comment is not None:
aliases[typename] += " # " + alias_node.comment
for required_import in alias_node.required_definition_imports:
required_imports.add(required_import)
output_path = Path(output_path) / root.export_name / "typing"
output_path.mkdir(parents=True, exist_ok=True)
required_imports: Set[str] = set()
aliases: Dict[str, str] = {}
# Resolve each node and register aliases
for node in PREDEFINED_TYPES.values():
node.resolve(root)
if isinstance(node, AliasTypeNode):
register_alias(node)
output_stream = StringIO()
_write_required_imports(required_imports, output_stream)
for alias_name, alias_type in aliases.items():
output_stream.write(alias_name)
output_stream.write(" = ")
output_stream.write(alias_type)
output_stream.write("\n")
(output_path / "__init__.pyi").write_text(output_stream.getvalue())
StubGenerator = Callable[[ASTNode, StringIO, int], None]
NODE_TYPE_TO_STUB_GENERATOR = {
ClassNode: _generate_class_stub,
ConstantNode: _generate_constant_stub,
EnumerationNode: _generate_enumeration_stub,
FunctionNode: _generate_function_stub
}

11
modules/python/src2/typing_stubs_generation/nodes/__init__.py Normal file
View File

@ -0,0 +1,11 @@
from .node import ASTNode
from .namespace_node import NamespaceNode
from .class_node import ClassNode, ClassProperty
from .function_node import FunctionNode
from .enumeration_node import EnumerationNode
from .constant_node import ConstantNode
from .type_node import (
TypeNode, OptionalTypeNode, UnionTypeNode, NoneTypeNode, TupleTypeNode,
ASTNodeTypeNode, AliasTypeNode, SequenceTypeNode, AnyTypeNode,
AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode,
)

181
modules/python/src2/typing_stubs_generation/nodes/class_node.py Normal file
View File

@ -0,0 +1,181 @@
from typing import Type, Sequence, NamedTuple, Optional, Tuple, Dict
import itertools
import weakref
from .node import ASTNode, ASTNodeType
from .function_node import FunctionNode
from .enumeration_node import EnumerationNode
from .constant_node import ConstantNode
from .type_node import TypeNode, TypeResolutionError
class ClassProperty(NamedTuple):
name: str
type_node: TypeNode
is_readonly: bool
@property
def typename(self) -> str:
return self.type_node.full_typename
def resolve_type_nodes(self, root: ASTNode) -> None:
try:
self.type_node.resolve(root)
except TypeResolutionError as e:
raise TypeResolutionError(
'Failed to resolve "{}" property'.format(self.name)
) from e
def relative_typename(self, full_node_name: str) -> str:
"""Typename relative to the passed AST node name.
Args:
full_node_name (str): Full export name of the AST node
Returns:
str: typename relative to the passed AST node name
"""
return self.type_node.relative_typename(full_node_name)
class ClassNode(ASTNode):
"""Represents a C++ class that is also a class in Python.
ClassNode can have functions (methods), enumerations, constants and other
classes as its children nodes.
Class properties are not treated as a part of AST for simplicity and have
extra handling if required.
"""
def __init__(self, name: str, parent: Optional[ASTNode] = None,
export_name: Optional[str] = None,
bases: Sequence["weakref.ProxyType[ClassNode]"] = (),
properties: Sequence[ClassProperty] = ()) -> None:
super().__init__(name, parent, export_name)
self.bases = list(bases)
self.properties = properties
@property
def weight(self) -> int:
return 1 + sum(base.weight for base in self.bases)
@property
def children_types(self) -> Tuple[Type[ASTNode], ...]:
return (ClassNode, FunctionNode, EnumerationNode, ConstantNode)
@property
def node_type(self) -> ASTNodeType:
return ASTNodeType.Class
@property
def classes(self) -> Dict[str, "ClassNode"]:
return self._children[ClassNode]
@property
def functions(self) -> Dict[str, FunctionNode]:
return self._children[FunctionNode]
@property
def enumerations(self) -> Dict[str, EnumerationNode]:
return self._children[EnumerationNode]
@property
def constants(self) -> Dict[str, ConstantNode]:
return self._children[ConstantNode]
def add_class(self, name: str,
bases: Sequence["weakref.ProxyType[ClassNode]"] = (),
properties: Sequence[ClassProperty] = ()) -> "ClassNode":
return self._add_child(ClassNode, name, bases=bases,
properties=properties)
def add_function(self, name: str, arguments: Sequence[FunctionNode.Arg] = (),
return_type: Optional[FunctionNode.RetType] = None,
is_static: bool = False) -> FunctionNode:
"""Adds function as a child node of a class.
Function is classified in 3 categories:
1. Instance method.
If function is an instance method then `self` argument is
inserted at the beginning of its arguments list.
2. Class method (or factory method)
If `is_static` flag is `True` and typename of the function
return type matches name of the class then function is treated
as class method.
If function is a class method then `cls` argument is inserted
at the beginning of its arguments list.
3. Static method
Args:
name (str): Name of the function.
arguments (Sequence[FunctionNode.Arg], optional): Function arguments.
Defaults to ().
return_type (Optional[FunctionNode.RetType], optional): Function
return type. Defaults to None.
is_static (bool, optional): Flag whenever function is static or not.
Defaults to False.
Returns:
FunctionNode: created function node.
"""
arguments = list(arguments)
if return_type is not None:
is_classmethod = return_type.typename == self.name
else:
is_classmethod = False
if not is_static:
arguments.insert(0, FunctionNode.Arg("self"))
elif is_classmethod:
is_static = False
arguments.insert(0, FunctionNode.Arg("cls"))
return self._add_child(FunctionNode, name, arguments=arguments,
return_type=return_type, is_static=is_static,
is_classmethod=is_classmethod)
def add_enumeration(self, name: str) -> EnumerationNode:
return self._add_child(EnumerationNode, name)
def add_constant(self, name: str, value: str) -> ConstantNode:
return self._add_child(ConstantNode, name, value=value)
def add_base(self, base_class_node: "ClassNode") -> None:
self.bases.append(weakref.proxy(base_class_node))
def resolve_type_nodes(self, root: ASTNode) -> None:
"""Resolves type nodes for all inner-classes, methods and properties
in 2 steps:
1. Resolve against `self` as a tree root
2. Resolve against `root` as a tree root
Type resolution errors are postponed until all children nodes are
examined.
Args:
root (Optional[ASTNode], optional): Root of the AST sub-tree.
Defaults to None.
"""
errors = []
for child in itertools.chain(self.properties,
self.functions.values(),
self.classes.values()):
try:
try:
# Give priority to narrowest scope (class-level scope in this case)
child.resolve_type_nodes(self) # type: ignore
except TypeResolutionError:
child.resolve_type_nodes(root) # type: ignore
except TypeResolutionError as e:
errors.append(str(e))
if len(errors) > 0:
raise TypeResolutionError(
'Failed to resolve "{}" class against "{}". Errors: {}'.format(
self.full_export_name, root.full_export_name, errors
)
)

30
modules/python/src2/typing_stubs_generation/nodes/constant_node.py Normal file
View File

@ -0,0 +1,30 @@
from typing import Type, Optional, Tuple
from .node import ASTNode, ASTNodeType
class ConstantNode(ASTNode):
"""Represents C++ constant that is also a constant in Python.
"""
def __init__(self, name: str, value: str,
parent: Optional[ASTNode] = None,
export_name: Optional[str] = None) -> None:
super().__init__(name, parent, export_name)
self.value = value
@property
def children_types(self) -> Tuple[Type[ASTNode], ...]:
return ()
@property
def node_type(self) -> ASTNodeType:
return ASTNodeType.Constant
@property
def value_type(self) -> str:
return 'int'
def __str__(self) -> str:
return "Constant('{}' exported as '{}': {})".format(
self.name, self.export_name, self.value
)

33
modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py Normal file
View File

@ -0,0 +1,33 @@
from typing import Type, Tuple, Optional, Dict
from .node import ASTNode, ASTNodeType
from .constant_node import ConstantNode
class EnumerationNode(ASTNode):
"""Represents C++ enumeration that treated as named set of constants in
Python.
EnumerationNode can have only constants as its children nodes.
"""
def __init__(self, name: str, is_scoped: bool = False,
parent: Optional[ASTNode] = None,
export_name: Optional[str] = None) -> None:
super().__init__(name, parent, export_name)
self.is_scoped = is_scoped
@property
def children_types(self) -> Tuple[Type[ASTNode], ...]:
return (ConstantNode, )
@property
def node_type(self) -> ASTNodeType:
return ASTNodeType.Enumeration
@property
def constants(self) -> Dict[str, ConstantNode]:
return self._children[ConstantNode]
def add_constant(self, name: str, value: str) -> ConstantNode:
return self._add_child(ConstantNode, name, value=value)

122
modules/python/src2/typing_stubs_generation/nodes/function_node.py Normal file
View File

@ -0,0 +1,122 @@
from typing import NamedTuple, Sequence, Type, Optional, Tuple, List
from .node import ASTNode, ASTNodeType
from .type_node import TypeNode, NoneTypeNode, TypeResolutionError
class FunctionNode(ASTNode):
"""Represents a function (or class method) in both C++ and Python.
This class defines an overload set rather then function itself, because
function without overloads is represented as FunctionNode with 1 overload.
"""
class Arg(NamedTuple):
name: str
type_node: Optional[TypeNode] = None
default_value: Optional[str] = None
@property
def typename(self) -> Optional[str]:
return getattr(self.type_node, "full_typename", None)
def relative_typename(self, root: str) -> Optional[str]:
if self.type_node is not None:
return self.type_node.relative_typename(root)
return None
class RetType(NamedTuple):
type_node: TypeNode = NoneTypeNode("void")
@property
def typename(self) -> str:
return self.type_node.full_typename
def relative_typename(self, root: str) -> Optional[str]:
return self.type_node.relative_typename(root)
class Overload(NamedTuple):
arguments: Sequence["FunctionNode.Arg"] = ()
return_type: Optional["FunctionNode.RetType"] = None
def __init__(self, name: str,
arguments: Optional[Sequence["FunctionNode.Arg"]] = None,
return_type: Optional["FunctionNode.RetType"] = None,
is_static: bool = False,
is_classmethod: bool = False,
parent: Optional[ASTNode] = None,
export_name: Optional[str] = None) -> None:
"""Function node initializer
Args:
name (str): Name of the function overload set
arguments (Optional[Sequence[FunctionNode.Arg]], optional): Function
arguments. If this argument is None, then no overloads are
added and node should be treated like a "function stub" rather
than function. This might be helpful if there is a knowledge
that function with the defined name exists, but information
about its interface is not available at that moment.
Defaults to None.
return_type (Optional[FunctionNode.RetType], optional): Function
return type. Defaults to None.
is_static (bool, optional): Flag pointing that function is
a static method of some class. Defaults to False.
is_classmethod (bool, optional): Flag pointing that function is
a class method of some class. Defaults to False.
parent (Optional[ASTNode], optional): Parent ASTNode of the function.
Can be class or namespace. Defaults to None.
export_name (Optional[str], optional): Export name of the function.
Defaults to None.
"""
super().__init__(name, parent, export_name)
self.overloads: List[FunctionNode.Overload] = []
self.is_static = is_static
self.is_classmethod = is_classmethod
if arguments is not None:
self.add_overload(arguments, return_type)
@property
def node_type(self) -> ASTNodeType:
return ASTNodeType.Function
@property
def children_types(self) -> Tuple[Type[ASTNode], ...]:
return ()
def add_overload(self, arguments: Sequence["FunctionNode.Arg"] = (),
return_type: Optional["FunctionNode.RetType"] = None):
self.overloads.append(FunctionNode.Overload(arguments, return_type))
def resolve_type_nodes(self, root: ASTNode):
"""Resolves type nodes in all overloads against `root`
Type resolution errors are postponed until all type nodes are examined.
Args:
root (ASTNode): Root of AST sub-tree used for type nodes resolution.
"""
def has_unresolved_type_node(item) -> bool:
return item.type_node is not None and not item.type_node.is_resolved
errors = []
for overload in self.overloads:
for arg in filter(has_unresolved_type_node, overload.arguments):
try:
arg.type_node.resolve(root) # type: ignore
except TypeResolutionError as e:
errors.append(
'Failed to resolve "{}" argument: {}'.format(arg.name, e)
)
if overload.return_type is not None and \
has_unresolved_type_node(overload.return_type):
try:
overload.return_type.type_node.resolve(root)
except TypeResolutionError as e:
errors.append('Failed to resolve return type: {}'.format(e))
if len(errors) > 0:
raise TypeResolutionError(
'Failed to resolve "{}" function against "{}". Errors: {}'.format(
self.full_export_name, root.full_export_name,
", ".join("[{}]: {}".format(i, e) for i, e in enumerate(errors))
)
)

103
modules/python/src2/typing_stubs_generation/nodes/namespace_node.py Normal file
View File

@ -0,0 +1,103 @@
from typing import Type, Iterable, Sequence, Tuple, Optional, Dict
import itertools
import weakref
from .node import ASTNode, ASTNodeType
from .class_node import ClassNode, ClassProperty
from .function_node import FunctionNode
from .enumeration_node import EnumerationNode
from .constant_node import ConstantNode
from .type_node import TypeResolutionError
class NamespaceNode(ASTNode):
"""Represents C++ namespace that treated as module in Python.
NamespaceNode can have other namespaces, classes, functions, enumerations
and global constants as its children nodes.
"""
@property
def node_type(self) -> ASTNodeType:
return ASTNodeType.Namespace
@property
def children_types(self) -> Tuple[Type[ASTNode], ...]:
return (NamespaceNode, ClassNode, FunctionNode,
EnumerationNode, ConstantNode)
@property
def namespaces(self) -> Dict[str, "NamespaceNode"]:
return self._children[NamespaceNode]
@property
def classes(self) -> Dict[str, ClassNode]:
return self._children[ClassNode]
@property
def functions(self) -> Dict[str, FunctionNode]:
return self._children[FunctionNode]
@property
def enumerations(self) -> Dict[str, EnumerationNode]:
return self._children[EnumerationNode]
@property
def constants(self) -> Dict[str, ConstantNode]:
return self._children[ConstantNode]
def add_namespace(self, name: str) -> "NamespaceNode":
return self._add_child(NamespaceNode, name)
def add_class(self, name: str,
bases: Sequence["weakref.ProxyType[ClassNode]"] = (),
properties: Sequence[ClassProperty] = ()) -> "ClassNode":
return self._add_child(ClassNode, name, bases=bases,
properties=properties)
def add_function(self, name: str, arguments: Sequence[FunctionNode.Arg] = (),
return_type: Optional[FunctionNode.RetType] = None) -> FunctionNode:
return self._add_child(FunctionNode, name, arguments=arguments,
return_type=return_type)
def add_enumeration(self, name: str) -> EnumerationNode:
return self._add_child(EnumerationNode, name)
def add_constant(self, name: str, value: str) -> ConstantNode:
return self._add_child(ConstantNode, name, value=value)
def resolve_type_nodes(self, root: Optional[ASTNode] = None) -> None:
"""Resolves type nodes for all children nodes in 2 steps:
1. Resolve against `self` as a tree root
2. Resolve against `root` as a tree root
Type resolution errors are postponed until all children nodes are
examined.
Args:
root (Optional[ASTNode], optional): Root of the AST sub-tree.
Defaults to None.
"""
errors = []
for child in itertools.chain(self.functions.values(),
self.classes.values(),
self.namespaces.values()):
try:
try:
child.resolve_type_nodes(self) # type: ignore
except TypeResolutionError:
if root is not None:
child.resolve_type_nodes(root) # type: ignore
else:
raise
except TypeResolutionError as e:
errors.append(str(e))
if len(errors) > 0:
raise TypeResolutionError(
'Failed to resolve "{}" namespace against "{}". '
'Errors: {}'.format(
self.full_export_name,
root if root is None else root.full_export_name,
errors
)
)

233
modules/python/src2/typing_stubs_generation/nodes/node.py Normal file
View File

@ -0,0 +1,233 @@
import abc
import enum
import itertools
from typing import (Iterator, Type, TypeVar, Iterable, Dict,
Optional, Tuple, DefaultDict)
from collections import defaultdict
import weakref
ASTNodeSubtype = TypeVar("ASTNodeSubtype", bound="ASTNode")
NodeType = Type["ASTNode"]
NameToNode = Dict[str, ASTNodeSubtype]
class ASTNodeType(enum.Enum):
Namespace = enum.auto()
Class = enum.auto()
Function = enum.auto()
Enumeration = enum.auto()
Constant = enum.auto()
class ASTNode:
"""Represents an element of the Abstract Syntax Tree produced by parsing
public C++ headers.
NOTE: Every node manages a lifetime of its children nodes. Children nodes
contain only weak references to their direct parents, so there are no
circular dependencies.
"""
def __init__(self, name: str, parent: Optional["ASTNode"] = None,
export_name: Optional[str] = None) -> None:
"""ASTNode initializer
Args:
name (str): name of the node, should be unique inside enclosing
context (There can't be 2 classes with the same name defined
in the same namespace).
parent (ASTNode, optional): parent node expressing node context.
None corresponds to globally defined object e.g. root namespace
or function without namespace. Defaults to None.
export_name (str, optional): export name of the node used to resolve
issues in languages without proper overload resolution and
provide more meaningful naming. Defaults to None.
"""
FORBIDDEN_SYMBOLS = ";,*&#/|\\@!()[]^% "
for forbidden_symbol in FORBIDDEN_SYMBOLS:
assert forbidden_symbol not in name, \
"Invalid node identifier '{}' - contains 1 or more "\
"forbidden symbols: ({})".format(name, FORBIDDEN_SYMBOLS)
assert ":" not in name, \
"Name '{}' contains C++ scope symbols (':'). Convert the name to "\
"Python style and create appropriate parent nodes".format(name)
assert "." not in name, \
"Trying to create a node with '.' symbols in its name ({}). " \
"Dots are supposed to be a scope delimiters, so create all nodes in ('{}') " \
"and add '{}' as a last child node".format(
name,
"->".join(name.split('.')[:-1]),
name.rsplit('.', maxsplit=1)[-1]
)
self.__name = name
self.export_name = name if export_name is None else export_name
self._parent: Optional["ASTNode"] = None
self.parent = parent
self.is_exported = True
self._children: DefaultDict[NodeType, NameToNode] = defaultdict(dict)
def __str__(self) -> str:
return "{}('{}' exported as '{}')".format(
type(self).__name__.replace("Node", ""), self.name, self.export_name
)
def __repr__(self) -> str:
return str(self)
@abc.abstractproperty
def children_types(self) -> Tuple[Type["ASTNode"], ...]:
"""Set of ASTNode types that are allowed to be children of this node
Returns:
Tuple[Type[ASTNode], ...]: Types of children nodes
"""
pass
@abc.abstractproperty
def node_type(self) -> ASTNodeType:
"""Type of the ASTNode that can be used to distinguish nodes without
importing all subclasses of ASTNode
Returns:
ASTNodeType: Current node type
"""
pass
@property
def name(self) -> str:
return self.__name
@property
def native_name(self) -> str:
return self.full_name.replace(".", "::")
@property
def full_name(self) -> str:
return self._construct_full_name("name")
@property
def full_export_name(self) -> str:
return self._construct_full_name("export_name")
@property
def parent(self) -> Optional["ASTNode"]:
return self._parent
@parent.setter
def parent(self, value: Optional["ASTNode"]) -> None:
assert value is None or isinstance(value, ASTNode), \
"ASTNode.parent should be None or another ASTNode, " \
"but got: {}".format(type(value))
if value is not None:
value.__check_child_before_add(type(self), self.name)
# Detach from previous parent
if self._parent is not None:
self._parent._children[type(self)].pop(self.name)
if value is None:
self._parent = None
return
# Set a weak reference to a new parent and add self to its children
self._parent = weakref.proxy(value)
value._children[type(self)][self.name] = self
def __check_child_before_add(self, child_type: Type[ASTNodeSubtype],
name: str) -> None:
assert len(self.children_types) > 0, \
"Trying to add child node '{}::{}' to node '{}::{}' " \
"that can't have children nodes".format(child_type.__name__, name,
type(self).__name__,
self.name)
assert child_type in self.children_types, \
"Trying to add child node '{}::{}' to node '{}::{}' " \
"that supports only ({}) as its children types".format(
child_type.__name__, name, type(self).__name__, self.name,
",".join(t.__name__ for t in self.children_types)
)
if self._find_child(child_type, name) is not None:
raise ValueError(
"Node '{}::{}' already has a child '{}::{}'".format(
type(self).__name__, self.name, child_type.__name__, name
)
)
def _add_child(self, child_type: Type[ASTNodeSubtype], name: str,
**kwargs) -> ASTNodeSubtype:
"""Creates a child of the node with the given type and performs common
validation checks:
- Node can have children of the provided type
- Node doesn't have child with the same name
NOTE: Shouldn't be used directly by a user.
Args:
child_type (Type[ASTNodeSubtype]): Type of the child to create.
name (str): Name of the child.
**kwargs: Extra keyword arguments supplied to child_type.__init__
method.
Returns:
ASTNodeSubtype: Created ASTNode
"""
self.__check_child_before_add(child_type, name)
return child_type(name, parent=self, **kwargs)
def _find_child(self, child_type: Type[ASTNodeSubtype],
name: str) -> Optional[ASTNodeSubtype]:
"""Looks for child node with the given type and name.
Args:
child_type (Type[ASTNodeSubtype]): Type of the child node.
name (str): Name of the child node.
Returns:
Optional[ASTNodeSubtype]: child node if it can be found, None
otherwise.
"""
if child_type not in self._children:
return None
return self._children[child_type].get(name, None)
def _construct_full_name(self, property_name: str) -> str:
"""Traverses nodes hierarchy upright to the root node and constructs a
full name of the node using original or export names depending on the
provided `property_name` argument.
Args:
property_name (str): Name of the property to quire from node to get
its name. Should be `name` or `export_name`.
Returns:
str: full node name where each node part is divided with a dot.
"""
def get_name(node: ASTNode) -> str:
return getattr(node, property_name)
assert property_name in ('name', 'export_name'), 'Invalid name property'
name_parts = [get_name(self), ]
parent = self.parent
while parent is not None:
name_parts.append(get_name(parent))
parent = parent.parent
return ".".join(reversed(name_parts))
def __iter__(self) -> Iterator["ASTNode"]:
return iter(itertools.chain.from_iterable(
node
# Iterate over mapping between node type and nodes dict
for children_nodes in self._children.values()
# Iterate over mapping between node name and node
for node in children_nodes.values()
))

762
modules/python/src2/typing_stubs_generation/nodes/type_node.py Normal file
View File

@ -0,0 +1,762 @@
from typing import Sequence, Generator, Tuple, Optional, Union
import weakref
import abc
from .node import ASTNode, ASTNodeType
class TypeResolutionError(Exception):
pass
class TypeNode(abc.ABC):
"""This class and its derivatives used for construction parts of AST that
otherwise can't be constructed from the information provided by header
parser, because this information is either not available at that moment of
time or not available at all:
- There is no possible way to derive correspondence between C++ type
and its Python equivalent if it is not exposed from library
e.g. `cv::Rect`.
- There is no information about types visibility (see `ASTNodeTypeNode`).
"""
def __init__(self, ctype_name: str) -> None:
self.ctype_name = ctype_name
@abc.abstractproperty
def typename(self) -> str:
"""Short name of the type node used that should be used in the same
module (or a file) where type is defined.
Returns:
str: short name of the type node.
"""
pass
@property
def full_typename(self) -> str:
"""Full name of the type node including full module name starting from
the package.
Example: 'cv2.Algorithm', 'cv2.gapi.ie.PyParams'.
Returns:
str: full name of the type node.
"""
return self.typename
@property
def required_definition_imports(self) -> Generator[str, None, None]:
"""Generator filled with import statements required for type
node definition (especially used by `AliasTypeNode`).
Example:
```python
# Alias defined in the `cv2.typing.__init__.pyi`
Callback = typing.Callable[[cv2.GMat, float], None]
# alias definition
callback_alias = AliasTypeNode.callable_(
'Callback',
arg_types=(ASTNodeTypeNode('GMat'), PrimitiveTypeNode.float_())
)
# Required definition imports
for required_import in callback_alias.required_definition_imports:
print(required_import)
# Outputs:
# 'import typing'
# 'import cv2'
```
Yields:
Generator[str, None, None]: generator filled with import statements
required for type node definition.
"""
yield from ()
@property
def required_usage_imports(self) -> Generator[str, None, None]:
"""Generator filled with import statements required for type node
usage.
Example:
```python
# Alias defined in the `cv2.typing.__init__.pyi`
Callback = typing.Callable[[cv2.GMat, float], None]
# alias definition
callback_alias = AliasTypeNode.callable_(
'Callback',
arg_types=(ASTNodeTypeNode('GMat'), PrimitiveTypeNode.float_())
)
# Required usage imports
for required_import in callback_alias.required_usage_imports:
print(required_import)
# Outputs:
# 'import cv2.typing'
```
Yields:
Generator[str, None, None]: generator filled with import statements
required for type node definition.
"""
yield from ()
@property
def is_resolved(self) -> bool:
return True
def relative_typename(self, module_full_export_name: str) -> str:
"""Type name relative to the provided module.
Args:
module_full_export_name (str): Full export name of the module to
get relative name to.
Returns:
str: If module name of the type node doesn't match `module`, then
returns class scopes + `self.typename`, otherwise
`self.full_typename`.
"""
return self.full_typename
def resolve(self, root: ASTNode) -> None:
"""Resolves all references to AST nodes using a top-down search
for nodes with corresponding export names. See `_resolve_symbol` for
more details.
Args:
root (ASTNode): Node pointing to the root of a subtree in AST
representing search scope of the symbol.
Most of the symbols don't have full paths in their names, so
scopes should be examined in bottom-up manner starting
with narrowest one.
Raises:
TypeResolutionError: if at least 1 reference to AST node can't
be resolved in the subtree pointed by the root.
"""
pass
class NoneTypeNode(TypeNode):
"""Type node representing a None (or `void` in C++) type.
"""
@property
def typename(self) -> str:
return "None"
class AnyTypeNode(TypeNode):
"""Type node representing any type (most of the time it means unknown).
"""
@property
def typename(self) -> str:
return "typing.Any"
@property
def required_usage_imports(self) -> Generator[str, None, None]:
yield "import typing"
class PrimitiveTypeNode(TypeNode):
"""Type node representing a primitive built-in types e.g. int, float, str.
"""
def __init__(self, ctype_name: str, typename: Optional[str] = None) -> None:
super().__init__(ctype_name)
self._typename = typename if typename is not None else ctype_name
@property
def typename(self) -> str:
return self._typename
@classmethod
def int_(cls, ctype_name: Optional[str] = None):
if ctype_name is None:
ctype_name = "int"
return PrimitiveTypeNode(ctype_name, typename="int")
@classmethod
def float_(cls, ctype_name: Optional[str] = None):
if ctype_name is None:
ctype_name = "float"
return PrimitiveTypeNode(ctype_name, typename="float")
@classmethod
def bool_(cls, ctype_name: Optional[str] = None):
if ctype_name is None:
ctype_name = "bool"
return PrimitiveTypeNode(ctype_name, typename="bool")
@classmethod
def str_(cls, ctype_name: Optional[str] = None):
if ctype_name is None:
ctype_name = "string"
return PrimitiveTypeNode(ctype_name, "str")
class AliasRefTypeNode(TypeNode):
"""Type node representing an alias referencing another alias. Example:
```python
Point2i = tuple[int, int]
Point = Point2i
```
During typing stubs generation procedure above code section might be defined
as follows
```python
AliasTypeNode.tuple_("Point2i",
items=(
PrimitiveTypeNode.int_(),
PrimitiveTypeNode.int_()
))
AliasTypeNode.ref_("Point", "Point2i")
```
"""
def __init__(self, alias_ctype_name: str,
alias_export_name: Optional[str] = None):
super().__init__(alias_ctype_name)
if alias_export_name is None:
self.alias_export_name = alias_ctype_name
else:
self.alias_export_name = alias_export_name
@property
def typename(self) -> str:
return self.alias_export_name
@property
def full_typename(self) -> str:
return "cv2.typing." + self.typename
class AliasTypeNode(TypeNode):
"""Type node representing an alias to another type.
Example:
```python
Point2i = tuple[int, int]
```
can be defined as
```python
AliasTypeNode.tuple_("Point2i",
items=(
PrimitiveTypeNode.int_(),
PrimitiveTypeNode.int_()
))
```
Under the hood it is implemented as a container of another type node.
"""
def __init__(self, ctype_name: str, value: TypeNode,
export_name: Optional[str] = None,
comment: Optional[str] = None) -> None:
super().__init__(ctype_name)
self.value = value
self._export_name = export_name
self.comment = comment
@property
def typename(self) -> str:
if self._export_name is not None:
return self._export_name
return self.ctype_name
@property
def full_typename(self) -> str:
return "cv2.typing." + self.typename
@property
def required_definition_imports(self) -> Generator[str, None, None]:
return self.value.required_usage_imports
@property
def required_usage_imports(self) -> Generator[str, None, None]:
yield "import cv2.typing"
@property
def is_resolved(self) -> bool:
return self.value.is_resolved
def resolve(self, root: ASTNode):
try:
self.value.resolve(root)
except TypeResolutionError as e:
raise TypeResolutionError(
'Failed to resolve alias "{}" exposed as "{}"'.format(
self.ctype_name, self.typename
)
) from e
@classmethod
def int_(cls, ctype_name: str, export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, PrimitiveTypeNode.int_(), export_name, comment)
@classmethod
def float_(cls, ctype_name: str, export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, PrimitiveTypeNode.float_(), export_name, comment)
@classmethod
def array_(cls, ctype_name: str, shape: Optional[Tuple[int, ...]],
dtype: Optional[str] = None, export_name: Optional[str] = None,
comment: Optional[str] = None):
if comment is None:
comment = "Shape: " + str(shape)
else:
comment += ". Shape: " + str(shape)
return cls(ctype_name, NDArrayTypeNode(ctype_name, shape, dtype),
export_name, comment)
@classmethod
def union_(cls, ctype_name: str, items: Tuple[TypeNode, ...],
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, UnionTypeNode(ctype_name, items),
export_name, comment)
@classmethod
def optional_(cls, ctype_name: str, item: TypeNode,
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, OptionalTypeNode(item), export_name, comment)
@classmethod
def sequence_(cls, ctype_name: str, item: TypeNode,
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, SequenceTypeNode(ctype_name, item),
export_name, comment)
@classmethod
def tuple_(cls, ctype_name: str, items: Tuple[TypeNode, ...],
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, TupleTypeNode(ctype_name, items),
export_name, comment)
@classmethod
def class_(cls, ctype_name: str, class_name: str,
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name, ASTNodeTypeNode(class_name),
export_name, comment)
@classmethod
def callable_(cls, ctype_name: str,
arg_types: Union[TypeNode, Sequence[TypeNode]],
ret_type: TypeNode = NoneTypeNode("void"),
export_name: Optional[str] = None,
comment: Optional[str] = None):
return cls(ctype_name,
CallableTypeNode(ctype_name, arg_types, ret_type),
export_name, comment)
@classmethod
def ref_(cls, ctype_name: str, alias_ctype_name: str,
alias_export_name: Optional[str] = None,
export_name: Optional[str] = None, comment: Optional[str] = None):
return cls(ctype_name,
AliasRefTypeNode(alias_ctype_name, alias_export_name),
export_name, comment)
@classmethod
def dict_(cls, ctype_name: str, key_type: TypeNode, value_type: TypeNode,
export_name: Optional[str] = None, comment: Optional[str] = None):
return cls(ctype_name, DictTypeNode(ctype_name, key_type, value_type),
export_name, comment)
class NDArrayTypeNode(TypeNode):
"""Type node representing NumPy ndarray.
"""
def __init__(self, ctype_name: str, shape: Optional[Tuple[int, ...]] = None,
dtype: Optional[str] = None) -> None:
super().__init__(ctype_name)
self.shape = shape
self.dtype = dtype
@property
def typename(self) -> str:
return "numpy.ndarray[{shape}, numpy.dtype[{dtype}]]".format(
# NOTE: Shape is not fully supported yet
# shape=self.shape if self.shape is not None else "typing.Any",
shape="typing.Any",
dtype=self.dtype if self.dtype is not None else "numpy.generic"
)
@property
def required_usage_imports(self) -> Generator[str, None, None]:
yield "import numpy"
# if self.shape is None:
yield "import typing"
class ASTNodeTypeNode(TypeNode):
"""Type node representing a lazy ASTNode corresponding to type of
function argument or its return type or type of class property.
Introduced laziness nature resolves the types visibility issue - all types
should be known during function declaration to select an appropriate node
from the AST. Such knowledge leads to evaluation of all preprocessor
directives (`#include` particularly) for each processed header and might be
too expensive and error prone.
"""
def __init__(self, ctype_name: str, typename: Optional[str] = None,
module_name: Optional[str] = None) -> None:
super().__init__(ctype_name)
self._typename = typename if typename is not None else ctype_name
self._module_name = module_name
self._ast_node: Optional[weakref.ProxyType[ASTNode]] = None
@property
def typename(self) -> str:
if self._ast_node is None:
return self._typename
typename = self._ast_node.export_name
if self._ast_node.node_type is not ASTNodeType.Enumeration:
return typename
# NOTE: Special handling for enums
parent = self._ast_node.parent
while parent.node_type is ASTNodeType.Class:
typename = parent.export_name + "_" + typename
parent = parent.parent
return typename
@property
def full_typename(self) -> str:
if self._ast_node is not None:
if self._ast_node.node_type is not ASTNodeType.Enumeration:
return self._ast_node.full_export_name
# NOTE: enumerations are exported to module scope
typename = self._ast_node.export_name
parent = self._ast_node.parent
while parent.node_type is ASTNodeType.Class:
typename = parent.export_name + "_" + typename
parent = parent.parent
return parent.full_export_name + "." + typename
if self._module_name is not None:
return self._module_name + "." + self._typename
return self._typename
@property
def required_usage_imports(self) -> Generator[str, None, None]:
if self._module_name is None:
assert self._ast_node is not None, \
"Can't find a module for class '{}' exported as '{}'".format(
self.ctype_name, self.typename,
)
module = self._ast_node.parent
while module.node_type is not ASTNodeType.Namespace:
module = module.parent
yield "import " + module.full_export_name
else:
yield "import " + self._module_name
@property
def is_resolved(self) -> bool:
return self._ast_node is not None or self._module_name is not None
def resolve(self, root: ASTNode):
if self.is_resolved:
return
node = _resolve_symbol(root, self.typename)
if node is None:
raise TypeResolutionError('Failed to resolve "{}" exposed as "{}"'.format(
self.ctype_name, self.typename
))
self._ast_node = weakref.proxy(node)
def relative_typename(self, module: str) -> str:
assert self._ast_node is not None or self._module_name is not None, \
"'{}' exported as '{}' is not resolved yet".format(self.ctype_name,
self.typename)
if self._module_name is None:
type_module = self._ast_node.parent # type: ignore
while type_module.node_type is not ASTNodeType.Namespace:
type_module = type_module.parent
module_name = type_module.full_export_name
else:
module_name = self._module_name
if module_name != module:
return self.full_typename
return self.full_typename[len(module_name) + 1:]
class AggregatedTypeNode(TypeNode):
"""Base type node for type nodes representing an aggregation of another
type nodes e.g. tuple, sequence or callable."""
def __init__(self, ctype_name: str, items: Sequence[TypeNode]) -> None:
super().__init__(ctype_name)
self.items = list(items)
@property
def is_resolved(self) -> bool:
return all(item.is_resolved for item in self.items)
def resolve(self, root: ASTNode) -> None:
errors = []
for item in filter(lambda item: not item.is_resolved, self):
try:
item.resolve(root)
except TypeResolutionError as e:
errors.append(str(e))
if len(errors) > 0:
raise TypeResolutionError(
'Failed to resolve one of "{}" items. Errors: {}'.format(
self.full_typename, errors
)
)
def __iter__(self):
return iter(self.items)
def __len__(self) -> int:
return len(self.items)
@property
def required_definition_imports(self) -> Generator[str, None, None]:
for item in self:
yield from item.required_definition_imports
@property
def required_usage_imports(self) -> Generator[str, None, None]:
for item in self:
yield from item.required_usage_imports
class ContainerTypeNode(AggregatedTypeNode):
"""Base type node for all type nodes representing a container type.
"""
@property
def typename(self) -> str:
return self.type_format.format(self.types_separator.join(
item.typename for item in self
))
@property
def full_typename(self) -> str:
return self.type_format.format(self.types_separator.join(
item.full_typename for item in self
))
def relative_typename(self, module: str) -> str:
return self.type_format.format(self.types_separator.join(
item.relative_typename(module) for item in self
))
@abc.abstractproperty
def type_format(self) -> str:
pass
@abc.abstractproperty
def types_separator(self) -> str:
pass
class SequenceTypeNode(ContainerTypeNode):
"""Type node representing a homogeneous collection of elements with
possible unknown length.
"""
def __init__(self, ctype_name: str, item: TypeNode) -> None:
super().__init__(ctype_name, (item, ))
@property
def type_format(self):
return "typing.Sequence[{}]"
@property
def types_separator(self):
return ", "
@property
def required_definition_imports(self) -> Generator[str, None, None]:
yield "import typing"
yield from super().required_definition_imports
@property
def required_usage_imports(self) -> Generator[str, None, None]:
yield "import typing"
yield from super().required_usage_imports
class TupleTypeNode(ContainerTypeNode):
"""Type node representing possibly heterogenous collection of types with
possibly unspecified length.
"""
@property
def type_format(self):
return "tuple[{}]"
@property
def types_separator(self) -> str:
return ", "
class UnionTypeNode(ContainerTypeNode):
"""Type node representing type that can be one of the predefined set of types.
"""
@property
def type_format(self):
return "{}"
@property
def types_separator(self):
return " | "
class OptionalTypeNode(UnionTypeNode):
"""Type node representing optional type which is effectively is a union
of value type node and None.
"""
def __init__(self, value: TypeNode) -> None:
super().__init__(value.ctype_name, (value, NoneTypeNode(value.ctype_name)))
class DictTypeNode(ContainerTypeNode):
"""Type node representing a homogeneous key-value mapping.
"""
def __init__(self, ctype_name: str, key_type: TypeNode,
value_type: TypeNode) -> None:
super().__init__(ctype_name, (key_type, value_type))
@property
def key_type(self) -> TypeNode:
return self.items[0]
@property
def value_type(self) -> TypeNode:
return self.items[1]
@property
def type_format(self):
return "dict[{}]"
@property
def types_separator(self):
return ", "
class CallableTypeNode(AggregatedTypeNode):
"""Type node representing a callable type (most probably a function).
```python
CallableTypeNode(
'image_reading_callback',
arg_types=(ASTNodeTypeNode('Image'), PrimitiveTypeNode.float_())
)
```
defines a callable type node representing a function with the same
interface as the following
```python
def image_reading_callback(image: Image, timestamp: float) -> None: ...
```
"""
def __init__(self, ctype_name: str,
arg_types: Union[TypeNode, Sequence[TypeNode]],
ret_type: TypeNode = NoneTypeNode("void")) -> None:
if isinstance(arg_types, TypeNode):
super().__init__(ctype_name, (arg_types, ret_type))
else:
super().__init__(ctype_name, (*arg_types, ret_type))
@property
def arg_types(self) -> Sequence[TypeNode]:
return self.items[:-1]
@property
def ret_type(self) -> TypeNode:
return self.items[-1]
@property
def typename(self) -> str:
return 'typing.Callable[[{}], {}]'.format(
', '.join(arg.typename for arg in self.arg_types),
self.ret_type.typename
)
@property
def full_typename(self) -> str:
return 'typing.Callable[[{}], {}]'.format(
', '.join(arg.full_typename for arg in self.arg_types),
self.ret_type.full_typename
)
def relative_typename(self, module: str) -> str:
return 'typing.Callable[[{}], {}]'.format(
', '.join(arg.relative_typename(module) for arg in self.arg_types),
self.ret_type.relative_typename(module)
)
@property
def required_definition_imports(self) -> Generator[str, None, None]:
yield "import typing"
yield from super().required_definition_imports
@property
def required_usage_imports(self) -> Generator[str, None, None]:
yield "import typing"
yield from super().required_usage_imports
def _resolve_symbol(root: Optional[ASTNode], full_symbol_name: str) -> Optional[ASTNode]:
"""Searches for a symbol with the given full export name in the AST
starting from the `root`.
Args:
root (Optional[ASTNode]): Root of the examining AST.
full_symbol_name (str): Full export name of the symbol to find. Path
components can be divided by '.' or '_'.
Returns:
Optional[ASTNode]: ASTNode with full export name equal to
`full_symbol_name`, None otherwise.
>>> root = NamespaceNode('cv')
>>> cls = root.add_class('Algorithm').add_class('Params')
>>> _resolve_symbol(root, 'cv.Algorithm.Params') == cls
True
>>> root = NamespaceNode('cv')
>>> enum = root.add_namespace('detail').add_enumeration('AlgorithmType')
>>> _resolve_symbol(root, 'cv_detail_AlgorithmType') == enum
True
>>> root = NamespaceNode('cv')
>>> _resolve_symbol(root, 'cv.detail.Algorithm')
None
>>> root = NamespaceNode('cv')
>>> enum = root.add_namespace('detail').add_enumeration('AlgorithmType')
>>> _resolve_symbol(root, 'AlgorithmType')
None
"""
def search_down_symbol(scope: Optional[ASTNode],
scope_sep: str) -> Optional[ASTNode]:
parts = full_symbol_name.split(scope_sep, maxsplit=1)
while len(parts) == 2:
# Try to find narrow scope
scope = _resolve_symbol(scope, parts[0])
if scope is None:
return None
# and resolve symbol in it
node = _resolve_symbol(scope, parts[1])
if node is not None:
return node
# symbol is not found, but narrowed scope is valid - diving further
parts = parts[1].split(scope_sep, maxsplit=1)
return None
assert root is not None, \
"Can't resolve symbol '{}' from NONE root".format(full_symbol_name)
# Looking for exact symbol match
for attr in filter(lambda attr: hasattr(root, attr),
("namespaces", "classes", "enumerations")):
nodes_dict = getattr(root, attr)
node = nodes_dict.get(full_symbol_name, None)
if node is not None:
return node
# Symbol is not found, looking for more fine-grained scope if possible
for scope_sep in ("_", "."):
node = search_down_symbol(root, scope_sep)
if node is not None:
return node
return None

195
modules/python/src2/typing_stubs_generation/predefined_types.py Normal file
View File

@ -0,0 +1,195 @@
from .nodes.type_node import (
AliasTypeNode, AliasRefTypeNode, PrimitiveTypeNode,
ASTNodeTypeNode, NDArrayTypeNode, NoneTypeNode, SequenceTypeNode,
TupleTypeNode, UnionTypeNode, AnyTypeNode
)
# Set of predefined types used to cover cases when library doesn't
# directly exports a type and equivalent one should be used instead.
# Example: Instead of C++ `cv::Rect(1, 1, 5, 6)` in Python any sequence type
# with length 4 can be used: tuple `(1, 1, 5, 6)` or list `[1, 1, 5, 6]`.
# Predefined type might be:
# - alias - defines a Python synonym for a native type name.
# Example: `cv::Rect` and `cv::Size` are both `Sequence[int]` in Python, but
# with different length constraints (4 and 2 accordingly).
# - direct substitution - just a plain type replacement without any credits to
# native type. Example:
# * `std::vector<uchar>` is `np.ndarray` with `dtype == np.uint8` in Python
# * `double` is a Python `float`
# * `std::string` is a Python `str`
_PREDEFINED_TYPES = (
PrimitiveTypeNode.int_("int"),
PrimitiveTypeNode.int_("uchar"),
PrimitiveTypeNode.int_("unsigned"),
PrimitiveTypeNode.int_("int64"),
PrimitiveTypeNode.int_("size_t"),
PrimitiveTypeNode.float_("float"),
PrimitiveTypeNode.float_("double"),
PrimitiveTypeNode.bool_("bool"),
PrimitiveTypeNode.str_("string"),
PrimitiveTypeNode.str_("char"),
PrimitiveTypeNode.str_("String"),
PrimitiveTypeNode.str_("c_string"),
NoneTypeNode("void"),
AliasTypeNode.int_("void*", "IntPointer", "Represents an arbitrary pointer"),
AliasTypeNode.union_(
"Mat",
items=(ASTNodeTypeNode("Mat", module_name="cv2.mat_wrapper"),
NDArrayTypeNode("Mat")),
export_name="MatLike"
),
AliasTypeNode.sequence_("MatShape", PrimitiveTypeNode.int_()),
AliasTypeNode.sequence_("Size", PrimitiveTypeNode.int_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Size2f", PrimitiveTypeNode.float_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Scalar", PrimitiveTypeNode.float_(),
comment="Required length is at most 4"),
AliasTypeNode.sequence_("Point", PrimitiveTypeNode.int_(),
comment="Required length is 2"),
AliasTypeNode.ref_("Point2i", "Point"),
AliasTypeNode.sequence_("Point2f", PrimitiveTypeNode.float_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Point2d", PrimitiveTypeNode.float_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Point3i", PrimitiveTypeNode.int_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Point3f", PrimitiveTypeNode.float_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Point3d", PrimitiveTypeNode.float_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Range", PrimitiveTypeNode.int_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Rect", PrimitiveTypeNode.int_(),
comment="Required length is 4"),
AliasTypeNode.sequence_("Rect2i", PrimitiveTypeNode.int_(),
comment="Required length is 4"),
AliasTypeNode.sequence_("Rect2d", PrimitiveTypeNode.float_(),
comment="Required length is 4"),
AliasTypeNode.dict_("Moments", PrimitiveTypeNode.str_("Moments::key"),
PrimitiveTypeNode.float_("Moments::value")),
AliasTypeNode.tuple_("RotatedRect",
items=(AliasRefTypeNode("Point2f"),
AliasRefTypeNode("Size"),
PrimitiveTypeNode.float_()),
comment="Any type providing sequence protocol is supported"),
AliasTypeNode.tuple_("TermCriteria",
items=(
ASTNodeTypeNode("TermCriteria.Type"),
PrimitiveTypeNode.int_(),
PrimitiveTypeNode.float_()),
comment="Any type providing sequence protocol is supported"),
AliasTypeNode.sequence_("Vec2i", PrimitiveTypeNode.int_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Vec2f", PrimitiveTypeNode.float_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Vec2d", PrimitiveTypeNode.float_(),
comment="Required length is 2"),
AliasTypeNode.sequence_("Vec3i", PrimitiveTypeNode.int_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Vec3f", PrimitiveTypeNode.float_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Vec3d", PrimitiveTypeNode.float_(),
comment="Required length is 3"),
AliasTypeNode.sequence_("Vec4i", PrimitiveTypeNode.int_(),
comment="Required length is 4"),
AliasTypeNode.sequence_("Vec4f", PrimitiveTypeNode.float_(),
comment="Required length is 4"),
AliasTypeNode.sequence_("Vec4d", PrimitiveTypeNode.float_(),
comment="Required length is 4"),
AliasTypeNode.sequence_("Vec6f", PrimitiveTypeNode.float_(),
comment="Required length is 6"),
AliasTypeNode.class_("FeatureDetector", "Feature2D",
export_name="FeatureDetector"),
AliasTypeNode.class_("DescriptorExtractor", "Feature2D",
export_name="DescriptorExtractor"),
AliasTypeNode.class_("FeatureExtractor", "Feature2D",
export_name="FeatureExtractor"),
AliasTypeNode.union_("GProtoArg",
items=(AliasRefTypeNode("Scalar"),
ASTNodeTypeNode("GMat"),
ASTNodeTypeNode("GOpaqueT"),
ASTNodeTypeNode("GArrayT"))),
SequenceTypeNode("GProtoArgs", AliasRefTypeNode("GProtoArg")),
AliasTypeNode.sequence_("GProtoInputArgs", AliasRefTypeNode("GProtoArg")),
AliasTypeNode.sequence_("GProtoOutputArgs", AliasRefTypeNode("GProtoArg")),
AliasTypeNode.union_(
"GRunArg",
items=(AliasRefTypeNode("Mat", "MatLike"),
AliasRefTypeNode("Scalar"),
ASTNodeTypeNode("GOpaqueT"),
ASTNodeTypeNode("GArrayT"),
SequenceTypeNode("GRunArg", AnyTypeNode("GRunArg")),
NoneTypeNode("GRunArg"))
),
AliasTypeNode.optional_("GOptRunArg", AliasRefTypeNode("GRunArg")),
AliasTypeNode.union_("GMetaArg",
items=(ASTNodeTypeNode("GMat"),
AliasRefTypeNode("Scalar"),
ASTNodeTypeNode("GOpaqueT"),
ASTNodeTypeNode("GArrayT"))),
AliasTypeNode.union_("Prim",
items=(ASTNodeTypeNode("gapi.wip.draw.Text"),
ASTNodeTypeNode("gapi.wip.draw.Circle"),
ASTNodeTypeNode("gapi.wip.draw.Image"),
ASTNodeTypeNode("gapi.wip.draw.Line"),
ASTNodeTypeNode("gapi.wip.draw.Rect"),
ASTNodeTypeNode("gapi.wip.draw.Mosaic"),
ASTNodeTypeNode("gapi.wip.draw.Poly"))),
SequenceTypeNode("Prims", AliasRefTypeNode("Prim")),
AliasTypeNode.array_("Matx33f", (3, 3), "numpy.float32"),
AliasTypeNode.array_("Matx33d", (3, 3), "numpy.float64"),
AliasTypeNode.array_("Matx44f", (4, 4), "numpy.float32"),
AliasTypeNode.array_("Matx44d", (4, 4), "numpy.float64"),
NDArrayTypeNode("vector<uchar>", dtype="numpy.uint8"),
NDArrayTypeNode("vector_uchar", dtype="numpy.uint8"),
TupleTypeNode("GMat2", items=(ASTNodeTypeNode("GMat"),
ASTNodeTypeNode("GMat"))),
ASTNodeTypeNode("GOpaque", "GOpaqueT"),
ASTNodeTypeNode("GArray", "GArrayT"),
AliasTypeNode.union_("GTypeInfo",
items=(ASTNodeTypeNode("GMat"),
AliasRefTypeNode("Scalar"),
ASTNodeTypeNode("GOpaqueT"),
ASTNodeTypeNode("GArrayT"))),
SequenceTypeNode("GCompileArgs", ASTNodeTypeNode("GCompileArg")),
SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")),
SequenceTypeNode("GRunArgs", AliasRefTypeNode("GRunArg")),
SequenceTypeNode("GMetaArgs", AliasRefTypeNode("GMetaArg")),
SequenceTypeNode("GOptRunArgs", AliasRefTypeNode("GOptRunArg")),
AliasTypeNode.callable_(
"detail_ExtractArgsCallback",
arg_types=SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")),
ret_type=SequenceTypeNode("GRunArgs", AliasRefTypeNode("GRunArg")),
export_name="ExtractArgsCallback"
),
AliasTypeNode.callable_(
"detail_ExtractMetaCallback",
arg_types=SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")),
ret_type=SequenceTypeNode("GMetaArgs", AliasRefTypeNode("GMetaArg")),
export_name="ExtractMetaCallback"
),
AliasTypeNode.class_("LayerId", "DictValue"),
PrimitiveTypeNode.int_("cvflann_flann_distance_t"),
PrimitiveTypeNode.int_("flann_flann_distance_t"),
PrimitiveTypeNode.int_("cvflann_flann_algorithm_t"),
PrimitiveTypeNode.int_("flann_flann_algorithm_t"),
AliasTypeNode.dict_("flann_IndexParams",
key_type=PrimitiveTypeNode.str_(),
value_type=UnionTypeNode("flann_IndexParams::value", items=(
PrimitiveTypeNode.bool_(),
PrimitiveTypeNode.int_(),
PrimitiveTypeNode.float_(),
PrimitiveTypeNode.str_())
), export_name="IndexParams"),
AliasTypeNode.dict_("flann_SearchParams",
key_type=PrimitiveTypeNode.str_(),
value_type=UnionTypeNode("flann_IndexParams::value", items=(
PrimitiveTypeNode.bool_(),
PrimitiveTypeNode.int_(),
PrimitiveTypeNode.float_(),
PrimitiveTypeNode.str_())
), export_name="SearchParams"),
)
PREDEFINED_TYPES = dict(zip((t.ctype_name for t in _PREDEFINED_TYPES), _PREDEFINED_TYPES))

333
modules/python/src2/typing_stubs_generation/types_conversion.py Normal file
View File

@ -0,0 +1,333 @@
from typing import Tuple, List, Optional
from .predefined_types import PREDEFINED_TYPES
from .nodes.type_node import (
TypeNode, UnionTypeNode, SequenceTypeNode, ASTNodeTypeNode, TupleTypeNode
)
def replace_template_parameters_with_placeholders(string: str) \
-> Tuple[str, Tuple[str, ...]]:
"""Replaces template parameters with `format` placeholders for all template
instantiations in provided string.
Only outermost template parameters are replaced.
Args:
string (str): input string containing C++ template instantiations
Returns:
tuple[str, tuple[str, ...]]: string with '{}' placeholders template
instead of instantiation types and a tuple of extracted types.
>>> template_string, args = replace_template_parameters_with_placeholders(
... "std::vector<cv::Point<int>>, test<int>"
... )
>>> template_string.format(*args) == "std::vector<cv::Point<int>>, test<int>"
True
>>> replace_template_parameters_with_placeholders(
... "cv::util::variant<cv::GRunArgs, cv::GOptRunArgs>"
... )
('cv::util::variant<{}>', ('cv::GRunArgs, cv::GOptRunArgs',))
>>> replace_template_parameters_with_placeholders("vector<Point<int>>")
('vector<{}>', ('Point<int>',))
>>> replace_template_parameters_with_placeholders(
... "vector<Point<int>>, vector<float>"
... )
('vector<{}>, vector<{}>', ('Point<int>', 'float'))
>>> replace_template_parameters_with_placeholders("string without templates")
('string without templates', ())
"""
template_brackets_indices = []
template_instantiations_count = 0
template_start_index = 0
for i, c in enumerate(string):
if c == "<":
template_instantiations_count += 1
if template_instantiations_count == 1:
# + 1 - because left bound is included in substring range
template_start_index = i + 1
elif c == ">":
template_instantiations_count -= 1
assert template_instantiations_count >= 0, \
"Provided string is ill-formed. There are more '>' than '<'."
if template_instantiations_count == 0:
template_brackets_indices.append((template_start_index, i))
assert template_instantiations_count == 0, \
"Provided string is ill-formed. There are more '<' than '>'."
template_args: List[str] = []
# Reversed loop is required to preserve template start/end indices
for i, j in reversed(template_brackets_indices):
template_args.insert(0, string[i:j])
string = string[:i] + "{}" + string[j:]
return string, tuple(template_args)
def get_template_instantiation_type(typename: str) -> str:
"""Extracts outermost template instantiation type from provided string
Args:
typename (str): String containing C++ template instantiation.
Returns:
str: String containing template instantiation type
>>> get_template_instantiation_type("std::vector<cv::Point<int>>")
'cv::Point<int>'
>>> get_template_instantiation_type("std::vector<uchar>")
'uchar'
>>> get_template_instantiation_type("std::map<int, float>")
'int, float'
>>> get_template_instantiation_type("uchar")
Traceback (most recent call last):
...
ValueError: typename ('uchar') doesn't contain template instantiations
>>> get_template_instantiation_type("std::vector<int>, std::vector<float>")
Traceback (most recent call last):
...
ValueError: typename ('std::vector<int>, std::vector<float>') contains more than 1 template instantiation
"""
_, args = replace_template_parameters_with_placeholders(typename)
if len(args) == 0:
raise ValueError(
"typename ('{}') doesn't contain template instantiations".format(typename)
)
if len(args) > 1:
raise ValueError(
"typename ('{}') contains more than 1 template instantiation".format(typename)
)
return args[0]
def normalize_ctype_name(typename: str) -> str:
"""Normalizes C++ name by removing unnecessary namespace prefixes and possible
pointer/reference qualification. '::' are replaced with '_'.
NOTE: Pointer decay for 'void*' is not performed.
Args:
typename (str): Name of the C++ type for normalization
Returns:
str: Normalized C++ type name.
>>> normalize_ctype_name('std::vector<cv::Point2f>&')
'vector<cv_Point2f>'
>>> normalize_ctype_name('AKAZE::DescriptorType')
'AKAZE_DescriptorType'
>>> normalize_ctype_name('std::vector<Mat>')
'vector<Mat>'
>>> normalize_ctype_name('std::string')
'string'
>>> normalize_ctype_name('void*') # keep void* as is - special case
'void*'
>>> normalize_ctype_name('Ptr<AKAZE>')
'AKAZE'
>>> normalize_ctype_name('Algorithm_Ptr')
'Algorithm'
"""
for prefix_to_remove in ("cv", "std"):
if typename.startswith(prefix_to_remove):
typename = typename[len(prefix_to_remove):]
typename = typename.replace("::", "_").lstrip("_")
if typename.endswith('&'):
typename = typename[:-1]
typename = typename.strip()
if typename == 'void*':
return typename
if is_pointer_type(typename):
# Case for "type*", "type_Ptr", "typePtr"
for suffix in ("*", "_Ptr", "Ptr"):
if typename.endswith(suffix):
return typename[:-len(suffix)]
# Case Ptr<Type>
if _is_template_instantiation(typename):
return normalize_ctype_name(
get_template_instantiation_type(typename)
)
# Case Ptr_Type
return typename.split("_", maxsplit=1)[-1]
# special normalization for several G-API Types
if typename.startswith("GArray_") or typename.startswith("GArray<"):
return "GArrayT"
if typename.startswith("GOpaque_") or typename.startswith("GOpaque<"):
return "GOpaqueT"
if typename == "GStreamerPipeline" or typename.startswith("GStreamerSource"):
return "gst_" + typename
return typename
def is_tuple_type(typename: str) -> bool:
return typename.startswith("tuple") or typename.startswith("pair")
def is_sequence_type(typename: str) -> bool:
return typename.startswith("vector")
def is_pointer_type(typename: str) -> bool:
return typename.endswith("Ptr") or typename.endswith("*") \
or typename.startswith("Ptr")
def is_union_type(typename: str) -> bool:
return typename.startswith('util_variant')
def _is_template_instantiation(typename: str) -> bool:
"""Fast, but unreliable check whenever provided typename is a template
instantiation.
Args:
typename (str): typename to check against template instantiation.
Returns:
bool: True if provided `typename` contains template instantiation,
False otherwise
"""
if "<" in typename:
assert ">" in typename, \
"Wrong template class instantiation: {}. '>' is missing".format(typename)
return True
return False
def create_type_nodes_from_template_arguments(template_args_str: str) \
-> List[TypeNode]:
"""Creates a list of type nodes corresponding to the argument types
used for template instantiation.
This method correctly addresses the situation when arguments of the input
template are also templates.
Example:
if `create_type_node` is called with
`std::tuple<std::variant<int, Point2i>, int, std::vector<int>>`
this function will be called with
`std::variant<int, Point<int>>, int, std::vector<int>`
that produces the following order of types resolution
`std::variant` ~ `Union`
`std::variant<int, Point2i>` -> `int` ~ `int` -> `Union[int, Point2i]`
`Point2i` ~ `Point2i`
`int` -> `int`
`std::vector<int>` -> `std::vector` ~ `Sequence` -> `Sequence[int]`
`int` ~ `int`
Returns:
List[TypeNode]: set of type nodes used for template instantiation.
List is empty if input string doesn't contain template instantiation.
"""
type_nodes = []
template_args_str, templated_args_types = replace_template_parameters_with_placeholders(
template_args_str
)
template_index = 0
# For each template argument
for template_arg in template_args_str.split(","):
template_arg = template_arg.strip()
# Check if argument requires type substitution
if _is_template_instantiation(template_arg):
# Reconstruct the original type
template_arg = template_arg.format(templated_args_types[template_index])
template_index += 1
# create corresponding type node
type_nodes.append(create_type_node(template_arg))
return type_nodes
def create_type_node(typename: str,
original_ctype_name: Optional[str] = None) -> TypeNode:
"""Converts C++ type name to appropriate type used in Python library API.
Conversion procedure:
1. Normalize typename: remove redundant prefixes, unify name
components delimiters, remove reference qualifications.
2. Check whenever typename has a known predefined conversion or exported
as alias e.g.
- C++ `double` -> Python `float`
- C++ `cv::Rect` -> Python `Sequence[int]`
- C++ `std::vector<char>` -> Python `np.ndarray`
return TypeNode corresponding to the appropriate type.
3. Check whenever typename is a container of types e.g. variant,
sequence or tuple. If so, select appropriate Python container type
and perform arguments conversion.
4. Create a type node corresponding to the AST node passing normalized
typename as its name.
Args:
typename (str): C++ type name to convert.
original_ctype_name (Optional[str]): Original C++ name of the type.
`original_ctype_name` == `typename` if provided argument is None.
Default is None.
Returns:
TypeNode: type node that wraps C++ type exposed to Python
>>> create_type_node('Ptr<AKAZE>').typename
'AKAZE'
>>> create_type_node('std::vector<Ptr<cv::Algorithm>>').typename
'typing.Sequence[Algorithm]'
"""
if original_ctype_name is None:
original_ctype_name = typename
typename = normalize_ctype_name(typename.strip())
# if typename is a known alias or has explicitly defined substitution
type_node = PREDEFINED_TYPES.get(typename)
if type_node is not None:
type_node.ctype_name = original_ctype_name
return type_node
# If typename is a known exported alias name (e.g. IndexParams or SearchParams)
for alias in PREDEFINED_TYPES.values():
if alias.typename == typename:
return alias
if is_union_type(typename):
union_types = get_template_instantiation_type(typename)
return UnionTypeNode(
original_ctype_name,
items=create_type_nodes_from_template_arguments(union_types)
)
# if typename refers to a sequence type e.g. vector<int>
if is_sequence_type(typename):
# Recursively convert sequence element type
if _is_template_instantiation(typename):
inner_sequence_type = create_type_node(
get_template_instantiation_type(typename)
)
else:
# Handle vector_Type cases
# maxsplit=1 is required to handle sequence of sequence e.g:
# vector_vector_Mat -> Sequence[Sequence[Mat]]
inner_sequence_type = create_type_node(typename.split("_", 1)[-1])
return SequenceTypeNode(original_ctype_name, inner_sequence_type)
# If typename refers to a heterogeneous container
# (can contain elements of different types)
if is_tuple_type(typename):
tuple_types = get_template_instantiation_type(typename)
return TupleTypeNode(
original_ctype_name,
items=create_type_nodes_from_template_arguments(tuple_types)
)
# If everything else is False, it means that input typename refers to a
# class or enum of the library.
return ASTNodeTypeNode(original_ctype_name, typename)
if __name__ == "__main__":
import doctest
doctest.testmod()

173
modules/python/src2/typing_stubs_generator.py Normal file
View File

@ -0,0 +1,173 @@
"""Contains a class used to resolve compatibility issues with old Python versions.
Typing stubs generation is available starting from Python 3.6 only.
For other versions all calls to functions are noop.
"""
import sys
import warnings
if sys.version_info >= (3, 6):
from contextlib import contextmanager
from typing import Dict, Set, Any, Sequence, Generator, Union
from pathlib import Path
from typing_stubs_generation import (
generate_typing_stubs,
NamespaceNode,
EnumerationNode,
SymbolName,
ClassNode,
create_function_node,
create_class_node,
find_class_node,
resolve_enum_scopes
)
import functools
class FailuresWrapper:
def __init__(self, exceptions_as_warnings=True):
self.has_failure = False
self.exceptions_as_warnings = exceptions_as_warnings
def wrap_exceptions_as_warnings(self, original_func=None,
ret_type_on_failure=None):
def parametrized_wrapper(func):
@functools.wraps(func)
def wrapped_func(*args, **kwargs):
if self.has_failure:
if ret_type_on_failure is None:
return None
return ret_type_on_failure()
try:
ret_type = func(*args, **kwargs)
except Exception as e:
self.has_failure = True
warnings.warn(
'Typing stubs generation has failed. Reason: {}'.format(e)
)
if ret_type_on_failure is None:
return None
return ret_type_on_failure()
return ret_type
if self.exceptions_as_warnings:
return wrapped_func
else:
return original_func
if original_func:
return parametrized_wrapper(original_func)
return parametrized_wrapper
@contextmanager
def delete_on_failure(self, file_path):
# type: (Path) -> Generator[None, None, None]
# There is no errors during stubs generation and file doesn't exist
if not self.has_failure and not file_path.is_file():
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.touch()
try:
# continue execution
yield
finally:
# If failure is occurred - delete file if exists
if self.has_failure and file_path.is_file():
file_path.unlink()
failures_wrapper = FailuresWrapper(exceptions_as_warnings=True)
class ClassNodeStub:
def add_base(self, base_node):
pass
class TypingStubsGenerator:
def __init__(self):
self.cv_root = NamespaceNode("cv", export_name="cv2")
self.exported_enums = {} # type: Dict[SymbolName, EnumerationNode]
self.type_hints_ignored_functions = set() # type: Set[str]
@failures_wrapper.wrap_exceptions_as_warnings
def add_enum(self, symbol_name, is_scoped_enum, entries):
# type: (SymbolName, bool, Dict[str, str]) -> None
if symbol_name in self.exported_enums:
assert symbol_name.name == "<unnamed>", \
"Trying to export 2 enums with same symbol " \
"name: {}".format(symbol_name)
enumeration_node = self.exported_enums[symbol_name]
else:
enumeration_node = EnumerationNode(symbol_name.name,
is_scoped_enum)
self.exported_enums[symbol_name] = enumeration_node
for entry_name, entry_value in entries.items():
enumeration_node.add_constant(entry_name, entry_value)
@failures_wrapper.wrap_exceptions_as_warnings
def add_ignored_function_name(self, function_name):
# type: (str) -> None
self.type_hints_ignored_functions.add(function_name)
@failures_wrapper.wrap_exceptions_as_warnings
def create_function_node(self, func_info):
# type: (Any) -> None
create_function_node(self.cv_root, func_info)
@failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub)
def find_class_node(self, class_info, namespaces):
# type: (Any, Sequence[str]) -> ClassNode
return find_class_node(self.cv_root, class_info.full_original_name, namespaces)
@failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub)
def create_class_node(self, class_info, namespaces):
# type: (Any, Sequence[str]) -> ClassNode
return create_class_node(self.cv_root, class_info, namespaces)
def generate(self, output_path):
# type: (Union[str, Path]) -> None
output_path = Path(output_path)
py_typed_path = output_path / self.cv_root.export_name / 'py.typed'
with failures_wrapper.delete_on_failure(py_typed_path):
self._generate(output_path)
@failures_wrapper.wrap_exceptions_as_warnings
def _generate(self, output_path):
# type: (Path) -> None
resolve_enum_scopes(self.cv_root, self.exported_enums)
generate_typing_stubs(self.cv_root, output_path)
else:
class ClassNode:
def add_base(self, base_node):
pass
class TypingStubsGenerator:
def __init__(self):
self.type_hints_ignored_functions = set() # type: Set[str]
print(
'WARNING! Typing stubs can be generated only with Python 3.6 or higher. '
'Current version {}'.format(sys.version_info)
)
def add_enum(self, symbol_name, is_scoped_enum, entries):
pass
def add_ignored_function_name(self, function_name):
pass
def create_function_node(self, func_info):
pass
def create_class_node(self, class_info, namespaces):
return ClassNode()
def find_class_node(self, class_info, namespaces):
return ClassNode()
def generate(self, output_path):
pass