doc: finalize Python signatures injection

doc/CMakeLists.txt

@@ -218,7 +218,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
     COMPONENT "docs" OPTIONAL
   )
 
-  if(BUILD_opencv_python2)
+  if(PYTHON2_EXECUTABLE)
     add_custom_target(doxygen_python
       COMMAND ${PYTHON2_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON2_SIGNATURES_FILE}" "python"
       DEPENDS "${doxygen_result}" gen_opencv_python2
@@ -226,7 +226,7 @@ if(BUILD_DOCS AND DOXYGEN_FOUND)
     add_custom_target(doxygen
       DEPENDS doxygen_cpp doxygen_python
     )
-  elseif(BUILD_opencv_python3)
+  elseif(PYTHON3_EXECUTABLE)
     add_custom_target(doxygen_python
       COMMAND ${PYTHON3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/tools/add_signatures.py" "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/" "${OPENCV_PYTHON3_SIGNATURES_FILE}" "python"
       DEPENDS "${doxygen_result}" gen_opencv_python3

doc/tools/add_signatures.py

@@ -12,10 +12,15 @@ TODO:
     http://docs.opencv.org/3.2.0/db/de0/group__core__utils.html#ga4910d7f86336cd4eff9dd05575667e41
 """
 from __future__ import print_function
-import os
-import re
 import sys
+sys.dont_write_bytecode = True  # Don't generate .pyc files / __pycache__ directories
+
+import os
+from pprint import pprint
+import re
 import logging
+import json
+
 import html_functions
 import doxygen_scan
 
@@ -23,37 +28,23 @@ loglevel=os.environ.get("LOGLEVEL", None)
 if loglevel:
     logging.basicConfig(level=loglevel)
 
-
 ROOT_DIR = sys.argv[1]
 PYTHON_SIGNATURES_FILE = sys.argv[2]
-JAVA_PYTHON = sys.argv[3]
+JAVA_OR_PYTHON = sys.argv[3]
 
 ADD_JAVA = False
 ADD_PYTHON = False
-if JAVA_PYTHON == "python":
+if JAVA_OR_PYTHON == "python":
     ADD_PYTHON = True
 
-import json
 python_signatures = dict()
 with open(PYTHON_SIGNATURES_FILE, "rt") as f:
     python_signatures = json.load(f)
     print("Loaded Python signatures: %d" % len(python_signatures))
 
-# only name -> class
-# name and ret -> constant
-# name, ret, arg-> function / class method
-
-class Configuration():
-    def __init__(self):
-        self.ADD_PYTHON = ADD_PYTHON
-        self.python_signatures = python_signatures
-        self.ADD_JAVA = ADD_JAVA
-
-config = Configuration()
-
 import xml.etree.ElementTree as ET
 root = ET.parse(ROOT_DIR + 'opencv.tag')
-files_dict = dict()
+files_dict = {}
 
 # constants and function from opencv.tag
 namespaces = root.findall("./compound[@kind='namespace']")
@@ -61,41 +52,48 @@ namespaces = root.findall("./compound[@kind='namespace']")
 for ns in namespaces:
     ns_name = ns.find("./name").text
     #print('NS: {}'.format(ns_name))
-
+    doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
-    files_dict = doxygen_scan.scan_namespace_constants(ns, ns_name, files_dict)
+    doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
-    files_dict = doxygen_scan.scan_namespace_functions(ns, ns_name, files_dict)
 
 # class methods from opencv.tag
 classes = root.findall("./compound[@kind='class']")
 #print("Found {} classes".format(len(classes)))
 for c in classes:
     c_name = c.find("./name").text
-    name = ns_name + '::' + c_name
     file = c.find("./filename").text
-    #print('Class: {} => {}'.format(name, file))
+    #print('Class: {} => {}'.format(c_name, file))
-    files_dict = doxygen_scan.scan_class_methods(c, c_name, files_dict)
+    doxygen_scan.scan_class_methods(c, c_name, files_dict)
+
+print('Doxygen files to scan: %s' % len(files_dict))
+
+files_processed = 0
+files_skipped = 0
+symbols_processed = 0
 
-# test
 for file in files_dict:
-    soup = html_functions.load_html_file(ROOT_DIR + file)
+    #if file != "dd/d9e/classcv_1_1VideoWriter.html":
-    if file == "dd/d9e/classcv_1_1VideoWriter.html":#"d4/d86/group__imgproc__filter.html":#"d4/d86/group__imgproc__filter.html":
+    #if file != "d4/d86/group__imgproc__filter.html":
-        anchor_list = files_dict[file]
+    #if file != "df/dfb/group__imgproc__object.html":
-        counter = 0
+    #    continue
-        anchor_tmp_list = []
+    #print('File: ' + file)
-        for anchor in anchor_list:
+
-            counter += 1
+    anchor_list = files_dict[file]
-            # if the next anchor shares the same C++ name (= same method/function), join them together
+    active_anchors = [a for a in anchor_list if a.cppname in python_signatures]
-            if counter < len(anchor_list) and anchor_list[counter].cppname == anchor.cppname:
+    if len(active_anchors) == 0: # no linked Python symbols
-                anchor_tmp_list.append(anchor)
+        #print('Skip: ' + file)
-                continue
+        files_skipped = files_skipped + 1
-            else:
+        continue
-                anchor_tmp_list.append(anchor)
+
-            # check if extists a python equivalent signature
+    active_anchors_dict = {a.anchor: a for a in active_anchors}
-            for signature in python_signatures: # signature is a key with the C++ name
+    if len(active_anchors_dict) != len(active_anchors):
-                if signature == anchor.cppname: # if available name in python
+        logging.info('Duplicate entries detected: %s -> %s (%s)' % (len(active_anchors), len(active_anchors_dict), file))
-                    # they should also have the same type
+
-                    soup = html_functions.append_python_signature(python_signatures[signature], anchor_tmp_list, soup)
+    files_processed = files_processed + 1
-                    #print(signature)
+
-            # reset anchor temporary list
+    #pprint(active_anchors)
-            anchor_tmp_list[:] = []
+    symbols_processed = symbols_processed + len(active_anchors_dict)
-        html_functions.update_html(ROOT_DIR + file, soup)
+
+    logging.info('File: %r' % file)
+    html_functions.insert_python_signatures(python_signatures, active_anchors_dict, ROOT_DIR + file)
+
+print('Done (processed files %d, symbols %d, skipped %d files)' % (files_processed, symbols_processed, files_skipped))

doc/tools/doxygen_scan.py

@@ -1,20 +1,20 @@
-class Anchor(object):
+import traceback
-    anchor = ""
-    type = ""
-    cppname = ""
 
+class Symbol(object):
     def __init__(self, anchor, type, cppname):
         self.anchor = anchor
         self.type = type
         self.cppname = cppname
+        #if anchor == 'ga586ebfb0a7fb604b35a23d85391329be':
+        #    print(repr(self))
+        #    traceback.print_stack()
+
+    def __repr__(self):
+        return '%s:%s@%s' % (self.type, self.cppname, self.anchor)
 
 def add_to_file(files_dict, file, anchor):
-    if file in files_dict:
+    anchors = files_dict.setdefault(file, [])
-        # if that file already exists as a key in the dictionary
+    anchors.append(anchor)
-        files_dict[file].append(anchor)
-    else:
-        files_dict[file] = [anchor]
-    return files_dict
 
 
 def scan_namespace_constants(ns, ns_name, files_dict):
@@ -25,8 +25,7 @@ def scan_namespace_constants(ns, ns_name, files_dict):
         file = c.find("./anchorfile").text
         anchor = c.find("./anchor").text
         #print('    CONST: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "const", name))
+        add_to_file(files_dict, file, Symbol(anchor, "const", name))
-    return files_dict
 
 def scan_namespace_functions(ns, ns_name, files_dict):
     functions = ns.findall("./member[@kind='function']")
@@ -36,8 +35,7 @@ def scan_namespace_functions(ns, ns_name, files_dict):
         file = f.find("./anchorfile").text
         anchor = f.find("./anchor").text
         #print('    FN: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "fn", name))
+        add_to_file(files_dict, file, Symbol(anchor, "fn", name))
-    return files_dict
 
 def scan_class_methods(c, c_name, files_dict):
     methods = c.findall("./member[@kind='function']")
@@ -47,5 +45,4 @@ def scan_class_methods(c, c_name, files_dict):
         file = m.find("./anchorfile").text
         anchor = m.find("./anchor").text
         #print('    Method: {} => {}#{}'.format(name, file, anchor))
-        files_dict = add_to_file(files_dict, file, Anchor(anchor, "method", name))
+        add_to_file(files_dict, file, Symbol(anchor, "method", name))
-    return files_dict

doc/tools/html_functions.py

@@ -1,7 +1,10 @@
 from __future__ import print_function
+import sys
+
 import logging
 import os
 from pprint import pprint
+import traceback
 
 try:
     import bs4
@@ -13,195 +16,110 @@ except ImportError:
 
 def load_html_file(file_dir):
     """ Uses BeautifulSoup to load an html """
-    with open(file_dir) as fp:
+    with open(file_dir, 'rb') as fp:
         soup = BeautifulSoup(fp, 'html.parser')
     return soup
 
-def add_item(soup, new_row, is_parameter, text):
-    """ Adds a new html tag for the table with the signature """
-    new_item = soup.new_tag('td')
-    if is_parameter:
-        new_item = soup.new_tag('td', **{'class': 'paramname'})
-    new_item.append(text)
-    new_row.append(new_item)
-    return new_row, soup
-
-def add_signature_to_table(soup, tmp_row, signature, language, type):
-    """ Add a signature to an html table"""
-    new_item = soup.new_tag('td', style="padding-left: 0.5cm;")
-
-    if str(signature.get('ret', None)) != "None":
-        new_item.append(signature.get('ret') + ' =')
-    tmp_row.append(new_item)
-
-    tmp_name = signature.get('name', None)
-    if type is not "method":
-        tmp_name = "cv2." + tmp_name
-    else:
-        tmp_name = "obj." + tmp_name
-    tmp_row, soup = add_item(soup, tmp_row, False, tmp_name + '(')
-    tmp_row, soup = add_item(soup, tmp_row, True, signature['arg'])
-    tmp_row, soup = add_item(soup, tmp_row, False, ')')
-    return tmp_row, soup
-
-
-def new_line(soup, tmp_table, new_row):
-    """ Adds a new line to the html table """
-    tmp_table.append(new_row)
-    new_row = soup.new_tag('tr')
-    return new_row, soup
-
-
-def add_bolded(soup, new_row, text):
-    """ Adds bolded text to the table """
-    new_item = soup.new_tag('th', style="text-align:left")
-    new_item.append(text)
-    new_row.append(new_item)
-    return new_row, soup
-
-
-def create_description(soup, language, signatures, type):
-    """ Insert the new Python / Java table after the current html c++ table """
-    assert signatures
-    tmp_table = soup.new_tag('table')
-    new_row = soup.new_tag('tr')
-    new_row, soup = add_bolded(soup, new_row, language)
-    new_row, soup = new_line(soup, tmp_table, new_row)
-    for s in signatures:
-        new_row, soup = new_line(soup, tmp_table, new_row)
-        new_row, soup = add_signature_to_table(soup, new_row, s, language, type)
-        new_row, soup = new_line(soup, tmp_table, new_row)
-    return tmp_table, soup
-
-
-def get_anchor_list(anchor, soup):
-    a_list = []
-    # go through all the links
-    for a in soup.find_all('a', href=True):
-        # find links with the same anchor
-        last_part_of_link = a['href'].rsplit('#', 1)[-1]
-        if last_part_of_link == anchor:
-            a_list.append(a)
-    return a_list
-
-def is_static_method(element):
-    if element.name == "table":
-        tmp_element = element.find('td', {'class': 'memname'})
-        if tmp_element is not None:
-            if 'static' in tmp_element.text:
-                return True
-    else:
-        if element['class'][0] == 'memItemRight':
-           if "static" in element.previousSibling.text:
-               return True
-    return False
-
-def append_python_signatures_to_table(soup, signatures, table, type):
-    if type == "method":
-        if is_static_method(table):
-            type = "static" + type
-    description, soup = create_description(soup, "Python:", signatures, type)
-    description['class'] = 'python_language'
-    soup = insert_or_replace(soup, table, description, "table", "python_language")
-    return soup
-
-def get_heading_text(a):
-    str = ""
-    element = a.parent
-    if element is not None:
-        childs = element.find_all('a')
-        # the anchor should not be an argument of a function / method
-        if childs.index(a) is not 0:
-            return str
-    element = element.parent
-    if element is not None:
-        if element.has_attr('class'):
-            tmp_class = element["class"][0]
-            if "memitem:" in tmp_class and "python" not in tmp_class:
-                str = element.parent.find("tr").text
-    return str
-
-def insert_or_replace(soup, element, description, tag_name, tag_class):
-    old = element.next_sibling
-    if old is not None:
-        if old.name != tag_name:
-            old = None
-        elif not tag_class in old.get('class', []):
-            old = None
-    # if already existed replace with the new
-    if old is None:
-        element.insert_after(description)
-    else:
-        old.replace_with(description)
-    return soup
-
-def new_heading_td(soup, s, href, type):
-    if href is None:
-        attrs = {'class': 'memItemLeft', 'valign': 'top', 'align': 'right'}
-        new_td = soup.new_tag('td', **attrs)
-        new_td.append(str(s.get('ret', None)))
-    else:
-        attrs = {'class': 'memItemRight', 'valign': 'bottom'}
-        new_td = soup.new_tag('td', **attrs)
-
-        # make the function name linkable
-        attrs_a = {'class': 'el', 'href': href}
-        new_a = soup.new_tag('a', **attrs_a)
-        tmp_name = str(s.get('name', None))
-        if type is not "method":
-            tmp_name = "cv2." + tmp_name
-        else:
-            tmp_name = "obj." + tmp_name
-        new_a.append(tmp_name)
-        new_td.append(new_a)
-
-        new_td.append("(" + s['arg'] +")")
-    return soup, new_td
-
-def append_python_signatures_to_heading(soup, signatures, element, href, type):
-    if type == "method":
-        if is_static_method(element):
-            type = "static" + type
-    for s in signatures:
-        attrs = {'class': 'memitem:python'}
-        new_tr = soup.new_tag('tr', **attrs)
-
-        soup, new_td_left = new_heading_td(soup, s, None, type)
-        new_tr.append(new_td_left)
-
-        soup, new_td_right = new_heading_td(soup, s, href, type)
-        new_tr.append(new_td_right)
-
-        soup = insert_or_replace(soup, element, new_tr, "tr", "memitem:python")
-    return soup
-
-def append_python_signature(function_variants, anchor_list, soup):
-    type = anchor_list[0].type
-    if type == "method" or type == "fn":
-        if len(anchor_list) == 1:
-            tmp_anchor = anchor_list[0].anchor
-            a_list = get_anchor_list(tmp_anchor, soup)
-            for a in a_list:
-                if a['href'] == "#" + tmp_anchor:
-                    tmp_element = a.parent
-                    # ignore the More... link <td class = mdescRight>
-                    if tmp_element is None or tmp_element['class'][0] == 'mdescRight':
-                        continue
-                    # Function Documentation (tables)
-                    table = a.findNext('table')
-                    if table is not None:
-                        soup = append_python_signatures_to_table(soup, function_variants, table, type)
-                else:
-                    str = get_heading_text(a)
-                    if "Functions" in str:
-                        soup = append_python_signatures_to_heading(soup, function_variants, a.parent, a['href'], type)
-    return soup
-
 def update_html(file, soup):
-    tmp_str = str(soup)
+    s = str(soup)
-    if os.name == 'nt': # if Windows
+    if os.name == 'nt' or sys.version_info[0] == 3: # if Windows
-        with open(file, "wb") as tmp_file:
+        s = s.encode('utf-8', 'ignore')
-            tmp_file.write(tmp_str.encode("ascii","ignore"))
+    with open(file, 'wb') as f:
+        f.write(s)
+
+
+def insert_python_signatures(python_signatures, symbols_dict, filepath):
+    soup = load_html_file(filepath)
+    entries = soup.find_all(lambda tag: tag.name == "a" and tag.has_attr('id'))
+    for e in entries:
+        anchor = e['id']
+        if anchor in symbols_dict:
+            s = symbols_dict[anchor]
+            logging.info('Process: %r' % s)
+            if s.type == 'fn' or s.type == 'method':
+                process_fn(soup, e, python_signatures[s.cppname], s)
+            elif s.type == 'const':
+                process_const(soup, e, python_signatures[s.cppname], s)
+            else:
+                logging.error('unsupported type: %s' % s);
+
+    update_html(filepath, soup)
+
+
+def process_fn(soup, anchor, python_signature, symbol):
+    try:
+        r = anchor.find_next_sibling(class_='memitem').find(class_='memproto').find('table')
+        insert_python_fn_signature(soup, r, python_signature, symbol)
+    except:
+        logging.error("Can't process: %s" % symbol)
+        traceback.print_exc()
+        pprint(anchor)
+
+
+def process_const(soup, anchor, python_signature, symbol):
+    try:
+        #pprint(anchor.parent)
+        description = append(soup.new_tag('div', **{'class' : ['python_language']}),
+            'Python: ' + python_signature[0]['name'])
+        old = anchor.find_next_sibling('div', class_='python_language')
+        if old is None:
+            anchor.parent.append(description)
+        else:
+            old.replace_with(description)
+        #pprint(anchor.parent)
+    except:
+        logging.error("Can't process: %s" % symbol)
+        traceback.print_exc()
+        pprint(anchor)
+
+
+def insert_python_fn_signature(soup, table, variants, symbol):
+    description = create_python_fn_description(soup, variants)
+    description['class'] = 'python_language'
+    soup = insert_or_replace(table, description, 'table', 'python_language')
+    return soup
+
+
+def create_python_fn_description(soup, variants):
+    language = 'Python:'
+    table = soup.new_tag('table')
+    heading_row = soup.new_tag('th')
+    table.append(
+        append(soup.new_tag('tr'),
+               append(soup.new_tag('th', colspan=999, style="text-align:left"), language)))
+    for v in variants:
+        #logging.debug(v)
+        add_signature_to_table(soup, table, v, language, type)
+    #print(table)
+    return table
+
+
+def add_signature_to_table(soup, table, signature, language, type):
+    """ Add a signature to an html table"""
+    row = soup.new_tag('tr')
+    row.append(soup.new_tag('td', style='width: 20px;'))
+
+    if 'ret' in signature:
+        row.append(append(soup.new_tag('td'), signature['ret']))
+        row.append(append(soup.new_tag('td'), '='))
     else:
-        with open(file, "w") as tmp_file:
+        row.append(soup.new_tag('td')) # return values
-            tmp_file.write(tmp_str)
+        row.append(soup.new_tag('td')) # '='
+
+    row.append(append(soup.new_tag('td'), signature['name'] + '('))
+    row.append(append(soup.new_tag('td', **{'class': 'paramname'}), signature['arg']))
+    row.append(append(soup.new_tag('td'), ')'))
+    table.append(row)
+
+
+def append(target, obj):
+    target.append(obj)
+    return target
+
+
+def insert_or_replace(element_before, new_element, tag, tag_class):
+    old = element_before.find_next_sibling(tag, class_=tag_class)
+    if old is None:
+        element_before.insert_after(new_element)
+    else:
+        old.replace_with(new_element)