Merge branch 'main' into JDWDIPLOPIA

.github/ISSUE_TEMPLATE.md

@@ -1,4 +1,4 @@
-Before you submit an issue, please review [the guidelines for this repository](https://github.com/tesseract-ocr/tesseract/blob/master/CONTRIBUTING.md).
+Before you submit an issue, please review [the guidelines for this repository](https://github.com/tesseract-ocr/tesseract/blob/main/CONTRIBUTING.md).
 
 Please report an issue only for a BUG, not for asking questions.
 

.github/workflows/autotools-macos.yml

@@ -5,6 +5,7 @@ on:
   #push:
   schedule:
     - cron: 0 20 * * *
+  workflow_dispatch:
 jobs:
 
   brew:
@@ -42,7 +43,7 @@ jobs:
 
     - name: Configure Tesseract
       run: |
-           ./configure '--disable-shared' '--disable-openmp' '--disable-doc' '--with-pic' 'CXX=${{ matrix.config.cxx }}' 'CXXFLAGS=-g -O2' "PKG_CONFIG_PATH=$(brew --prefix)/opt/icu4c/lib/pkgconfig:$(brew --prefix)/opt/libarchive/lib/pkgconfig:$(brew --prefix)/opt/libffi/lib/pkgconfig"
+           ./configure '--disable-shared' '--disable-openmp' '--disable-doc' '--with-pic' 'CXX=${{ matrix.config.cxx }}' 'CXXFLAGS=-g -O2'
 
     - name: Make and Install Tesseract
       run: |
@@ -92,7 +93,7 @@ jobs:
       run: |
            export "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig"
            cd test
-           ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp -I/usr/local/include -L/usr/local/lib `pkg-config --cflags --libs tesseract lept ` -pthread -std=c++11
+           ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp $(pkg-config --cflags --libs tesseract lept) -pthread -std=c++11 -framework accelerate
            ./basicapitest
 
     - name: Display Compiler Version
@@ -130,6 +131,9 @@ jobs:
     - name: Install Macports
       run: |
         curl -LO https://raw.githubusercontent.com/GiovanniBussi/macports-ci/master/macports-ci; source ./macports-ci install
+        # --remove-brew does not remove the Homebrew entries in bin,
+        # so remove them now.
+        rm -v $(brew --prefix)/bin/*
 
     - name: Install Dependencies
       run: |
@@ -189,7 +193,7 @@ jobs:
       run: |
            export "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig"
            cd test
-           ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp -I/opt/local/include -L/opt/local/lib -I/usr/local/include -L/usr/local/lib `pkg-config --cflags --libs tesseract lept ` -pthread -std=c++11
+           ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp -I/opt/local/include -L/opt/local/lib $(pkg-config --cflags --libs tesseract lept) -pthread -std=c++11 -framework Accelerate
            ./basicapitest
 
     - name: Display Compiler Version
@@ -202,4 +206,3 @@ jobs:
       run: |
            cat test-suite.log
       if: always()
-

.github/workflows/cifuzz.yml

@@ -0,0 +1,33 @@
+name: CIFuzz
+# OSS-Fuzz CI
+# See https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+on:
+  pull_request:
+    branches:
+    - main
+    paths:
+    - '**.cpp'
+    - '**.h'
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tesseract-ocr'
+        language: c++
+        dry-run: false
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'tesseract-ocr'
+        fuzz-seconds: 600
+        dry-run: false
+    - name: Upload Crash
+      uses: actions/upload-artifact@v1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/cmake-win64.yml

@@ -133,8 +133,8 @@ jobs:
           git clone --depth 1 https://github.com/tesseract-ocr/tessconfigs
           mkdir d:/a/local/share
           move tessconfigs d:/a/local/share
-          curl -L https://github.com/tesseract-ocr/tessdata/raw/master/eng.traineddata --output d:/a/local/share/tessconfigs/eng.traineddata
-          curl -L https://github.com/tesseract-ocr/tessdata/raw/master/osd.traineddata --output d:/a/local/share/tessconfigs/osd.traineddata
+          curl -L https://github.com/tesseract-ocr/tessdata/raw/main/eng.traineddata --output d:/a/local/share/tessconfigs/eng.traineddata
+          curl -L https://github.com/tesseract-ocr/tessdata/raw/main/osd.traineddata --output d:/a/local/share/tessconfigs/osd.traineddata
           set TESSDATA_PREFIX=d:/a/local/share/tessconfigs
           set PATH=d:/a/local/bin;%PATH%
           tesseract -v

.github/workflows/cmake.yml

@@ -17,9 +17,15 @@ jobs:
 
           - { name: macos-10.15-clang-12-cmake, os: macos-10.15, cxx: clang++ } # default
           - { name: macos-10.15-clang-11-cmake, os: macos-10.15, cxx: '$(brew --prefix llvm)/bin/clang++' }  #installed
-          - { name: macos-10.15-gcc-8-cmake, os: macos-10.15, cxx: g++-8 } #installed
           - { name: macos-10.15-gcc-9-cmake, os: macos-10.15, cxx: g++-9 } #installed
           - { name: macos-10.15-gcc-10-cmake, os: macos-10.15, cxx: g++-10 } #installed
+          - { name: macos-10.15-gcc-11-cmake, os: macos-10.15, cxx: g++-11 } #installed
+
+          - { name: macos-11-clang-12-cmake, os: macos-11, cxx: clang++ } # default
+          - { name: macos-11-clang-11-cmake, os: macos-11, cxx: '$(brew --prefix llvm)/bin/clang++' }  #installed
+          - { name: macos-11-gcc-9-cmake, os: macos-11, cxx: g++-9 } #installed
+          - { name: macos-11-gcc-10-cmake, os: macos-11, cxx: g++-10 } #installed
+          - { name: macos-11-gcc-11-cmake, os: macos-11, cxx: g++-11 } #installed
 
           - { name: ubuntu-18.04-clang-7-cmake, os: ubuntu-18.04, cxx: clang++-7 }
           - { name: ubuntu-18.04-clang-8-cmake, os: ubuntu-18.04, cxx: clang++-8 } #installed
@@ -47,20 +53,22 @@ jobs:
              sudo apt-get install ${{ matrix.config.cxx }} -y
         if: runner.os == 'Linux'
 
-# sudo apt-get install libarchive-dev libcurl4-openssl-dev libcurl4 curl -y
       - name: Install dependencies on Linux
         run: |
            sudo apt-get install autoconf libleptonica-dev -y
+           sudo apt-get install libarchive-dev libcurl4-openssl-dev -y
            sudo apt-get install libpango1.0-dev -y
            sudo apt-get install cabextract -y
            sudo apt-get install ninja-build -y
+           cmake --version
         if: runner.os == 'Linux'
 
       - name: Install dependencies on macOS
         run: |
            brew install autoconf automake
            brew install leptonica
-           brew install cairo pango icu4c
+           brew install libarchive
+           brew install pango
            brew install cabextract
            brew install ninja
            ninja --version
@@ -137,16 +145,24 @@ jobs:
              build/inst/bin/tesseract test/testing/eurotext.tif - -l fra --oem 1 --tessdata-dir ../tessdata_best
              build/inst/bin/tesseract test/testing/arabic.tif - -l ara --oem 1 --psm 6  --tessdata-dir ../tessdata
 
-      - name: Build and run basicapitest
+      - name: Build and run basicapitest (Linux)
         run: |
              export "PKG_CONFIG_PATH=$GITHUB_WORKSPACE/build/inst/lib/pkgconfig/:$PKG_CONFIG_PATH"
              cd test
-             ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp "-I$GITHUB_WORKSPACE/build/inst/include" "-L$GITHUB_WORKSPACE/build/inst/lib" `pkg-config --cflags --libs tesseract lept ` -pthread -std=c++11
+             ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp "-I$GITHUB_WORKSPACE/build/inst/include" "-L$GITHUB_WORKSPACE/build/inst/lib" $(pkg-config --cflags --libs tesseract lept libarchive libcurl) -pthread -std=c++11
              ./basicapitest
+        if: runner.os == 'Linux'
+
+      - name: Build and run basicapitest (macOS)
+        run: |
+             export "PKG_CONFIG_PATH=$GITHUB_WORKSPACE/build/inst/lib/pkgconfig/:$(brew --prefix)/opt/libarchive/lib/pkgconfig:$(brew --prefix)/Library/Homebrew/os/mac/pkgconfig/11:$PKG_CONFIG_PATH"
+             cd test
+             ${{ matrix.config.cxx }} -o basicapitest testing/basicapitest.cpp "-I$GITHUB_WORKSPACE/build/inst/include" "-L$GITHUB_WORKSPACE/build/inst/lib" $(pkg-config --cflags --libs tesseract lept libarchive libcurl) -pthread -std=c++11
+             ./basicapitest
+        if: runner.os == 'macOS'
 
       - name: Display Compiler Version
         run: |
              ${{ matrix.config.cxx }} --version
              git log -3 --pretty=format:'%h %ad %s | %an'
         if: always()
-

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,69 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '34 23 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: |
+           sudo apt-get update
+           sudo apt-get install autoconf libleptonica-dev -y
+           sudo apt-get install libpango1.0-dev -y
+           sudo apt-get install cabextract libarchive-dev -y
+           sudo apt-get install libcurl4-openssl-dev libcurl4 curl -y
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    - name: Build
+      run: |
+       ./autogen.sh
+       ./configure
+       make all training
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/msys2-4.1.1.yml

@@ -29,4 +29,3 @@ jobs:
            tesseract -v
            text2image -v
            lstmtraining -v
-           

.github/workflows/msys2.yml

@@ -1,5 +1,5 @@
 name: msys2
-# msys2 build for tesseract -head from master branch.
+# msys2 build for tesseract -head from main branch.
 on:
   #push:
   schedule:

.github/workflows/sw.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-20.04, macOS-latest]
+        os: [windows-latest, windows-2022, ubuntu-20.04, macOS-latest]
 
     steps:
     - uses: actions/checkout@v2
@@ -22,50 +22,50 @@ jobs:
     - uses: egorpugin/sw-action@master
 
     - name: build
-      if: matrix.os == 'windows-latest'
+      if: matrix.os == 'windows-latest' || matrix.os == 'windows-2022'
       run: ./sw -static -shared -platform x86,x64 -config d,r build
 
     - name: build
-      if: matrix.os != 'windows-latest'
+      if: matrix.os != 'windows-latest' && matrix.os != 'windows-2022'
       run: ./sw -static -shared -config d,r build -Dwith-tests=1
 
     - name: download test data
       run: git clone https://github.com/egorpugin/tessdata tessdata_unittest
 
     - name: copy fonts
-      if: matrix.os != 'windows-latest'
+      if: matrix.os != 'windows-latest' && matrix.os != 'windows-2022'
       run: cp tessdata_unittest/fonts/* test/testing/
 
     - name: copy fonts
-      if: matrix.os == 'windows-latest'
+      if: matrix.os == 'windows-latest' || matrix.os == 'windows-2022'
       run: Copy-Item -Path "tessdata_unittest\fonts\*" -Destination "test\testing" -Recurse
       shell: pwsh
 
     - name: test
-      if: matrix.os != 'windows-latest'
+      if: matrix.os != 'windows-latest' && matrix.os != 'windows-2022'
       run: ./sw -static -shared -config "d,r" test -Dwith-tests=1 "-Dskip-tests=lstm,lstm_recode"
       continue-on-error: true
 
     - name: test-nightly
-      if: matrix.os != 'windows-latest' && github.event.schedule=='0 0 * * *'
+      if: matrix.os != 'windows-latest' && matrix.os != 'windows-2022' && github.event.schedule=='0 0 * * *'
       run: ./sw -static -shared -config "d,r" test -Dwith-tests=1
       continue-on-error: true
 
     # windows tests hang here for some reason, investigate
     #- name: test
-      #if: matrix.os == 'windows-latest'
+      #if: matrix.os == 'windows-latest' || matrix.os == 'windows-2022'
       #run: ./sw test -Dwith-tests=1 "-Dskip-tests=lstm,lstm_recode"
       #continue-on-error: true
 
     - name: Upload Unit Test Results
-      if: always() && matrix.os != 'windows-latest'
+      if: always() && matrix.os != 'windows-latest' && matrix.os != 'windows-2022'
       uses: actions/upload-artifact@v2
       with:
         name: Test Results (${{ matrix.os }})
         path: .sw/test/results.xml
 
     - name: Publish Test Report
-      if: always() && matrix.os != 'windows-latest'
+      if: always() && matrix.os != 'windows-latest' && matrix.os != 'windows-2022'
       uses: mikepenz/action-junit-report@v1
       with:
         check_name: test (${{ matrix.os }})

.github/workflows/unittest-disablelegacy.yml

@@ -5,7 +5,7 @@ name: unittest-disablelegacy
 on:
   #push:
   schedule:
-    - cron: 0 0 1 * *
+    - cron: 0 10 * * *
 
 jobs:
   linux:
@@ -14,7 +14,8 @@ jobs:
       fail-fast: false
       matrix:
         compiler: [ g++, clang++ ]
-        os: [ ubuntu-18.04, ubuntu-20.04 ]
+        #os: [ ubuntu-18.04, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,7 +26,8 @@ jobs:
       run: |
            sudo apt-get install autoconf libleptonica-dev libpango1.0-dev -y
            sudo apt-get install cabextract -y
-
+           #sudo apt-get install libc++-7-dev libc++abi-7-dev -y
+           
     - name: Setup
       run: |
            mkdir -p m4

.github/workflows/unittest-macos.yml

@@ -25,7 +25,7 @@ jobs:
       run: |
            brew install autoconf automake libarchive
            brew install leptonica cairo pango
-           brew install cabextract abseil
+           brew install cabextract
 
     - name: Setup
       run: |
@@ -36,8 +36,7 @@ jobs:
       run: |
            ./configure '--disable-shared' '--with-pic' \
                 'CXX=${{ matrix.config.cxx }}' \
-                'CXXFLAGS=-g -O2 -fsanitize=address,undefined' \
-                "PKG_CONFIG_PATH=$(brew --prefix)/opt/icu4c/lib/pkgconfig:$(brew --prefix)/opt/libarchive/lib/pkgconfig:$(brew --prefix)/opt/libffi/lib/pkgconfig"
+                'CXXFLAGS=-g -O2 -fsanitize=address,undefined'
 
     - name: Make and Install Tesseract
       run: |

.github/workflows/vcpkg-4.1.1.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest ]
+        os: [ubuntu-latest, windows-latest]
 
     steps:
       - name: Checkout Tesseract Source (for test images)
@@ -56,7 +56,7 @@ jobs:
              git clone https://github.com/egorpugin/tessdata tessdata_unittest
              mv tessdata_unittest/* ../
         if: runner.os == 'Windows'
-        
+
       - name: Create CMakeLists.txt file for basicapitest
         shell: bash
         run: |
@@ -75,7 +75,7 @@ jobs:
              EOF
              cat CMakeLists.txt
         if: runner.os == 'Windows'
-        
+
       - name: Configure basicapitest
         run: |
              cd test
@@ -87,7 +87,7 @@ jobs:
              cd test
              cmake --build .  --config Release
         if: runner.os == 'Windows'
-        
+
       - name: Run basicapitest (Windows)
         run: |
              cd test

.github/workflows/vcpkg.yml

@@ -15,7 +15,7 @@ jobs:
         os: [windows-2019]
 
     steps:
-      - name: Checkout Tesseract Source (--head from master branch)
+      - name: Checkout Tesseract Source (--head from main branch)
         uses: actions/checkout@v2
         with:
           submodules: recursive
@@ -35,7 +35,7 @@ jobs:
         run: |
              vcpkg/vcpkg install leptonica:x64-windows
 
-      - name: Configure and Build Tesseract (--head from master branch) with cmake
+      - name: Configure and Build Tesseract (--head from main branch) with cmake
         run: |
              cmake . -B build -DCMAKE_BUILD_TYPE=Release -DSW_BUILD=OFF -DOPENMP_BUILD=OFF -DBUILD_TRAINING_TOOLS=OFF "-DCMAKE_TOOLCHAIN_FILE=${env:GITHUB_WORKSPACE}/vcpkg/scripts/buildsystems/vcpkg.cmake"
              cmake --build build --config Release --target install
@@ -57,10 +57,10 @@ jobs:
              include_directories(${Leptonica_INCLUDE_DIRS})
              add_executable( basicapitest testing/basicapitest.cpp )
              target_link_libraries(basicapitest ${Leptonica_LIBRARIES})
-             target_link_libraries(basicapitest ${Tesseract_LIBRARIES})
+             target_link_libraries(basicapitest Tesseract::libtesseract)
              add_library(libtesseract UNKNOWN IMPORTED)
              set_property(TARGET libtesseract PROPERTY IMPORTED_LOCATION D:/a/tesseract/tesseract/build/Release/tesseract50.lib)
-             target_link_libraries(basicapitest libtesseract)
+             target_link_libraries(basicapitest Tesseract::libtesseract)
              EOF
              cat CMakeLists.txt
 

.gitmodules

@@ -1,8 +1,5 @@
-[submodule "abseil"]
-	path = abseil
-	url = https://github.com/abseil/abseil-cpp.git
 [submodule "googletest"]
-	path = googletest
+	path = unittest/third_party/googletest
 	url = https://github.com/google/googletest.git
 [submodule "test"]
 	path = test

.travis.yml

@@ -21,7 +21,7 @@ cache:
 
 before_install:
   - sudo apt-get install libleptonica-dev libpango1.0-dev libtiff5-dev -y
-  
+
 install:
 
 script:
@@ -30,7 +30,7 @@ script:
   - cmake .. -DSW_BUILD=OFF
   - make
   - sudo make install
-  
+
 #after_script: # let those commands trigger build errors
   - tesseract -v
   - text2image -v

CONTRIBUTING.md

@@ -69,7 +69,7 @@ your question has been asked (and has been answered) many times before...
 
 You should always make sure your changes build and run successfully.
 
-For that, your clone needs to have all submodules (`abseil`, `googletest`, `test`) included. To do so, either specify `--recurse-submodules` during the initial clone, or run `git submodule update --init --recursive NAME` for each `NAME` later. If `configure` already created those directories (blocking the clone), remove them first (or `make distclean`), then clone and reconfigure.
+For that, your clone needs to have all submodules (`googletest`, `test`) included. To do so, either specify `--recurse-submodules` during the initial clone, or run `git submodule update --init --recursive NAME` for each `NAME` later. If `configure` already created those directories (blocking the clone), remove them first (or `make distclean`), then clone and reconfigure.
 
 Have a look at [the README](./README.md) and [testing README](./test/testing/README.md) and the [documentation](https://tesseract-ocr.github.io/tessdoc/Compiling-%E2%80%93-GitInstallation.html#unit-test-builds) on installation.
 

ChangeLog

@@ -1,3 +1,51 @@
+2022-01-06 - V5.0.1
+  * Add SPDX-License-Identifier to public include files.
+  * Support redirections when running OCR on a URL.
+  * Lots of fixes and improvements for cmake builds.
+    Distributions should use the autoconf build.
+  * Fix broken msys2 build with gcc 11.
+  * Fix parameter certainty_scale (was duplicated).
+  * Fix some compiler warnings and clean code.
+  * Correctly detect amd64 and i386 on FreeBSD.
+  * Add libarchive and libcurl in continuous integration actions.
+  * Update submodule googletest to release v1.11.0.
+
+2021-11-22 - V5.0.0
+  * Faster training and recognition by default (float instead of
+    double calculations)
+  * More options for binarization
+  * Improved support for ARM NEON
+  * Modernized code
+  * Removed proprietary data types like GenericVector and STRING
+    from public API
+  * pdf.ttf no longer needed, now integrated into the code
+  * Faster flat build with automake
+  * New options for combine_tessdata to show details of traineddata files
+  * Improved training messages
+  * Improved unit tests and fuzzing tests
+  * Lots of bug fixes
+
+2021-11-15 - V4.1.3
+  * Fix build regression for autoconf build
+
+2021-11-14 - V4.1.2
+  * Add RowAttributes getter to PageIterator
+  * Allow line images with larger width for training
+  * Fix memory leaks
+  * Improve build process
+  * Don't output empty ALTO sourceImageInformation (issue #2700)
+  * Extend URI support for Tesseract with libcurl
+  * Abort LSTM training with integer model (fixes issue #1573)
+  * Update documentation
+  * Make automake builds less noisy by default
+  * Don't use -march=native in automake builds
+
+2019-12-26 - V4.1.1
+  * Implemented sw build (cppan is depreciated)
+  * Improved cmake build
+  * Code cleanup and optimization
+  * A lot of bug fixes...
+
 2019-07-07 - V4.1.0
   * Added new renders Alto, LSTMBox, WordStrBox.
   * Added character boxes in hOCR output.
@@ -150,7 +198,7 @@
   * `OcrEngineMode` in `Init` replaces `AccuracyVSpeed` to control cube.
   * Greatly improved segmentation search with consequent accuracy and speed improvements, especially for Chinese.
   * Added `PageIterator` and `ResultIterator` as cleaner ways to get the full results out of Tesseract, that are not currently provided by any of the `TessBaseAPI::Get*` methods. All other methods, such as the `ETEXT_STRUCT` in particular are deprecated and will be deleted in the future.
-  * ApplyBoxes totally rewritten to make training easier. It can now cope with touching/overlapping training characters, and a new boxfile format allows word boxes instead of character boxes, BUT to use that you have to have already boostrapped the language with character boxes. "Cyclic dependency" on traineddata.
+  * ApplyBoxes totally rewritten to make training easier. It can now cope with touching/overlapping training characters, and a new boxfile format allows word boxes instead of character boxes, BUT to use that you have to have already bootstrapped the language with character boxes. "Cyclic dependency" on traineddata.
   * Auto orientation and script detection added to page layout analysis.
   * Deleted *lots* of dead code.
   * Fixxht module replaced with scalable data-driven module.

Makefile.am

@@ -1,16 +1,10 @@
 ## run autogen.sh to create Makefile.in from this file
-
-# Be less noisy by default. Can be overridden with `make V=1`.
-V = 0
-
 ACLOCAL_AMFLAGS = -I m4
 
 .PHONY: doc html install-langs ScrollView.jar install-jars pdf training
 
 CLEANFILES =
 
-DEFAULT_INCLUDES =
-
 SUBDIRS = . tessdata
 
 EXTRA_DIST = README.md LICENSE
@@ -80,7 +74,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/include
 AM_CPPFLAGS += -I$(top_builddir)/include
 if VISIBILITY
 AM_CPPFLAGS += -DTESS_EXPORTS
-AM_CPPFLAGS += -fvisibility=hidden -fvisibility-inlines-hidden
+AM_CPPFLAGS += -fvisibility=hidden -fvisibility-inlines-hidden -fPIC
 endif
 AM_CPPFLAGS += $(OPENCL_CPPFLAGS)
 
@@ -144,13 +138,15 @@ noinst_HEADERS += src/arch/simddetect.h
 noinst_LTLIBRARIES += libtesseract_native.la
 
 libtesseract_native_la_CXXFLAGS = -O3 -ffast-math
-if MARCH_NATIVE_OPT
-libtesseract_native_la_CXXFLAGS += -march=native -mtune=native
+if OPENMP_SIMD
+libtesseract_native_la_CXXFLAGS += -fopenmp-simd -DOPENMP_SIMD
 endif
+libtesseract_native_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_native_la_SOURCES = src/arch/dotproduct.cpp
 
 if HAVE_AVX
 libtesseract_avx_la_CXXFLAGS = -mavx
+libtesseract_avx_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_avx_la_SOURCES = src/arch/dotproductavx.cpp
 libtesseract_la_LIBADD += libtesseract_avx.la
 noinst_LTLIBRARIES += libtesseract_avx.la
@@ -158,6 +154,7 @@ endif
 
 if HAVE_AVX2
 libtesseract_avx2_la_CXXFLAGS = -mavx2
+libtesseract_avx2_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_avx2_la_SOURCES = src/arch/intsimdmatrixavx2.cpp
 libtesseract_la_LIBADD += libtesseract_avx2.la
 noinst_LTLIBRARIES += libtesseract_avx2.la
@@ -165,6 +162,7 @@ endif
 
 if HAVE_FMA
 libtesseract_fma_la_CXXFLAGS = -mfma
+libtesseract_fma_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_fma_la_SOURCES = src/arch/dotproductfma.cpp
 libtesseract_la_LIBADD += libtesseract_fma.la
 noinst_LTLIBRARIES += libtesseract_fma.la
@@ -172,6 +170,7 @@ endif
 
 if HAVE_SSE4_1
 libtesseract_sse_la_CXXFLAGS = -msse4.1
+libtesseract_sse_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_sse_la_SOURCES = src/arch/dotproductsse.cpp src/arch/intsimdmatrixsse.cpp
 libtesseract_la_LIBADD += libtesseract_sse.la
 noinst_LTLIBRARIES += libtesseract_sse.la
@@ -179,7 +178,13 @@ endif
 
 if HAVE_NEON
 libtesseract_neon_la_CXXFLAGS = $(NEON_CXXFLAGS)
+libtesseract_neon_la_CXXFLAGS += -O3
+if OPENMP_SIMD
+libtesseract_neon_la_CXXFLAGS += -fopenmp-simd -DOPENMP_SIMD
+endif
+libtesseract_neon_la_CXXFLAGS += -I$(top_srcdir)/src/ccutil
 libtesseract_neon_la_SOURCES = src/arch/intsimdmatrixneon.cpp
+libtesseract_neon_la_SOURCES += src/arch/dotproductneon.cpp
 libtesseract_la_LIBADD += libtesseract_neon.la
 noinst_LTLIBRARIES += libtesseract_neon.la
 endif
@@ -281,7 +286,6 @@ noinst_HEADERS += src/ccstruct/seam.h
 noinst_HEADERS += src/ccstruct/split.h
 noinst_HEADERS += src/ccstruct/statistc.h
 noinst_HEADERS += src/ccstruct/stepblob.h
-noinst_HEADERS += src/ccstruct/tabletransfer.h
 noinst_HEADERS += src/ccstruct/werd.h
 if !DISABLED_LEGACY_ENGINE
 noinst_HEADERS += src/ccstruct/fontinfo.h
@@ -334,7 +338,7 @@ endif
 libtesseract_ccutil_la_CPPFLAGS = $(AM_CPPFLAGS)
 libtesseract_ccutil_la_CPPFLAGS += $(libarchive_CFLAGS)
 if !NO_TESSDATA_PREFIX
-libtesseract_ccutil_la_CPPFLAGS += -DTESSDATA_PREFIX=@datadir@
+libtesseract_ccutil_la_CPPFLAGS += -DTESSDATA_PREFIX='"@datadir@"'
 endif
 
 noinst_HEADERS += src/ccutil/ccutil.h
@@ -389,7 +393,6 @@ if !DISABLED_LEGACY_ENGINE
 libtesseract_ccutil_la_SOURCES += src/ccutil/ambigs.cpp
 libtesseract_ccutil_la_SOURCES += src/ccutil/bitvector.cpp
 libtesseract_ccutil_la_SOURCES += src/ccutil/indexmapbidi.cpp
-libtesseract_ccutil_la_SOURCES += src/ccutil/universalambigs.cpp
 endif
 
 # Rules for src/classify.
@@ -501,7 +504,7 @@ libtesseract_lstm_la_CPPFLAGS += -DINCLUDE_TENSORFLOW
 libtesseract_lstm_la_CPPFLAGS += -I/usr/include/tensorflow
 endif
 if !NO_TESSDATA_PREFIX
-libtesseract_lstm_la_CPPFLAGS += -DTESSDATA_PREFIX=@datadir@
+libtesseract_lstm_la_CPPFLAGS += -DTESSDATA_PREFIX='"@datadir@"'
 endif
 
 noinst_HEADERS += src/lstm/convolve.h
@@ -712,13 +715,16 @@ endif
 # Rules for tesseract executable.
 
 bin_PROGRAMS = tesseract
-tesseract_SOURCES = src/api/tesseractmain.cpp
+tesseract_SOURCES = src/tesseract.cpp
 tesseract_CPPFLAGS =
 tesseract_CPPFLAGS += -I$(top_srcdir)/src/arch
 tesseract_CPPFLAGS += -I$(top_srcdir)/src/ccstruct
 tesseract_CPPFLAGS += -I$(top_srcdir)/src/ccutil
 tesseract_CPPFLAGS += -I$(top_srcdir)/src/dict
 tesseract_CPPFLAGS += -I$(top_srcdir)/src/viewer
+if OPENCL
+tesseract_CPPFLAGS += -I$(top_srcdir)/src/opencl
+endif
 tesseract_CPPFLAGS += $(AM_CPPFLAGS)
 if VISIBILITY
 tesseract_CPPFLAGS += -DTESS_IMPORTS
@@ -1143,65 +1149,34 @@ unittest_CPPFLAGS += $(pangocairo_CFLAGS)
 endif # ENABLE_TRAINING
 unittest_CPPFLAGS += -I$(top_srcdir)/src/viewer
 unittest_CPPFLAGS += -I$(top_srcdir)/src/wordrec
-unittest_CPPFLAGS += -I$(top_srcdir)/abseil
+unittest_CPPFLAGS += -I$(top_srcdir)/unittest
 if TENSORFLOW
 unittest_CPPFLAGS += -DINCLUDE_TENSORFLOW
-unittest_CPPFLAGS += -I$(top_srcdir)/unittest
 unittest_CPPFLAGS += -I/usr/include/tensorflow
 endif # TENSORFLOW
 
 # Build googletest:
 check_LTLIBRARIES = libgtest.la libgtest_main.la libgmock.la libgmock_main.la
-libgtest_la_SOURCES = googletest/googletest/src/gtest-all.cc
-libgtest_la_CPPFLAGS = -I$(top_srcdir)/googletest/googletest/include -I$(top_srcdir)/googletest/googletest -pthread
-libgtest_main_la_SOURCES = googletest/googletest/src/gtest_main.cc
+libgtest_la_SOURCES = unittest/third_party/googletest/googletest/src/gtest-all.cc
+libgtest_la_CPPFLAGS = -I$(top_srcdir)/unittest/third_party/googletest/googletest/include
+libgtest_la_CPPFLAGS += -I$(top_srcdir)/unittest/third_party/googletest/googletest
+libgtest_la_CPPFLAGS += -pthread
+libgtest_main_la_SOURCES = unittest/third_party/googletest/googletest/src/gtest_main.cc
 libgtest_main_la_CPPFLAGS = $(libgtest_la_CPPFLAGS)
 
-# Build Abseil (needed for some unit tests).
-check_LTLIBRARIES += libabseil.la
-libabseil_la_SOURCES =
-libabseil_la_SOURCES += abseil/absl/base/internal/cycleclock.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/raw_logging.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/spinlock.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/spinlock_wait.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/sysinfo.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/throw_delegate.cc
-libabseil_la_SOURCES += abseil/absl/base/internal/unscaledcycleclock.cc
-libabseil_la_SOURCES += abseil/absl/numeric/int128.cc
-libabseil_la_SOURCES += abseil/absl/strings/ascii.cc
-libabseil_la_SOURCES += abseil/absl/strings/charconv.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/charconv_bigint.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/charconv_parse.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/memutil.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/arg.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/bind.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/extension.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/float_conversion.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/output.cc
-libabseil_la_SOURCES += abseil/absl/strings/internal/str_format/parser.cc
-libabseil_la_SOURCES += abseil/absl/strings/match.cc
-libabseil_la_SOURCES += abseil/absl/strings/numbers.cc
-libabseil_la_SOURCES += abseil/absl/strings/str_cat.cc
-libabseil_la_SOURCES += abseil/absl/strings/str_split.cc
-libabseil_la_SOURCES += abseil/absl/strings/string_view.cc
-libabseil_la_SOURCES += abseil/absl/time/clock.cc
-libabseil_la_SOURCES += abseil/absl/time/duration.cc
-libabseil_la_CPPFLAGS = -I$(top_srcdir)/abseil
+GMOCK_INCLUDES = -I$(top_srcdir)/unittest/third_party/googletest/googlemock/include \
+                 -I$(top_srcdir)/unittest/third_party/googletest/googlemock \
+                 -I$(top_srcdir)/unittest/third_party/googletest/googletest/include \
+                 -I$(top_srcdir)/unittest/third_party/googletest/googletest
 
-GMOCK_INCLUDES = -I$(top_srcdir)/googletest/googlemock/include \
-                 -I$(top_srcdir)/googletest/googlemock \
-                 -I$(top_srcdir)/googletest/googletest/include \
-                 -I$(top_srcdir)/googletest/googletest
-
-libgmock_la_SOURCES = googletest/googlemock/src/gmock-all.cc
+libgmock_la_SOURCES = unittest/third_party/googletest/googlemock/src/gmock-all.cc
 libgmock_la_CPPFLAGS = $(GMOCK_INCLUDES) \
                        -pthread
-libgmock_main_la_SOURCES = googletest/googlemock/src/gmock_main.cc
+libgmock_main_la_SOURCES = unittest/third_party/googletest/googlemock/src/gmock_main.cc
 libgmock_main_la_CPPFLAGS = $(GMOCK_INCLUDES) \
                             -pthread
 
 # Build unittests
-ABSEIL_LIBS = libabseil.la
 GTEST_LIBS =  libgtest.la libgtest_main.la -lpthread
 GMOCK_LIBS =  libgmock.la libgmock_main.la
 TESS_LIBS = $(GTEST_LIBS)
@@ -1209,8 +1184,8 @@ TESS_LIBS += libtesseract.la $(libarchive_LIBS)
 TESS_LIBS += $(TENSORFLOW_LIBS)
 TRAINING_LIBS = libtesseract_training.la
 TRAINING_LIBS += $(TESS_LIBS)
-unittest_CPPFLAGS += -isystem $(top_srcdir)/googletest/googletest/include
-unittest_CPPFLAGS += -isystem $(top_srcdir)/googletest/googlemock/include
+unittest_CPPFLAGS += -isystem $(top_srcdir)/unittest/third_party/googletest/googletest/include
+unittest_CPPFLAGS += -isystem $(top_srcdir)/unittest/third_party/googletest/googlemock/include
 
 check_PROGRAMS = apiexample_test
 if ENABLE_TRAINING
@@ -1323,12 +1298,11 @@ endif # !DISABLED_LEGACY_ENGINE
 
 baseapi_test_SOURCES = unittest/baseapi_test.cc
 baseapi_test_CPPFLAGS = $(unittest_CPPFLAGS)
-baseapi_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+baseapi_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 
 baseapi_thread_test_SOURCES = unittest/baseapi_thread_test.cc
 baseapi_thread_test_CPPFLAGS = $(unittest_CPPFLAGS)
-baseapi_thread_test_LDADD = $(ABSEIL_LIBS)
-baseapi_thread_test_LDADD += $(TESS_LIBS) $(LEPTONICA_LIBS)
+baseapi_thread_test_LDADD = $(TESS_LIBS) $(LEPTONICA_LIBS)
 
 if !DISABLED_LEGACY_ENGINE
 bitvector_test_SOURCES = unittest/bitvector_test.cc
@@ -1364,7 +1338,7 @@ endif # !DISABLED_LEGACY_ENGINE
 
 fileio_test_SOURCES = unittest/fileio_test.cc
 fileio_test_CPPFLAGS = $(unittest_CPPFLAGS)
-fileio_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+fileio_test_LDADD = $(TRAINING_LIBS)
 
 heap_test_SOURCES = unittest/heap_test.cc
 heap_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1372,7 +1346,7 @@ heap_test_LDADD = $(TESS_LIBS)
 
 imagedata_test_SOURCES = unittest/imagedata_test.cc
 imagedata_test_CPPFLAGS = $(unittest_CPPFLAGS)
-imagedata_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+imagedata_test_LDADD = $(TRAINING_LIBS)
 
 if !DISABLED_LEGACY_ENGINE
 indexmapbidi_test_SOURCES = unittest/indexmapbidi_test.cc
@@ -1398,7 +1372,7 @@ intsimdmatrix_test_LDADD = $(TESS_LIBS)
 
 lang_model_test_SOURCES = unittest/lang_model_test.cc
 lang_model_test_CPPFLAGS = $(unittest_CPPFLAGS)
-lang_model_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+lang_model_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 layout_test_SOURCES = unittest/layout_test.cc
 layout_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1425,24 +1399,24 @@ loadlang_test_LDADD = $(TESS_LIBS) $(LEPTONICA_LIBS)
 
 lstm_recode_test_SOURCES = unittest/lstm_recode_test.cc
 lstm_recode_test_CPPFLAGS = $(unittest_CPPFLAGS)
-lstm_recode_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+lstm_recode_test_LDADD = $(TRAINING_LIBS)
 
 lstm_squashed_test_SOURCES = unittest/lstm_squashed_test.cc
 lstm_squashed_test_CPPFLAGS = $(unittest_CPPFLAGS)
-lstm_squashed_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+lstm_squashed_test_LDADD = $(TRAINING_LIBS)
 
 lstm_test_SOURCES = unittest/lstm_test.cc
 lstm_test_CPPFLAGS = $(unittest_CPPFLAGS)
-lstm_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+lstm_test_LDADD = $(TRAINING_LIBS)
 
 lstmtrainer_test_SOURCES = unittest/lstmtrainer_test.cc
 lstmtrainer_test_CPPFLAGS = $(unittest_CPPFLAGS)
-lstmtrainer_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+lstmtrainer_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 
 if !DISABLED_LEGACY_ENGINE
 mastertrainer_test_SOURCES = unittest/mastertrainer_test.cc
 mastertrainer_test_CPPFLAGS = $(unittest_CPPFLAGS)
-mastertrainer_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+mastertrainer_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 endif # !DISABLED_LEGACY_ENGINE
 
 matrix_test_SOURCES = unittest/matrix_test.cc
@@ -1459,7 +1433,7 @@ normstrngs_test_SOURCES += unittest/third_party/utf/rune.c
 normstrngs_test_SOURCES += unittest/util/utf8/unilib.cc
 endif # TENSORFLOW
 normstrngs_test_CPPFLAGS = $(unittest_CPPFLAGS)
-normstrngs_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+normstrngs_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 nthitem_test_SOURCES = unittest/nthitem_test.cc
 nthitem_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1482,14 +1456,14 @@ pango_font_info_test_SOURCES += unittest/util/utf8/unicodetext.cc
 pango_font_info_test_SOURCES += unittest/util/utf8/unilib.cc
 endif # TENSORFLOW
 pango_font_info_test_CPPFLAGS = $(unittest_CPPFLAGS)
-pango_font_info_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+pango_font_info_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 pango_font_info_test_LDADD += $(ICU_I18N_LIBS)
 pango_font_info_test_LDADD += $(pangocairo_LIBS)
 pango_font_info_test_LDADD += $(pangoft2_LIBS)
 
 paragraphs_test_SOURCES = unittest/paragraphs_test.cc
 paragraphs_test_CPPFLAGS = $(unittest_CPPFLAGS)
-paragraphs_test_LDADD = $(ABSEIL_LIBS) $(TESS_LIBS)
+paragraphs_test_LDADD = $(TESS_LIBS)
 
 if !DISABLED_LEGACY_ENGINE
 params_model_test_SOURCES = unittest/params_model_test.cc
@@ -1504,11 +1478,11 @@ progress_test_LDADD = $(GTEST_LIBS) $(GMOCK_LIBS) $(TESS_LIBS) $(LEPTONICA_LIBS)
 
 qrsequence_test_SOURCES = unittest/qrsequence_test.cc
 qrsequence_test_CPPFLAGS = $(unittest_CPPFLAGS)
-qrsequence_test_LDADD = $(ABSEIL_LIBS) $(TESS_LIBS)
+qrsequence_test_LDADD = $(TESS_LIBS)
 
 recodebeam_test_SOURCES = unittest/recodebeam_test.cc
 recodebeam_test_CPPFLAGS = $(unittest_CPPFLAGS)
-recodebeam_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+recodebeam_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 rect_test_SOURCES = unittest/rect_test.cc
 rect_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1516,7 +1490,7 @@ rect_test_LDADD = $(TESS_LIBS)
 
 resultiterator_test_SOURCES = unittest/resultiterator_test.cc
 resultiterator_test_CPPFLAGS = $(unittest_CPPFLAGS)
-resultiterator_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+resultiterator_test_LDADD = $(TRAINING_LIBS)
 resultiterator_test_LDADD += $(LEPTONICA_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 scanutils_test_SOURCES = unittest/scanutils_test.cc
@@ -1526,7 +1500,7 @@ scanutils_test_LDADD = $(TRAINING_LIBS)
 if !DISABLED_LEGACY_ENGINE
 shapetable_test_SOURCES = unittest/shapetable_test.cc
 shapetable_test_CPPFLAGS = $(unittest_CPPFLAGS)
-shapetable_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS)
+shapetable_test_LDADD = $(TRAINING_LIBS)
 endif # !DISABLED_LEGACY_ENGINE
 
 stats_test_SOURCES = unittest/stats_test.cc
@@ -1539,7 +1513,7 @@ stridemap_test_LDADD = $(TESS_LIBS)
 
 stringrenderer_test_SOURCES = unittest/stringrenderer_test.cc
 stringrenderer_test_CPPFLAGS = $(unittest_CPPFLAGS)
-stringrenderer_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+stringrenderer_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 stringrenderer_test_LDADD += $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 stringrenderer_test_LDADD += $(pangocairo_LIBS) $(pangoft2_LIBS)
 stringrenderer_test_LDADD += $(cairo_LIBS) $(pango_LIBS)
@@ -1557,17 +1531,15 @@ tabvector_test_CPPFLAGS = $(unittest_CPPFLAGS)
 tabvector_test_LDADD = $(TESS_LIBS)
 
 tatweel_test_SOURCES = unittest/tatweel_test.cc
-if TENSORFLOW
 tatweel_test_SOURCES += unittest/third_party/utf/rune.c
 tatweel_test_SOURCES += unittest/util/utf8/unicodetext.cc
 tatweel_test_SOURCES += unittest/util/utf8/unilib.cc
-endif # TENSORFLOW
 tatweel_test_CPPFLAGS = $(unittest_CPPFLAGS)
 tatweel_test_LDADD = $(TRAINING_LIBS)
 
 textlineprojection_test_SOURCES = unittest/textlineprojection_test.cc
 textlineprojection_test_CPPFLAGS = $(unittest_CPPFLAGS)
-textlineprojection_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(LEPTONICA_LIBS)
+textlineprojection_test_LDADD = $(TRAINING_LIBS) $(LEPTONICA_LIBS)
 
 tfile_test_SOURCES = unittest/tfile_test.cc
 tfile_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1579,7 +1551,7 @@ unichar_test_LDADD = $(TRAINING_LIBS) $(ICU_UC_LIBS)
 
 unicharcompress_test_SOURCES = unittest/unicharcompress_test.cc
 unicharcompress_test_CPPFLAGS = $(unittest_CPPFLAGS)
-unicharcompress_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_UC_LIBS)
+unicharcompress_test_LDADD = $(TRAINING_LIBS) $(ICU_UC_LIBS)
 
 unicharset_test_SOURCES = unittest/unicharset_test.cc
 unicharset_test_CPPFLAGS = $(unittest_CPPFLAGS)
@@ -1587,19 +1559,19 @@ unicharset_test_LDADD = $(TRAINING_LIBS) $(ICU_UC_LIBS)
 
 validate_grapheme_test_SOURCES = unittest/validate_grapheme_test.cc
 validate_grapheme_test_CPPFLAGS = $(unittest_CPPFLAGS)
-validate_grapheme_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+validate_grapheme_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 validate_indic_test_SOURCES = unittest/validate_indic_test.cc
 validate_indic_test_CPPFLAGS = $(unittest_CPPFLAGS)
-validate_indic_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+validate_indic_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 validate_khmer_test_SOURCES = unittest/validate_khmer_test.cc
 validate_khmer_test_CPPFLAGS = $(unittest_CPPFLAGS)
-validate_khmer_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+validate_khmer_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 validate_myanmar_test_SOURCES = unittest/validate_myanmar_test.cc
 validate_myanmar_test_CPPFLAGS = $(unittest_CPPFLAGS)
-validate_myanmar_test_LDADD = $(ABSEIL_LIBS) $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
+validate_myanmar_test_LDADD = $(TRAINING_LIBS) $(ICU_I18N_LIBS) $(ICU_UC_LIBS)
 
 validator_test_SOURCES = unittest/validator_test.cc
 validator_test_CPPFLAGS = $(unittest_CPPFLAGS)

README.md

@@ -8,9 +8,24 @@
 [![Total Alerts](https://img.shields.io/lgtm/alerts/g/tesseract-ocr/tesseract.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/tesseract-ocr/tesseract/alerts)
 [![OSS-Fuzz](https://img.shields.io/badge/oss--fuzz-fuzzing-brightgreen)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=2&q=proj:tesseract-ocr)
 <br/>
-[![GitHub license](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://raw.githubusercontent.com/tesseract-ocr/tesseract/master/LICENSE)
+[![GitHub license](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://raw.githubusercontent.com/tesseract-ocr/tesseract/main/LICENSE)
 [![Downloads](https://img.shields.io/badge/download-all%20releases-brightgreen.svg)](https://github.com/tesseract-ocr/tesseract/releases/)
 
+Table of Contents
+=================
+
+* [Tesseract OCR](#tesseract-ocr)
+   * [About](#about)
+   * [Brief history](#brief-history)
+   * [Installing Tesseract](#installing-tesseract)
+   * [Running Tesseract](#running-tesseract)
+   * [For developers](#for-developers)
+   * [Support](#support)
+   * [License](#license)
+   * [Dependencies](#dependencies)
+   * [Latest Version of README](#latest-version-of-readme)
+
+
 ## About
 
 This package contains an **OCR engine** - `libtesseract` and a **command line program** - `tesseract`.
@@ -22,12 +37,12 @@ It also needs [traineddata](https://tesseract-ocr.github.io/tessdoc/Data-Files.h
 those from the tessdata repository.
 
 The lead developer is Ray Smith. The maintainer is Zdenko Podobny.
-For a list of contributors see [AUTHORS](https://github.com/tesseract-ocr/tesseract/blob/master/AUTHORS)
+For a list of contributors see [AUTHORS](https://github.com/tesseract-ocr/tesseract/blob/main/AUTHORS)
 and GitHub's log of [contributors](https://github.com/tesseract-ocr/tesseract/graphs/contributors).
 
 Tesseract has **unicode (UTF-8) support**, and can **recognize more than 100 languages** "out of the box".
 
-Tesseract supports **various output formats**: plain text, hOCR (HTML), PDF, invisible-text-only PDF, TSV. The master branch also has experimental support for ALTO (XML) output.
+Tesseract supports **various output formats**: plain text, hOCR (HTML), PDF, invisible-text-only PDF, TSV. The main branch also has experimental support for ALTO (XML) output.
 
 You should note that in many cases, in order to get better OCR results,
 you'll need to **[improve the quality](https://tesseract-ocr.github.io/tessdoc/ImproveQuality.html) of the image** you are giving Tesseract.
@@ -45,23 +60,24 @@ at Hewlett-Packard Co, Greeley Colorado between 1985 and 1994, with some
 more changes made in 1996 to port to Windows, and some C++izing in 1998.
 In 2005 Tesseract was open sourced by HP. From 2006 until November 2018 it was developed by Google.
 
-The latest (LSTM based) stable version is **[4.1.1](https://github.com/tesseract-ocr/tesseract/releases/tag/4.1.1)**, released on December 26, 2019.
-Latest source code is available from [master branch on GitHub](https://github.com/tesseract-ocr/tesseract/tree/master).
+Major version 5 is the current stable version and started with release
+[5.0.0](https://github.com/tesseract-ocr/tesseract/releases/tag/5.0.0) on November 30, 2021.
+Newer minor versions and bugfix versions are available from
+[GitHub](https://github.com/tesseract-ocr/tesseract/releases/).
+
+Latest source code is available from [main branch on GitHub](https://github.com/tesseract-ocr/tesseract/tree/main).
 Open issues can be found in [issue tracker](https://github.com/tesseract-ocr/tesseract/issues),
 and [planning documentation](https://tesseract-ocr.github.io/tessdoc/Planning.html).
 
-The latest 3.0x version is **[3.05.02](https://github.com/tesseract-ocr/tesseract/releases/tag/3.05.02)**, released on June 19, 2018. Latest source code for 3.05 is available from [3.05 branch on GitHub](https://github.com/tesseract-ocr/tesseract/tree/3.05).
-There is no development for this version, but it can be used for special cases (e.g. see [Regression of features from 3.0x](https://tesseract-ocr.github.io/tessdoc/Planning.html#regression-of-features-from-30x)).
-
 See **[Release Notes](https://tesseract-ocr.github.io/tessdoc/ReleaseNotes.html)**
-and **[Change Log](https://github.com/tesseract-ocr/tesseract/blob/master/ChangeLog)** for more details of the releases.
+and **[Change Log](https://github.com/tesseract-ocr/tesseract/blob/main/ChangeLog)** for more details of the releases.
 
 ## Installing Tesseract
 
 You can either [Install Tesseract via pre-built binary package](https://tesseract-ocr.github.io/tessdoc/Home.html)
 or [build it from source](https://tesseract-ocr.github.io/tessdoc/Compiling.html).
 
-C++17 support is required for building.
+A C++ compiler with good C++17 support is required for building Tesseract from source.
 
 ## Running Tesseract
 
@@ -75,8 +91,8 @@ Examples can be found in the [documentation](https://tesseract-ocr.github.io/tes
 
 ## For developers
 
-Developers can use `libtesseract` [C](https://github.com/tesseract-ocr/tesseract/blob/master/include/tesseract/capi.h) or
-[C++](https://github.com/tesseract-ocr/tesseract/blob/master/include/tesseract/baseapi.h) API to build their own application.
+Developers can use `libtesseract` [C](https://github.com/tesseract-ocr/tesseract/blob/main/include/tesseract/capi.h) or
+[C++](https://github.com/tesseract-ocr/tesseract/blob/main/include/tesseract/baseapi.h) API to build their own application.
 If you need bindings to `libtesseract` for other programming languages, please see the
 [wrapper](https://tesseract-ocr.github.io/tessdoc/AddOns.html#tesseract-wrappers) section in the AddOns documentation.
 
@@ -84,7 +100,7 @@ Documentation of Tesseract generated from source code by doxygen can be found on
 
 ## Support
 
-Before you submit an issue, please review **[the guidelines for this repository](https://github.com/tesseract-ocr/tesseract/blob/master/CONTRIBUTING.md)**.
+Before you submit an issue, please review **[the guidelines for this repository](https://github.com/tesseract-ocr/tesseract/blob/main/CONTRIBUTING.md)**.
 
 For support, first read the [documentation](https://tesseract-ocr.github.io/tessdoc/),
 particularly the [FAQ](https://tesseract-ocr.github.io/tessdoc/FAQ.html) to see if your problem is addressed there.
@@ -127,4 +143,4 @@ It is suggested to use leptonica with built-in support for [zlib](https://zlib.n
 
 For the latest online version of the README.md see:
 
-https://github.com/tesseract-ocr/tesseract/blob/master/README.md
+https://github.com/tesseract-ocr/tesseract/blob/main/README.md

VERSION

@@ -1 +1 @@
-5.0.0-alpha-20210401
+5.0.1

abseil

@@ -1 +0,0 @@
-Subproject commit e1d388e7e74803050423d035e4374131b9b57919

autogen.sh

@@ -82,6 +82,10 @@ echo "Running $LIBTOOLIZE"
 $LIBTOOLIZE -f -c || bail_out
 $LIBTOOLIZE --automake || bail_out
 
+# Run aclocal a 2nd time because glibtoolize created additional m4 files.
+echo "Running aclocal"
+aclocal -I config || bail_out
+
 # --- Step 3: Generate configure and include/miaconfig.h from:
 #             . configure.ac
 #

cmake/AddCompilerFlag.cmake

@@ -1,130 +0,0 @@
-# - Add a given compiler flag to flags variables.
-# AddCompilerFlag(<flag> [<var>])
-# or
-# AddCompilerFlag(<flag> [C_FLAGS <var>] [CXX_FLAGS <var>] [C_RESULT <var>]
-#                        [CXX_RESULT <var>])
-
-#=============================================================================
-# Copyright 2010-2015 Matthias Kretz <kretz@kde.org>
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#  * Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-#
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-#  * Neither the names of contributing organizations nor the
-#    names of its contributors may be used to endorse or promote products
-#    derived from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS ``AS IS''
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#=============================================================================
-
-get_filename_component(_currentDir "${CMAKE_CURRENT_LIST_FILE}" PATH)
-include(CheckCCompilerFlag)
-include(CheckCXXCompilerFlag)
-
-macro(AddCompilerFlag _flag)
-   string(REGEX REPLACE "[-.+/:= ]" "_" _flag_esc "${_flag}")
-
-   set(_c_flags "CMAKE_C_FLAGS")
-   set(_cxx_flags "CMAKE_CXX_FLAGS")
-   set(_c_result tmp)
-   set(_cxx_result tmp)
-   if(${ARGC} EQUAL 2)
-      message(WARNING "Deprecated use of the AddCompilerFlag macro.")
-      unset(_c_result)
-      set(_cxx_result ${ARGV1})
-   elseif(${ARGC} GREATER 2)
-      set(state 0)
-      unset(_c_flags)
-      unset(_cxx_flags)
-      unset(_c_result)
-      unset(_cxx_result)
-      foreach(_arg ${ARGN})
-         if("x${_arg}" STREQUAL "xC_FLAGS")
-            set(state 1)
-            if(NOT DEFINED _c_result)
-               set(_c_result tmp0)
-            endif()
-         elseif("x${_arg}" STREQUAL "xCXX_FLAGS")
-            set(state 2)
-            if(NOT DEFINED _cxx_result)
-               set(_cxx_result tmp1)
-            endif()
-         elseif("x${_arg}" STREQUAL "xC_RESULT")
-            set(state 3)
-         elseif("x${_arg}" STREQUAL "xCXX_RESULT")
-            set(state 4)
-         elseif(state EQUAL 1)
-            set(_c_flags "${_arg}")
-         elseif(state EQUAL 2)
-            set(_cxx_flags "${_arg}")
-         elseif(state EQUAL 3)
-            set(_c_result "${_arg}")
-         elseif(state EQUAL 4)
-            set(_cxx_result "${_arg}")
-         else()
-            message(FATAL_ERROR "Syntax error for AddCompilerFlag")
-         endif()
-      endforeach()
-   endif()
-
-   set(_c_code "int main() { return 0; }")
-   set(_cxx_code "int main() { return 0; }")
-   if("${_flag}" STREQUAL "-mfma")
-      # Compiling with FMA3 support may fail only at the assembler level.
-      # In that case we need to have such an instruction in the test code
-      set(_c_code "#include <immintrin.h>
-      __m128 foo(__m128 x) { return _mm_fmadd_ps(x, x, x); }
-      int main() { return 0; }")
-      set(_cxx_code "${_c_code}")
-   elseif("${_flag}" STREQUAL "-stdlib=libc++")
-      # Compiling with libc++ not only requires a compiler that understands it, but also
-      # the libc++ headers itself
-      set(_cxx_code "#include <iostream>
-      #include <cstdio>
-      int main() { return 0; }")
-   else()
-      set(_cxx_code "#include <cstdio>
-      int main() { return 0; }")
-   endif()
-
-   if(DEFINED _c_result)
-      check_c_compiler_flag("${_flag}" check_c_compiler_flag_${_flag_esc} "${_c_code}")
-      set(${_c_result} ${check_c_compiler_flag_${_flag_esc}})
-   endif()
-   if(DEFINED _cxx_result)
-      check_cxx_compiler_flag("${_flag}" check_cxx_compiler_flag_${_flag_esc} "${_cxx_code}")
-      set(${_cxx_result} ${check_cxx_compiler_flag_${_flag_esc}})
-   endif()
-
-   macro(my_append _list _flag _special)
-      if("x${_list}" STREQUAL "x${_special}")
-         set(${_list} "${${_list}} ${_flag}")
-      else()
-         list(APPEND ${_list} "${_flag}")
-      endif()
-   endmacro()
-
-   if(check_c_compiler_flag_${_flag_esc} AND DEFINED _c_flags)
-      my_append(${_c_flags} "${_flag}" CMAKE_C_FLAGS)
-   endif()
-   if(check_cxx_compiler_flag_${_flag_esc} AND DEFINED _cxx_flags)
-      my_append(${_cxx_flags} "${_flag}" CMAKE_CXX_FLAGS)
-   endif()
-endmacro(AddCompilerFlag)

cmake/Configure.cmake

@@ -94,7 +94,6 @@ set(include_files_list
     CL/cl.h
     OpenCL/cl.h
     pango-1.0/pango/pango-features.h
-    tiffio.h
     unicode/uchar.h
 )
 check_includes(include_files_list)
@@ -105,18 +104,27 @@ set(types_list
 )
 check_types(types_list)
 
+list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE)
+list(APPEND CMAKE_REQUIRED_LIBRARIES -lm)
+set(functions_list
+    feenableexcept
+)
+check_functions(functions_list)
+
 file(APPEND ${AUTOCONFIG_SRC} "
 /* Version number */
 #cmakedefine PACKAGE_VERSION \"${PACKAGE_VERSION}\"
 #cmakedefine GRAPHICS_DISABLED ${GRAPHICS_DISABLED}
+#cmakedefine FAST_FLOAT ${FAST_FLOAT}
 #cmakedefine DISABLED_LEGACY_ENGINE ${DISABLED_LEGACY_ENGINE}
+#cmakedefine HAVE_TIFFIO_H ${HAVE_TIFFIO_H}
 #cmakedefine HAVE_LIBARCHIVE ${HAVE_LIBARCHIVE}
+#cmakedefine HAVE_LIBCURL ${HAVE_LIBCURL}
 ")
 
 if(TESSDATA_PREFIX)
- add_definitions(-DTESSDATA_PREFIX=${TESSDATA_PREFIX})
  file(APPEND ${AUTOCONFIG_SRC} "
-#cmakedefine TESSDATA_PREFIX ${TESSDATA_PREFIX}
+#cmakedefine TESSDATA_PREFIX \"${TESSDATA_PREFIX}\"
 ")
 endif()
 

cmake/OptimizeForArchitecture.cmake

@@ -1,581 +0,0 @@
-# Determine the host CPU feature set and determine the best set of compiler
-# flags to enable all supported SIMD relevant features. Alternatively, the
-# target CPU can be explicitly selected (for generating more generic binaries
-# or for targeting a different system).
-# Compilers provide e.g. the -march=native flag to achieve a similar result.
-# This fails to address the need for building for a different microarchitecture
-# than the current host.
-# The script tries to deduce all settings from the model and family numbers of
-# the CPU instead of reading the CPUID flags from e.g. /proc/cpuinfo. This makes
-# the detection more independent from the CPUID code in the kernel (e.g. avx2 is
-# not listed on older kernels).
-#
-# Usage:
-# OptimizeForArchitecture()
-# If either of Vc_SSE_INTRINSICS_BROKEN, Vc_AVX_INTRINSICS_BROKEN,
-# Vc_AVX2_INTRINSICS_BROKEN is defined and set, the OptimizeForArchitecture
-# macro will consequently disable the relevant features via compiler flags.
-
-#=============================================================================
-# Copyright 2010-2016 Matthias Kretz <kretz@kde.org>
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#  * Redistributions of source code must retain the above copyright notice,
-#    this list of conditions and the following disclaimer.
-#  * Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#  * Neither the names of contributing organizations nor the
-#    names of its contributors may be used to endorse or promote products
-#    derived from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS ``AS IS''
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#=============================================================================
-
-get_filename_component(_currentDir "${CMAKE_CURRENT_LIST_FILE}" PATH)
-include("${_currentDir}/AddCompilerFlag.cmake")
-include(CheckIncludeFileCXX)
-
-macro(_my_find _list _value _ret)
-   list(FIND ${_list} "${_value}" _found)
-   if(_found EQUAL -1)
-      set(${_ret} FALSE)
-   else(_found EQUAL -1)
-      set(${_ret} TRUE)
-   endif(_found EQUAL -1)
-endmacro(_my_find)
-
-macro(AutodetectHostArchitecture)
-   set(TARGET_ARCHITECTURE "generic")
-   set(Vc_ARCHITECTURE_FLAGS)
-   set(_vendor_id)
-   set(_cpu_family)
-   set(_cpu_model)
-   if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-      file(READ "/proc/cpuinfo" _cpuinfo)
-      string(REGEX REPLACE ".*vendor_id[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _vendor_id "${_cpuinfo}")
-      string(REGEX REPLACE ".*cpu family[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _cpu_family "${_cpuinfo}")
-      string(REGEX REPLACE ".*model[ \t]*:[ \t]+([a-zA-Z0-9_-]+).*" "\\1" _cpu_model "${_cpuinfo}")
-      string(REGEX REPLACE ".*flags[ \t]*:[ \t]+([^\n]+).*" "\\1" _cpu_flags "${_cpuinfo}")
-   elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-      exec_program("/usr/sbin/sysctl -n machdep.cpu.vendor machdep.cpu.model machdep.cpu.family machdep.cpu.features" OUTPUT_VARIABLE _sysctl_output_string)
-      string(REPLACE "\n" ";" _sysctl_output ${_sysctl_output_string})
-      list(GET _sysctl_output 0 _vendor_id)
-      list(GET _sysctl_output 1 _cpu_model)
-      list(GET _sysctl_output 2 _cpu_family)
-      list(GET _sysctl_output 3 _cpu_flags)
-
-      string(TOLOWER "${_cpu_flags}" _cpu_flags)
-      string(REPLACE "." "_" _cpu_flags "${_cpu_flags}")
-   elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
-      get_filename_component(_vendor_id "[HKEY_LOCAL_MACHINE\\Hardware\\Description\\System\\CentralProcessor\\0;VendorIdentifier]" NAME CACHE)
-      get_filename_component(_cpu_id "[HKEY_LOCAL_MACHINE\\Hardware\\Description\\System\\CentralProcessor\\0;Identifier]" NAME CACHE)
-      mark_as_advanced(_vendor_id _cpu_id)
-      string(REGEX REPLACE ".* Family ([0-9]+) .*" "\\1" _cpu_family "${_cpu_id}")
-      string(REGEX REPLACE ".* Model ([0-9]+) .*" "\\1" _cpu_model "${_cpu_id}")
-   endif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-   if(_vendor_id STREQUAL "GenuineIntel")
-      if(_cpu_family EQUAL 6)
-         # taken from the Intel ORM
-         # http://www.intel.com/content/www/us/en/processors/architectures-software-developer-manuals.html
-         # CPUID Signature Values of Of Recent Intel Microarchitectures
-         # 4E 5E       | Skylake microarchitecture
-         # 3D 47 56    | Broadwell microarchitecture
-         # 3C 45 46 3F | Haswell microarchitecture
-         # 3A 3E       | Ivy Bridge microarchitecture
-         # 2A 2D       | Sandy Bridge microarchitecture
-         # 25 2C 2F    | Intel microarchitecture Westmere
-         # 1A 1E 1F 2E | Intel microarchitecture Nehalem
-         # 17 1D       | Enhanced Intel Core microarchitecture
-         # 0F          | Intel Core microarchitecture
-         #
-         # Intel SDM Vol. 3C 35-1 / December 2016:
-         # 57          | Xeon Phi 3200, 5200, 7200  [Knights Landing]
-         # 85          | Future Xeon Phi
-         # 8E 9E       | 7th gen. Core              [Kaby Lake]
-         # 55          | Future Xeon                [Skylake w/ AVX512]
-         # 4E 5E       | 6th gen. Core / E3 v5      [Skylake w/o AVX512]
-         # 56          | Xeon D-1500                [Broadwell]
-         # 4F          | Xeon E5 v4, E7 v4, i7-69xx [Broadwell]
-         # 47          | 5th gen. Core / Xeon E3 v4 [Broadwell]
-         # 3D          | M-5xxx / 5th gen.          [Broadwell]
-         # 3F          | Xeon E5 v3, E7 v3, i7-59xx [Haswell-E]
-         # 3C 45 46    | 4th gen. Core, Xeon E3 v3  [Haswell]
-         # 3E          | Xeon E5 v2, E7 v2, i7-49xx [Ivy Bridge-E]
-         # 3A          | 3rd gen. Core, Xeon E3 v2  [Ivy Bridge]
-         # 2D          | Xeon E5, i7-39xx           [Sandy Bridge]
-         # 2F          | Xeon E7
-         # 2A          | Xeon E3, 2nd gen. Core     [Sandy Bridge]
-         # 2E          | Xeon 7500, 6500 series
-         # 25 2C       | Xeon 3600, 5600 series, Core i7, i5 and i3
-         #
-         # Values from the Intel SDE:
-         # 5C | Goldmont
-         # 5A | Silvermont
-         # 57 | Knights Landing
-         # 66 | Cannonlake
-         # 55 | Skylake Server
-         # 4E | Skylake Client
-         # 3C | Broadwell (likely a bug in the SDE)
-         # 3C | Haswell
-         if(_cpu_model EQUAL 87) # 57
-            set(TARGET_ARCHITECTURE "knl")  # Knights Landing
-         elseif(_cpu_model EQUAL 92)
-            set(TARGET_ARCHITECTURE "goldmont")
-         elseif(_cpu_model EQUAL 90 OR _cpu_model EQUAL 76)
-            set(TARGET_ARCHITECTURE "silvermont")
-         elseif(_cpu_model EQUAL 102)
-            set(TARGET_ARCHITECTURE "cannonlake")
-         elseif(_cpu_model EQUAL 142 OR _cpu_model EQUAL 158) # 8E, 9E
-            set(TARGET_ARCHITECTURE "kaby-lake")
-         elseif(_cpu_model EQUAL 85) # 55
-            set(TARGET_ARCHITECTURE "skylake-avx512")
-         elseif(_cpu_model EQUAL 78 OR _cpu_model EQUAL 94) # 4E, 5E
-            set(TARGET_ARCHITECTURE "skylake")
-         elseif(_cpu_model EQUAL 61 OR _cpu_model EQUAL 71 OR _cpu_model EQUAL 79 OR _cpu_model EQUAL 86) # 3D, 47, 4F, 56
-            set(TARGET_ARCHITECTURE "broadwell")
-         elseif(_cpu_model EQUAL 60 OR _cpu_model EQUAL 69 OR _cpu_model EQUAL 70 OR _cpu_model EQUAL 63)
-            set(TARGET_ARCHITECTURE "haswell")
-         elseif(_cpu_model EQUAL 58 OR _cpu_model EQUAL 62)
-            set(TARGET_ARCHITECTURE "ivy-bridge")
-         elseif(_cpu_model EQUAL 42 OR _cpu_model EQUAL 45)
-            set(TARGET_ARCHITECTURE "sandy-bridge")
-         elseif(_cpu_model EQUAL 37 OR _cpu_model EQUAL 44 OR _cpu_model EQUAL 47)
-            set(TARGET_ARCHITECTURE "westmere")
-         elseif(_cpu_model EQUAL 26 OR _cpu_model EQUAL 30 OR _cpu_model EQUAL 31 OR _cpu_model EQUAL 46)
-            set(TARGET_ARCHITECTURE "nehalem")
-         elseif(_cpu_model EQUAL 23 OR _cpu_model EQUAL 29)
-            set(TARGET_ARCHITECTURE "penryn")
-         elseif(_cpu_model EQUAL 15)
-            set(TARGET_ARCHITECTURE "merom")
-         elseif(_cpu_model EQUAL 28)
-            set(TARGET_ARCHITECTURE "atom")
-         elseif(_cpu_model EQUAL 14)
-            set(TARGET_ARCHITECTURE "core")
-         elseif(_cpu_model LESS 14)
-            message(WARNING "Your CPU (family ${_cpu_family}, model ${_cpu_model}) is not known. Auto-detection of optimization flags failed and will use the generic CPU settings with SSE2.")
-            set(TARGET_ARCHITECTURE "generic")
-         else()
-            message(WARNING "Your CPU (family ${_cpu_family}, model ${_cpu_model}) is not known. Auto-detection of optimization flags failed and will use the 65nm Core 2 CPU settings.")
-            set(TARGET_ARCHITECTURE "merom")
-         endif()
-      elseif(_cpu_family EQUAL 7) # Itanium (not supported)
-         message(WARNING "Your CPU (Itanium: family ${_cpu_family}, model ${_cpu_model}) is not supported by OptimizeForArchitecture.cmake.")
-      elseif(_cpu_family EQUAL 15) # NetBurst
-         list(APPEND _available_vector_units_list "sse" "sse2")
-         if(_cpu_model GREATER 2) # Not sure whether this must be 3 or even 4 instead
-            list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
-         endif(_cpu_model GREATER 2)
-      endif(_cpu_family EQUAL 6)
-   elseif(_vendor_id STREQUAL "AuthenticAMD")
-      if(_cpu_family EQUAL 23)
-         set(TARGET_ARCHITECTURE "zen")
-      elseif(_cpu_family EQUAL 22) # 16h
-         set(TARGET_ARCHITECTURE "AMD 16h")
-      elseif(_cpu_family EQUAL 21) # 15h
-         if(_cpu_model LESS 2)
-            set(TARGET_ARCHITECTURE "bulldozer")
-         else()
-            set(TARGET_ARCHITECTURE "piledriver")
-         endif()
-      elseif(_cpu_family EQUAL 20) # 14h
-         set(TARGET_ARCHITECTURE "AMD 14h")
-      elseif(_cpu_family EQUAL 18) # 12h
-      elseif(_cpu_family EQUAL 16) # 10h
-         set(TARGET_ARCHITECTURE "barcelona")
-      elseif(_cpu_family EQUAL 15)
-         set(TARGET_ARCHITECTURE "k8")
-         if(_cpu_model GREATER 64) # I don't know the right number to put here. This is just a guess from the hardware I have access to
-            set(TARGET_ARCHITECTURE "k8-sse3")
-         endif(_cpu_model GREATER 64)
-      endif()
-   endif(_vendor_id STREQUAL "GenuineIntel")
-endmacro()
-
-macro(OptimizeForArchitecture)
-   if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "(x86|AMD64)")
-      OptimizeForArchitectureX86()
-   else()
-      message(STATUS "No support for auto-detection of the target instruction set/extension")
-      set(TARGET_ARCHITECTURE "unused" CACHE STRING "CPU architecture to optimize for. (unused)")
-   endif()
-endmacro()
-
-macro(OptimizeForArchitectureX86)
-   set(TARGET_ARCHITECTURE "auto" CACHE STRING "CPU architecture to optimize for. \
-Using an incorrect setting here can result in crashes of the resulting binary because of invalid instructions used. \
-Setting the value to \"auto\" will try to optimize for the architecture where cmake is called. \
-Other supported values are: \"none\", \"generic\", \"core\", \"merom\" (65nm Core2), \
-\"penryn\" (45nm Core2), \"nehalem\", \"westmere\", \"sandy-bridge\", \"ivy-bridge\", \
-\"haswell\", \"broadwell\", \"skylake\", \"skylake-xeon\", \"kaby-lake\", \"cannonlake\", \"silvermont\", \
-\"goldmont\", \"knl\" (Knights Landing), \"atom\", \"k8\", \"k8-sse3\", \"barcelona\", \
-\"istanbul\", \"magny-cours\", \"bulldozer\", \"interlagos\", \"piledriver\", \
-\"AMD 14h\", \"AMD 16h\", \"zen\".")
-   set(_force)
-   if(NOT _last_target_arch STREQUAL "${TARGET_ARCHITECTURE}")
-      message(STATUS "target changed from \"${_last_target_arch}\" to \"${TARGET_ARCHITECTURE}\"")
-      set(_force FORCE)
-   endif()
-   set(_last_target_arch "${TARGET_ARCHITECTURE}" CACHE STRING "" FORCE)
-   mark_as_advanced(_last_target_arch)
-   string(TOLOWER "${TARGET_ARCHITECTURE}" TARGET_ARCHITECTURE)
-
-   set(_march_flag_list)
-   set(_available_vector_units_list)
-
-   if(TARGET_ARCHITECTURE STREQUAL "auto")
-      AutodetectHostArchitecture()
-      message(STATUS "Detected CPU: ${TARGET_ARCHITECTURE}")
-   endif(TARGET_ARCHITECTURE STREQUAL "auto")
-
-   macro(_nehalem)
-      list(APPEND _march_flag_list "nehalem")
-      list(APPEND _march_flag_list "corei7")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4.1" "sse4.2")
-   endmacro()
-   macro(_westmere)
-      list(APPEND _march_flag_list "westmere")
-      _nehalem()
-   endmacro()
-   macro(_sandybridge)
-      list(APPEND _march_flag_list "sandybridge")
-      list(APPEND _march_flag_list "corei7-avx")
-      _westmere()
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4.1" "sse4.2" "avx")
-   endmacro()
-   macro(_ivybridge)
-      list(APPEND _march_flag_list "ivybridge")
-      list(APPEND _march_flag_list "core-avx-i")
-      _sandybridge()
-      list(APPEND _available_vector_units_list "rdrnd" "f16c")
-   endmacro()
-   macro(_haswell)
-      list(APPEND _march_flag_list "haswell")
-      list(APPEND _march_flag_list "core-avx2")
-      _ivybridge()
-      list(APPEND _available_vector_units_list "avx2" "fma" "bmi" "bmi2")
-   endmacro()
-   macro(_broadwell)
-      list(APPEND _march_flag_list "broadwell")
-      _haswell()
-   endmacro()
-   macro(_skylake)
-      list(APPEND _march_flag_list "skylake")
-      _broadwell()
-   endmacro()
-   macro(_skylake_avx512)
-      list(APPEND _march_flag_list "skylake-avx512")
-      _skylake()
-      list(APPEND _available_vector_units_list "avx512f" "avx512cd" "avx512dq" "avx512bw" "avx512vl")
-   endmacro()
-   macro(_cannonlake)
-      list(APPEND _march_flag_list "cannonlake")
-      _skylake_avx512()
-      list(APPEND _available_vector_units_list "avx512ifma" "avx512vbmi")
-   endmacro()
-   macro(_knightslanding)
-      list(APPEND _march_flag_list "knl")
-      _broadwell()
-      list(APPEND _available_vector_units_list "avx512f" "avx512pf" "avx512er" "avx512cd")
-   endmacro()
-   macro(_silvermont)
-      list(APPEND _march_flag_list "silvermont")
-      _westmere()
-      list(APPEND _available_vector_units_list "rdrnd")
-   endmacro()
-   macro(_goldmont)
-      list(APPEND _march_flag_list "goldmont")
-      _silvermont()
-   endmacro()
-
-   if(TARGET_ARCHITECTURE STREQUAL "core")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
-   elseif(TARGET_ARCHITECTURE STREQUAL "merom")
-      list(APPEND _march_flag_list "merom")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
-   elseif(TARGET_ARCHITECTURE STREQUAL "penryn")
-      list(APPEND _march_flag_list "penryn")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
-      message(STATUS "Sadly the Penryn architecture exists in variants with SSE4.1 and without SSE4.1.")
-      if(_cpu_flags MATCHES "sse4_1")
-         message(STATUS "SSE4.1: enabled (auto-detected from this computer's CPU flags)")
-         list(APPEND _available_vector_units_list "sse4.1")
-      else()
-         message(STATUS "SSE4.1: disabled (auto-detected from this computer's CPU flags)")
-      endif()
-   elseif(TARGET_ARCHITECTURE STREQUAL "knl")
-      _knightslanding()
-   elseif(TARGET_ARCHITECTURE STREQUAL "cannonlake")
-      _cannonlake()
-   elseif(TARGET_ARCHITECTURE STREQUAL "kaby-lake")
-      _skylake()
-   elseif(TARGET_ARCHITECTURE STREQUAL "skylake-xeon" OR TARGET_ARCHITECTURE STREQUAL "skylake-avx512")
-      _skylake_avx512()
-   elseif(TARGET_ARCHITECTURE STREQUAL "skylake")
-      _skylake()
-   elseif(TARGET_ARCHITECTURE STREQUAL "broadwell")
-      _broadwell()
-   elseif(TARGET_ARCHITECTURE STREQUAL "haswell")
-      _haswell()
-   elseif(TARGET_ARCHITECTURE STREQUAL "ivy-bridge")
-      _ivybridge()
-   elseif(TARGET_ARCHITECTURE STREQUAL "sandy-bridge")
-      _sandybridge()
-   elseif(TARGET_ARCHITECTURE STREQUAL "westmere")
-      _westmere()
-   elseif(TARGET_ARCHITECTURE STREQUAL "nehalem")
-      _nehalem()
-   elseif(TARGET_ARCHITECTURE STREQUAL "goldmont")
-      _goldmont()
-   elseif(TARGET_ARCHITECTURE STREQUAL "silvermont")
-      _silvermont()
-   elseif(TARGET_ARCHITECTURE STREQUAL "atom")
-      list(APPEND _march_flag_list "atom")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3")
-   elseif(TARGET_ARCHITECTURE STREQUAL "k8")
-      list(APPEND _march_flag_list "k8")
-      list(APPEND _available_vector_units_list "sse" "sse2")
-   elseif(TARGET_ARCHITECTURE STREQUAL "k8-sse3")
-      list(APPEND _march_flag_list "k8-sse3")
-      list(APPEND _march_flag_list "k8")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3")
-   elseif(TARGET_ARCHITECTURE STREQUAL "AMD 16h")
-      list(APPEND _march_flag_list "btver2")
-      list(APPEND _march_flag_list "btver1")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "f16c")
-   elseif(TARGET_ARCHITECTURE STREQUAL "AMD 14h")
-      list(APPEND _march_flag_list "btver1")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a")
-   elseif(TARGET_ARCHITECTURE STREQUAL "zen")
-      list(APPEND _march_flag_list "znver1")
-      _skylake()
-      list(APPEND _available_vector_units_list "sse4a")
-   elseif(TARGET_ARCHITECTURE STREQUAL "piledriver")
-      list(APPEND _march_flag_list "bdver2")
-      list(APPEND _march_flag_list "bdver1")
-      list(APPEND _march_flag_list "bulldozer")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "xop" "fma4" "fma" "f16c")
-   elseif(TARGET_ARCHITECTURE STREQUAL "interlagos")
-      list(APPEND _march_flag_list "bdver1")
-      list(APPEND _march_flag_list "bulldozer")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "xop" "fma4")
-   elseif(TARGET_ARCHITECTURE STREQUAL "bulldozer")
-      list(APPEND _march_flag_list "bdver1")
-      list(APPEND _march_flag_list "bulldozer")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "ssse3" "sse4a" "sse4.1" "sse4.2" "avx" "xop" "fma4")
-   elseif(TARGET_ARCHITECTURE STREQUAL "barcelona")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
-   elseif(TARGET_ARCHITECTURE STREQUAL "istanbul")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
-   elseif(TARGET_ARCHITECTURE STREQUAL "magny-cours")
-      list(APPEND _march_flag_list "barcelona")
-      list(APPEND _march_flag_list "core2")
-      list(APPEND _available_vector_units_list "sse" "sse2" "sse3" "sse4a")
-   elseif(TARGET_ARCHITECTURE STREQUAL "generic")
-      list(APPEND _march_flag_list "generic")
-   elseif(TARGET_ARCHITECTURE STREQUAL "none")
-      # add this clause to remove it from the else clause
-   else(TARGET_ARCHITECTURE STREQUAL "core")
-      message(FATAL_ERROR "Unknown target architecture: \"${TARGET_ARCHITECTURE}\". Please set TARGET_ARCHITECTURE to a supported value.")
-   endif(TARGET_ARCHITECTURE STREQUAL "core")
-
-   if(NOT TARGET_ARCHITECTURE STREQUAL "none")
-      set(_disable_vector_unit_list)
-      set(_enable_vector_unit_list)
-      if(DEFINED Vc_AVX_INTRINSICS_BROKEN AND Vc_AVX_INTRINSICS_BROKEN)
-         UserWarning("AVX disabled per default because of old/broken toolchain")
-         set(_avx_broken true)
-         set(_avx2_broken true)
-         set(_fma4_broken true)
-         set(_xop_broken true)
-      else()
-         set(_avx_broken false)
-         if(DEFINED Vc_FMA4_INTRINSICS_BROKEN AND Vc_FMA4_INTRINSICS_BROKEN)
-            UserWarning("FMA4 disabled per default because of old/broken toolchain")
-            set(_fma4_broken true)
-         else()
-            set(_fma4_broken false)
-         endif()
-         if(DEFINED Vc_XOP_INTRINSICS_BROKEN AND Vc_XOP_INTRINSICS_BROKEN)
-            UserWarning("XOP disabled per default because of old/broken toolchain")
-            set(_xop_broken true)
-         else()
-            set(_xop_broken false)
-         endif()
-         if(DEFINED Vc_AVX2_INTRINSICS_BROKEN AND Vc_AVX2_INTRINSICS_BROKEN)
-            UserWarning("AVX2 disabled per default because of old/broken toolchain")
-            set(_avx2_broken true)
-         else()
-            set(_avx2_broken false)
-         endif()
-      endif()
-
-      macro(_enable_or_disable _name _flag _documentation _broken)
-         if(_broken)
-            set(_found false)
-         else()
-            _my_find(_available_vector_units_list "${_flag}" _found)
-         endif()
-         set(USE_${_name} ${_found} CACHE BOOL "${documentation}" ${_force})
-         mark_as_advanced(USE_${_name})
-         if(USE_${_name})
-            list(APPEND _enable_vector_unit_list "${_flag}")
-         else()
-            list(APPEND _disable_vector_unit_list "${_flag}")
-         endif()
-      endmacro()
-      _enable_or_disable(SSE2 "sse2" "Use SSE2. If SSE2 instructions are not enabled the SSE implementation will be disabled." false)
-      _enable_or_disable(SSE3 "sse3" "Use SSE3. If SSE3 instructions are not enabled they will be emulated." false)
-      _enable_or_disable(SSSE3 "ssse3" "Use SSSE3. If SSSE3 instructions are not enabled they will be emulated." false)
-      _enable_or_disable(SSE4_1 "sse4.1" "Use SSE4.1. If SSE4.1 instructions are not enabled they will be emulated." false)
-      _enable_or_disable(SSE4_2 "sse4.2" "Use SSE4.2. If SSE4.2 instructions are not enabled they will be emulated." false)
-      _enable_or_disable(SSE4a "sse4a" "Use SSE4a. If SSE4a instructions are not enabled they will be emulated." false)
-      _enable_or_disable(AVX "avx" "Use AVX. This will all floating-point vector sizes relative to SSE." _avx_broken)
-      _enable_or_disable(FMA "fma" "Use FMA." _avx_broken)
-      _enable_or_disable(BMI2 "bmi2" "Use BMI2." _avx_broken)
-      _enable_or_disable(AVX2 "avx2" "Use AVX2. This will double all of the vector sizes relative to SSE." _avx2_broken)
-      _enable_or_disable(XOP "xop" "Use XOP." _xop_broken)
-      _enable_or_disable(FMA4 "fma4" "Use FMA4." _fma4_broken)
-      _enable_or_disable(AVX512F "avx512f" "Use AVX512F. This will double all floating-point vector sizes relative to AVX2." false)
-      _enable_or_disable(AVX512VL "avx512vl" "Use AVX512VL. This enables 128- and 256-bit vector length instructions with EVEX coding (improved write-masking & more vector registers)." _avx2_broken)
-      _enable_or_disable(AVX512PF "avx512pf" "Use AVX512PF. This enables prefetch instructions for gathers and scatters." false)
-      _enable_or_disable(AVX512ER "avx512er" "Use AVX512ER. This enables exponential and reciprocal instructions." false)
-      _enable_or_disable(AVX512CD "avx512cd" "Use AVX512CD." false)
-      _enable_or_disable(AVX512DQ "avx512dq" "Use AVX512DQ." false)
-      _enable_or_disable(AVX512BW "avx512bw" "Use AVX512BW." false)
-      _enable_or_disable(AVX512IFMA "avx512ifma" "Use AVX512IFMA." false)
-      _enable_or_disable(AVX512VBMI "avx512vbmi" "Use AVX512VBMI." false)
-
-      if(MSVC)
-         # MSVC on 32 bit can select /arch:SSE2 (since 2010 also /arch:AVX)
-         # MSVC on 64 bit cannot select anything (should have changed with MSVC 2010)
-         _my_find(_enable_vector_unit_list "avx2" _found)
-         if(_found)
-            AddCompilerFlag("/arch:AVX2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS CXX_RESULT _found)
-         endif()
-         if(NOT _found)
-            _my_find(_enable_vector_unit_list "avx" _found)
-            if(_found)
-               AddCompilerFlag("/arch:AVX" CXX_FLAGS Vc_ARCHITECTURE_FLAGS CXX_RESULT _found)
-            endif()
-         endif()
-         if(NOT _found)
-            _my_find(_enable_vector_unit_list "sse2" _found)
-            if(_found)
-               AddCompilerFlag("/arch:SSE2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
-            endif()
-         endif()
-         foreach(_flag ${_enable_vector_unit_list})
-            string(TOUPPER "${_flag}" _flag)
-            string(REPLACE "." "_" _flag "__${_flag}__")
-            add_definitions("-D${_flag}")
-         endforeach(_flag)
-      elseif(CMAKE_CXX_COMPILER MATCHES "/(icpc|icc)$") # ICC (on Linux)
-         set(OFA_map_knl "-xMIC-AVX512")
-         set(OFA_map_cannonlake "-xCORE-AVX512")
-         set(OFA_map_skylake-avx512 "-xCORE-AVX512")
-         set(OFA_map_skylake "-xCORE-AVX2")
-         set(OFA_map_broadwell "-xCORE-AVX2")
-         set(OFA_map_haswell "-xCORE-AVX2")
-         set(OFA_map_ivybridge "-xCORE-AVX-I")
-         set(OFA_map_sandybridge "-xAVX")
-         set(OFA_map_westmere "-xSSE4.2")
-         set(OFA_map_nehalem "-xSSE4.2")
-         set(OFA_map_penryn "-xSSSE3")
-         set(OFA_map_merom "-xSSSE3")
-         set(OFA_map_core2 "-xSSE3")
-         set(_ok FALSE)
-         foreach(arch ${_march_flag_list})
-            if(DEFINED OFA_map_${arch})
-               AddCompilerFlag(${OFA_map_${arch}} CXX_FLAGS Vc_ARCHITECTURE_FLAGS CXX_RESULT _ok)
-               if(_ok)
-                  break()
-               endif()
-            endif()
-         endforeach()
-         if(NOT _ok)
-            # This is the Intel compiler, so SSE2 is a very reasonable baseline.
-            message(STATUS "Did not recognize the requested architecture flag, falling back to SSE2")
-            AddCompilerFlag("-xSSE2" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
-         endif()
-      else() # not MSVC and not ICC => GCC, Clang, Open64
-         foreach(_flag ${_march_flag_list})
-            AddCompilerFlag("-march=${_flag}" CXX_RESULT _good CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
-            if(_good)
-               break()
-            endif(_good)
-         endforeach(_flag)
-         foreach(_flag ${_enable_vector_unit_list})
-            AddCompilerFlag("-m${_flag}" CXX_RESULT _result)
-            if(_result)
-               set(_header FALSE)
-               if(_flag STREQUAL "sse3")
-                  set(_header "pmmintrin.h")
-               elseif(_flag STREQUAL "ssse3")
-                  set(_header "tmmintrin.h")
-               elseif(_flag STREQUAL "sse4.1")
-                  set(_header "smmintrin.h")
-               elseif(_flag STREQUAL "sse4.2")
-                  set(_header "smmintrin.h")
-               elseif(_flag STREQUAL "sse4a")
-                  set(_header "ammintrin.h")
-               elseif(_flag STREQUAL "avx")
-                  set(_header "immintrin.h")
-               elseif(_flag STREQUAL "avx2")
-                  set(_header "immintrin.h")
-               elseif(_flag STREQUAL "fma4")
-                  set(_header "x86intrin.h")
-               elseif(_flag STREQUAL "xop")
-                  set(_header "x86intrin.h")
-               endif()
-               set(_resultVar "HAVE_${_header}")
-               string(REPLACE "." "_" _resultVar "${_resultVar}")
-               if(_header)
-                  CHECK_INCLUDE_FILE_CXX("${_header}" ${_resultVar} "-m${_flag}")
-                  if(NOT ${_resultVar})
-                     set(_useVar "USE_${_flag}")
-                     string(TOUPPER "${_useVar}" _useVar)
-                     string(REPLACE "." "_" _useVar "${_useVar}")
-                     message(STATUS "disabling ${_useVar} because ${_header} is missing")
-                     set(${_useVar} FALSE)
-                     list(APPEND _disable_vector_unit_list "${_flag}")
-                  endif()
-               endif()
-               if(NOT _header OR ${_resultVar})
-                  list(APPEND Vc_ARCHITECTURE_FLAGS "-m${_flag}")
-               endif()
-            endif()
-         endforeach(_flag)
-         foreach(_flag ${_disable_vector_unit_list})
-            AddCompilerFlag("-mno-${_flag}" CXX_FLAGS Vc_ARCHITECTURE_FLAGS)
-         endforeach(_flag)
-      endif()
-   endif()
-endmacro()

cmake/templates/TesseractConfig.cmake.in

@@ -10,8 +10,13 @@
 #    target_link_libraries(MY_TARGET_NAME Tesseract::libtesseract)
 #
 #    This file will define the following variables:
-#      - Tesseract_LIBRARIES             : The list of all imported targets for OpenCV modules.
+#      - Tesseract_LIBRARIES             : The list of all imported targets.
 #      - Tesseract_INCLUDE_DIRS          : The Tesseract include directories.
+#      - Tesseract_LIBRARY_DIRS          : The Tesseract library directories.
+#      - Tesseract_VERSION               : The version of this Tesseract build: "@VERSION_PLAIN@"
+#      - Tesseract_VERSION_MAJOR         : Major version part of Tesseract_VERSION: "@VERSION_MAJOR@"
+#      - Tesseract_VERSION_MINOR         : Minor version part of Tesseract_VERSION: "@VERSION_MINOR@"
+#      - Tesseract_VERSION_PATCH         : Patch version part of Tesseract_VERSION: "@VERSION_PATCH@"
 #
 # ===================================================================================
 
@@ -22,7 +27,13 @@ include(${CMAKE_CURRENT_LIST_DIR}/TesseractTargets.cmake)
 
 @PACKAGE_INIT@
 
+SET(Tesseract_VERSION           @VERSION_PLAIN@)
+SET(Tesseract_VERSION_MAJOR     @VERSION_MAJOR@)
+SET(Tesseract_VERSION_MINOR     @VERSION_MINOR@)
+SET(Tesseract_VERSION_PATCH     @VERSION_PATCH@)
+
 set_and_check(Tesseract_INCLUDE_DIRS "@PACKAGE_INCLUDE_DIR@")
-set(Tesseract_LIBRARIES libtesseract)
+set_and_check(Tesseract_LIBRARY_DIRS "@PACKAGE_LIBRARY_DIRS@")
+set(Tesseract_LIBRARIES @tesseract_OUTPUT_NAME@)
 
 check_required_components(Tesseract)

configure.ac

@@ -5,7 +5,7 @@
 # ----------------------------------------
 # Initialization
 # ----------------------------------------
-AC_PREREQ([2.63])
+AC_PREREQ([2.69])
 AC_INIT([tesseract],
         [m4_esyscmd_s([test -d .git && git describe --abbrev=4 || cat VERSION])],
         [https://github.com/tesseract-ocr/tesseract/issues],,
@@ -20,16 +20,16 @@ AC_LANG_COMPILER_REQUIRE
 CXXFLAGS=${CXXFLAGS:-""}
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_AUX_DIR([config])
-AC_CONFIG_SRCDIR([src/api/tesseractmain.cpp])
+AC_CONFIG_SRCDIR([src/tesseract.cpp])
 AC_PREFIX_DEFAULT([/usr/local])
 
 # Automake configuration. Do not require README file (we use README.md).
-AM_INIT_AUTOMAKE([foreign subdir-objects])
+AM_INIT_AUTOMAKE([foreign subdir-objects nostdinc])
 
 # Define date of package, etc. Could be useful in auto-generated
 # documentation.
-PACKAGE_YEAR=2021
-PACKAGE_DATE="04/01"
+PACKAGE_YEAR=2022
+PACKAGE_DATE="01/07"
 
 abs_top_srcdir=`AS_DIRNAME([$0])`
 
@@ -75,6 +75,10 @@ AM_CONDITIONAL([T_WIN], false)
 AM_CONDITIONAL([GRAPHICS_DISABLED], false)
 AC_SUBST([AM_CPPFLAGS])
 
+# Be less noisy by default.
+# Can be overridden with `configure --disable-silent-rules` or with `make V=1`.
+AM_SILENT_RULES([yes])
+
 OPENCL_INC="/opt/AMDAPP/include"
 OPENCL_LIBS="-lOpenCL"
 #############################
@@ -184,8 +188,11 @@ case "${host_cpu}" in
 
 esac
 
-AX_CHECK_COMPILE_FLAG([-march=native], [arch_native=true], [arch_native=false], [$WERROR])
-AM_CONDITIONAL([MARCH_NATIVE_OPT], $arch_native)
+# check whether feenableexcept is supported. some C libraries (e.g. uclibc) don't.
+AC_CHECK_FUNCS([feenableexcept])
+
+AX_CHECK_COMPILE_FLAG([-fopenmp-simd], [openmp_simd=true], [openmp_simd=false], [$WERROR])
+AM_CONDITIONAL([OPENMP_SIMD], $openmp_simd)
 
 AC_ARG_WITH([extra-includes],
             [AS_HELP_STRING([--with-extra-includes=DIR],
@@ -205,6 +212,14 @@ AC_ARG_WITH([extra-libraries],
                AC_MSG_ERROR([Cannot stat directory $withval])
              fi])
 
+AC_MSG_CHECKING([--enable-float32 argument])
+AC_ARG_ENABLE([float32],
+	      AS_HELP_STRING([--disable-float32], [disable float and enable double for LSTM]))
+AC_MSG_RESULT([$enable_float32])
+if test "$enable_float32" != "no"; then
+  AC_DEFINE([FAST_FLOAT], [1], [Enable float for LSTM])
+fi
+
 AC_MSG_CHECKING([--enable-graphics argument])
 AC_ARG_ENABLE([graphics],
   AS_HELP_STRING([--disable-graphics], [disable graphics (ScrollView)]))
@@ -284,7 +299,7 @@ m4_define([MY_CHECK_FRAMEWORK],
     ])
    if test "$my_cv_framework_$1"="yes"; then
      AC_DEFINE(AS_TR_CPP([HAVE_FRAMEWORK_$1]), 1,
-            [Define if you have the  $1 framework])
+            [Define if you have the $1 framework])
      AS_TR_CPP([FRAMEWORK_$1])="-framework $1"
      AC_SUBST(AS_TR_CPP([FRAMEWORK_$1]))
    fi]
@@ -295,13 +310,14 @@ OPENCL_CPPFLAGS=''
 OPENCL_LDFLAGS=''
 case "${host_os}" in
   *darwin* | *-macos10*)
-    echo "checking for OpenCL framework"
-    MY_CHECK_FRAMEWORK([OpenCL])
-    if test $my_cv_framework_OpenCL = yes; then
-       have_opencl_lib=true
+    MY_CHECK_FRAMEWORK([Accelerate])
+    if test $my_cv_framework_Accelerate = yes; then
+      AM_CPPFLAGS="-DHAVE_FRAMEWORK_ACCELERATE $AM_CPPFLAGS"
+      AM_LDFLAGS="$AM_LDFLAGS -framework Accelerate"
     fi
+    MY_CHECK_FRAMEWORK([OpenCL])
     if test "$enable_opencl" = "yes"; then
-      if !($have_opencl_lib); then
+      if test $my_cv_framework_OpenCL = no; then
         AC_MSG_ERROR([Required OpenCL library not found!])
       fi
       AM_CPPFLAGS="-DUSE_OPENCL $AM_CPPFLAGS"
@@ -430,6 +446,15 @@ esac
 
 AC_SEARCH_LIBS([pthread_create], [pthread])
 
+# Set PKG_CONFIG_PATH for MacOS with Homebrew unless it is already set.
+AC_CHECK_PROG([have_brew], brew, true, false)
+if $have_brew; then
+  brew_prefix=$(brew --prefix)
+  if test -z "$PKG_CONFIG_PATH"; then
+    PKG_CONFIG_PATH=$brew_prefix/opt/icu4c/lib/pkgconfig:$brew_prefix/opt/libarchive/lib/pkgconfig
+    export PKG_CONFIG_PATH
+  fi
+fi
 
 # ----------------------------------------
 # Check for programs needed to build documentation.
@@ -449,9 +474,7 @@ AS_IF([test "$enable_doc" != "no"], [
   if $have_asciidoc && $have_xsltproc; then
     AM_CONDITIONAL([ASCIIDOC], true)
     XML_CATALOG_FILES=
-    AC_CHECK_PROG([have_brew], brew, true, false)
     if $have_brew; then
-      brew_prefix=$(brew --prefix)
       catalog_file=$brew_prefix/etc/xml/catalog
       if test -f $catalog_file; then
         AM_CONDITIONAL([HAVE_XML_CATALOG_FILES], true)

doc/combine_tessdata.1.asc

@@ -177,7 +177,7 @@ lang.lstm-recoder::
 lang.version::
   (Optional) Version string for the traineddata file.
   First appeared in version 4.0 of Tesseract.
-  Old version of traineddata files will report Version string:Pre-4.0.0.
+  Old version of traineddata files will report Version:Pre-4.0.0.
   4.0 version of traineddata files may include the network spec
   used for LSTM training as part of version string.
 

doc/tesseract.1.asc

@@ -107,7 +107,7 @@ OPTIONS
   * *pdf* -- Output PDF ('OUTPUTBASE'`.pdf`).
   * *tsv* -- Output TSV ('OUTPUTBASE'`.tsv`).
   * *txt* -- Output plain text ('OUTPUTBASE'`.txt`).
-  * *get.images* -- Write processed input images to file (`tessinput.tif`).
+  * *get.images* -- Write processed input images to file ('OUTPUTBASE'`.processedPAGENUMBER.tif`).
   * *logfile* -- Redirect debug messages to file (`tesseract.log`).
   * *lstm.train* -- Output files used by LSTM training ('OUTPUTBASE'`.lstmf`).
   * *makebox* -- Write box file ('OUTPUTBASE'`.box`).
@@ -432,7 +432,7 @@ Version 2.00 brought Unicode (UTF-8) support, six languages, and the ability
 to train Tesseract.
 
 Tesseract was included in UNLV's Fourth Annual Test of OCR Accuracy.
-See <https://github.com/tesseract-ocr/docs/blob/master/AT-1995.pdf>.
+See <https://github.com/tesseract-ocr/docs/blob/main/AT-1995.pdf>.
 Since Tesseract 2.00,
 scripts are now included to allow anyone to reproduce some of these tests.
 See <https://tesseract-ocr.github.io/tessdoc/TestingTesseract.html> for more
@@ -482,7 +482,7 @@ Romano, Ray Smith, Rika Antonova, Robert Moss, Samuel Charron, Sheelagh
 Lloyd, Shobhit Saxena, and Thomas Kielbus.
 
 For a list of contributors see
-<https://github.com/tesseract-ocr/tesseract/blob/master/AUTHORS>.
+<https://github.com/tesseract-ocr/tesseract/blob/main/AUTHORS>.
 
 COPYING
 -------

googletest

@@ -1 +0,0 @@
-Subproject commit 703bd9caab50b139428cea1aaff9974ebee5742e

include/tesseract/baseapi.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        baseapi.h
 // Description: Simple API for calling tesseract.
 // Author:      Ray Smith
@@ -13,8 +13,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_API_BASEAPI_H_
 #define TESSERACT_API_BASEAPI_H_
@@ -32,7 +30,6 @@
 #include <tesseract/version.h>
 
 #include <cstdio>
-#include <tuple>  // for std::tuple
 #include <vector> // for std::vector
 
 struct Pix;
@@ -155,7 +152,7 @@ public:
   /**
    * Print Tesseract fonts table to the given file.
    */
-  void PrintFontsTable(FILE* fp) const;
+  void PrintFontsTable(FILE *fp) const;
 
 #endif
 
@@ -248,14 +245,6 @@ public:
    */
   void GetAvailableLanguagesAsVector(std::vector<std::string> *langs) const;
 
-  /**
-   * Init only the lang model component of Tesseract. The only functions
-   * that work after this init are SetVariable and IsValidWord.
-   * WARNING: temporary! This function will be removed from here and placed
-   * in a separate API at some future time.
-   */
-  int InitLangMod(const char *datapath, const char *language);
-
   /**
    * Init only for page layout analysis. Use only for calls to SetImage and
    * AnalysePage. Calls that attempt recognition will generate an error.
@@ -539,31 +528,6 @@ public:
    */
   char *GetUTF8Text();
 
-  size_t GetNumberOfTables() const;
-
-  /// Return the i-th table bounding box coordinates
-  ///
-  /// Gives the (top_left.x, top_left.y, bottom_right.x, bottom_right.y)
-  /// coordinates of the i-th table.
-  std::tuple<int, int, int, int> GetTableBoundingBox(
-      unsigned
-          i ///< Index of the table, for upper limit \see GetNumberOfTables()
-  );
-
-  /// Get bounding boxes of the rows of a table
-  /// return values are (top_left.x, top_left.y, bottom_right.x, bottom_right.y)
-  std::vector<std::tuple<int, int, int, int> > GetTableRows(
-      unsigned
-          i ///< Index of the table, for upper limit \see GetNumberOfTables()
-  );
-
-  /// Get bounding boxes of the cols of a table
-  /// return values are (top_left.x, top_left.y, bottom_right.x, bottom_right.y)
-  std::vector<std::tuple<int, int, int, int> > GetTableCols(
-      unsigned
-          i ///< Index of the table, for upper limit \see GetNumberOfTables()
-  );
-
   /**
    * Make a HTML-formatted string with hOCR markup from the internal
    * data structures.

include/tesseract/capi.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        capi.h
 // Description: C-API TessBaseAPI
 //
@@ -12,8 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef API_CAPI_H_
 #define API_CAPI_H_
@@ -240,8 +238,6 @@ TESS_API char **TessBaseAPIGetLoadedLanguagesAsVector(
 TESS_API char **TessBaseAPIGetAvailableLanguagesAsVector(
     const TessBaseAPI *handle);
 
-TESS_API int TessBaseAPIInitLangMod(TessBaseAPI *handle, const char *datapath,
-                                    const char *language);
 TESS_API void TessBaseAPIInitForAnalysePage(TessBaseAPI *handle);
 
 TESS_API void TessBaseAPIReadConfigFile(TessBaseAPI *handle,

include/tesseract/export.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        export.h
 // Description: Place holder
 //
@@ -12,8 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_PLATFORM_H_
 #define TESSERACT_PLATFORM_H_

include/tesseract/ltrresultiterator.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        ltrresultiterator.h
 // Description: Iterator for tesseract results in strict left-to-right
 //              order that avoids using tesseract internal data structures.
@@ -14,8 +14,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCMAIN_LTR_RESULT_ITERATOR_H_
 #define TESSERACT_CCMAIN_LTR_RESULT_ITERATOR_H_
@@ -93,10 +91,6 @@ public:
   // The number should be interpreted as a percent probability. (0.0f-100.0f)
   float Confidence(PageIteratorLevel level) const;
 
-  // Returns the attributes of the current row.
-  void RowAttributes(float *row_height, float *descenders,
-                     float *ascenders) const;
-
   // ============= Functions that refer to words only ============.
 
   // Returns the font attributes of the current word. If iterating at a higher

include/tesseract/ocrclass.h

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: Apache-2.0
 /**********************************************************************
  * File:        ocrclass.h
  * Description: Class definitions and constants for the OCR API.

include/tesseract/osdetect.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        osdetect.h
 // Description: Orientation and script detection.
 // Author:      Samuel Charron
@@ -14,8 +14,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCMAIN_OSDETECT_H_
 #define TESSERACT_CCMAIN_OSDETECT_H_

include/tesseract/pageiterator.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        pageiterator.h
 // Description: Iterator for tesseract page structure that avoids using
 //              tesseract internal data structures.
@@ -14,8 +14,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCMAIN_PAGEITERATOR_H_
 #define TESSERACT_CCMAIN_PAGEITERATOR_H_
@@ -263,6 +261,10 @@ public:
   bool Baseline(PageIteratorLevel level, int *x1, int *y1, int *x2,
                 int *y2) const;
 
+  // Returns the attributes of the current row.
+  void RowAttributes(float *row_height, float *descenders,
+                     float *ascenders) const;
+
   /**
    * Returns orientation for the block the iterator points to.
    *   orientation, writing_direction, textline_order: see publictypes.h

include/tesseract/publictypes.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        publictypes.h
 // Description: Types used in both the API and internally
 // Author:      Ray Smith
@@ -13,8 +13,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCSTRUCT_PUBLICTYPES_H_
 #define TESSERACT_CCSTRUCT_PUBLICTYPES_H_
@@ -278,18 +276,6 @@ enum OcrEngineMode {
   OEM_COUNT                    // Number of OEMs
 };
 
-/**
- * Except when Otsu is chosen
- * Leptonica is used for thresholding
- */
-enum class ThresholdMethod {
-  Otsu,         // Legacy Tesseract's Otsu thresholding
-  AdaptiveOtsu,
-  Sauvola,
-
-  Max,        // Number of Thresholding methods
-};
-
 } // namespace tesseract.
 
 #endif // TESSERACT_CCSTRUCT_PUBLICTYPES_H_

include/tesseract/renderer.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        renderer.h
 // Description: Rendering interface to inject into TessBaseAPI
 //
@@ -12,8 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_API_RENDERER_H_
 #define TESSERACT_API_RENDERER_H_
@@ -141,13 +139,12 @@ protected:
   void AppendData(const char *s, int len);
 
 private:
+  TessResultRenderer *next_;   // Can link multiple renderers together
+  FILE *fout_;                 // output file pointer
   const char *file_extension_; // standard extension for generated output
   std::string title_;          // title of document being rendered
   int imagenum_;               // index of last image added
-
-  FILE *fout_;               // output file pointer
-  TessResultRenderer *next_; // Can link multiple renderers together
-  bool happy_;               // I get grumpy when the disk fills up, etc.
+  bool happy_;                 // I get grumpy when the disk fills up, etc.
 };
 
 /**
@@ -189,6 +186,9 @@ protected:
   bool BeginDocumentHandler() override;
   bool AddImageHandler(TessBaseAPI *api) override;
   bool EndDocumentHandler() override;
+
+private:
+  bool begin_document;
 };
 
 /**

include/tesseract/resultiterator.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        resultiterator.h
 // Description: Iterator for tesseract results that is capable of
 //              iterating in proper reading order over Bi Directional
@@ -15,8 +15,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCMAIN_RESULT_ITERATOR_H_
 #define TESSERACT_CCMAIN_RESULT_ITERATOR_H_

include/tesseract/unichar.h

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        unichar.h
 // Description: Unicode character/ligature class.
 // Author:      Ray Smith
@@ -13,8 +13,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_CCUTIL_UNICHAR_H_
 #define TESSERACT_CCUTIL_UNICHAR_H_
@@ -32,7 +30,6 @@ namespace tesseract {
 // at least 4. Must not exceed 31 without changing the coding of length.
 #define UNICHAR_LEN 30
 
-// TODO(rays) Move these to the tesseract namespace.
 // A UNICHAR_ID is the unique id of a unichar.
 using UNICHAR_ID = int;
 
@@ -100,10 +97,10 @@ public:
   //   for (UNICHAR::const_iterator it = UNICHAR::begin(str, str_len);
   //        it != UNICHAR::end(str, len);
   //        ++it) {
-  //     tprintf("UCS-4 symbol code = %d\n", *it);
+  //     printf("UCS-4 symbol code = %d\n", *it);
   //     char buf[5];
   //     int char_len = it.get_utf8(buf); buf[char_len] = '\0';
-  //     tprintf("Char = %s\n", buf);
+  //     printf("Char = %s\n", buf);
   //   }
   class TESS_API const_iterator {
     using CI = const_iterator;

include/tesseract/version.h.in

@@ -1,4 +1,4 @@
-///////////////////////////////////////////////////////////////////////
+// SPDX-License-Identifier: Apache-2.0
 // File:        version.h
 // Description: Version information
 //
@@ -12,8 +12,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
-///////////////////////////////////////////////////////////////////////
 
 #ifndef TESSERACT_API_VERSION_H_
 #define TESSERACT_API_VERSION_H_

src/api/altorenderer.cpp

@@ -55,7 +55,17 @@ static void AddBoxToAlto(const ResultIterator *it, PageIteratorLevel level,
 /// Append the ALTO XML for the beginning of the document
 ///
 bool TessAltoRenderer::BeginDocumentHandler() {
-  AppendString(
+  // Delay the XML output because we need the name of the image file.
+  begin_document = true;
+  return true;
+}
+
+///
+/// Append the ALTO XML for the layout of the image
+///
+bool TessAltoRenderer::AddImageHandler(TessBaseAPI *api) {
+  if (begin_document) {
+    AppendString(
       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
       "<alto xmlns=\"http://www.loc.gov/standards/alto/ns-v3#\" "
       "xmlns:xlink=\"http://www.w3.org/1999/xlink\" "
@@ -67,31 +77,26 @@ bool TessAltoRenderer::BeginDocumentHandler() {
       "\t\t<sourceImageInformation>\n"
       "\t\t\t<fileName>");
 
-  AppendString(title());
+    AppendString(api->GetInputName());
 
-  AppendString(
+    AppendString(
       "</fileName>\n"
       "\t\t</sourceImageInformation>\n"
       "\t\t<OCRProcessing ID=\"OCR_0\">\n"
       "\t\t\t<ocrProcessingStep>\n"
       "\t\t\t\t<processingSoftware>\n"
       "\t\t\t\t\t<softwareName>tesseract ");
-  AppendString(TessBaseAPI::Version());
-  AppendString(
+    AppendString(TessBaseAPI::Version());
+    AppendString(
       "</softwareName>\n"
       "\t\t\t\t</processingSoftware>\n"
       "\t\t\t</ocrProcessingStep>\n"
       "\t\t</OCRProcessing>\n"
       "\t</Description>\n"
       "\t<Layout>\n");
+    begin_document = false;
+  }
 
-  return true;
-}
-
-///
-/// Append the ALTO XML for the layout of the image
-///
-bool TessAltoRenderer::AddImageHandler(TessBaseAPI *api) {
   const std::unique_ptr<const char[]> text(api->GetAltoText(imagenum()));
   if (text == nullptr) {
     return false;
@@ -112,7 +117,8 @@ bool TessAltoRenderer::EndDocumentHandler() {
 }
 
 TessAltoRenderer::TessAltoRenderer(const char *outputbase)
-    : TessResultRenderer(outputbase, "xml") {}
+    : TessResultRenderer(outputbase, "xml"),
+      begin_document(false) {}
 
 ///
 /// Make an XML-formatted string with ALTO markup from the internal

src/api/baseapi.cpp

@@ -30,8 +30,8 @@
 #include "elst.h"       // for ELIST_ITERATOR, ELISTIZE, ELISTIZEH
 #include "environ.h"    // for l_uint8
 #ifndef DISABLED_LEGACY_ENGINE
-#  include "equationdetect.h" // for EquationDetect
-#endif
+#include "equationdetect.h" // for EquationDetect, destructor of equ_detect_
+#endif // ndef DISABLED_LEGACY_ENGINE
 #include "errcode.h" // for ASSERT_HOST
 #include "helpers.h" // for IntCastRounded, chomp_string
 #include "host.h"    // for MAX_PATH
@@ -56,7 +56,6 @@
 #include "tesseractclass.h"  // for Tesseract
 #include "tprintf.h"         // for tprintf
 #include "werd.h"            // for WERD, WERD_IT, W_FUZZY_NON, W_FUZZY_SP
-#include "tabletransfer.h"   // for detected tables from tablefind.h
 #include "thresholder.h"     // for ImageThresholder
 
 #include <tesseract/baseapi.h>
@@ -100,6 +99,9 @@ namespace tesseract {
 
 static BOOL_VAR(stream_filelist, false, "Stream a filelist from stdin");
 static STRING_VAR(document_title, "", "Title of output document (used for hOCR and PDF output)");
+#ifdef HAVE_LIBCURL
+static INT_VAR(curl_timeout, 0, "Timeout for curl in seconds");
+#endif
 
 /** Minimum sensible image size to be worth running tesseract. */
 const int kMinRectSize = 10;
@@ -109,17 +111,17 @@ const char kTesseractReject = '~';
 const char kUNLVReject = '~';
 /** Character used by UNLV as a suspect marker. */
 const char kUNLVSuspect = '^';
-/**
- * Filename used for input image file, from which to derive a name to search
- * for a possible UNLV zone file, if none is specified by SetInputName.
- */
-static const char *kInputFile = "noname.tif";
 /**
  * Temp file used for storing current parameters before applying retry values.
  */
 static const char *kOldVarsFile = "failed_vars.txt";
 
 #ifndef DISABLED_LEGACY_ENGINE
+/**
+ * Filename used for input image file, from which to derive a name to search
+ * for a possible UNLV zone file, if none is specified by SetInputName.
+ */
+static const char *kInputFile = "noname.tif";
 static const char kUnknownFontName[] = "UnknownFont";
 
 static STRING_VAR(classify_font_name, kUnknownFontName,
@@ -377,9 +379,8 @@ int TessBaseAPI::Init(const char *data, int data_size, const char *language, Ocr
                       char **configs, int configs_size, const std::vector<std::string> *vars_vec,
                       const std::vector<std::string> *vars_values, bool set_only_non_debug_params,
                       FileReader reader) {
-  // Default language is "eng".
   if (language == nullptr) {
-    language = "eng";
+    language = "";
   }
   if (data == nullptr) {
     data = "";
@@ -420,7 +421,7 @@ int TessBaseAPI::Init(const char *data, int data_size, const char *language, Ocr
 
   // Update datapath and language requested for the last valid initialization.
   datapath_ = datapath;
-  if ((strcmp(datapath_.c_str(), "") == 0) && (strcmp(tesseract_->datadir.c_str(), "") != 0)) {
+  if (datapath_.empty() && !tesseract_->datadir.empty()) {
     datapath_ = tesseract_->datadir;
   }
 
@@ -475,25 +476,6 @@ void TessBaseAPI::GetAvailableLanguagesAsVector(std::vector<std::string> *langs)
   }
 }
 
-// TODO(amit): Adapt to lstm
-#ifndef DISABLED_LEGACY_ENGINE
-/**
- * Init only the lang model component of Tesseract. The only functions
- * that work after this init are SetVariable and IsValidWord.
- * WARNING: temporary! This function will be removed from here and placed
- * in a separate API at some future time.
- */
-int TessBaseAPI::InitLangMod(const char *datapath, const char *language) {
-  if (tesseract_ == nullptr) {
-    tesseract_ = new Tesseract;
-  } else {
-    ParamUtils::ResetToDefaults(tesseract_->params());
-  }
-  TessdataManager mgr;
-  return tesseract_->init_tesseract_lm(datapath, nullptr, language, &mgr);
-}
-#endif // ndef DISABLED_LEGACY_ENGINE
-
 /**
  * Init only for page layout analysis. Use only for calls to SetImage and
  * AnalysePage. Calls that attempt recognition will generate an error.
@@ -1143,7 +1125,7 @@ bool TessBaseAPI::ProcessPagesInternal(const char *filename, const char *retry_c
   if (stdInput) {
     buf.assign((std::istreambuf_iterator<char>(std::cin)), (std::istreambuf_iterator<char>()));
     data = reinterpret_cast<const l_uint8 *>(buf.data());
-  } else if (strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0) {
+  } else if (strstr(filename, "://") != nullptr) {
     // Get image or image list by URL.
 #ifdef HAVE_LIBCURL
     CURL *curl = curl_easy_init();
@@ -1161,6 +1143,27 @@ bool TessBaseAPI::ProcessPagesInternal(const char *filename, const char *retry_c
       if (curlcode != CURLE_OK) {
         return error("curl_easy_setopt");
       }
+      // Follow HTTP, HTTPS, FTP and FTPS redirects.
+      curlcode = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+      if (curlcode != CURLE_OK) {
+        return error("curl_easy_setopt");
+      }
+      // Allow no more than 8 redirections to prevent endless loops.
+      curlcode = curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 8);
+      if (curlcode != CURLE_OK) {
+        return error("curl_easy_setopt");
+      }
+      int timeout = curl_timeout;
+      if (timeout > 0) {
+        curlcode = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+        if (curlcode != CURLE_OK) {
+          return error("curl_easy_setopt");
+        }
+        curlcode = curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+        if (curlcode != CURLE_OK) {
+          return error("curl_easy_setopt");
+        }
+      }
       curlcode = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
       if (curlcode != CURLE_OK) {
         return error("curl_easy_setopt");
@@ -1277,7 +1280,13 @@ bool TessBaseAPI::ProcessPage(Pix *pix, int page_index, const char *filename,
 
   if (tesseract_->tessedit_write_images) {
     Pix *page_pix = GetThresholdedImage();
-    pixWrite("tessinput.tif", page_pix, IFF_TIFF_G4);
+    std::string output_filename = output_file_ + ".processed";
+    if (page_index > 0) {
+      output_filename += std::to_string(page_index);
+    }
+    output_filename += ".tif";
+    pixWrite(output_filename.c_str(), page_pix, IFF_TIFF_G4);
+    pixDestroy(&page_pix);
   }
 
   if (failed && retry_config != nullptr && retry_config[0] != '\0') {
@@ -1370,66 +1379,6 @@ char *TessBaseAPI::GetUTF8Text() {
   return result;
 }
 
-size_t TessBaseAPI::GetNumberOfTables() const
-{
-  return constUniqueInstance<std::vector<TessTable>>().size();
-}
-
-std::tuple<int,int,int,int> TessBaseAPI::GetTableBoundingBox(unsigned i)
-{
-  const auto &t = constUniqueInstance<std::vector<TessTable>>();
-
-  if (i >= t.size()) {
-    return std::tuple<int, int, int, int>(0, 0, 0, 0);
-  }
-
-  const int height = tesseract_->ImageHeight();
-
-  return std::make_tuple<int,int,int,int>(
-    t[i].box.left(), height - t[i].box.top(),
-    t[i].box.right(), height - t[i].box.bottom());
-}
-
-std::vector<std::tuple<int,int,int,int>> TessBaseAPI::GetTableRows(unsigned i)
-{
-  const auto &t = constUniqueInstance<std::vector<TessTable>>();
-
-  if (i >= t.size()) {
-    return std::vector<std::tuple<int, int, int, int>>();
-  }
-
-  std::vector<std::tuple<int,int,int,int>> rows(t[i].rows.size());
-  const int height = tesseract_->ImageHeight();
-
-  for (unsigned j = 0; j < t[i].rows.size(); ++j) {
-    rows[j] =
-        std::make_tuple<int, int, int, int>(t[i].rows[j].left(), height - t[i].rows[j].top(),
-                                            t[i].rows[j].right(), height - t[i].rows[j].bottom());
-  }
-
-  return rows;
-}
-
-std::vector<std::tuple<int,int,int,int> > TessBaseAPI::GetTableCols(unsigned i)
-{
-  const auto &t = constUniqueInstance<std::vector<TessTable>>();
-
-  if (i >= t.size()) {
-    return std::vector<std::tuple<int, int, int, int>>();
-  }
-
-  std::vector<std::tuple<int,int,int,int>> cols(t[i].cols.size());
-  const int height = tesseract_->ImageHeight();
-
-  for (unsigned j = 0; j < t[i].cols.size(); ++j) {
-    cols[j] =
-        std::make_tuple<int, int, int, int>(t[i].cols[j].left(), height - t[i].cols[j].top(),
-                                            t[i].cols[j].right(), height - t[i].cols[j].bottom());
-  }
-
-  return cols;
-}
-
 static void AddBoxToTSV(const PageIterator *it, PageIteratorLevel level, std::string &text) {
   int left, top, right, bottom;
   it->BoundingBox(level, &left, &top, &right, &bottom);
@@ -1966,15 +1915,17 @@ void TessBaseAPI::End() {
     delete paragraph_models_;
     paragraph_models_ = nullptr;
   }
+#ifndef DISABLED_LEGACY_ENGINE
   if (osd_tesseract_ == tesseract_) {
     osd_tesseract_ = nullptr;
   }
-  delete tesseract_;
-  tesseract_ = nullptr;
   delete osd_tesseract_;
   osd_tesseract_ = nullptr;
   delete equ_detect_;
   equ_detect_ = nullptr;
+#endif // ndef DISABLED_LEGACY_ENGINE
+  delete tesseract_;
+  tesseract_ = nullptr;
   input_file_.clear();
   output_file_.clear();
   datapath_.clear();
@@ -2127,7 +2078,7 @@ bool TessBaseAPI::Threshold(Pix **pix) {
       tesseract_->set_pix_grey(nullptr);
     }
   } else {
-    auto [ok, pix_grey, pix_binary, pix_thresholds] = thresholder_->Threshold(thresholding_method);
+    auto [ok, pix_grey, pix_binary, pix_thresholds] = thresholder_->Threshold(this, thresholding_method);
 
     if (!ok) {
       return false;
@@ -2196,6 +2147,7 @@ int TessBaseAPI::FindLines() {
 
   Tesseract *osd_tess = osd_tesseract_;
   OSResults osr;
+#ifndef DISABLED_LEGACY_ENGINE
   if (PSM_OSD_ENABLED(tesseract_->tessedit_pageseg_mode) && osd_tess == nullptr) {
     if (strcmp(language_.c_str(), "osd") == 0) {
       osd_tess = tesseract_;
@@ -2221,6 +2173,7 @@ int TessBaseAPI::FindLines() {
       }
     }
   }
+#endif // ndef DISABLED_LEGACY_ENGINE
 
   if (tesseract_->SegmentPage(input_file_.c_str(), block_list_, osd_tess, &osr) < 0) {
     return -1;
@@ -2252,8 +2205,6 @@ void TessBaseAPI::ClearResults() {
     delete paragraph_models_;
     paragraph_models_ = nullptr;
   }
-
-  uniqueInstance<std::vector<TessTable>>().clear();
 }
 
 /**

src/api/capi.cpp

@@ -254,12 +254,6 @@ char **TessBaseAPIGetAvailableLanguagesAsVector(const TessBaseAPI *handle) {
   return arr;
 }
 
-#ifndef DISABLED_LEGACY_ENGINE
-int TessBaseAPIInitLangMod(TessBaseAPI *handle, const char *datapath, const char *language) {
-  return handle->InitLangMod(datapath, language);
-}
-#endif
-
 void TessBaseAPIInitForAnalysePage(TessBaseAPI *handle) {
   handle->InitForAnalysePage();
 }

src/api/hocrrenderer.cpp

@@ -37,7 +37,8 @@ static tesseract::Orientation GetBlockTextOrientation(const PageIterator *it) {
   tesseract::WritingDirection writing_direction;
   tesseract::TextlineOrder textline_order;
   float deskew_angle;
-  it->Orientation(&orientation, &writing_direction, &textline_order, &deskew_angle);
+  it->Orientation(&orientation, &writing_direction, &textline_order,
+                  &deskew_angle);
   return orientation;
 }
 
@@ -49,7 +50,8 @@ static tesseract::Orientation GetBlockTextOrientation(const PageIterator *it) {
  * method currently only inserts a 'textangle' property to indicate the rotation
  * direction and does not add any baseline information to the hocr string.
  */
-static void AddBaselineCoordsTohOCR(const PageIterator *it, PageIteratorLevel level,
+static void AddBaselineCoordsTohOCR(const PageIterator *it,
+                                    PageIteratorLevel level,
                                     std::stringstream &hocr_str) {
   tesseract::Orientation orientation = GetBlockTextOrientation(it);
   if (orientation != ORIENTATION_PAGE_UP) {
@@ -82,7 +84,8 @@ static void AddBaselineCoordsTohOCR(const PageIterator *it, PageIteratorLevel le
   double p1 = (y2 - y1) / static_cast<double>(x2 - x1);
   double p0 = y1 - p1 * x1;
 
-  hocr_str << "; baseline " << round(p1 * 1000.0) / 1000.0 << " " << round(p0 * 1000.0) / 1000.0;
+  hocr_str << "; baseline " << round(p1 * 1000.0) / 1000.0 << " "
+           << round(p0 * 1000.0) / 1000.0;
 }
 
 static void AddBoxTohOCR(const ResultIterator *it, PageIteratorLevel level,
@@ -91,7 +94,8 @@ static void AddBoxTohOCR(const ResultIterator *it, PageIteratorLevel level,
   it->BoundingBox(level, &left, &top, &right, &bottom);
   // This is the only place we use double quotes instead of single quotes,
   // but it may too late to change for consistency
-  hocr_str << " title=\"bbox " << left << " " << top << " " << right << " " << bottom;
+  hocr_str << " title=\"bbox " << left << " " << top << " " << right << " "
+           << bottom;
   // Add baseline coordinates & heights for textlines only.
   if (level == RIL_TEXTLINE) {
     AddBaselineCoordsTohOCR(it, level, hocr_str);
@@ -99,8 +103,8 @@ static void AddBoxTohOCR(const ResultIterator *it, PageIteratorLevel level,
     float row_height, descenders, ascenders; // row attributes
     it->RowAttributes(&row_height, &descenders, &ascenders);
     // TODO(rays): Do we want to limit these to a single decimal place?
-    hocr_str << "; x_size " << row_height << "; x_descenders " << -descenders << "; x_ascenders "
-             << ascenders;
+    hocr_str << "; x_size " << row_height << "; x_descenders " << -descenders
+             << "; x_ascenders " << ascenders;
   }
   hocr_str << "\">";
 }
@@ -128,7 +132,8 @@ char *TessBaseAPI::GetHOCRText(int page_number) {
  * Returned string must be freed with the delete [] operator.
  */
 char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
-  if (tesseract_ == nullptr || (page_res_ == nullptr && Recognize(monitor) < 0)) {
+  if (tesseract_ == nullptr ||
+      (page_res_ == nullptr && Recognize(monitor) < 0)) {
     return nullptr;
   }
 
@@ -147,13 +152,16 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
 
 #ifdef _WIN32
   // convert input name from ANSI encoding to utf-8
-  int str16_len = MultiByteToWideChar(CP_ACP, 0, input_file_.c_str(), -1, nullptr, 0);
+  int str16_len =
+      MultiByteToWideChar(CP_ACP, 0, input_file_.c_str(), -1, nullptr, 0);
   wchar_t *uni16_str = new WCHAR[str16_len];
-  str16_len = MultiByteToWideChar(CP_ACP, 0, input_file_.c_str(), -1, uni16_str, str16_len);
-  int utf8_len =
-      WideCharToMultiByte(CP_UTF8, 0, uni16_str, str16_len, nullptr, 0, nullptr, nullptr);
+  str16_len = MultiByteToWideChar(CP_ACP, 0, input_file_.c_str(), -1, uni16_str,
+                                  str16_len);
+  int utf8_len = WideCharToMultiByte(CP_UTF8, 0, uni16_str, str16_len, nullptr,
+                                     0, nullptr, nullptr);
   char *utf8_str = new char[utf8_len];
-  WideCharToMultiByte(CP_UTF8, 0, uni16_str, str16_len, utf8_str, utf8_len, nullptr, nullptr);
+  WideCharToMultiByte(CP_UTF8, 0, uni16_str, str16_len, utf8_str, utf8_len,
+                      nullptr, nullptr);
   input_file_ = utf8_str;
   delete[] uni16_str;
   delete[] utf8_str;
@@ -173,8 +181,11 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
   } else {
     hocr_str << "unknown";
   }
-  hocr_str << "\"; bbox " << rect_left_ << " " << rect_top_ << " " << rect_width_ << " "
-           << rect_height_ << "; ppageno " << page_number << "'>\n";
+
+  hocr_str << "\"; bbox " << rect_left_ << " " << rect_top_ << " "
+           << rect_width_ << " " << rect_height_ << "; ppageno " << page_number
+           << "; scan_res " << GetSourceYResolution() << " "
+           << GetSourceYResolution() << "'>\n";
 
   std::unique_ptr<ResultIterator> res_it(GetIterator());
   while (!res_it->Empty(RIL_BLOCK)) {
@@ -227,7 +238,8 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
 
     // Now, process the word...
     int32_t lstm_choice_mode = tesseract_->lstm_choice_mode;
-    std::vector<std::vector<std::vector<std::pair<const char *, float>>>> *rawTimestepMap = nullptr;
+    std::vector<std::vector<std::vector<std::pair<const char *, float>>>>
+        *rawTimestepMap = nullptr;
     std::vector<std::vector<std::pair<const char *, float>>> *CTCMap = nullptr;
     if (lstm_choice_mode) {
       CTCMap = res_it->GetBestLSTMSymbolChoices();
@@ -241,10 +253,12 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
     int pointsize, font_id;
     const char *font_name;
     res_it->BoundingBox(RIL_WORD, &left, &top, &right, &bottom);
-    font_name = res_it->WordFontAttributes(&bold, &italic, &underlined, &monospace, &serif,
-                                           &smallcaps, &pointsize, &font_id);
-    hocr_str << " title='bbox " << left << " " << top << " " << right << " " << bottom
-             << "; x_wconf " << static_cast<int>(res_it->Confidence(RIL_WORD));
+    font_name =
+        res_it->WordFontAttributes(&bold, &italic, &underlined, &monospace,
+                                   &serif, &smallcaps, &pointsize, &font_id);
+    hocr_str << " title='bbox " << left << " " << top << " " << right << " "
+             << bottom << "; x_wconf "
+             << static_cast<int>(res_it->Confidence(RIL_WORD));
     if (font_info) {
       if (font_name) {
         hocr_str << "; x_font " << HOcrEscape(font_name).c_str();
@@ -284,31 +298,36 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
       hocr_str << "<em>";
     }
     do {
-      const std::unique_ptr<const char[]> grapheme(res_it->GetUTF8Text(RIL_SYMBOL));
+      const std::unique_ptr<const char[]> grapheme(
+          res_it->GetUTF8Text(RIL_SYMBOL));
       if (grapheme && grapheme[0] != 0) {
         if (hocr_boxes) {
           res_it->BoundingBox(RIL_SYMBOL, &left, &top, &right, &bottom);
-          hocr_str << "\n       <span class='ocrx_cinfo' title='x_bboxes " << left << " " << top
-                   << " " << right << " " << bottom << "; x_conf " << res_it->Confidence(RIL_SYMBOL)
-                   << "'>";
+          hocr_str << "\n       <span class='ocrx_cinfo' title='x_bboxes "
+                   << left << " " << top << " " << right << " " << bottom
+                   << "; x_conf " << res_it->Confidence(RIL_SYMBOL) << "'>";
         }
         hocr_str << HOcrEscape(grapheme.get()).c_str();
         if (hocr_boxes) {
           hocr_str << "</span>";
           tesseract::ChoiceIterator ci(*res_it);
           if (lstm_choice_mode == 1 && ci.Timesteps() != nullptr) {
-            std::vector<std::vector<std::pair<const char *, float>>> *symbol = ci.Timesteps();
+            std::vector<std::vector<std::pair<const char *, float>>> *symbol =
+                ci.Timesteps();
             hocr_str << "\n        <span class='ocr_symbol'"
                      << " id='"
-                     << "symbol_" << page_id << "_" << wcnt << "_" << scnt << "'>";
-            for (auto timestep : *symbol) {
+                     << "symbol_" << page_id << "_" << wcnt << "_" << scnt
+                     << "'>";
+            for (const auto &timestep : *symbol) {
               hocr_str << "\n         <span class='ocrx_cinfo'"
                        << " id='"
-                       << "timestep" << page_id << "_" << wcnt << "_" << tcnt << "'>";
+                       << "timestep" << page_id << "_" << wcnt << "_" << tcnt
+                       << "'>";
               for (auto conf : timestep) {
                 hocr_str << "\n          <span class='ocrx_cinfo'"
                          << " id='"
-                         << "choice_" << page_id << "_" << wcnt << "_" << ccnt << "'"
+                         << "choice_" << page_id << "_" << wcnt << "_" << ccnt
+                         << "'"
                          << " title='x_confs " << int(conf.second * 100) << "'>"
                          << HOcrEscape(conf.first).c_str() << "</span>";
                 ++ccnt;
@@ -319,19 +338,20 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
             hocr_str << "\n        </span>";
             ++scnt;
           } else if (lstm_choice_mode == 2) {
-            tesseract::ChoiceIterator ci(*res_it);
             hocr_str << "\n        <span class='ocrx_cinfo'"
                      << " id='"
-                     << "lstm_choices_" << page_id << "_" << wcnt << "_" << tcnt << "'>";
+                     << "lstm_choices_" << page_id << "_" << wcnt << "_" << tcnt
+                     << "'>";
             do {
               const char *choice = ci.GetUTF8Text();
               float choiceconf = ci.Confidence();
               if (choice != nullptr) {
                 hocr_str << "\n         <span class='ocrx_cinfo'"
                          << " id='"
-                         << "choice_" << page_id << "_" << wcnt << "_" << ccnt << "'"
-                         << " title='x_confs " << choiceconf << "'>" << HOcrEscape(choice).c_str()
-                         << "</span>";
+                         << "choice_" << page_id << "_" << wcnt << "_" << ccnt
+                         << "'"
+                         << " title='x_confs " << choiceconf << "'>"
+                         << HOcrEscape(choice).c_str() << "</span>";
                 ccnt++;
               }
             } while (ci.Next());
@@ -350,18 +370,20 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
     }
     // If the lstm choice mode is required it is added here
     if (lstm_choice_mode == 1 && !hocr_boxes && rawTimestepMap != nullptr) {
-      for (auto symbol : *rawTimestepMap) {
+      for (const auto &symbol : *rawTimestepMap) {
         hocr_str << "\n       <span class='ocr_symbol'"
                  << " id='"
                  << "symbol_" << page_id << "_" << wcnt << "_" << scnt << "'>";
-        for (auto timestep : symbol) {
+        for (const auto &timestep : symbol) {
           hocr_str << "\n        <span class='ocrx_cinfo'"
                    << " id='"
-                   << "timestep" << page_id << "_" << wcnt << "_" << tcnt << "'>";
+                   << "timestep" << page_id << "_" << wcnt << "_" << tcnt
+                   << "'>";
           for (auto conf : timestep) {
             hocr_str << "\n         <span class='ocrx_cinfo'"
                      << " id='"
-                     << "choice_" << page_id << "_" << wcnt << "_" << ccnt << "'"
+                     << "choice_" << page_id << "_" << wcnt << "_" << ccnt
+                     << "'"
                      << " title='x_confs " << int(conf.second * 100) << "'>"
                      << HOcrEscape(conf.first).c_str() << "</span>";
             ++ccnt;
@@ -373,11 +395,12 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
         ++scnt;
       }
     } else if (lstm_choice_mode == 2 && !hocr_boxes && CTCMap != nullptr) {
-      for (auto timestep : *CTCMap) {
+      for (const auto &timestep : *CTCMap) {
         if (timestep.size() > 0) {
           hocr_str << "\n       <span class='ocrx_cinfo'"
                    << " id='"
-                   << "lstm_choices_" << page_id << "_" << wcnt << "_" << tcnt << "'>";
+                   << "lstm_choices_" << page_id << "_" << wcnt << "_" << tcnt
+                   << "'>";
           for (auto &j : timestep) {
             float conf = 100 - tesseract_->lstm_rating_coefficient * j.second;
             if (conf < 0.0f) {
@@ -388,9 +411,10 @@ char *TessBaseAPI::GetHOCRText(ETEXT_DESC *monitor, int page_number) {
             }
             hocr_str << "\n        <span class='ocrx_cinfo'"
                      << " id='"
-                     << "choice_" << page_id << "_" << wcnt << "_" << ccnt << "'"
-                     << " title='x_confs " << conf << "'>" << HOcrEscape(j.first).c_str()
-                     << "</span>";
+                     << "choice_" << page_id << "_" << wcnt << "_" << ccnt
+                     << "'"
+                     << " title='x_confs " << conf << "'>"
+                     << HOcrEscape(j.first).c_str() << "</span>";
             ccnt++;
           }
           hocr_str << "</span>";

src/api/renderer.cpp

@@ -31,11 +31,11 @@ namespace tesseract {
  * Base Renderer interface implementation
  **********************************************************************/
 TessResultRenderer::TessResultRenderer(const char *outputbase, const char *extension)
-    : file_extension_(extension)
+    : next_(nullptr)
+    , fout_(stdout)
+    , file_extension_(extension)
     , title_("")
     , imagenum_(-1)
-    , fout_(stdout)
-    , next_(nullptr)
     , happy_(true) {
   if (strcmp(outputbase, "-") && strcmp(outputbase, "stdout")) {
     std::string outfile = std::string(outputbase) + "." + extension;
@@ -139,13 +139,13 @@ bool TessTextRenderer::AddImageHandler(TessBaseAPI *api) {
     return false;
   }
 
-  AppendString(utf8.get());
-
   const char *pageSeparator = api->GetStringVariable("page_separator");
-  if (pageSeparator != nullptr && *pageSeparator != '\0') {
+  if (pageSeparator != nullptr && *pageSeparator != '\0' && imagenum() > 0) {
     AppendString(pageSeparator);
   }
 
+  AppendString(utf8.get());
+
   return true;
 }
 

src/arch/dotproduct.cpp

@@ -1,5 +1,5 @@
 ///////////////////////////////////////////////////////////////////////
-// File:        dotproduct.h
+// File:        dotproduct.cpp
 // Description: Native dot product function.
 //
 // (C) Copyright 2018, Google Inc.
@@ -19,9 +19,12 @@
 namespace tesseract {
 
 // Computes and returns the dot product of the two n-vectors u and v.
-double DotProductNative(const double *u, const double *v, int n) {
-  double total = 0.0;
-  for (int k = 0; k < n; ++k) {
+TFloat DotProductNative(const TFloat *u, const TFloat *v, int n) {
+  TFloat total = 0;
+#if defined(OPENMP_SIMD) || defined(_OPENMP)
+#pragma omp simd reduction(+:total)
+#endif
+  for (int k = 0; k < n; k++) {
     total += u[k] * v[k];
   }
   return total;

src/arch/dotproduct.h

@@ -17,19 +17,24 @@
 #ifndef TESSERACT_ARCH_DOTPRODUCT_H_
 #define TESSERACT_ARCH_DOTPRODUCT_H_
 
+#include "tesstypes.h"
+
 namespace tesseract {
 
 // Computes and returns the dot product of the n-vectors u and v.
-double DotProductNative(const double *u, const double *v, int n);
+TFloat DotProductNative(const TFloat *u, const TFloat *v, int n);
 
 // Uses Intel AVX intrinsics to access the SIMD instruction set.
-double DotProductAVX(const double *u, const double *v, int n);
+TFloat DotProductAVX(const TFloat *u, const TFloat *v, int n);
 
 // Use Intel FMA.
-double DotProductFMA(const double *u, const double *v, int n);
+TFloat DotProductFMA(const TFloat *u, const TFloat *v, int n);
 
 // Uses Intel SSE intrinsics to access the SIMD instruction set.
-double DotProductSSE(const double *u, const double *v, int n);
+TFloat DotProductSSE(const TFloat *u, const TFloat *v, int n);
+
+// Use NEON intrinsics.
+TFloat DotProductNEON(const TFloat *u, const TFloat *v, int n);
 
 } // namespace tesseract.
 

src/arch/dotproductavx.cpp

@@ -29,6 +29,28 @@ namespace tesseract {
 
 // Computes and returns the dot product of the n-vectors u and v.
 // Uses Intel AVX intrinsics to access the SIMD instruction set.
+#if defined(FAST_FLOAT)
+float DotProductAVX(const float *u, const float *v, int n) {
+  const unsigned quot = n / 8;
+  const unsigned rem = n % 8;
+  __m256 t0 = _mm256_setzero_ps();
+  for (unsigned k = 0; k < quot; k++) {
+    __m256 f0 = _mm256_loadu_ps(u);
+    __m256 f1 = _mm256_loadu_ps(v);
+    f0 = _mm256_mul_ps(f0, f1);
+    t0 = _mm256_add_ps(t0, f0);
+    u += 8;
+    v += 8;
+  }
+  alignas(32) float tmp[8];
+  _mm256_store_ps(tmp, t0);
+  float result = tmp[0] + tmp[1] + tmp[2] + tmp[3] + tmp[4] + tmp[5] + tmp[6] + tmp[7];
+  for (unsigned k = 0; k < rem; k++) {
+    result += *u++ * *v++;
+  }
+  return result;
+}
+#else
 double DotProductAVX(const double *u, const double *v, int n) {
   const unsigned quot = n / 8;
   const unsigned rem = n % 8;
@@ -57,6 +79,7 @@ double DotProductAVX(const double *u, const double *v, int n) {
   }
   return result;
 }
+#endif
 
 } // namespace tesseract.
 

src/arch/dotproductfma.cpp

@@ -29,6 +29,34 @@ namespace tesseract {
 
 // Computes and returns the dot product of the n-vectors u and v.
 // Uses Intel FMA intrinsics to access the SIMD instruction set.
+#if defined(FAST_FLOAT)
+float DotProductFMA(const float *u, const float *v, int n) {
+  const unsigned quot = n / 16;
+  const unsigned rem = n % 16;
+  __m256 t0 = _mm256_setzero_ps();
+  __m256 t1 = _mm256_setzero_ps();
+  for (unsigned k = 0; k < quot; k++) {
+    __m256 f0 = _mm256_loadu_ps(u);
+    __m256 f1 = _mm256_loadu_ps(v);
+    t0 = _mm256_fmadd_ps(f0, f1, t0);
+    u += 8;
+    v += 8;
+    __m256 f2 = _mm256_loadu_ps(u);
+    __m256 f3 = _mm256_loadu_ps(v);
+    t1 = _mm256_fmadd_ps(f2, f3, t1);
+    u += 8;
+    v += 8;
+  }
+  t0 = _mm256_hadd_ps(t0, t1);
+  alignas(32) float tmp[8];
+  _mm256_store_ps(tmp, t0);
+  float result = tmp[0] + tmp[1] + tmp[2] + tmp[3] + tmp[4] + tmp[5] + tmp[6] + tmp[7];
+  for (unsigned k = 0; k < rem; k++) {
+    result += *u++ * *v++;
+  }
+  return result;
+}
+#else
 double DotProductFMA(const double *u, const double *v, int n) {
   const unsigned quot = n / 8;
   const unsigned rem = n % 8;
@@ -55,6 +83,7 @@ double DotProductFMA(const double *u, const double *v, int n) {
   }
   return result;
 }
+#endif
 
 } // namespace tesseract.
 

src/arch/dotproductneon.cpp

@@ -0,0 +1,71 @@
+///////////////////////////////////////////////////////////////////////
+// File:        dotproductneon.cpp
+// Description: Dot product function for ARM NEON.
+// Author:      Stefan Weil
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+///////////////////////////////////////////////////////////////////////
+
+#if defined(__ARM_NEON)
+
+#include <arm_neon.h>
+#include "dotproduct.h"
+
+namespace tesseract {
+
+// Documentation:
+// https://developer.arm.com/architectures/instruction-sets/intrinsics/
+
+#if defined(FAST_FLOAT) && defined(__ARM_ARCH_ISA_A64)
+
+float DotProductNEON(const float *u, const float *v, int n) {
+  float32x4_t result0123 = vdupq_n_f32(0.0f);
+  float32x4_t result4567 = vdupq_n_f32(0.0f);
+  while (n > 7) {
+    // Calculate 8 dot products per iteration.
+    float32x4_t u0 = vld1q_f32(u);
+    float32x4_t v0 = vld1q_f32(v);
+    float32x4_t u4 = vld1q_f32(u + 4);
+    float32x4_t v4 = vld1q_f32(v + 4);
+    result0123 = vfmaq_f32(result0123, u0, v0);
+    result4567 = vfmaq_f32(result4567, u4, v4);
+    u += 8;
+    v += 8;
+    n -= 8;
+  }
+  float total = vaddvq_f32(result0123);
+  total += vaddvq_f32(result4567);
+  while (n > 0) {
+    total += *u++ * *v++;
+    n--;
+  }
+  return total;
+}
+
+#else
+
+// Computes and returns the dot product of the two n-vectors u and v.
+TFloat DotProductNEON(const TFloat *u, const TFloat *v, int n) {
+  TFloat total = 0;
+#if defined(OPENMP_SIMD) || defined(_OPENMP)
+#pragma omp simd reduction(+:total)
+#endif
+  for (int k = 0; k < n; k++) {
+    total += u[k] * v[k];
+  }
+  return total;
+}
+
+#endif
+
+} // namespace tesseract
+
+#endif /* __ARM_NEON */

src/arch/dotproductsse.cpp

@@ -30,6 +30,66 @@ namespace tesseract {
 
 // Computes and returns the dot product of the n-vectors u and v.
 // Uses Intel SSE intrinsics to access the SIMD instruction set.
+#if defined(FAST_FLOAT)
+float DotProductSSE(const float *u, const float *v, int n) {
+  int max_offset = n - 4;
+  int offset = 0;
+  // Accumulate a set of 4 sums in sum, by loading pairs of 4 values from u and
+  // v, and multiplying them together in parallel.
+  __m128 sum = _mm_setzero_ps();
+  if (offset <= max_offset) {
+    offset = 4;
+    // Aligned load is reputedly faster but requires 16 byte aligned input.
+    if ((reinterpret_cast<uintptr_t>(u) & 15) == 0 &&
+        (reinterpret_cast<uintptr_t>(v) & 15) == 0) {
+      // Use aligned load.
+      sum = _mm_load_ps(u);
+      __m128 floats2 = _mm_load_ps(v);
+      // Multiply.
+      sum = _mm_mul_ps(sum, floats2);
+      while (offset <= max_offset) {
+        __m128 floats1 = _mm_load_ps(u + offset);
+        floats2 = _mm_load_ps(v + offset);
+        floats1 = _mm_mul_ps(floats1, floats2);
+        sum = _mm_add_ps(sum, floats1);
+        offset += 4;
+      }
+    } else {
+      // Use unaligned load.
+      sum = _mm_loadu_ps(u);
+      __m128 floats2 = _mm_loadu_ps(v);
+      // Multiply.
+      sum = _mm_mul_ps(sum, floats2);
+      while (offset <= max_offset) {
+        __m128 floats1 = _mm_loadu_ps(u + offset);
+        floats2 = _mm_loadu_ps(v + offset);
+        floats1 = _mm_mul_ps(floats1, floats2);
+        sum = _mm_add_ps(sum, floats1);
+        offset += 4;
+      }
+    }
+  }
+  // Add the 4 sums in sum horizontally.
+#if 0
+	alignas(32) float tmp[4];
+	_mm_store_ps(tmp, sum);
+	float result = tmp[0] + tmp[1] + tmp[2] + tmp[3];
+#else
+  __m128 zero = _mm_setzero_ps();
+  // https://www.felixcloutier.com/x86/haddps
+  sum = _mm_hadd_ps(sum, zero);
+  sum = _mm_hadd_ps(sum, zero);
+  // Extract the low result.
+  float result = _mm_cvtss_f32(sum);
+#endif
+  // Add on any left-over products.
+  while (offset < n) {
+    result += u[offset] * v[offset];
+    ++offset;
+  }
+  return result;
+}
+#else
 double DotProductSSE(const double *u, const double *v, int n) {
   int max_offset = n - 2;
   int offset = 0;
@@ -39,7 +99,8 @@ double DotProductSSE(const double *u, const double *v, int n) {
   if (offset <= max_offset) {
     offset = 2;
     // Aligned load is reputedly faster but requires 16 byte aligned input.
-    if ((reinterpret_cast<uintptr_t>(u) & 15) == 0 && (reinterpret_cast<uintptr_t>(v) & 15) == 0) {
+    if ((reinterpret_cast<uintptr_t>(u) & 15) == 0 &&
+        (reinterpret_cast<uintptr_t>(v) & 15) == 0) {
       // Use aligned load.
       sum = _mm_load_pd(u);
       __m128d floats2 = _mm_load_pd(v);
@@ -78,6 +139,7 @@ double DotProductSSE(const double *u, const double *v, int n) {
   }
   return result;
 }
+#endif
 
 } // namespace tesseract.
 

src/arch/intsimdmatrix.cpp

@@ -76,7 +76,7 @@ void IntSimdMatrix::Init(const GENERIC_2D_ARRAY<int8_t> &w, std::vector<int8_t>
 // u is imagined to have an extra element at the end with value 1, to
 // implement the bias, but it doesn't actually have it.
 void IntSimdMatrix::MatrixDotVector(const GENERIC_2D_ARRAY<int8_t> &w,
-                                    const std::vector<double> &scales, const int8_t *u, double *v) {
+                                    const std::vector<TFloat> &scales, const int8_t *u, TFloat *v) {
   int num_out = w.dim1();
   int num_in = w.dim2() - 1;
   // Base implementation.

src/arch/intsimdmatrix.h

@@ -23,6 +23,8 @@
 #include <cstdint>
 #include <vector>
 
+#include "tesstypes.h"
+
 namespace tesseract {
 
 template <class T>
@@ -78,8 +80,8 @@ struct TESS_API IntSimdMatrix {
   // u is imagined to have an extra element at the end with value 1, to
   // implement the bias, but it doesn't actually have it.
   // Computes the base C++ implementation.
-  static void MatrixDotVector(const GENERIC_2D_ARRAY<int8_t> &w, const std::vector<double> &scales,
-                              const int8_t *u, double *v);
+  static void MatrixDotVector(const GENERIC_2D_ARRAY<int8_t> &w, const std::vector<TFloat> &scales,
+                              const int8_t *u, TFloat *v);
 
   // Rounds the input up to a multiple of the given factor.
   static int Roundup(int input, int factor) {
@@ -95,8 +97,8 @@ struct TESS_API IntSimdMatrix {
   // RoundInputs above.
   // The input will be over-read to the extent of the padding. There are no
   // alignment requirements.
-  using MatrixDotVectorFunction = void (*)(int, int, const int8_t *, const double *, const int8_t *,
-                                           double *);
+  using MatrixDotVectorFunction = void (*)(int, int, const int8_t *, const TFloat *, const int8_t *,
+                                           TFloat *);
   MatrixDotVectorFunction matrixDotVectorFunction;
 
   // Number of 32 bit outputs held in each register.
@@ -113,7 +115,7 @@ struct TESS_API IntSimdMatrix {
   static const IntSimdMatrix *intSimdMatrix;
   // Only available with NEON.
   static const IntSimdMatrix intSimdMatrixNEON;
-  // Only available with AVX2 / SSE.
+  // Only available with AVX2 / AVX / FMA / SSE.
   static const IntSimdMatrix intSimdMatrixAVX2;
   static const IntSimdMatrix intSimdMatrixSSE;
 };

src/arch/intsimdmatrixavx2.cpp

@@ -15,14 +15,13 @@
 // limitations under the License.
 ///////////////////////////////////////////////////////////////////////
 
+#include "intsimdmatrix.h"
+
 #if !defined(__AVX2__)
 #  if defined(__i686__) || defined(__x86_64__)
 #    error Implementation only for AVX2 capable architectures
 #  endif
 #else
-
-#  include "intsimdmatrix.h"
-
 #  include <immintrin.h>
 #  include <algorithm>
 #  include <cstdint>
@@ -86,6 +85,243 @@ static inline __m128i load64_to_128(const int8_t *wi_) {
   return _mm_set_epi64x(0, wi[0]);
 }
 
+#if defined(FAST_FLOAT)
+
+static inline void ExtractResults8(__m256i result, const int8_t *wi,
+                                   const float *scales, float *v) {
+  __m128i w128 = load64_to_128(wi); // 8x8bit vals in bottom of 128bit reg
+  __m256i w256 = _mm256_cvtepi8_epi32(w128); // 8x32bit vals in 256bit reg
+  __m256i bias_scale = _mm256_set_epi32(127, 127, 127, 127, 127, 127, 127, 127);
+  __m256 scale01234567 = _mm256_loadu_ps(scales);
+  w256 = _mm256_mullo_epi32(w256, bias_scale); // 8x32 <bias * 127>
+  result = _mm256_add_epi32(result, w256);     // result += bias * 127
+  __m256 res01234567 = _mm256_cvtepi32_ps(result);
+  result = _mm256_permute4x64_epi64(result, 2 + (3 << 2));
+  res01234567 = _mm256_mul_ps(res01234567, scale01234567);
+  _mm256_storeu_ps(v, res01234567);
+}
+
+static inline void ExtractResults16(__m256i result0, __m256i result1,
+                                    const int8_t *&wi, const float *&scales,
+                                    float *&v) {
+  __m128i w8 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(wi));
+  // 8x8bit vals in bottom of 128bit reg
+  const __m256i bias_scale =
+      _mm256_set_epi32(127, 127, 127, 127, 127, 127, 127, 127);
+  __m256i w256 = _mm256_cvtepi8_epi32(w8); // 8x32bit vals in 256bit reg
+  __m256 scale01234567 = _mm256_loadu_ps(scales);
+  w256 = _mm256_mullo_epi32(w256, bias_scale); // 8x32 <bias * 127>
+  result0 = _mm256_add_epi32(result0, w256);   // result += bias * 127
+  __m256 res01234567 = _mm256_cvtepi32_ps(result0);
+  result0 = _mm256_permute4x64_epi64(result0, 2 + (3 << 2));
+  res01234567 = _mm256_mul_ps(res01234567, scale01234567);
+  _mm256_storeu_ps(v, res01234567);
+  w8 = _mm_shuffle_epi32(w8, 2 + (3 << 2));
+  w256 = _mm256_cvtepi8_epi32(w8); // 8x32bit vals in 256bit reg
+  scale01234567 = _mm256_loadu_ps(scales + 8);
+  w256 = _mm256_mullo_epi32(w256, bias_scale); // 8x32 <bias * 127>
+  result1 = _mm256_add_epi32(result1, w256);   // result += bias * 127
+  res01234567 = _mm256_cvtepi32_ps(result1);
+  result1 = _mm256_permute4x64_epi64(result1, 2 + (3 << 2));
+  res01234567 = _mm256_mul_ps(res01234567, scale01234567);
+  _mm256_storeu_ps(v + 8, res01234567);
+  wi += 16;
+  scales += 16;
+  v += 16;
+}
+
+// Computes part of matrix.vector v = Wu. Computes N=64 results.
+// The weights *must* be arranged so that consecutive reads from wi
+// provides (num_in/kNumInputsPerGroup groups of (N output dim groups of
+// (kNumInputsPerGroup inputs))). After that there must be N consecutive
+// bias weights, before continuing with any more weights.
+// u must be padded out with zeros to
+// kNumInputsPerGroup*ceil(num_in/kNumInputsPerGroup) elements.
+static void PartialMatrixDotVector64(const int8_t *wi, const float *scales, const int8_t *u,
+                                     int num_in, float *v) {
+  // Register containing 16-bit ones for horizontal add with 16->32 bit
+  // conversion.
+  __m256i ones = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+  __m256i shift_id = _mm256_set_epi32(0, 7, 6, 5, 4, 3, 2, 1);
+  // Initialize all the results to 0.
+  __m256i result0 = _mm256_setzero_si256();
+  __m256i result1 = _mm256_setzero_si256();
+  __m256i result2 = _mm256_setzero_si256();
+  __m256i result3 = _mm256_setzero_si256();
+  __m256i result4 = _mm256_setzero_si256();
+  __m256i result5 = _mm256_setzero_si256();
+  __m256i result6 = _mm256_setzero_si256();
+  __m256i result7 = _mm256_setzero_si256();
+  // Iterate over the input (u), one registerful at a time.
+  for (int j = 0; j < num_in;) {
+    __m256i inputs = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(u + j));
+    // Inputs are processed in groups of kNumInputsPerGroup, replicated
+    // kNumInputGroups times.
+    for (int ig = 0; ig < kNumInputGroups && j < num_in; ++ig, j += kNumInputsPerGroup) {
+      // Replicate the low 32 bits (4 inputs) 8 times.
+      __m256i rep_input = _mm256_broadcastd_epi32(_mm256_castsi256_si128(inputs));
+      // Rotate the inputs in groups of 4, so the next 4 inputs are ready.
+      inputs = _mm256_permutevar8x32_epi32(inputs, shift_id);
+      __m256i weights, reps;
+      // Mul-add, with horizontal add of the 4 inputs to each of the results.
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result0);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result1);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result2);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result3);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result4);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result5);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result6);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result7);
+    }
+  }
+  ExtractResults16(result0, result1, wi, scales, v);
+  ExtractResults16(result2, result3, wi, scales, v);
+  ExtractResults16(result4, result5, wi, scales, v);
+  ExtractResults16(result6, result7, wi, scales, v);
+}
+
+// Computes part of matrix.vector v = Wu. Computes N=32 results.
+// For details see PartialMatrixDotVector64 with N=32.
+static void PartialMatrixDotVector32(const int8_t *wi, const float *scales, const int8_t *u,
+                                     int num_in, float *v) {
+  // Register containing 16-bit ones for horizontal add with 16->32 bit
+  // conversion.
+  __m256i ones = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+  __m256i shift_id = _mm256_set_epi32(0, 7, 6, 5, 4, 3, 2, 1);
+  // Initialize all the results to 0.
+  __m256i result0 = _mm256_setzero_si256();
+  __m256i result1 = _mm256_setzero_si256();
+  __m256i result2 = _mm256_setzero_si256();
+  __m256i result3 = _mm256_setzero_si256();
+  // Iterate over the input (u), one registerful at a time.
+  for (int j = 0; j < num_in;) {
+    __m256i inputs = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(u + j));
+    // Inputs are processed in groups of kNumInputsPerGroup, replicated
+    // kNumInputGroups times.
+    for (int ig = 0; ig < kNumInputGroups && j < num_in; ++ig, j += kNumInputsPerGroup) {
+      // Replicate the low 32 bits (4 inputs) 8 times.
+      __m256i rep_input = _mm256_broadcastd_epi32(_mm256_castsi256_si128(inputs));
+      // Rotate the inputs in groups of 4, so the next 4 inputs are ready.
+      inputs = _mm256_permutevar8x32_epi32(inputs, shift_id);
+      __m256i weights, reps;
+      // Mul-add, with horizontal add of the 4 inputs to each of the results.
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result0);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result1);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result2);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result3);
+    }
+  }
+  ExtractResults16(result0, result1, wi, scales, v);
+  ExtractResults16(result2, result3, wi, scales, v);
+}
+
+// Computes part of matrix.vector v = Wu. Computes N=16 results.
+// For details see PartialMatrixDotVector64 with N=16.
+static void PartialMatrixDotVector16(const int8_t *wi, const float *scales, const int8_t *u,
+                                     int num_in, float *v) {
+  // Register containing 16-bit ones for horizontal add with 16->32 bit
+  // conversion.
+  __m256i ones = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+  __m256i shift_id = _mm256_set_epi32(0, 7, 6, 5, 4, 3, 2, 1);
+  // Initialize all the results to 0.
+  __m256i result0 = _mm256_setzero_si256();
+  __m256i result1 = _mm256_setzero_si256();
+  // Iterate over the input (u), one registerful at a time.
+  for (int j = 0; j < num_in;) {
+    __m256i inputs = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(u + j));
+    // Inputs are processed in groups of kNumInputsPerGroup, replicated
+    // kNumInputGroups times.
+    for (int ig = 0; ig < kNumInputGroups && j < num_in; ++ig, j += kNumInputsPerGroup) {
+      // Replicate the low 32 bits (4 inputs) 8 times.
+      __m256i rep_input = _mm256_broadcastd_epi32(_mm256_castsi256_si128(inputs));
+      // Rotate the inputs in groups of 4, so the next 4 inputs are ready.
+      inputs = _mm256_permutevar8x32_epi32(inputs, shift_id);
+      __m256i weights, reps;
+      // Mul-add, with horizontal add of the 4 inputs to each of the results.
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result0);
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result1);
+    }
+  }
+  ExtractResults16(result0, result1, wi, scales, v);
+}
+
+// Computes part of matrix.vector v = Wu. Computes N=8 results.
+// For details see PartialMatrixDotVector64 with N=8.
+static inline void PartialMatrixDotVector8(const int8_t *wi, const float *scales, const int8_t *u,
+                                           int num_in, float *v) {
+  // Register containing 16-bit ones for horizontal add with 16->32 bit
+  // conversion.
+  __m256i ones = _mm256_set_epi16(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+  __m256i shift_id = _mm256_set_epi32(0, 7, 6, 5, 4, 3, 2, 1);
+  // Initialize all the results to 0.
+  __m256i result0 = _mm256_setzero_si256();
+  // Iterate over the input (u), one registerful at a time.
+  for (int j = 0; j < num_in;) {
+    __m256i inputs = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(u + j));
+    // Inputs are processed in groups of kNumInputsPerGroup, replicated
+    // kNumInputGroups times.
+    for (int ig = 0; ig < kNumInputGroups && j < num_in; ++ig, j += kNumInputsPerGroup) {
+      // Replicate the low 32 bits (4 inputs) 8 times.
+      __m256i rep_input = _mm256_broadcastd_epi32(_mm256_castsi256_si128(inputs));
+      // Rotate the inputs in groups of 4, so the next 4 inputs are ready.
+      inputs = _mm256_permutevar8x32_epi32(inputs, shift_id);
+      __m256i weights, reps;
+      // Mul-add, with horizontal add of the 4 inputs to each of the results.
+      MultiplyGroup(rep_input, ones, wi, weights, reps, result0);
+    }
+  }
+  ExtractResults8(result0, wi, scales, v);
+}
+
+static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const float *scales,
+                            const int8_t *u, float *v) {
+  const int num_out = dim1;
+  const int num_in = dim2 - 1;
+  // Each call to a partial_func_ produces group_size outputs, except the
+  // last one, which can produce less.
+  const int rounded_num_in = IntSimdMatrix::Roundup(num_in, kNumInputsPerGroup);
+  const int rounded_num_out = IntSimdMatrix::Roundup(num_out, kNumOutputsPerRegister);
+  int group_size = kNumOutputsPerRegister * kMaxOutputRegisters;
+  int output = 0;
+
+  int w_step = (rounded_num_in + 1) * group_size;
+
+  // Run with this group size, until it would produce too much output, then
+  // switch to a smaller size.
+  for (; output + group_size <= rounded_num_out; output += group_size) {
+    PartialMatrixDotVector64(wi, scales, u, rounded_num_in, v);
+    wi += w_step;
+    scales += group_size;
+    v += group_size;
+  }
+  group_size /= 2;
+  w_step /= 2;
+
+  if (output + group_size <= rounded_num_out) {
+    PartialMatrixDotVector32(wi, scales, u, rounded_num_in, v);
+    wi += w_step;
+    scales += group_size;
+    v += group_size;
+    output += group_size;
+  }
+  group_size /= 2;
+  w_step /= 2;
+
+  if (output + group_size <= rounded_num_out) {
+    PartialMatrixDotVector16(wi, scales, u, rounded_num_in, v);
+    wi += w_step;
+    scales += group_size;
+    v += group_size;
+    output += group_size;
+  }
+  group_size /= 2;
+  w_step /= 2;
+
+  if (output + group_size <= rounded_num_out) {
+    PartialMatrixDotVector8(wi, scales, u, rounded_num_in, v);
+  }
+}
+#else
 static inline void ExtractResults8(__m256i result, const int8_t *wi, const double *scales,
                                    double *v) {
   __m128i w128 = load64_to_128(wi);          // 8x8bit vals in bottom of 128bit reg
@@ -330,6 +566,7 @@ static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const double *
     PartialMatrixDotVector8(wi, scales, u, rounded_num_in, v);
   }
 }
+#endif
 
 const IntSimdMatrix IntSimdMatrix::intSimdMatrixAVX2 = {
     // Function.
@@ -341,7 +578,8 @@ const IntSimdMatrix IntSimdMatrix::intSimdMatrixAVX2 = {
     // Number of 8 bit inputs in the inputs register.
     kNumInputsPerRegister,
     // Number of inputs in each weight group.
-    kNumInputsPerGroup};
+    kNumInputsPerGroup
+};
 
 } // namespace tesseract.
 

src/arch/intsimdmatrixneon.cpp

@@ -19,6 +19,7 @@
 #if defined(__ARM_NEON)
 
 #  include "intsimdmatrix.h"
+#  include "tesstypes.h"
 
 #  include <algorithm>
 #  include <cstdint>
@@ -52,9 +53,9 @@ constexpr int kNumInputsPerGroup = 8;
 // u must be padded out with zeros to
 // kNumInputsPerGroup*ceil(num_in/kNumInputsPerGroup) elements.
 static inline void PartialMatrixDotVector8(const int8_t *__restrict wi,
-                                           const double *__restrict scales,
+                                           const TFloat *__restrict scales,
                                            const int8_t *__restrict u, int num_in,
-                                           double *__restrict v, int num_out) {
+                                           TFloat *__restrict v, int num_out) {
   // Initialize all the results to 0.
   int32x4_t result0123 = {0, 0, 0, 0};
   int32x4_t result4567 = {0, 0, 0, 0};
@@ -163,8 +164,8 @@ static inline void PartialMatrixDotVector8(const int8_t *__restrict wi,
   }
 }
 
-static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const double *scales,
-                            const int8_t *u, double *v) {
+static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const TFloat *scales,
+                            const int8_t *u, TFloat *v) {
   const int num_out = dim1;
   const int num_in = dim2 - 1;
   // Each call to a partial_func_ produces group_size outputs, except the
@@ -196,7 +197,8 @@ const IntSimdMatrix IntSimdMatrix::intSimdMatrixNEON = {
     // Number of 8 bit inputs in the inputs register.
     kNumInputsPerRegister,
     // Number of inputs in each weight group.
-    kNumInputsPerGroup};
+    kNumInputsPerGroup
+};
 
 } // namespace tesseract.
 

src/arch/intsimdmatrixsse.cpp

@@ -69,15 +69,15 @@ static int32_t IntDotProductSSE(const int8_t *u, const int8_t *v, int n) {
 }
 
 // Computes part of matrix.vector v = Wu. Computes 1 result.
-static void PartialMatrixDotVector1(const int8_t *wi, const double *scales, const int8_t *u,
-                                    int num_in, double *v) {
-  double total = IntDotProductSSE(u, wi, num_in);
+static void PartialMatrixDotVector1(const int8_t *wi, const TFloat *scales, const int8_t *u,
+                                    int num_in, TFloat *v) {
+  TFloat total = IntDotProductSSE(u, wi, num_in);
   // Add in the bias and correct for integer values.
   *v = (total + wi[num_in] * INT8_MAX) * *scales;
 }
 
-static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const double *scales,
-                            const int8_t *u, double *v) {
+static void matrixDotVector(int dim1, int dim2, const int8_t *wi, const TFloat *scales,
+                            const int8_t *u, TFloat *v) {
   const int num_out = dim1;
   const int num_in = dim2 - 1;
   int output = 0;
@@ -99,7 +99,8 @@ const IntSimdMatrix IntSimdMatrix::intSimdMatrixSSE = {
     // Number of 8 bit inputs in the inputs register.
     1,
     // Number of inputs in each weight group.
-    1};
+    1
+};
 
 } // namespace tesseract.
 

src/arch/simddetect.cpp

@@ -25,6 +25,21 @@
 #include "simddetect.h"
 #include "tprintf.h" // for tprintf
 
+#if !defined(__clang__) && defined(__GNUC__) && (__GNUC__ < 12)
+// The GNU compiler g++ fails to compile with the Accelerate framework
+// (tested with versions 10 and 11), so unconditionally disable it.
+#undef HAVE_FRAMEWORK_ACCELERATE
+#endif
+
+#if defined(HAVE_FRAMEWORK_ACCELERATE)
+
+// Use Apple Accelerate framework.
+// https://developer.apple.com/documentation/accelerate/simd
+
+#include <Accelerate/Accelerate.h>
+
+#endif
+
 #if defined(HAVE_AVX) || defined(HAVE_AVX2) || defined(HAVE_FMA) || defined(HAVE_SSE4_1)
 #  define HAS_CPUID
 #endif
@@ -83,9 +98,22 @@ bool SIMDDetect::fma_available_;
 bool SIMDDetect::sse_available_;
 #endif
 
+#if defined(HAVE_FRAMEWORK_ACCELERATE)
+static TFloat DotProductAccelerate(const TFloat* u, const TFloat* v, int n) {
+  TFloat total = 0;
+  const int stride = 1;
+#if defined(FAST_FLOAT)
+  vDSP_dotpr(u, stride, v, stride, &total, n);
+#else
+  vDSP_dotprD(u, stride, v, stride, &total, n);
+#endif
+  return total;
+}
+#endif
+
 // Computes and returns the dot product of the two n-vectors u and v.
-static double DotProductGeneric(const double *u, const double *v, int n) {
-  double total = 0.0;
+static TFloat DotProductGeneric(const TFloat *u, const TFloat *v, int n) {
+  TFloat total = 0;
   for (int k = 0; k < n; ++k) {
     total += u[k] * v[k];
   }
@@ -93,8 +121,8 @@ static double DotProductGeneric(const double *u, const double *v, int n) {
 }
 
 // Compute dot product using std::inner_product.
-static double DotProductStdInnerProduct(const double *u, const double *v, int n) {
-  return std::inner_product(u, u + n, v, 0.0);
+static TFloat DotProductStdInnerProduct(const TFloat *u, const TFloat *v, int n) {
+  return std::inner_product(u, u + n, v, static_cast<TFloat>(0));
 }
 
 static void SetDotProduct(DotProductFunction f, const IntSimdMatrix *m = nullptr) {
@@ -215,64 +243,90 @@ SIMDDetect::SIMDDetect() {
 #if defined(HAVE_NEON) || defined(__aarch64__)
   } else if (neon_available_) {
     // NEON detected.
-    SetDotProduct(DotProduct, &IntSimdMatrix::intSimdMatrixNEON);
+    SetDotProduct(DotProductNEON, &IntSimdMatrix::intSimdMatrixNEON);
 #endif
   }
+
+  const char *dotproduct_env = getenv("DOTPRODUCT");
+  if (dotproduct_env != nullptr) {
+    // Override automatic settings by value from environment variable.
+    dotproduct = dotproduct_env;
+    Update();
+  }
 }
 
 void SIMDDetect::Update() {
   // Select code for calculation of dot product based on the
   // value of the config variable if that value is not empty.
   const char *dotproduct_method = "generic";
-  if (!strcmp(dotproduct.c_str(), "auto")) {
+  if (dotproduct == "auto") {
     // Automatic detection. Nothing to be done.
-  } else if (!strcmp(dotproduct.c_str(), "generic")) {
+  } else if (dotproduct == "generic") {
     // Generic code selected by config variable.
     SetDotProduct(DotProductGeneric);
     dotproduct_method = "generic";
-  } else if (!strcmp(dotproduct.c_str(), "native")) {
+  } else if (dotproduct == "native") {
     // Native optimized code selected by config variable.
-    SetDotProduct(DotProductNative);
+    SetDotProduct(DotProductNative, IntSimdMatrix::intSimdMatrix);
     dotproduct_method = "native";
 #if defined(HAVE_AVX2)
-  } else if (!strcmp(dotproduct.c_str(), "avx2")) {
+  } else if (dotproduct == "avx2") {
     // AVX2 selected by config variable.
     SetDotProduct(DotProductAVX, &IntSimdMatrix::intSimdMatrixAVX2);
     dotproduct_method = "avx2";
 #endif
 #if defined(HAVE_AVX)
-  } else if (!strcmp(dotproduct.c_str(), "avx")) {
+  } else if (dotproduct == "avx") {
     // AVX selected by config variable.
     SetDotProduct(DotProductAVX, &IntSimdMatrix::intSimdMatrixSSE);
     dotproduct_method = "avx";
 #endif
 #if defined(HAVE_FMA)
-  } else if (!strcmp(dotproduct.c_str(), "fma")) {
+  } else if (dotproduct == "fma") {
     // FMA selected by config variable.
     SetDotProduct(DotProductFMA, IntSimdMatrix::intSimdMatrix);
     dotproduct_method = "fma";
 #endif
 #if defined(HAVE_SSE4_1)
-  } else if (!strcmp(dotproduct.c_str(), "sse")) {
+  } else if (dotproduct == "sse") {
     // SSE selected by config variable.
     SetDotProduct(DotProductSSE, &IntSimdMatrix::intSimdMatrixSSE);
     dotproduct_method = "sse";
 #endif
-  } else if (!strcmp(dotproduct.c_str(), "std::inner_product")) {
+#if defined(HAVE_FRAMEWORK_ACCELERATE)
+  } else if (dotproduct == "accelerate") {
+    SetDotProduct(DotProductAccelerate, IntSimdMatrix::intSimdMatrix);
+#endif
+#if defined(HAVE_NEON) || defined(__aarch64__)
+  } else if (dotproduct == "neon" && neon_available_) {
+    // NEON selected by config variable.
+    SetDotProduct(DotProductNEON, &IntSimdMatrix::intSimdMatrixNEON);
+    dotproduct_method = "neon";
+#endif
+  } else if (dotproduct == "std::inner_product") {
     // std::inner_product selected by config variable.
-    SetDotProduct(DotProductStdInnerProduct);
+    SetDotProduct(DotProductStdInnerProduct, IntSimdMatrix::intSimdMatrix);
     dotproduct_method = "std::inner_product";
   } else {
     // Unsupported value of config variable.
     tprintf("Warning, ignoring unsupported config variable value: dotproduct=%s\n",
             dotproduct.c_str());
     tprintf(
-        "Support values for dotproduct: auto generic native"
+        "Supported values for dotproduct: auto generic native"
+#if defined(HAVE_AVX2)
+        " avx2"
+#endif
 #if defined(HAVE_AVX)
         " avx"
 #endif
+#if defined(HAVE_FMA)
+        " fma"
+#endif
 #if defined(HAVE_SSE4_1)
         " sse"
+#endif
+#if defined(HAVE_FRAMEWORK_ACCELERATE)
+        " accelerate"
 #endif
         " std::inner_product.\n");
   }

src/arch/simddetect.h

@@ -18,11 +18,12 @@
 #define TESSERACT_ARCH_SIMDDETECT_H_
 
 #include <tesseract/export.h>
+#include "tesstypes.h"
 
 namespace tesseract {
 
 // Function pointer for best calculation of dot product.
-using DotProductFunction = double (*)(const double *, const double *, int);
+using DotProductFunction = TFloat (*)(const TFloat *, const TFloat *, int);
 extern DotProductFunction DotProduct;
 
 // Architecture detector. Add code here to detect any other architectures for

src/ccmain/applybox.cpp

@@ -243,7 +243,7 @@ void Tesseract::MaximallyChopWord(const std::vector<TBOX> &boxes, BLOCK *block,
   std::vector<BLOB_CHOICE *> blob_choices;
   ASSERT_HOST(!word_res->chopped_word->blobs.empty());
   auto rating = static_cast<float>(INT8_MAX);
-  for (int i = 0; i < word_res->chopped_word->NumBlobs(); ++i) {
+  for (unsigned i = 0; i < word_res->chopped_word->NumBlobs(); ++i) {
     // The rating and certainty are not quite arbitrary. Since
     // select_blob_to_chop uses the worst certainty to choose, they all have
     // to be different, so starting with INT8_MAX, subtract 1/8 for each blob
@@ -257,7 +257,7 @@ void Tesseract::MaximallyChopWord(const std::vector<TBOX> &boxes, BLOCK *block,
     rating -= 0.125f;
   }
   const double e = exp(1.0); // The base of natural logs.
-  int blob_number;
+  unsigned blob_number;
   int right_chop_index = 0;
   if (!assume_fixed_pitch_char_segment) {
     // We only chop if the language is not fixed pitch like CJK.
@@ -613,8 +613,8 @@ bool Tesseract::FindSegmentation(const std::vector<UNICHAR_ID> &target_text, WER
 /// @param best_rating
 /// @param best_segmentation
 void Tesseract::SearchForText(const std::vector<BLOB_CHOICE_LIST *> *choices, int choices_pos,
-                              int choices_length, const std::vector<UNICHAR_ID> &target_text,
-                              int text_index, float rating, std::vector<int> *segmentation,
+                              unsigned choices_length, const std::vector<UNICHAR_ID> &target_text,
+                              unsigned text_index, float rating, std::vector<int> *segmentation,
                               float *best_rating, std::vector<int> *best_segmentation) {
   const UnicharAmbigsVector &table = getDict().getUnicharAmbigs().dang_ambigs();
   for (unsigned length = 1; length <= choices[choices_pos].size(); ++length) {
@@ -625,12 +625,12 @@ void Tesseract::SearchForText(const std::vector<BLOB_CHOICE_LIST *> *choices, in
     for (choice_it.mark_cycle_pt(); !choice_it.cycled_list(); choice_it.forward()) {
       const BLOB_CHOICE *choice = choice_it.data();
       choice_rating = choice->rating();
-      UNICHAR_ID class_id = choice->unichar_id();
+      auto class_id = choice->unichar_id();
       if (class_id == target_text[text_index]) {
         break;
       }
       // Search ambigs table.
-      if (class_id < table.size() && table[class_id] != nullptr) {
+      if (static_cast<size_t>(class_id) < table.size() && table[class_id] != nullptr) {
         AmbigSpec_IT spec_it(table[class_id]);
         for (spec_it.mark_cycle_pt(); !spec_it.cycled_list(); spec_it.forward()) {
           const AmbigSpec *ambig_spec = spec_it.data();

src/ccmain/control.cpp

@@ -45,9 +45,11 @@
 #include "werdit.h"
 
 const char *const kBackUpConfigFile = "tempconfigdata.config";
+#ifndef DISABLED_LEGACY_ENGINE
 // Min believable x-height for any text when refitting as a fraction of
 // original x-height
 const double kMinRefitXHeightFraction = 0.5;
+#endif // ! DISABLED_LEGACY_ENGINE
 
 /**
  * Make a word from the selected blobs and run Tess on them.
@@ -227,7 +229,7 @@ bool Tesseract::RecogAllWordsPassN(int pass_n, ETEXT_DESC *monitor, PAGE_RES_IT
       }
     }
     if (word->word->tess_failed) {
-      int s;
+      unsigned s;
       for (s = 0; s < word->lang_words.size() && word->lang_words[s]->tess_failed; ++s) {
       }
       // If all are failed, skip it. Image words are skipped by this test.
@@ -727,7 +729,7 @@ void Tesseract::script_pos_pass(PAGE_RES *page_res) {
       // Scan for upper/lower.
       int num_upper = 0;
       int num_lower = 0;
-      for (int i = 0; i < word->best_choice->length(); ++i) {
+      for (unsigned i = 0; i < word->best_choice->length(); ++i) {
         if (word->uch_set->get_isupper(word->best_choice->unichar_id(i))) {
           ++num_upper;
         } else if (word->uch_set->get_islower(word->best_choice->unichar_id(i))) {
@@ -743,7 +745,7 @@ void Tesseract::script_pos_pass(PAGE_RES *page_res) {
 }
 
 // Helper finds the gap between the index word and the next.
-static void WordGap(const PointerVector<WERD_RES> &words, int index, int *right, int *next_left) {
+static void WordGap(const PointerVector<WERD_RES> &words, unsigned index, int *right, int *next_left) {
   *right = -INT32_MAX;
   *next_left = INT32_MAX;
   if (index < words.size()) {
@@ -756,13 +758,13 @@ static void WordGap(const PointerVector<WERD_RES> &words, int index, int *right,
 
 // Factored helper computes the rating, certainty, badness and validity of
 // the permuter of the words in [first_index, end_index).
-static void EvaluateWordSpan(const PointerVector<WERD_RES> &words, int first_index, int end_index,
+static void EvaluateWordSpan(const PointerVector<WERD_RES> &words, unsigned first_index, unsigned end_index,
                              float *rating, float *certainty, bool *bad, bool *valid_permuter) {
   if (end_index <= first_index) {
     *bad = true;
     *valid_permuter = false;
   }
-  for (int index = first_index; index < end_index && index < words.size(); ++index) {
+  for (unsigned index = first_index; index < end_index && index < words.size(); ++index) {
     WERD_CHOICE *choice = words[index]->best_choice;
     if (choice == nullptr) {
       *bad = true;
@@ -790,11 +792,11 @@ static int SelectBestWords(double rating_ratio, double certainty_margin, bool de
   // boundary at the end.
   std::vector<WERD_RES *> out_words;
   // Index into each word vector (best, new).
-  int b = 0, n = 0;
+  unsigned b = 0, n = 0;
   int num_best = 0, num_new = 0;
   while (b < best_words->size() || n < new_words->size()) {
     // Start of the current run in each.
-    int start_b = b, start_n = n;
+    auto start_b = b, start_n = n;
     while (b < best_words->size() || n < new_words->size()) {
       int b_right = -INT32_MAX;
       int next_b_left = INT32_MAX;
@@ -884,7 +886,7 @@ int Tesseract::RetryWithLanguage(const WordData &word_data, WordRecognizer recog
     *in_word = nullptr;
   }
   if (debug) {
-    for (int i = 0; i < new_words.size(); ++i) {
+    for (unsigned i = 0; i < new_words.size(); ++i) {
       new_words[i]->DebugTopChoice("Lang result");
     }
   }
@@ -896,7 +898,7 @@ int Tesseract::RetryWithLanguage(const WordData &word_data, WordRecognizer recog
 
 // Helper returns true if all the words are acceptable.
 static bool WordsAcceptable(const PointerVector<WERD_RES> &words) {
-  for (int w = 0; w < words.size(); ++w) {
+  for (unsigned w = 0; w < words.size(); ++w) {
     if (words[w]->tess_failed || !words[w]->tess_accepted) {
       return false;
     }
@@ -982,9 +984,12 @@ void Tesseract::AssignDiacriticsToOverlappingBlobs(const std::vector<C_OUTLINE *
                                                    std::vector<bool> *overlapped_any_blob,
                                                    std::vector<C_BLOB *> *target_blobs) {
   std::vector<bool> blob_wanted;
-  word_wanted->resize(outlines.size(), false);
-  overlapped_any_blob->resize(outlines.size(), false);
-  target_blobs->resize(outlines.size(), nullptr);
+  word_wanted->clear();
+  word_wanted->resize(outlines.size());
+  overlapped_any_blob->clear();
+  overlapped_any_blob->resize(outlines.size());
+  target_blobs->clear();
+  target_blobs->resize(outlines.size());
   // For each real blob, find the outlines that seriously overlap it.
   // A single blob could be several merged characters, so there can be quite
   // a few outlines overlapping, and the full engine needs to be used to chop
@@ -993,7 +998,8 @@ void Tesseract::AssignDiacriticsToOverlappingBlobs(const std::vector<C_OUTLINE *
   for (blob_it.mark_cycle_pt(); !blob_it.cycled_list(); blob_it.forward()) {
     C_BLOB *blob = blob_it.data();
     const TBOX blob_box = blob->bounding_box();
-    blob_wanted.resize(outlines.size(), false);
+    blob_wanted.clear();
+    blob_wanted.resize(outlines.size());
     int num_blob_outlines = 0;
     for (unsigned i = 0; i < outlines.size(); ++i) {
       if (blob_box.major_x_overlap(outlines[i]->bounding_box()) && !(*word_wanted)[i]) {
@@ -1032,15 +1038,18 @@ void Tesseract::AssignDiacriticsToNewBlobs(const std::vector<C_OUTLINE *> &outli
                                            std::vector<bool> *word_wanted,
                                            std::vector<C_BLOB *> *target_blobs) {
   std::vector<bool> blob_wanted;
-  word_wanted->resize(outlines.size(), false);
-  target_blobs->resize(outlines.size(), nullptr);
+  word_wanted->clear();
+  word_wanted->resize(outlines.size());
+  target_blobs->clear();
+  target_blobs->resize(outlines.size());
   // Check for outlines that need to be turned into stand-alone blobs.
   for (unsigned i = 0; i < outlines.size(); ++i) {
     if (outlines[i] == nullptr) {
       continue;
     }
     // Get a set of adjacent outlines that don't overlap any existing blob.
-    blob_wanted.resize(outlines.size(), false);
+    blob_wanted.clear();
+    blob_wanted.resize(outlines.size());
     int num_blob_outlines = 0;
     TBOX total_ol_box(outlines[i]->bounding_box());
     while (i < outlines.size() && outlines[i] != nullptr) {
@@ -1590,10 +1599,10 @@ void Tesseract::match_word_pass_n(int pass_n, WERD_RES *word, ROW *row, BLOCK *b
         word->fix_hyphens();
       }
       /* Don't trust fix_quotes! - though I think I've fixed the bug */
-      if (word->best_choice->length() != word->box_word->length()) {
+      if (static_cast<unsigned>(word->best_choice->length()) != word->box_word->length()) {
         tprintf(
             "POST FIX_QUOTES FAIL String:\"%s\"; Strlen=%d;"
-            " #Blobs=%d\n",
+            " #Blobs=%u\n",
             word->best_choice->debug_string().c_str(), word->best_choice->length(),
             word->box_word->length());
       }
@@ -1614,7 +1623,7 @@ void Tesseract::match_word_pass_n(int pass_n, WERD_RES *word, ROW *row, BLOCK *b
 static BLOB_CHOICE *FindBestMatchingChoice(UNICHAR_ID char_id, WERD_RES *word_res) {
   // Find the corresponding best BLOB_CHOICE from any position in the word_res.
   BLOB_CHOICE *best_choice = nullptr;
-  for (int i = 0; i < word_res->best_choice->length(); ++i) {
+  for (unsigned i = 0; i < word_res->best_choice->length(); ++i) {
     BLOB_CHOICE *choice = FindMatchingChoice(char_id, word_res->GetBlobChoices(i));
     if (choice != nullptr) {
       if (best_choice == nullptr || choice->rating() < best_choice->rating()) {
@@ -1630,7 +1639,7 @@ static BLOB_CHOICE *FindBestMatchingChoice(UNICHAR_ID char_id, WERD_RES *word_re
 // in the best_choice.
 static void CorrectRepcharChoices(BLOB_CHOICE *blob_choice, WERD_RES *word_res) {
   WERD_CHOICE *word = word_res->best_choice;
-  for (int i = 0; i < word_res->best_choice->length(); ++i) {
+  for (unsigned i = 0; i < word_res->best_choice->length(); ++i) {
     BLOB_CHOICE *choice =
         FindMatchingChoice(blob_choice->unichar_id(), word_res->GetBlobChoices(i));
     if (choice == nullptr) {
@@ -1639,7 +1648,7 @@ static void CorrectRepcharChoices(BLOB_CHOICE *blob_choice, WERD_RES *word_res)
     }
   }
   // Correct any incorrect results in word.
-  for (int i = 0; i < word->length(); ++i) {
+  for (unsigned i = 0; i < word->length(); ++i) {
     if (word->unichar_id(i) != blob_choice->unichar_id()) {
       word->set_unichar_id(blob_choice->unichar_id(), i);
     }
@@ -1659,7 +1668,7 @@ void Tesseract::fix_rep_char(PAGE_RES_IT *page_res_it) {
 
   // Find the frequency of each unique character in the word.
   SortHelper<UNICHAR_ID> rep_ch(word.length());
-  for (int i = 0; i < word.length(); ++i) {
+  for (unsigned i = 0; i < word.length(); ++i) {
     rep_ch.Add(word.unichar_id(i), 1);
   }
 
@@ -1888,6 +1897,7 @@ bool Tesseract::check_debug_pt(WERD_RES *word, int location) {
  *
  * Find the modal font and remove from the stats.
  */
+#ifndef DISABLED_LEGACY_ENGINE
 static void find_modal_font( // good chars in word
     STATS *fonts,            // font stats
     int16_t *font_out,       // output font
@@ -1907,6 +1917,7 @@ static void find_modal_font( // good chars in word
     *font_count = 0;
   }
 }
+#endif // ! DISABLED_LEGACY_ENGINE
 
 /**
  * set_word_fonts
@@ -1944,7 +1955,7 @@ void Tesseract::set_word_fonts(WERD_RES *word) {
   if (tessedit_debug_fonts) {
     tprintf("Examining fonts in %s\n", word->best_choice->debug_string().c_str());
   }
-  for (int b = 0; b < word->best_choice->length(); ++b) {
+  for (unsigned b = 0; b < word->best_choice->length(); ++b) {
     const BLOB_CHOICE *choice = word->GetBlobChoice(b);
     if (choice == nullptr) {
       continue;

src/ccmain/docqual.cpp

@@ -64,7 +64,7 @@ int16_t Tesseract::word_outline_errs(WERD_RES *word) {
   int16_t err_count = 0;
 
   if (word->rebuild_word != nullptr) {
-    for (int b = 0; b < word->rebuild_word->NumBlobs(); ++b) {
+    for (unsigned b = 0; b < word->rebuild_word->NumBlobs(); ++b) {
       TBLOB *blob = word->rebuild_word->blobs[b];
       err_count += count_outline_errs(word->best_choice->unichar_string()[i], blob->NumOutlines());
       i++;
@@ -911,7 +911,7 @@ bool Tesseract::noise_outlines(TWERD *word) {
   int16_t max_dimension;
   float small_limit = kBlnXHeight * crunch_small_outlines_size;
 
-  for (int b = 0; b < word->NumBlobs(); ++b) {
+  for (unsigned b = 0; b < word->NumBlobs(); ++b) {
     TBLOB *blob = word->blobs[b];
     for (TESSLINE *ol = blob->outlines; ol != nullptr; ol = ol->next) {
       outline_count++;

src/ccmain/equationdetect.cpp

@@ -35,6 +35,7 @@
 
 #include <algorithm>
 #include <cfloat>
+#include <cmath>
 #include <limits>
 #include <memory>
 
@@ -189,11 +190,11 @@ void EquationDetect::IdentifySpecialText(BLOBNBOX *blobnbox, const int height_th
   const float kConfScoreTh = -5.0f, kConfDiffTh = 1.8;
   // The scores here are negative, so the max/min == fabs(min/max).
   // float ratio = fmax(lang_score, equ_score) / fmin(lang_score, equ_score);
-  const float diff = fabs(lang_score - equ_score);
+  const float diff = std::fabs(lang_score - equ_score);
   BlobSpecialTextType type = BSTT_NONE;
 
   // Classification.
-  if (fmax(lang_score, equ_score) < kConfScoreTh) {
+  if (std::fmax(lang_score, equ_score) < kConfScoreTh) {
     // If both score are very small, then mark it as unclear.
     type = BSTT_UNCLEAR;
   } else if (diff > kConfDiffTh && equ_score > lang_score) {
@@ -727,7 +728,7 @@ int EquationDetect::CountAlignment(const std::vector<int> &sorted_vec, const int
   if (sorted_vec.empty()) {
     return 0;
   }
-  const int kDistTh = static_cast<int>(round(0.03f * resolution_));
+  const int kDistTh = static_cast<int>(std::round(0.03f * resolution_));
   auto pos = std::upper_bound(sorted_vec.begin(), sorted_vec.end(), val);
   if (pos > sorted_vec.begin()) {
     --pos;
@@ -742,7 +743,7 @@ int EquationDetect::CountAlignment(const std::vector<int> &sorted_vec, const int
 
   // Search right side.
   index = pos + 1 - sorted_vec.begin();
-  while (index < sorted_vec.size() && sorted_vec[index++] - val < kDistTh) {
+  while (static_cast<size_t>(index) < sorted_vec.size() && sorted_vec[index++] - val < kDistTh) {
     count++;
   }
 
@@ -772,7 +773,7 @@ void EquationDetect::IdentifyInlinePartsHorizontal() {
   ASSERT_HOST(cps_super_bbox_);
   std::vector<ColPartition *> new_seeds;
   const int kMarginDiffTh = IntCastRounded(0.5 * lang_tesseract_->source_resolution());
-  const int kGapTh = static_cast<int>(round(1.0f * lang_tesseract_->source_resolution()));
+  const int kGapTh = static_cast<int>(std::round(1.0f * lang_tesseract_->source_resolution()));
   ColPartitionGridSearch search(part_grid_);
   search.SetUniqueMode(true);
   // The center x coordinate of the cp_super_bbox_.
@@ -923,8 +924,8 @@ bool EquationDetect::IsInline(const bool search_bottom, const int textparts_line
     // Check if neighbor and part is inline similar.
     const float kHeightRatioTh = 0.5;
     const int kYGapTh = textparts_linespacing > 0
-                            ? textparts_linespacing + static_cast<int>(round(0.02f * resolution_))
-                            : static_cast<int>(round(0.05f * resolution_)); // Default value.
+                            ? textparts_linespacing + static_cast<int>(std::round(0.02f * resolution_))
+                            : static_cast<int>(std::round(0.05f * resolution_)); // Default value.
     if (part_box.x_overlap(neighbor_box) &&                                 // Location feature.
         part_box.y_gap(neighbor_box) <= kYGapTh &&                          // Line spacing.
         // Geo feature.
@@ -978,9 +979,9 @@ EquationDetect::IndentType EquationDetect::IsIndented(ColPartition *part) {
   ColPartitionGridSearch search(part_grid_);
   ColPartition *neighbor = nullptr;
   const TBOX &part_box(part->bounding_box());
-  const int kXGapTh = static_cast<int>(round(0.5f * resolution_));
-  const int kRadiusTh = static_cast<int>(round(3.0f * resolution_));
-  const int kYGapTh = static_cast<int>(round(0.5f * resolution_));
+  const int kXGapTh = static_cast<int>(std::round(0.5f * resolution_));
+  const int kRadiusTh = static_cast<int>(std::round(3.0f * resolution_));
+  const int kYGapTh = static_cast<int>(std::round(0.5f * resolution_));
 
   // Here we use a simple approximation algorithm: from the center of part, We
   // perform the radius search, and check if we can find a neighboring partition
@@ -1080,7 +1081,7 @@ void EquationDetect::ExpandSeedHorizontal(const bool search_left, ColPartition *
                                           std::vector<ColPartition *> *parts_to_merge) {
   ASSERT_HOST(seed != nullptr && parts_to_merge != nullptr);
   const float kYOverlapTh = 0.6;
-  const int kXGapTh = static_cast<int>(round(0.2f * resolution_));
+  const int kXGapTh = static_cast<int>(std::round(0.2f * resolution_));
 
   ColPartitionGridSearch search(part_grid_);
   const TBOX &seed_box(seed->bounding_box());
@@ -1132,7 +1133,7 @@ void EquationDetect::ExpandSeedVertical(const bool search_bottom, ColPartition *
                                         std::vector<ColPartition *> *parts_to_merge) {
   ASSERT_HOST(seed != nullptr && parts_to_merge != nullptr && cps_super_bbox_ != nullptr);
   const float kXOverlapTh = 0.4;
-  const int kYGapTh = static_cast<int>(round(0.2f * resolution_));
+  const int kYGapTh = static_cast<int>(std::round(0.2f * resolution_));
 
   ColPartitionGridSearch search(part_grid_);
   const TBOX &seed_box(seed->bounding_box());
@@ -1210,8 +1211,8 @@ void EquationDetect::ExpandSeedVertical(const bool search_bottom, ColPartition *
 }
 
 bool EquationDetect::IsNearSmallNeighbor(const TBOX &seed_box, const TBOX &part_box) const {
-  const int kXGapTh = static_cast<int>(round(0.25f * resolution_));
-  const int kYGapTh = static_cast<int>(round(0.05f * resolution_));
+  const int kXGapTh = static_cast<int>(std::round(0.25f * resolution_));
+  const int kYGapTh = static_cast<int>(std::round(0.05f * resolution_));
 
   // Check geometric feature.
   if (part_box.height() > seed_box.height() || part_box.width() > seed_box.width()) {
@@ -1266,7 +1267,7 @@ void EquationDetect::ProcessMathBlockSatelliteParts() {
   int med_height = text_box.height();
   if (text_parts.size() % 2 == 0 && text_parts.size() > 1) {
     const TBOX &text_box = text_parts[text_parts.size() / 2 - 1]->bounding_box();
-    med_height = static_cast<int>(round(0.5f * (text_box.height() + med_height)));
+    med_height = static_cast<int>(std::round(0.5f * (text_box.height() + med_height)));
   }
 
   // Iterate every text_parts and check if it is a math block satellite.
@@ -1348,7 +1349,7 @@ bool EquationDetect::IsMathBlockSatellite(ColPartition *part,
 ColPartition *EquationDetect::SearchNNVertical(const bool search_bottom, const ColPartition *part) {
   ASSERT_HOST(part);
   ColPartition *nearest_neighbor = nullptr, *neighbor = nullptr;
-  const int kYGapTh = static_cast<int>(round(resolution_ * 0.5f));
+  const int kYGapTh = static_cast<int>(std::round(resolution_ * 0.5f));
 
   ColPartitionGridSearch search(part_grid_);
   search.SetUniqueMode(true);
@@ -1383,7 +1384,7 @@ bool EquationDetect::IsNearMathNeighbor(const int y_gap, const ColPartition *nei
   if (!neighbor) {
     return false;
   }
-  const int kYGapTh = static_cast<int>(round(resolution_ * 0.1f));
+  const int kYGapTh = static_cast<int>(std::round(resolution_ * 0.1f));
   return neighbor->type() == PT_EQUATION && y_gap <= kYGapTh;
 }
 

src/ccmain/fixspace.cpp

@@ -262,7 +262,7 @@ int16_t Tesseract::eval_word_spacing(WERD_RES_LIST &word_res_list) {
   int16_t total_score = 0;
   int16_t word_count = 0;
   int16_t done_word_count = 0;
-  int16_t i;
+  int i;
   int16_t offset;
   int16_t prev_word_score = 0;
   bool prev_word_done = false;
@@ -684,7 +684,6 @@ void Tesseract::break_noisiest_blob_word(WERD_RES_LIST &words) {
 
 int16_t Tesseract::worst_noise_blob(WERD_RES *word_res, float *worst_noise_score) {
   float noise_score[512];
-  int i;
   int min_noise_blob; // 1st contender
   int max_noise_blob; // last contender
   int non_noise_count;
@@ -697,7 +696,7 @@ int16_t Tesseract::worst_noise_blob(WERD_RES *word_res, float *worst_noise_score
   }
 
   // Normalised.
-  int blob_count = word_res->box_word->length();
+  auto blob_count = word_res->box_word->length();
   ASSERT_HOST(blob_count <= 512);
   if (blob_count < 5) {
     return -1; // too short to split
@@ -712,7 +711,7 @@ int16_t Tesseract::worst_noise_blob(WERD_RES *word_res, float *worst_noise_score
   }
 #endif
 
-  for (i = 0; i < blob_count && i < word_res->rebuild_word->NumBlobs(); i++) {
+  for (unsigned i = 0; i < blob_count && i < word_res->rebuild_word->NumBlobs(); i++) {
     TBLOB *blob = word_res->rebuild_word->blobs[i];
     if (word_res->reject_map[i].accepted()) {
       noise_score[i] = non_noise_limit;
@@ -731,7 +730,8 @@ int16_t Tesseract::worst_noise_blob(WERD_RES *word_res, float *worst_noise_score
   /* Now find the worst one which is far enough away from the end of the word */
 
   non_noise_count = 0;
-  for (i = 0; i < blob_count && non_noise_count < fixsp_non_noise_limit; i++) {
+  int i;
+  for (i = 0; static_cast<unsigned>(i) < blob_count && non_noise_count < fixsp_non_noise_limit; i++) {
     if (noise_score[i] >= non_noise_limit) {
       non_noise_count++;
     }
@@ -760,7 +760,7 @@ int16_t Tesseract::worst_noise_blob(WERD_RES *word_res, float *worst_noise_score
 
   *worst_noise_score = small_limit;
   worst_noise_blob = -1;
-  for (i = min_noise_blob; i <= max_noise_blob; i++) {
+  for (auto i = min_noise_blob; i <= max_noise_blob; i++) {
     if (noise_score[i] < *worst_noise_score) {
       worst_noise_blob = i;
       *worst_noise_score = noise_score[i];
@@ -838,7 +838,6 @@ int16_t Tesseract::fp_eval_word_spacing(WERD_RES_LIST &word_res_list) {
   WERD_RES_IT word_it(&word_res_list);
   WERD_RES *word;
   int16_t score = 0;
-  int16_t i;
   float small_limit = kBlnXHeight * fixsp_small_outlines_size;
 
   for (word_it.mark_cycle_pt(); !word_it.cycled_list(); word_it.forward()) {
@@ -849,9 +848,9 @@ int16_t Tesseract::fp_eval_word_spacing(WERD_RES_LIST &word_res_list) {
     if (word->done || word->tess_accepted || word->best_choice->permuter() == SYSTEM_DAWG_PERM ||
         word->best_choice->permuter() == FREQ_DAWG_PERM ||
         word->best_choice->permuter() == USER_DAWG_PERM || safe_dict_word(word) > 0) {
-      int num_blobs = word->rebuild_word->NumBlobs();
+      auto num_blobs = word->rebuild_word->NumBlobs();
       UNICHAR_ID space = word->uch_set->unichar_to_id(" ");
-      for (i = 0; i < word->best_choice->length() && i < num_blobs; ++i) {
+      for (unsigned i = 0; i < word->best_choice->length() && i < num_blobs; ++i) {
         TBLOB *blob = word->rebuild_word->blobs[i];
         if (word->best_choice->unichar_id(i) == space || blob_noise_score(blob) < small_limit) {
           score -= 1; // penalise possibly erroneous non-space

src/ccmain/fixxht.cpp

@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cmath>
 #include <cstring>
 
 namespace tesseract {
@@ -205,7 +206,7 @@ float Tesseract::ComputeCompatibleXheight(WERD_RES *word_res, float *baseline_sh
             new_xht / word_res->denorm.y_scale());
   }
   // The xheight must change by at least x_ht_min_change to be used.
-  if (fabs(new_xht - kBlnXHeight) >= x_ht_min_change) {
+  if (std::fabs(new_xht - kBlnXHeight) >= x_ht_min_change) {
     return new_xht / word_res->denorm.y_scale();
   } else {
     return bottom_shift != 0 ? word_res->x_height : 0.0f;

src/ccmain/linerec.cpp

@@ -151,8 +151,7 @@ ImageData *Tesseract::GetLineData(const TBOX &line_box, const std::vector<TBOX>
     line_boxes.push_back(box);
     line_texts.push_back(texts[b]);
   }
-  std::vector<int> page_numbers;
-  page_numbers.resize(line_boxes.size(), applybox_page);
+  std::vector<int> page_numbers(line_boxes.size(), applybox_page);
   image_data->AddBoxes(line_boxes, line_texts, page_numbers);
   return image_data;
 }
@@ -270,22 +269,14 @@ void Tesseract::SearchWords(PointerVector<WERD_RES> *words) {
   if (stopper_dict == nullptr) {
     stopper_dict = &getDict();
   }
-  bool any_nonspace_delimited = false;
-  for (int w = 0; w < words->size(); ++w) {
-    WERD_RES *word = (*words)[w];
-    if (word->best_choice != nullptr && word->best_choice->ContainsAnyNonSpaceDelimited()) {
-      any_nonspace_delimited = true;
-      break;
-    }
-  }
-  for (int w = 0; w < words->size(); ++w) {
+  for (unsigned w = 0; w < words->size(); ++w) {
     WERD_RES *word = (*words)[w];
     if (word->best_choice == nullptr) {
       // It is a dud.
       word->SetupFake(lstm_recognizer_->GetUnicharset());
     } else {
       // Set the best state.
-      for (int i = 0; i < word->best_choice->length(); ++i) {
+      for (unsigned i = 0; i < word->best_choice->length(); ++i) {
         int length = word->best_choice->state(i);
         word->best_state.push_back(length);
       }

src/ccmain/ltrresultiterator.cpp

@@ -147,14 +147,6 @@ float LTRResultIterator::Confidence(PageIteratorLevel level) const {
   return 0.0f;
 }
 
-void LTRResultIterator::RowAttributes(float *row_height, float *descenders,
-                                      float *ascenders) const {
-  *row_height =
-      it_->row()->row->x_height() + it_->row()->row->ascenders() - it_->row()->row->descenders();
-  *descenders = it_->row()->row->descenders();
-  *ascenders = it_->row()->row->ascenders();
-}
-
 // Returns the font attributes of the current word. If iterating at a higher
 // level object than words, eg textlines, then this will return the
 // attributes of the first word in that textline.
@@ -335,10 +327,10 @@ char *LTRResultIterator::WordNormedUTF8Text() const {
   WERD_CHOICE *best_choice = it_->word()->best_choice;
   const UNICHARSET *unicharset = it_->word()->uch_set;
   ASSERT_HOST(best_choice != nullptr);
-  for (int i = 0; i < best_choice->length(); ++i) {
+  for (unsigned i = 0; i < best_choice->length(); ++i) {
     ocr_text += unicharset->get_normed_unichar(best_choice->unichar_id(i));
   }
-  int length = ocr_text.length() + 1;
+  auto length = ocr_text.length() + 1;
   char *result = new char[length];
   strncpy(result, ocr_text.c_str(), length);
   return result;
@@ -404,7 +396,7 @@ ChoiceIterator::ChoiceIterator(const LTRResultIterator &result_it) {
         strcmp(word_res_->CTC_symbol_choices[0][0].first, " ")) {
       blanks_before_word_ = 0;
     }
-    auto index = *tstep_index_;
+    unsigned index = *tstep_index_;
     index += blanks_before_word_;
     if (index < word_res_->CTC_symbol_choices.size()) {
       LSTM_choices_ = &word_res_->CTC_symbol_choices[index];
@@ -432,7 +424,8 @@ ChoiceIterator::~ChoiceIterator() {
 // are none left.
 bool ChoiceIterator::Next() {
   if (oemLSTM_ && LSTM_choices_ != nullptr && !LSTM_choices_->empty()) {
-    if (LSTM_choice_it_ != LSTM_choices_->end() && next(LSTM_choice_it_) == LSTM_choices_->end()) {
+    if (LSTM_choice_it_ == LSTM_choices_->end() ||
+        next(LSTM_choice_it_) == LSTM_choices_->end()) {
       return false;
     } else {
       ++LSTM_choice_it_;
@@ -484,7 +477,7 @@ float ChoiceIterator::Confidence() const {
 
 // Returns the set of timesteps which belong to the current symbol
 std::vector<std::vector<std::pair<const char *, float>>> *ChoiceIterator::Timesteps() const {
-  int offset = *tstep_index_ + blanks_before_word_;
+  unsigned offset = *tstep_index_ + blanks_before_word_;
   if (offset >= word_res_->segmented_timesteps.size() || !oemLSTM_) {
     return nullptr;
   }

src/ccmain/osdetect.cpp

@@ -381,7 +381,7 @@ bool OrientationDetector::detect_blob(BLOB_CHOICE_LIST *scores) {
         for (choice_it.mark_cycle_pt(); !choice_it.cycled_list() && choice == nullptr;
              choice_it.forward()) {
           int choice_script = choice_it.data()->script_id();
-          int s = 0;
+          unsigned s = 0;
           for (s = 0; s < allowed_scripts_->size(); ++s) {
             if ((*allowed_scripts_)[s] == choice_script) {
               choice = choice_it.data();
@@ -428,7 +428,7 @@ bool OrientationDetector::detect_blob(BLOB_CHOICE_LIST *scores) {
   // Normalize the orientation scores for the blob and use them to
   // update the aggregated orientation score.
   for (int i = 0; total_blob_o_score != 0 && i < 4; ++i) {
-    osr_->orientations[i] += log(blob_o_score[i] / total_blob_o_score);
+    osr_->orientations[i] += std::log(blob_o_score[i] / total_blob_o_score);
   }
 
   // TODO(ranjith) Add an early exit test, based on min_orientation_margin,
@@ -477,7 +477,7 @@ void ScriptDetector::detect_blob(BLOB_CHOICE_LIST *scores) {
       int id = choice->script_id();
       if (allowed_scripts_ != nullptr && !allowed_scripts_->empty()) {
         // Check that the choice is in an allowed script.
-        int s = 0;
+        size_t s = 0;
         for (s = 0; s < allowed_scripts_->size(); ++s) {
           if ((*allowed_scripts_)[s] == id) {
             break;

src/ccmain/output.cpp

@@ -101,7 +101,6 @@ void Tesseract::write_results(PAGE_RES_IT &page_res_it,
                               bool force_eol) {  // override tilde crunch?
   WERD_RES *word = page_res_it.word();
   const UNICHARSET &uchset = *word->uch_set;
-  int i;
   bool need_reject = false;
   UNICHAR_ID space = uchset.unichar_to_id(" ");
 
@@ -181,7 +180,7 @@ void Tesseract::write_results(PAGE_RES_IT &page_res_it,
   if (!word->word->flag(W_REP_CHAR) || !tessedit_write_rep_codes) {
     if (tessedit_zero_rejection) {
       /* OVERRIDE ALL REJECTION MECHANISMS - ONLY REJECT TESS FAILURES */
-      for (i = 0; i < word->best_choice->length(); ++i) {
+      for (unsigned i = 0; i < word->best_choice->length(); ++i) {
         if (word->reject_map[i].rejected()) {
           word->reject_map[i].setrej_minimal_rej_accept();
         }
@@ -189,7 +188,7 @@ void Tesseract::write_results(PAGE_RES_IT &page_res_it,
     }
     if (tessedit_minimal_rejection) {
       /* OVERRIDE ALL REJECTION MECHANISMS - ONLY REJECT TESS FAILURES */
-      for (i = 0; i < word->best_choice->length(); ++i) {
+      for (unsigned i = 0; i < word->best_choice->length(); ++i) {
         if ((word->best_choice->unichar_id(i) != space) && word->reject_map[i].rejected()) {
           word->reject_map[i].setrej_minimal_rej_accept();
         }
@@ -365,7 +364,7 @@ void Tesseract::set_unlv_suspects(WERD_RES *word_res) {
 
 int16_t Tesseract::count_alphas(const WERD_CHOICE &word) {
   int count = 0;
-  for (int i = 0; i < word.length(); ++i) {
+  for (unsigned i = 0; i < word.length(); ++i) {
     if (word.unicharset()->get_isalpha(word.unichar_id(i))) {
       count++;
     }
@@ -375,7 +374,7 @@ int16_t Tesseract::count_alphas(const WERD_CHOICE &word) {
 
 int16_t Tesseract::count_alphanums(const WERD_CHOICE &word) {
   int count = 0;
-  for (int i = 0; i < word.length(); ++i) {
+  for (unsigned i = 0; i < word.length(); ++i) {
     if (word.unicharset()->get_isalpha(word.unichar_id(i)) ||
         word.unicharset()->get_isdigit(word.unichar_id(i))) {
       count++;

src/ccmain/pageiterator.cpp

@@ -27,22 +27,23 @@
 
 namespace tesseract {
 
-PageIterator::PageIterator(PAGE_RES *page_res, Tesseract *tesseract, int scale, int scaled_yres,
-                           int rect_left, int rect_top, int rect_width, int rect_height)
-    : page_res_(page_res)
-    , tesseract_(tesseract)
-    , word_(nullptr)
-    , word_length_(0)
-    , blob_index_(0)
-    , cblob_it_(nullptr)
-    , include_upper_dots_(false)
-    , include_lower_dots_(false)
-    , scale_(scale)
-    , scaled_yres_(scaled_yres)
-    , rect_left_(rect_left)
-    , rect_top_(rect_top)
-    , rect_width_(rect_width)
-    , rect_height_(rect_height) {
+PageIterator::PageIterator(PAGE_RES *page_res, Tesseract *tesseract, int scale,
+                           int scaled_yres, int rect_left, int rect_top,
+                           int rect_width, int rect_height)
+    : page_res_(page_res),
+      tesseract_(tesseract),
+      word_(nullptr),
+      word_length_(0),
+      blob_index_(0),
+      cblob_it_(nullptr),
+      include_upper_dots_(false),
+      include_lower_dots_(false),
+      scale_(scale),
+      scaled_yres_(scaled_yres),
+      rect_left_(rect_left),
+      rect_top_(rect_top),
+      rect_width_(rect_width),
+      rect_height_(rect_height) {
   it_ = new PAGE_RES_IT(page_res);
   PageIterator::Begin();
 }
@@ -58,20 +59,20 @@ PageIterator::~PageIterator() {
  * objects at a higher level.
  */
 PageIterator::PageIterator(const PageIterator &src)
-    : page_res_(src.page_res_)
-    , tesseract_(src.tesseract_)
-    , word_(nullptr)
-    , word_length_(src.word_length_)
-    , blob_index_(src.blob_index_)
-    , cblob_it_(nullptr)
-    , include_upper_dots_(src.include_upper_dots_)
-    , include_lower_dots_(src.include_lower_dots_)
-    , scale_(src.scale_)
-    , scaled_yres_(src.scaled_yres_)
-    , rect_left_(src.rect_left_)
-    , rect_top_(src.rect_top_)
-    , rect_width_(src.rect_width_)
-    , rect_height_(src.rect_height_) {
+    : page_res_(src.page_res_),
+      tesseract_(src.tesseract_),
+      word_(nullptr),
+      word_length_(src.word_length_),
+      blob_index_(src.blob_index_),
+      cblob_it_(nullptr),
+      include_upper_dots_(src.include_upper_dots_),
+      include_lower_dots_(src.include_lower_dots_),
+      scale_(src.scale_),
+      scaled_yres_(src.scaled_yres_),
+      rect_left_(src.rect_left_),
+      rect_top_(src.rect_top_),
+      rect_width_(src.rect_width_),
+      rect_height_(src.rect_height_) {
   it_ = new PAGE_RES_IT(*src.it_);
   BeginWord(src.blob_index_);
 }
@@ -201,8 +202,9 @@ bool PageIterator::IsAtBeginningOf(PageIteratorLevel level) const {
     case RIL_BLOCK:
       return blob_index_ == 0 && it_->block() != it_->prev_block();
     case RIL_PARA:
-      return blob_index_ == 0 && (it_->block() != it_->prev_block() ||
-                                  it_->row()->row->para() != it_->prev_row()->row->para());
+      return blob_index_ == 0 &&
+             (it_->block() != it_->prev_block() ||
+              it_->row()->row->para() != it_->prev_row()->row->para());
     case RIL_TEXTLINE:
       return blob_index_ == 0 && it_->row() != it_->prev_row();
     case RIL_WORD:
@@ -217,7 +219,8 @@ bool PageIterator::IsAtBeginningOf(PageIteratorLevel level) const {
  * Returns whether the iterator is positioned at the last element in a
  * given level. (e.g. the last word in a line, the last line in a block)
  */
-bool PageIterator::IsAtFinalElement(PageIteratorLevel level, PageIteratorLevel element) const {
+bool PageIterator::IsAtFinalElement(PageIteratorLevel level,
+                                    PageIteratorLevel element) const {
   if (Empty(element)) {
     return true; // Already at the end!
   }
@@ -280,7 +283,8 @@ int PageIterator::Cmp(const PageIterator &other) const {
  * See comment on coordinate system above.
  * Returns false if there is no such object at the current position.
  */
-bool PageIterator::BoundingBoxInternal(PageIteratorLevel level, int *left, int *top, int *right,
+bool PageIterator::BoundingBoxInternal(PageIteratorLevel level, int *left,
+                                       int *top, int *right,
                                        int *bottom) const {
   if (Empty(level)) {
     return false;
@@ -289,16 +293,19 @@ bool PageIterator::BoundingBoxInternal(PageIteratorLevel level, int *left, int *
   PARA *para = nullptr;
   switch (level) {
     case RIL_BLOCK:
-      box = it_->block()->block->restricted_bounding_box(include_upper_dots_, include_lower_dots_);
+      box = it_->block()->block->restricted_bounding_box(include_upper_dots_,
+                                                         include_lower_dots_);
       break;
     case RIL_PARA:
       para = it_->row()->row->para();
       // Fall through.
     case RIL_TEXTLINE:
-      box = it_->row()->row->restricted_bounding_box(include_upper_dots_, include_lower_dots_);
+      box = it_->row()->row->restricted_bounding_box(include_upper_dots_,
+                                                     include_lower_dots_);
       break;
     case RIL_WORD:
-      box = it_->word()->word->restricted_bounding_box(include_upper_dots_, include_lower_dots_);
+      box = it_->word()->word->restricted_bounding_box(include_upper_dots_,
+                                                       include_lower_dots_);
       break;
     case RIL_SYMBOL:
       if (cblob_it_ == nullptr) {
@@ -311,8 +318,10 @@ bool PageIterator::BoundingBoxInternal(PageIteratorLevel level, int *left, int *
     PageIterator other = *this;
     other.Begin();
     do {
-      if (other.it_->block() && other.it_->block()->block == it_->block()->block &&
-          other.it_->row() && other.it_->row()->row && other.it_->row()->row->para() == para) {
+      if (other.it_->block() &&
+          other.it_->block()->block == it_->block()->block &&
+          other.it_->row() && other.it_->row()->row &&
+          other.it_->row()->row->para() == para) {
         box = box.bounding_union(other.it_->row()->row->bounding_box());
       }
     } while (other.Next(RIL_TEXTLINE));
@@ -337,23 +346,26 @@ bool PageIterator::BoundingBoxInternal(PageIteratorLevel level, int *left, int *
  * See comment on coordinate system above.
  * Returns false if there is no such object at the current position.
  */
-bool PageIterator::BoundingBox(PageIteratorLevel level, int *left, int *top, int *right,
-                               int *bottom) const {
+bool PageIterator::BoundingBox(PageIteratorLevel level, int *left, int *top,
+                               int *right, int *bottom) const {
   return BoundingBox(level, 0, left, top, right, bottom);
 }
 
-bool PageIterator::BoundingBox(PageIteratorLevel level, const int padding, int *left, int *top,
-                               int *right, int *bottom) const {
+bool PageIterator::BoundingBox(PageIteratorLevel level, const int padding,
+                               int *left, int *top, int *right,
+                               int *bottom) const {
   if (!BoundingBoxInternal(level, left, top, right, bottom)) {
     return false;
   }
   // Convert to the coordinate system of the original image.
-  *left = ClipToRange(*left / scale_ + rect_left_ - padding, rect_left_, rect_left_ + rect_width_);
-  *top = ClipToRange(*top / scale_ + rect_top_ - padding, rect_top_, rect_top_ + rect_height_);
-  *right = ClipToRange((*right + scale_ - 1) / scale_ + rect_left_ + padding, *left,
-                       rect_left_ + rect_width_);
-  *bottom = ClipToRange((*bottom + scale_ - 1) / scale_ + rect_top_ + padding, *top,
-                        rect_top_ + rect_height_);
+  *left = ClipToRange(*left / scale_ + rect_left_ - padding, rect_left_,
+                      rect_left_ + rect_width_);
+  *top = ClipToRange(*top / scale_ + rect_top_ - padding, rect_top_,
+                     rect_top_ + rect_height_);
+  *right = ClipToRange((*right + scale_ - 1) / scale_ + rect_left_ + padding,
+                       *left, rect_left_ + rect_width_);
+  *bottom = ClipToRange((*bottom + scale_ - 1) / scale_ + rect_top_ + padding,
+                        *top, rect_top_ + rect_height_);
   return true;
 }
 
@@ -440,7 +452,8 @@ Pix *PageIterator::GetBinaryImage(PageIteratorLevel level) const {
   if (!BoundingBoxInternal(level, &left, &top, &right, &bottom)) {
     return nullptr;
   }
-  if (level == RIL_SYMBOL && cblob_it_ != nullptr && cblob_it_->data()->area() != 0) {
+  if (level == RIL_SYMBOL && cblob_it_ != nullptr &&
+      cblob_it_->data()->area() != 0) {
     return cblob_it_->data()->render();
   }
   Box *box = boxCreate(left, top, right - left, bottom - top);
@@ -453,9 +466,9 @@ Pix *PageIterator::GetBinaryImage(PageIteratorLevel level) const {
     int mask_x = left - mask_box.left();
     int mask_y = top - (tesseract_->ImageHeight() - mask_box.top());
     // AND the mask and pix, putting the result in pix.
-    pixRasterop(pix, std::max(0, -mask_x), std::max(0, -mask_y), pixGetWidth(pix),
-                pixGetHeight(pix), PIX_SRC & PIX_DST, mask, std::max(0, mask_x),
-                std::max(0, mask_y));
+    pixRasterop(pix, std::max(0, -mask_x), std::max(0, -mask_y),
+                pixGetWidth(pix), pixGetHeight(pix), PIX_SRC & PIX_DST, mask,
+                std::max(0, mask_x), std::max(0, mask_y));
     mask.destroy();
   }
   return pix;
@@ -472,8 +485,8 @@ Pix *PageIterator::GetBinaryImage(PageIteratorLevel level) const {
  * If you do not supply an original image, you will get a binary one.
  * Use pixDestroy to delete the image after use.
  */
-Pix *PageIterator::GetImage(PageIteratorLevel level, int padding, Pix *original_img, int *left,
-                            int *top) const {
+Pix *PageIterator::GetImage(PageIteratorLevel level, int padding,
+                            Pix *original_img, int *left, int *top) const {
   int right, bottom;
   if (!BoundingBox(level, left, top, &right, &bottom)) {
     return nullptr;
@@ -500,10 +513,12 @@ Pix *PageIterator::GetImage(PageIteratorLevel level, int padding, Pix *original_
     int width = pixGetWidth(grey_pix);
     int height = pixGetHeight(grey_pix);
     Image resized_mask = pixCreate(width, height, 1);
-    pixRasterop(resized_mask, std::max(0, -mask_x), std::max(0, -mask_y), width, height, PIX_SRC,
-                mask, std::max(0, mask_x), std::max(0, mask_y));
+    pixRasterop(resized_mask, std::max(0, -mask_x), std::max(0, -mask_y), width,
+                height, PIX_SRC, mask, std::max(0, mask_x),
+                std::max(0, mask_y));
     mask.destroy();
-    pixDilateBrick(resized_mask, resized_mask, 2 * padding + 1, 2 * padding + 1);
+    pixDilateBrick(resized_mask, resized_mask, 2 * padding + 1,
+                   2 * padding + 1);
     pixInvert(resized_mask, resized_mask);
     pixSetMasked(grey_pix, resized_mask, UINT32_MAX);
     resized_mask.destroy();
@@ -516,14 +531,15 @@ Pix *PageIterator::GetImage(PageIteratorLevel level, int padding, Pix *original_
  * The baseline is the line that passes through (x1, y1) and (x2, y2).
  * WARNING: with vertical text, baselines may be vertical!
  */
-bool PageIterator::Baseline(PageIteratorLevel level, int *x1, int *y1, int *x2, int *y2) const {
+bool PageIterator::Baseline(PageIteratorLevel level, int *x1, int *y1, int *x2,
+                            int *y2) const {
   if (it_->word() == nullptr) {
     return false; // Already at the end!
   }
   ROW *row = it_->row()->row;
   WERD *word = it_->word()->word;
-  TBOX box =
-      (level == RIL_WORD || level == RIL_SYMBOL) ? word->bounding_box() : row->bounding_box();
+  TBOX box = (level == RIL_WORD || level == RIL_SYMBOL) ? word->bounding_box()
+                                                        : row->bounding_box();
   int left = box.left();
   ICOORD startpt(left, static_cast<int16_t>(row->base_line(left) + 0.5));
   int right = box.right();
@@ -538,6 +554,14 @@ bool PageIterator::Baseline(PageIteratorLevel level, int *x1, int *y1, int *x2,
   return true;
 }
 
+void PageIterator::RowAttributes(float *row_height, float *descenders,
+                                 float *ascenders) const {
+  *row_height = it_->row()->row->x_height() + it_->row()->row->ascenders() -
+                it_->row()->row->descenders();
+  *descenders = it_->row()->row->descenders();
+  *ascenders = it_->row()->row->ascenders();
+}
+
 void PageIterator::Orientation(tesseract::Orientation *orientation,
                                tesseract::WritingDirection *writing_direction,
                                tesseract::TextlineOrder *textline_order,
@@ -564,23 +588,26 @@ void PageIterator::Orientation(tesseract::Orientation *orientation,
   // Writing direction
   bool is_vertical_text = (block->classify_rotation().x() == 0.0);
   bool right_to_left = block->right_to_left();
-  *writing_direction = is_vertical_text ? WRITING_DIRECTION_TOP_TO_BOTTOM
-                                        : (right_to_left ? WRITING_DIRECTION_RIGHT_TO_LEFT
-                                                         : WRITING_DIRECTION_LEFT_TO_RIGHT);
+  *writing_direction = is_vertical_text
+                           ? WRITING_DIRECTION_TOP_TO_BOTTOM
+                           : (right_to_left ? WRITING_DIRECTION_RIGHT_TO_LEFT
+                                            : WRITING_DIRECTION_LEFT_TO_RIGHT);
 
   // Textline Order
   const bool is_mongolian = false; // TODO(eger): fix me
-  *textline_order = is_vertical_text ? (is_mongolian ? TEXTLINE_ORDER_LEFT_TO_RIGHT
-                                                     : TEXTLINE_ORDER_RIGHT_TO_LEFT)
-                                     : TEXTLINE_ORDER_TOP_TO_BOTTOM;
+  *textline_order = is_vertical_text
+                        ? (is_mongolian ? TEXTLINE_ORDER_LEFT_TO_RIGHT
+                                        : TEXTLINE_ORDER_RIGHT_TO_LEFT)
+                        : TEXTLINE_ORDER_TOP_TO_BOTTOM;
 
   // Deskew angle
   FCOORD skew = block->skew(); // true horizontal for textlines
   *deskew_angle = -skew.angle();
 }
 
-void PageIterator::ParagraphInfo(tesseract::ParagraphJustification *just, bool *is_list_item,
-                                 bool *is_crown, int *first_line_indent) const {
+void PageIterator::ParagraphInfo(tesseract::ParagraphJustification *just,
+                                 bool *is_list_item, bool *is_crown,
+                                 int *first_line_indent) const {
   *just = tesseract::JUSTIFICATION_UNKNOWN;
   if (!it_->row() || !it_->row()->row || !it_->row()->row->para() ||
       !it_->row()->row->para()->model) {
@@ -612,12 +639,14 @@ void PageIterator::BeginWord(int offset) {
     // is already baseline denormalized.
     word_length_ = word_res->best_choice->length();
     if (word_res->box_word != nullptr) {
-      if (word_res->box_word->length() != word_length_) {
-        tprintf("Corrupted word! best_choice[len=%d] = %s, box_word[len=%d]: ", word_length_,
-                word_res->best_choice->unichar_string().c_str(), word_res->box_word->length());
+      if (word_res->box_word->length() != static_cast<unsigned>(word_length_)) {
+        tprintf("Corrupted word! best_choice[len=%d] = %s, box_word[len=%d]: ",
+                word_length_, word_res->best_choice->unichar_string().c_str(),
+                word_res->box_word->length());
         word_res->box_word->bounding_box().print();
       }
-      ASSERT_HOST(word_res->box_word->length() == word_length_);
+      ASSERT_HOST(word_res->box_word->length() ==
+                  static_cast<unsigned>(word_length_));
     }
     word_ = nullptr;
     // We will be iterating the box_word.

src/ccmain/par_control.cpp

@@ -40,10 +40,10 @@ void Tesseract::PrerecAllWordsPar(const std::vector<WordData> &words) {
   std::vector<BlobData> blobs;
   for (const auto &w : words) {
     if (w.word->ratings != nullptr && w.word->ratings->get(0, 0) == nullptr) {
-      for (int s = 0; s < w.lang_words.size(); ++s) {
+      for (size_t s = 0; s < w.lang_words.size(); ++s) {
         Tesseract *sub = s < sub_langs_.size() ? sub_langs_[s] : this;
         const WERD_RES &word = *w.lang_words[s];
-        for (int b = 0; b < word.chopped_word->NumBlobs(); ++b) {
+        for (unsigned b = 0; b < word.chopped_word->NumBlobs(); ++b) {
           blobs.emplace_back(b, sub, word);
         }
       }

src/ccmain/paragraphs.cpp

@@ -73,7 +73,7 @@ static int Epsilon(int space_pix) {
 static bool AcceptableRowArgs(int debug_level, int min_num_rows, const char *function_name,
                               const std::vector<RowScratchRegisters> *rows, int row_start,
                               int row_end) {
-  if (row_start < 0 || row_end > rows->size() || row_start > row_end) {
+  if (row_start < 0 || static_cast<size_t>(row_end) > rows->size() || row_start > row_end) {
     tprintf("Invalid arguments rows[%d, %d) while rows is of size %zu.\n", row_start, row_end,
             rows->size());
     return false;
@@ -94,8 +94,8 @@ static bool AcceptableRowArgs(int debug_level, int min_num_rows, const char *fun
 static void PrintTable(const std::vector<std::vector<std::string>> &rows, const char *colsep) {
   std::vector<int> max_col_widths;
   for (const auto &row : rows) {
-    int num_columns = row.size();
-    for (int c = 0; c < num_columns; c++) {
+    auto num_columns = row.size();
+    for (size_t c = 0; c < num_columns; c++) {
       int num_unicodes = 0;
       for (char i : row[c]) {
         if ((i & 0xC0) != 0x80) {
@@ -113,6 +113,7 @@ static void PrintTable(const std::vector<std::vector<std::string>> &rows, const
   }
 
   std::vector<std::string> col_width_patterns;
+  col_width_patterns.reserve(max_col_widths.size());
   for (int max_col_width : max_col_widths) {
     col_width_patterns.push_back(std::string("%-") + std::to_string(max_col_width) + "s");
   }
@@ -285,7 +286,7 @@ bool AsciiLikelyListItem(const std::string &word) {
 // ========== Brain Dead Language Model (Tesseract Version) ================
 
 // Return the first Unicode Codepoint from werd[pos].
-int UnicodeFor(const UNICHARSET *u, const WERD_CHOICE *werd, int pos) {
+static int UnicodeFor(const UNICHARSET *u, const WERD_CHOICE *werd, unsigned pos) {
   if (!u || !werd || pos > werd->length()) {
     return 0;
   }
@@ -297,33 +298,32 @@ int UnicodeFor(const UNICHARSET *u, const WERD_CHOICE *werd, int pos) {
 class UnicodeSpanSkipper {
 public:
   UnicodeSpanSkipper(const UNICHARSET *unicharset, const WERD_CHOICE *word)
-      : u_(unicharset), word_(word) {
-    wordlen_ = word->length();
+      : u_(unicharset), word_(word), wordlen_(word->length()) {
   }
 
   // Given an input position, return the first position >= pos not punc.
-  int SkipPunc(int pos);
+  unsigned SkipPunc(unsigned pos);
   // Given an input position, return the first position >= pos not digit.
-  int SkipDigits(int pos);
+  unsigned SkipDigits(unsigned pos);
   // Given an input position, return the first position >= pos not roman.
-  int SkipRomans(int pos);
+  unsigned SkipRomans(unsigned pos);
   // Given an input position, return the first position >= pos not alpha.
-  int SkipAlpha(int pos);
+  unsigned SkipAlpha(unsigned pos);
 
 private:
   const UNICHARSET *u_;
   const WERD_CHOICE *word_;
-  int wordlen_;
+  unsigned wordlen_;
 };
 
-int UnicodeSpanSkipper::SkipPunc(int pos) {
+unsigned UnicodeSpanSkipper::SkipPunc(unsigned pos) {
   while (pos < wordlen_ && u_->get_ispunctuation(word_->unichar_id(pos))) {
     pos++;
   }
   return pos;
 }
 
-int UnicodeSpanSkipper::SkipDigits(int pos) {
+unsigned UnicodeSpanSkipper::SkipDigits(unsigned pos) {
   while (pos < wordlen_ &&
          (u_->get_isdigit(word_->unichar_id(pos)) || IsDigitLike(UnicodeFor(u_, word_, pos)))) {
     pos++;
@@ -331,7 +331,7 @@ int UnicodeSpanSkipper::SkipDigits(int pos) {
   return pos;
 }
 
-int UnicodeSpanSkipper::SkipRomans(int pos) {
+unsigned UnicodeSpanSkipper::SkipRomans(unsigned pos) {
   const char *kRomans = "ivxlmdIVXLMD";
   while (pos < wordlen_) {
     int ch = UnicodeFor(u_, word_, pos);
@@ -343,7 +343,7 @@ int UnicodeSpanSkipper::SkipRomans(int pos) {
   return pos;
 }
 
-int UnicodeSpanSkipper::SkipAlpha(int pos) {
+unsigned UnicodeSpanSkipper::SkipAlpha(unsigned pos) {
   while (pos < wordlen_ && u_->get_isalpha(word_->unichar_id(pos))) {
     pos++;
   }
@@ -386,13 +386,13 @@ static bool UniLikelyListItem(const UNICHARSET *u, const WERD_CHOICE *werd) {
 
   UnicodeSpanSkipper m(u, werd);
   int num_segments = 0;
-  int pos = 0;
+  unsigned pos = 0;
   while (pos < werd->length() && num_segments < 3) {
-    int numeral_start = m.SkipPunc(pos);
+    auto numeral_start = m.SkipPunc(pos);
     if (numeral_start > pos + 1) {
       break;
     }
-    int numeral_end = m.SkipRomans(numeral_start);
+    auto numeral_end = m.SkipRomans(numeral_start);
     if (numeral_end == numeral_start) {
       numeral_end = m.SkipDigits(numeral_start);
       if (numeral_end == numeral_start) {
@@ -2314,14 +2314,14 @@ void CanonicalizeDetectionResults(std::vector<PARA *> *row_owners, PARA_LIST *pa
 void DetectParagraphs(int debug_level, std::vector<RowInfo> *row_infos,
                       std::vector<PARA *> *row_owners, PARA_LIST *paragraphs,
                       std::vector<ParagraphModel *> *models) {
-  std::vector<RowScratchRegisters> rows;
   ParagraphTheory theory(models);
 
   // Initialize row_owners to be a bunch of nullptr pointers.
+  row_owners->clear();
   row_owners->resize(row_infos->size());
 
   // Set up row scratch registers for the main algorithm.
-  rows.resize(row_infos->size(), RowScratchRegisters());
+  std::vector<RowScratchRegisters> rows(row_infos->size());
   for (unsigned i = 0; i < row_infos->size(); i++) {
     rows[i].Init((*row_infos)[i]);
   }
@@ -2353,7 +2353,7 @@ void DetectParagraphs(int debug_level, std::vector<RowInfo> *row_infos,
     LeftoverSegments(rows, &leftovers2, leftover.begin, leftover.end);
     bool pass2a_was_useful =
         leftovers2.size() > 1 ||
-        (leftovers2.size() == 1 && (leftovers2[0].begin != 0 || leftovers2[0].end != rows.size()));
+        (leftovers2.size() == 1 && (leftovers2[0].begin != 0 || static_cast<size_t>(leftovers2[0].end) != rows.size()));
     if (pass2a_was_useful) {
       for (auto &leftover2 : leftovers2) {
         StrongEvidenceClassify(debug_level, &rows, leftover2.begin, leftover2.end, &theory);

src/ccmain/paragraphs_internal.h

@@ -34,9 +34,6 @@ class WERD_CHOICE;
 TESS_API
 bool AsciiLikelyListItem(const std::string &word);
 
-// Return the first Unicode Codepoint from werd[pos].
-int UnicodeFor(const UNICHARSET *u, const WERD_CHOICE *werd, int pos);
-
 // Set right word attributes given either a unicharset and werd or a utf8
 // string.
 TESS_API

src/ccmain/paramsd.cpp

@@ -278,7 +278,7 @@ void ParamsEditor::Notify(const SVEvent *sve) {
     } else {
       ParamContent *vc = ParamContent::GetParamContentById(sve->command_id);
       vc->SetValue(param);
-      sv_window_->AddMessage("Setting %s to %s", vc->GetName(), vc->GetValue().c_str());
+      sv_window_->AddMessageF("Setting %s to %s", vc->GetName(), vc->GetValue().c_str());
     }
   }
 }
@@ -336,11 +336,7 @@ void ParamsEditor::WriteParams(char *filename, bool changes_only) {
 
   fp = fopen(filename, "wb"); // can we write to it?
   if (fp == nullptr) {
-    sv_window_->AddMessage(
-        "Can't write to file "
-        "%s"
-        "",
-        filename);
+    sv_window_->AddMessageF("Can't write to file %s", filename);
     return;
   }
   for (auto &iter : vcMap) {

src/ccmain/pgedit.cpp

@@ -122,13 +122,6 @@ INT_VAR(editor_image_ypos, 10, "Editor image Y Pos");
 static INT_VAR(editor_image_menuheight, 50, "Add to image height for menu bar");
 INT_VAR(editor_image_word_bb_color, ScrollView::BLUE, "Word bounding box colour");
 INT_VAR(editor_image_blob_bb_color, ScrollView::YELLOW, "Blob bounding box colour");
-INT_VAR(editor_image_text_color, ScrollView::WHITE, "Correct text colour");
-
-STRING_VAR(editor_dbwin_name, "EditorDBWin", "Editor debug window name");
-INT_VAR(editor_dbwin_xpos, 50, "Editor debug window X Pos");
-INT_VAR(editor_dbwin_ypos, 500, "Editor debug window Y Pos");
-INT_VAR(editor_dbwin_height, 24, "Editor debug window height");
-INT_VAR(editor_dbwin_width, 80, "Editor debug window width");
 
 STRING_VAR(editor_word_name, "BlnWords", "BL normalized word window");
 INT_VAR(editor_word_xpos, 60, "Word window X Pos");

src/ccmain/pgedit.h

@@ -43,25 +43,16 @@ private:
 #endif // !GRAPHICS_DISABLED
 
 extern BLOCK_LIST *current_block_list;
-extern STRING_VAR_H(editor_image_win_name, "EditorImage", "Editor image window name");
-extern INT_VAR_H(editor_image_xpos, 590, "Editor image X Pos");
-extern INT_VAR_H(editor_image_ypos, 10, "Editor image Y Pos");
-extern INT_VAR_H(editor_image_height, 680, "Editor image height");
-extern INT_VAR_H(editor_image_width, 655, "Editor image width");
-extern INT_VAR_H(editor_image_word_bb_color, BLUE, "Word bounding box colour");
-extern INT_VAR_H(editor_image_blob_bb_color, YELLOW, "Blob bounding box colour");
-extern INT_VAR_H(editor_image_text_color, WHITE, "Correct text colour");
-extern STRING_VAR_H(editor_dbwin_name, "EditorDBWin", "Editor debug window name");
-extern INT_VAR_H(editor_dbwin_xpos, 50, "Editor debug window X Pos");
-extern INT_VAR_H(editor_dbwin_ypos, 500, "Editor debug window Y Pos");
-extern INT_VAR_H(editor_dbwin_height, 24, "Editor debug window height");
-extern INT_VAR_H(editor_dbwin_width, 80, "Editor debug window width");
-extern STRING_VAR_H(editor_word_name, "BlnWords", "BL normalised word window");
-extern INT_VAR_H(editor_word_xpos, 60, "Word window X Pos");
-extern INT_VAR_H(editor_word_ypos, 510, "Word window Y Pos");
-extern INT_VAR_H(editor_word_height, 240, "Word window height");
-extern INT_VAR_H(editor_word_width, 655, "Word window width");
-extern double_VAR_H(editor_smd_scale_factor, 1.0, "Scaling for smd image");
+extern STRING_VAR_H(editor_image_win_name);
+extern INT_VAR_H(editor_image_xpos);
+extern INT_VAR_H(editor_image_ypos);
+extern INT_VAR_H(editor_image_word_bb_color);
+extern INT_VAR_H(editor_image_blob_bb_color);
+extern STRING_VAR_H(editor_word_name);
+extern INT_VAR_H(editor_word_xpos);
+extern INT_VAR_H(editor_word_ypos);
+extern INT_VAR_H(editor_word_height);
+extern INT_VAR_H(editor_word_width);
 
 } // namespace tesseract
 

src/ccmain/reject.cpp

@@ -94,9 +94,6 @@ void Tesseract::set_done(WERD_RES *word, int16_t pass) {
  * Sets a reject map for the word.
  *************************************************************************/
 void Tesseract::make_reject_map(WERD_RES *word, ROW *row, int16_t pass) {
-  int i;
-  int offset;
-
   flip_0O(word);
   check_debug_pt(word, -1); // For trap only
   set_done(word, pass);     // Set acceptance
@@ -145,7 +142,7 @@ void Tesseract::make_reject_map(WERD_RES *word, ROW *row, int16_t pass) {
           // PASSED TEST
         } else if (best_choice->permuter() == NUMBER_PERM) {
           if (rej_alphas_in_number_perm) {
-            for (i = 0, offset = 0; best_choice->unichar_string()[offset] != '\0';
+            for (int i = 0, offset = 0; best_choice->unichar_string()[offset] != '\0';
                  offset += best_choice->unichar_lengths()[i++]) {
               if (word->reject_map[i].accepted() &&
                   word->uch_set->get_isalpha(best_choice->unichar_string().c_str() + offset,
@@ -210,7 +207,7 @@ void Tesseract::reject_I_1_L(WERD_RES *word) {
 
 void reject_poor_matches(WERD_RES *word) {
   float threshold = compute_reject_threshold(word->best_choice);
-  for (int i = 0; i < word->best_choice->length(); ++i) {
+  for (unsigned i = 0; i < word->best_choice->length(); ++i) {
     if (word->best_choice->unichar_id(i) == UNICHAR_SPACE) {
       word->reject_map[i].setrej_tess_failure();
     } else if (word->best_choice->certainty(i) < threshold) {
@@ -232,16 +229,16 @@ float compute_reject_threshold(WERD_CHOICE *word) {
   float bestgap = 0.0f; // biggest gap
   float gapstart;       // bottom of gap
 
-  int blob_count = word->length();
+  auto blob_count = word->length();
   std::vector<float> ratings;
   ratings.reserve(blob_count);
-  for (int i = 0; i < blob_count; ++i) {
+  for (unsigned i = 0; i < blob_count; ++i) {
     ratings.push_back(word->certainty(i));
   }
   std::sort(ratings.begin(), ratings.end());
   gapstart = ratings[0] - 1; // all reject if none better
   if (blob_count >= 3) {
-    for (int index = 0; index < blob_count - 1; index++) {
+    for (unsigned index = 0; index < blob_count - 1; index++) {
       if (ratings[index + 1] - ratings[index] > bestgap) {
         bestgap = ratings[index + 1] - ratings[index];
         // find biggest
@@ -514,14 +511,12 @@ bool Tesseract::word_contains_non_1_digit(const char *word, const char *word_len
  * Don't unreject LONE accepted 1Il conflict set chars
  *************************************************************************/
 void Tesseract::dont_allow_1Il(WERD_RES *word) {
-  int i = 0;
-  int offset;
   int word_len = word->reject_map.length();
   const char *s = word->best_choice->unichar_string().c_str();
   const char *lengths = word->best_choice->unichar_lengths().c_str();
   bool accepted_1Il = false;
 
-  for (i = 0, offset = 0; i < word_len; offset += word->best_choice->unichar_lengths()[i++]) {
+  for (int i = 0, offset = 0; i < word_len; offset += word->best_choice->unichar_lengths()[i++]) {
     if (word->reject_map[i].accepted()) {
       if (conflict_set_I_l_1.contains(s[offset])) {
         accepted_1Il = true;
@@ -537,7 +532,7 @@ void Tesseract::dont_allow_1Il(WERD_RES *word) {
     return; // Nothing to worry about
   }
 
-  for (i = 0, offset = 0; i < word_len; offset += word->best_choice->unichar_lengths()[i++]) {
+  for (int i = 0, offset = 0; i < word_len; offset += word->best_choice->unichar_lengths()[i++]) {
     if (conflict_set_I_l_1.contains(s[offset]) && word->reject_map[i].accepted()) {
       word->reject_map[i].setrej_postNN_1Il();
     }
@@ -547,7 +542,7 @@ void Tesseract::dont_allow_1Il(WERD_RES *word) {
 int16_t Tesseract::count_alphanums(WERD_RES *word_res) {
   int count = 0;
   const WERD_CHOICE *best_choice = word_res->best_choice;
-  for (int i = 0; i < word_res->reject_map.length(); ++i) {
+  for (unsigned i = 0; i < word_res->reject_map.length(); ++i) {
     if ((word_res->reject_map[i].accepted()) &&
         (word_res->uch_set->get_isalpha(best_choice->unichar_id(i)) ||
          word_res->uch_set->get_isdigit(best_choice->unichar_id(i)))) {
@@ -568,9 +563,6 @@ void Tesseract::reject_mostly_rejects(WERD_RES *word) {
 }
 
 bool Tesseract::repeated_nonalphanum_wd(WERD_RES *word, ROW *row) {
-  int16_t char_quality;
-  int16_t accepted_char_quality;
-
   if (word->best_choice->unichar_lengths().length() <= 1) {
     return false;
   }
@@ -580,15 +572,17 @@ bool Tesseract::repeated_nonalphanum_wd(WERD_RES *word, ROW *row) {
   }
 
   UNICHAR_ID uch_id = word->best_choice->unichar_id(0);
-  for (int i = 1; i < word->best_choice->length(); ++i) {
+  for (unsigned i = 1; i < word->best_choice->length(); ++i) {
     if (word->best_choice->unichar_id(i) != uch_id) {
       return false;
     }
   }
 
+  int16_t char_quality;
+  int16_t accepted_char_quality;
   word_char_quality(word, &char_quality, &accepted_char_quality);
 
-  if ((word->best_choice->unichar_lengths().length() == char_quality) &&
+  if ((word->best_choice->unichar_lengths().length() == static_cast<size_t>(char_quality)) &&
       (char_quality == accepted_char_quality)) {
     return true;
   } else {
@@ -607,7 +601,6 @@ int16_t Tesseract::safe_dict_word(const WERD_RES *werd_res) {
 // in word_res->best_choice.
 void Tesseract::flip_hyphens(WERD_RES *word_res) {
   WERD_CHOICE *best_choice = word_res->best_choice;
-  int i;
   int prev_right = -9999;
   int next_left;
   TBOX out_box;
@@ -617,9 +610,9 @@ void Tesseract::flip_hyphens(WERD_RES *word_res) {
     return;
   }
 
-  int num_blobs = word_res->rebuild_word->NumBlobs();
+  auto num_blobs = word_res->rebuild_word->NumBlobs();
   UNICHAR_ID unichar_dash = word_res->uch_set->unichar_to_id("-");
-  for (i = 0; i < best_choice->length() && i < num_blobs; ++i) {
+  for (unsigned i = 0; i < best_choice->length() && i < num_blobs; ++i) {
     TBLOB *blob = word_res->rebuild_word->blobs[i];
     out_box = blob->bounding_box();
     if (i + 1 == num_blobs) {
@@ -666,15 +659,14 @@ void Tesseract::flip_hyphens(WERD_RES *word_res) {
 // in word_res->best_choice.
 void Tesseract::flip_0O(WERD_RES *word_res) {
   WERD_CHOICE *best_choice = word_res->best_choice;
-  int i;
   TBOX out_box;
 
   if (!tessedit_flip_0O) {
     return;
   }
 
-  int num_blobs = word_res->rebuild_word->NumBlobs();
-  for (i = 0; i < best_choice->length() && i < num_blobs; ++i) {
+  auto num_blobs = word_res->rebuild_word->NumBlobs();
+  for (unsigned i = 0; i < best_choice->length() && i < num_blobs; ++i) {
     TBLOB *blob = word_res->rebuild_word->blobs[i];
     if (word_res->uch_set->get_isupper(best_choice->unichar_id(i)) ||
         word_res->uch_set->get_isdigit(best_choice->unichar_id(i))) {
@@ -691,7 +683,7 @@ void Tesseract::flip_0O(WERD_RES *word_res) {
       unichar_O == INVALID_UNICHAR_ID || !word_res->uch_set->get_enabled(unichar_O)) {
     return; // 0 or O are not present/enabled in unicharset
   }
-  for (i = 1; i < best_choice->length(); ++i) {
+  for (unsigned i = 1; i < best_choice->length(); ++i) {
     if (best_choice->unichar_id(i) == unichar_0 || best_choice->unichar_id(i) == unichar_O) {
       /* A0A */
       if ((i + 1) < best_choice->length() &&

src/ccmain/resultiterator.cpp

@@ -228,7 +228,7 @@ void ResultIterator::CalculateBlobOrder(std::vector<int> *blob_indices) const {
       i = j;
     }
   }
-  ASSERT_HOST(blob_indices->size() == word_length_);
+  ASSERT_HOST(blob_indices->size() == static_cast<size_t>(word_length_));
 }
 
 static void PrintScriptDirs(const std::vector<StrongScriptDirection> &dirs) {
@@ -501,7 +501,7 @@ bool ResultIterator::Next(PageIteratorLevel level) {
     case RIL_SYMBOL: {
       std::vector<int> blob_order;
       CalculateBlobOrder(&blob_order);
-      int next_blob = 0;
+      unsigned next_blob = 0;
       while (next_blob < blob_order.size() && blob_index_ != blob_order[next_blob]) {
         next_blob++;
       }
@@ -731,10 +731,12 @@ void ResultIterator::IterateAndAppendUTF8TextlineText(std::string *text) {
     std::vector<int> textline_order;
     std::vector<StrongScriptDirection> dirs;
     CalculateTextlineOrder(current_paragraph_is_ltr_, *this, &dirs, &textline_order);
-    tprintf("Strong Script dirs     [%p/P=%s]: ", it_->row(),
+    tprintf("Strong Script dirs     [%p/P=%s]: ",
+            static_cast<void *>(it_->row()),
             current_paragraph_is_ltr_ ? "ltr" : "rtl");
     PrintScriptDirs(dirs);
-    tprintf("Logical textline order [%p/P=%s]: ", it_->row(),
+    tprintf("Logical textline order [%p/P=%s]: ",
+            static_cast<void *>(it_->row()),
             current_paragraph_is_ltr_ ? "ltr" : "rtl");
     for (int i : textline_order) {
       tprintf("%d ", i);

src/ccmain/superscript.cpp

@@ -502,13 +502,13 @@ WERD_RES *Tesseract::TrySuperscriptSplits(int num_chopped_leading, float leading
  */
 bool Tesseract::BelievableSuperscript(bool debug, const WERD_RES &word, float certainty_threshold,
                                       int *left_ok, int *right_ok) const {
-  int initial_ok_run_count = 0;
-  int ok_run_count = 0;
+  unsigned initial_ok_run_count = 0;
+  unsigned ok_run_count = 0;
   float worst_certainty = 0.0f;
   const WERD_CHOICE &wc = *word.best_choice;
 
   const UnicityTable<FontInfo> &fontinfo_table = get_fontinfo_table();
-  for (int i = 0; i < wc.length(); i++) {
+  for (unsigned i = 0; i < wc.length(); i++) {
     TBLOB *blob = word.rebuild_word->blobs[i];
     UNICHAR_ID unichar_id = wc.unichar_id(i);
     float char_certainty = wc.certainty(i);

src/ccmain/tessedit.cpp

@@ -23,6 +23,8 @@
 #  include "config_auto.h"
 #endif
 
+#include <regex> // for std::regex_match
+
 #include "control.h"
 #include "matchdefs.h"
 #include "pageres.h"
@@ -73,15 +75,12 @@ void Tesseract::read_config_file(const char *filename, SetParamConstraint constr
 // from the language-specific config file (stored in [lang].traineddata), from
 // the config files specified on the command line or left as the default
 // OEM_TESSERACT_ONLY if none of the configs specify this variable.
-bool Tesseract::init_tesseract_lang_data(const std::string &arg0, const std::string &textbase,
+bool Tesseract::init_tesseract_lang_data(const std::string &arg0,
                                          const std::string &language, OcrEngineMode oem,
                                          char **configs, int configs_size,
                                          const std::vector<std::string> *vars_vec,
                                          const std::vector<std::string> *vars_values,
                                          bool set_only_non_debug_params, TessdataManager *mgr) {
-  // Set the basename, compute the data directory.
-  main_setup(arg0, textbase);
-
   // Set the language data path prefix
   lang = !language.empty() ? language : "eng";
   language_data_path_prefix = datadir;
@@ -247,6 +246,15 @@ static bool IsStrInList(const std::string &str, const std::vector<std::string> &
 void Tesseract::ParseLanguageString(const std::string &lang_str, std::vector<std::string> *to_load,
                                     std::vector<std::string> *not_to_load) {
   std::string remains(lang_str);
+  // Look whether the model file uses a prefix which must be applied to
+  // included model files as well.
+  std::regex e("(.*)/[^/]*");
+  std::cmatch cm;
+  std::string prefix;
+  if (std::regex_match(lang.c_str(), cm, e, std::regex_constants::match_default)) {
+    // A prefix was found.
+    prefix = cm[1].str() + "/";
+  }
   while (!remains.empty()) {
     // Find the start of the lang code and which vector to add to.
     const char *start = remains.c_str();
@@ -268,6 +276,7 @@ void Tesseract::ParseLanguageString(const std::string &lang_str, std::vector<std
     lang_code.resize(end);
     std::string next(start + end);
     remains = next;
+    lang_code = prefix + lang_code;
     // Check whether lang_code is already in the target vector and add.
     if (!IsStrInList(lang_code, *target)) {
       target->push_back(lang_code);
@@ -291,19 +300,26 @@ int Tesseract::init_tesseract(const std::string &arg0, const std::string &textba
   for (auto *lang : sub_langs_) {
     delete lang;
   }
+
+  // Set the basename, compute the data directory.
+  main_setup(arg0, textbase);
+
   sub_langs_.clear();
   // Find the first loadable lang and load into this.
   // Add any languages that this language requires
   bool loaded_primary = false;
   // Load the rest into sub_langs_.
-  for (unsigned lang_index = 0; lang_index < langs_to_load.size(); ++lang_index) {
-    if (!IsStrInList(langs_to_load[lang_index], langs_not_to_load)) {
-      const char *lang_str = langs_to_load[lang_index].c_str();
+  // A range based for loop does not work here because langs_to_load
+  // might be changed in the loop when a new submodel is found.
+  for (auto &lang_to_load : langs_to_load) {
+    if (!IsStrInList(lang_to_load, langs_not_to_load)) {
+      const char *lang_str = lang_to_load.c_str();
       Tesseract *tess_to_init;
       if (!loaded_primary) {
         tess_to_init = this;
       } else {
         tess_to_init = new Tesseract;
+        tess_to_init->main_setup(arg0, textbase);
       }
 
       int result = tess_to_init->init_tesseract_internal(arg0, textbase, lang_str, oem, configs,
@@ -316,7 +332,7 @@ int Tesseract::init_tesseract(const std::string &arg0, const std::string &textba
         if (result < 0) {
           tprintf("Failed loading language '%s'\n", lang_str);
         } else {
-          ParseLanguageString(tess_to_init->tessedit_load_sublangs.c_str(), &langs_to_load,
+          ParseLanguageString(tess_to_init->tessedit_load_sublangs, &langs_to_load,
                               &langs_not_to_load);
           loaded_primary = true;
         }
@@ -327,13 +343,13 @@ int Tesseract::init_tesseract(const std::string &arg0, const std::string &textba
         } else {
           sub_langs_.push_back(tess_to_init);
           // Add any languages that this language requires
-          ParseLanguageString(tess_to_init->tessedit_load_sublangs.c_str(), &langs_to_load,
+          ParseLanguageString(tess_to_init->tessedit_load_sublangs, &langs_to_load,
                               &langs_not_to_load);
         }
       }
     }
   }
-  if (!loaded_primary) {
+  if (!loaded_primary && !langs_to_load.empty()) {
     tprintf("Tesseract couldn't load any languages!\n");
     return -1; // Couldn't load any language!
   }
@@ -384,7 +400,7 @@ int Tesseract::init_tesseract_internal(const std::string &arg0, const std::strin
                                        const std::vector<std::string> *vars_vec,
                                        const std::vector<std::string> *vars_values,
                                        bool set_only_non_debug_params, TessdataManager *mgr) {
-  if (!init_tesseract_lang_data(arg0, textbase, language, oem, configs, configs_size, vars_vec,
+  if (!init_tesseract_lang_data(arg0, language, oem, configs, configs_size, vars_vec,
                                 vars_values, set_only_non_debug_params, mgr)) {
     return -1;
   }
@@ -412,7 +428,7 @@ static void CollectFonts(const UnicityTable<FontInfo> &new_fonts,
 // Helper assigns an id to lang_fonts using the index in all_fonts table.
 static void AssignIds(const UnicityTable<FontInfo> &all_fonts, UnicityTable<FontInfo> *lang_fonts) {
   for (int i = 0; i < lang_fonts->size(); ++i) {
-    int index = all_fonts.get_id(lang_fonts->at(i));
+    auto index = all_fonts.get_index(lang_fonts->at(i));
     lang_fonts->at(i).universal_id = index;
   }
 }
@@ -438,19 +454,6 @@ void Tesseract::SetupUniversalFontIds() {
   font_table_size_ = all_fonts.size();
 }
 
-// init the LM component
-int Tesseract::init_tesseract_lm(const std::string &arg0, const std::string &textbase,
-                                 const std::string &language, TessdataManager *mgr) {
-  if (!init_tesseract_lang_data(arg0, textbase, language, OEM_TESSERACT_ONLY, nullptr, 0, nullptr,
-                                nullptr, false, mgr)) {
-    return -1;
-  }
-  getDict().SetupForLoad(Dict::GlobalDawgCache());
-  getDict().Load(lang, mgr);
-  getDict().FinishLoad();
-  return 0;
-}
-
 #endif // ndef DISABLED_LEGACY_ENGINE
 
 void Tesseract::end_tesseract() {

src/ccmain/tesseractclass.cpp

@@ -46,6 +46,7 @@
 #  include "equationdetect.h"
 #endif
 #include "lstmrecognizer.h"
+#include "thresholder.h" // for ThresholdMethod
 
 namespace tesseract {
 
@@ -75,10 +76,40 @@ Tesseract::Tesseract()
                " (Values from PageSegMode enum in tesseract/publictypes.h)",
                this->params())
     , INT_MEMBER(thresholding_method,
-                 static_cast<int>(tesseract::ThresholdMethod::Otsu),
-                 "Thresholding "
-                 "method: 0 = Otsu, 1 = Adaptive Otsu, 2 = Sauvola",
+                 static_cast<int>(ThresholdMethod::Otsu),
+                 "Thresholding method: 0 = Otsu, 1 = LeptonicaOtsu, 2 = "
+                 "Sauvola",
                  this->params())
+    , BOOL_MEMBER(thresholding_debug, false,
+                  "Debug the thresholding process",
+                  this->params())
+    , double_MEMBER(thresholding_window_size, 0.33,
+                    "Window size for measuring local statistics (to be "
+                    "multiplied by image DPI). "
+                    "This parameter is used by the Sauvola thresolding method",
+                    this->params())
+    , double_MEMBER(thresholding_kfactor, 0.34,
+                    "Factor for reducing threshold due to variance. "
+                    "This parameter is used by the Sauvola thresolding method."
+                    " Normal range: 0.2-0.5",
+                    this->params())
+    , double_MEMBER(thresholding_tile_size, 0.33,
+                    "Desired tile size (to be multiplied by image DPI). "
+                    "This parameter is used by the LeptonicaOtsu thresolding "
+                    "method",
+                    this->params())
+    , double_MEMBER(thresholding_smooth_kernel_size, 0.0,
+                    "Size of convolution kernel applied to threshold array "
+                    "(to be multiplied by image DPI). Use 0 for no smoothing. "
+                    "This parameter is used by the LeptonicaOtsu thresolding "
+                    "method",
+                    this->params())
+    , double_MEMBER(thresholding_score_fraction, 0.1,
+                    "Fraction of the max Otsu score. "
+                    "This parameter is used by the LeptonicaOtsu thresolding "
+                    "method. "
+                    "For standard Otsu use 0.0, otherwise 0.1 is recommended",
+                    this->params())
     , INT_INIT_MEMBER(tessedit_ocr_engine_mode, tesseract::OEM_DEFAULT,
                       "Which OCR engine(s) to run (Tesseract, LSTM, both)."
                       " Defaults to loading and running the most accurate"
@@ -369,7 +400,9 @@ Tesseract::Tesseract()
                        "instance is not going to be used for OCR but say only "
                        "for layout analysis.",
                        this->params())
+#ifndef DISABLED_LEGACY_ENGINE
     , BOOL_MEMBER(textord_equation_detect, false, "Turn on equation detector", this->params())
+#endif // ndef DISABLED_LEGACY_ENGINE
     , BOOL_MEMBER(textord_tabfind_vertical_text, true, "Enable vertical detection", this->params())
     , BOOL_MEMBER(textord_tabfind_force_vertical_text, false, "Force using vertical text page mode",
                   this->params())
@@ -403,7 +436,7 @@ Tesseract::Tesseract()
                     "information is lost due to the cut off at 0. The standard value is "
                     "5",
                     this->params())
-    , BOOL_MEMBER(pageseg_apply_music_mask, true,
+    , BOOL_MEMBER(pageseg_apply_music_mask, false,
                   "Detect music staff and remove intersecting components", this->params())
     ,
 
@@ -421,7 +454,9 @@ Tesseract::Tesseract()
     , reskew_(1.0f, 0.0f)
     , most_recently_used_(this)
     , font_table_size_(0)
+#ifndef DISABLED_LEGACY_ENGINE
     , equ_detect_(nullptr)
+#endif // ndef DISABLED_LEGACY_ENGINE
     , lstm_recognizer_(nullptr)
     , train_line_page_num_(0) {}
 

src/ccmain/tesseractclass.h

@@ -69,7 +69,9 @@ class WERD_RES;
 
 class ColumnFinder;
 class DocumentData;
+#ifndef DISABLED_LEGACY_ENGINE
 class EquationDetect;
+#endif // ndef DISABLED_LEGACY_ENGINE
 class ImageData;
 class LSTMRecognizer;
 class Tesseract;
@@ -110,7 +112,7 @@ class Tesseract;
 // NOTE: that each level contains members that correspond to global
 // data that is defined (and used) at that level, not necessarily where
 // the type is defined so for instance:
-// BOOL_VAR_H(textord_show_blobs, false, "Display unsorted blobs");
+// BOOL_VAR_H(textord_show_blobs);
 // goes inside the Textord class, not the cc_util class.
 
 // A collection of various variables for statistics and debugging.
@@ -189,8 +191,10 @@ public:
   // Clear the document dictionary for this and all subclassifiers.
   void ResetDocumentDictionary();
 
+#ifndef DISABLED_LEGACY_ENGINE
   // Set the equation detector.
   void SetEquationDetect(EquationDetect *detector);
+#endif // ndef DISABLED_LEGACY_ENGINE
 
   // Simple accessors.
   const FCOORD &reskew() const {
@@ -523,13 +527,10 @@ public:
   // instances of the same font loaded.
   void SetupUniversalFontIds();
 
-  int init_tesseract_lm(const std::string &arg0, const std::string &textbase,
-                        const std::string &language, TessdataManager *mgr);
-
   void recognize_page(std::string &image_name);
   void end_tesseract();
 
-  bool init_tesseract_lang_data(const std::string &arg0, const std::string &textbase,
+  bool init_tesseract_lang_data(const std::string &arg0,
                                 const std::string &language, OcrEngineMode oem, char **configs,
                                 int configs_size, const std::vector<std::string> *vars_vec,
                                 const std::vector<std::string> *vars_values,
@@ -593,7 +594,7 @@ public:
   void recog_word_recursive(WERD_RES *word);
   void recog_word(WERD_RES *word);
   void split_and_recog_word(WERD_RES *word);
-  void split_word(WERD_RES *word, int split_pt, WERD_RES **right_piece,
+  void split_word(WERD_RES *word, unsigned split_pt, WERD_RES **right_piece,
                   BlamerBundle **orig_blamer_bundle) const;
   void join_words(WERD_RES *word, WERD_RES *word2, BlamerBundle *orig_bb) const;
   //// fixspace.cpp ///////////////////////////////////////////////////////
@@ -722,8 +723,8 @@ public:
   // vector holding classification results for a sequence of consecutive
   // blobs, with index 0 being a single blob, index 1 being 2 blobs etc.
   void SearchForText(const std::vector<BLOB_CHOICE_LIST *> *choices, int choices_pos,
-                     int choices_length, const std::vector<UNICHAR_ID> &target_text,
-                     int text_index, float rating, std::vector<int> *segmentation,
+                     unsigned choices_length, const std::vector<UNICHAR_ID> &target_text,
+                     unsigned text_index, float rating, std::vector<int> *segmentation,
                      float *best_rating, std::vector<int> *best_segmentation);
   // Counts up the labelled words and the blobs within.
   // Deletes all unused or emptied words, counting the unused ones.
@@ -748,282 +749,217 @@ public:
   float ComputeCompatibleXheight(WERD_RES *word_res, float *baseline_shift);
   //// Data members ///////////////////////////////////////////////////////
   // TODO(ocr-team): Find and remove obsolete parameters.
-  BOOL_VAR_H(tessedit_resegment_from_boxes, false, "Take segmentation and labeling from box file");
-  BOOL_VAR_H(tessedit_resegment_from_line_boxes, false,
-             "Conversion of word/line box file to char box file");
-  BOOL_VAR_H(tessedit_train_from_boxes, false, "Generate training data from boxed chars");
-  BOOL_VAR_H(tessedit_make_boxes_from_boxes, false, "Generate more boxes from boxed chars");
-  BOOL_VAR_H(tessedit_train_line_recognizer, false,
-             "Break input into lines and remap boxes if present");
-  BOOL_VAR_H(tessedit_dump_pageseg_images, false,
-             "Dump intermediate images made during page segmentation");
-  BOOL_VAR_H(tessedit_do_invert, true, "Try inverting the image in `LSTMRecognizeWord`");
-  INT_VAR_H(tessedit_pageseg_mode, PSM_SINGLE_BLOCK,
-            "Page seg mode: 0=osd only, 1=auto+osd, 2=auto, 3=col, 4=block,"
-            " 5=line, 6=word, 7=char"
-            " (Values from PageSegMode enum in tesseract/publictypes.h)");
-  INT_VAR_H(thresholding_method,
-            static_cast<int>(tesseract::ThreshMethod::Otsu), "Thresholding "
-            "method: 0 = Otsu, 1 = Adaptive Otsu, 2 = Sauvola");
-  INT_VAR_H(tessedit_ocr_engine_mode, tesseract::OEM_DEFAULT,
-            "Which OCR engine(s) to run (Tesseract, LSTM, both). Defaults"
-            " to loading and running the most accurate available.");
-  STRING_VAR_H(tessedit_char_blacklist, "", "Blacklist of chars not to recognize");
-  STRING_VAR_H(tessedit_char_whitelist, "", "Whitelist of chars to recognize");
-  STRING_VAR_H(tessedit_char_unblacklist, "", "List of chars to override tessedit_char_blacklist");
-  BOOL_VAR_H(tessedit_ambigs_training, false, "Perform training for ambiguities");
-  INT_VAR_H(pageseg_devanagari_split_strategy, tesseract::ShiroRekhaSplitter::NO_SPLIT,
-            "Whether to use the top-line splitting process for Devanagari "
-            "documents while performing page-segmentation.");
-  INT_VAR_H(ocr_devanagari_split_strategy, tesseract::ShiroRekhaSplitter::NO_SPLIT,
-            "Whether to use the top-line splitting process for Devanagari "
-            "documents while performing ocr.");
-  STRING_VAR_H(tessedit_write_params_to_file, "", "Write all parameters to the given file.");
-  BOOL_VAR_H(tessedit_adaption_debug, false, "Generate and print debug information for adaption");
-  INT_VAR_H(bidi_debug, 0, "Debug level for BiDi");
-  INT_VAR_H(applybox_debug, 1, "Debug level");
-  INT_VAR_H(applybox_page, 0, "Page number to apply boxes from");
-  STRING_VAR_H(applybox_exposure_pattern, ".exp",
-               "Exposure value follows this pattern in the image"
-               " filename. The name of the image files are expected"
-               " to be in the form [lang].[fontname].exp[num].tif");
-  BOOL_VAR_H(applybox_learn_chars_and_char_frags_mode, false,
-             "Learn both character fragments (as is done in the"
-             " special low exposure mode) as well as unfragmented"
-             " characters.");
-  BOOL_VAR_H(applybox_learn_ngrams_mode, false,
-             "Each bounding box is assumed to contain ngrams. Only"
-             " learn the ngrams whose outlines overlap horizontally.");
-  BOOL_VAR_H(tessedit_display_outwords, false, "Draw output words");
-  BOOL_VAR_H(tessedit_dump_choices, false, "Dump char choices");
-  BOOL_VAR_H(tessedit_timing_debug, false, "Print timing stats");
-  BOOL_VAR_H(tessedit_fix_fuzzy_spaces, true, "Try to improve fuzzy spaces");
-  BOOL_VAR_H(tessedit_unrej_any_wd, false, "Don't bother with word plausibility");
-  BOOL_VAR_H(tessedit_fix_hyphens, true, "Crunch double hyphens?");
-  BOOL_VAR_H(tessedit_enable_doc_dict, true, "Add words to the document dictionary");
-  BOOL_VAR_H(tessedit_debug_fonts, false, "Output font info per char");
-  INT_VAR_H(tessedit_font_id, 0, "Disable font detection and use the font"
-                                 " corresponding to the ID specified instead");
-  BOOL_VAR_H(tessedit_debug_block_rejection, false, "Block and Row stats");
-  BOOL_VAR_H(tessedit_enable_bigram_correction, true,
-             "Enable correction based on the word bigram dictionary.");
-  BOOL_VAR_H(tessedit_enable_dict_correction, false,
-             "Enable single word correction based on the dictionary.");
-  INT_VAR_H(tessedit_bigram_debug, 0,
-            "Amount of debug output for bigram "
-            "correction.");
-  BOOL_VAR_H(enable_noise_removal, true,
-             "Remove and conditionally reassign small outlines when they"
-             " confuse layout analysis, determining diacritics vs noise");
-  INT_VAR_H(debug_noise_removal, 0, "Debug reassignment of small outlines");
+  BOOL_VAR_H(tessedit_resegment_from_boxes);
+  BOOL_VAR_H(tessedit_resegment_from_line_boxes);
+  BOOL_VAR_H(tessedit_train_from_boxes);
+  BOOL_VAR_H(tessedit_make_boxes_from_boxes);
+  BOOL_VAR_H(tessedit_train_line_recognizer);
+  BOOL_VAR_H(tessedit_dump_pageseg_images);
+  BOOL_VAR_H(tessedit_do_invert);
+  INT_VAR_H(tessedit_pageseg_mode);
+  INT_VAR_H(thresholding_method);
+  BOOL_VAR_H(thresholding_debug);
+  double_VAR_H(thresholding_window_size);
+  double_VAR_H(thresholding_kfactor);
+  double_VAR_H(thresholding_tile_size);
+  double_VAR_H(thresholding_smooth_kernel_size);
+  double_VAR_H(thresholding_score_fraction);
+  INT_VAR_H(tessedit_ocr_engine_mode);
+  STRING_VAR_H(tessedit_char_blacklist);
+  STRING_VAR_H(tessedit_char_whitelist);
+  STRING_VAR_H(tessedit_char_unblacklist);
+  BOOL_VAR_H(tessedit_ambigs_training);
+  INT_VAR_H(pageseg_devanagari_split_strategy);
+  INT_VAR_H(ocr_devanagari_split_strategy);
+  STRING_VAR_H(tessedit_write_params_to_file);
+  BOOL_VAR_H(tessedit_adaption_debug);
+  INT_VAR_H(bidi_debug);
+  INT_VAR_H(applybox_debug);
+  INT_VAR_H(applybox_page);
+  STRING_VAR_H(applybox_exposure_pattern);
+  BOOL_VAR_H(applybox_learn_chars_and_char_frags_mode);
+  BOOL_VAR_H(applybox_learn_ngrams_mode);
+  BOOL_VAR_H(tessedit_display_outwords);
+  BOOL_VAR_H(tessedit_dump_choices);
+  BOOL_VAR_H(tessedit_timing_debug);
+  BOOL_VAR_H(tessedit_fix_fuzzy_spaces);
+  BOOL_VAR_H(tessedit_unrej_any_wd);
+  BOOL_VAR_H(tessedit_fix_hyphens);
+  BOOL_VAR_H(tessedit_enable_doc_dict);
+  BOOL_VAR_H(tessedit_debug_fonts);
+  INT_VAR_H(tessedit_font_id);
+  BOOL_VAR_H(tessedit_debug_block_rejection);
+  BOOL_VAR_H(tessedit_enable_bigram_correction);
+  BOOL_VAR_H(tessedit_enable_dict_correction);
+  INT_VAR_H(tessedit_bigram_debug);
+  BOOL_VAR_H(enable_noise_removal);
+  INT_VAR_H(debug_noise_removal);
   // Worst (min) certainty, for which a diacritic is allowed to make the base
   // character worse and still be included.
-  double_VAR_H(noise_cert_basechar, -8.0, "Hingepoint for base char certainty");
+  double_VAR_H(noise_cert_basechar);
   // Worst (min) certainty, for which a non-overlapping diacritic is allowed to
   // make the base character worse and still be included.
-  double_VAR_H(noise_cert_disjoint, -2.5, "Hingepoint for disjoint certainty");
+  double_VAR_H(noise_cert_disjoint);
   // Worst (min) certainty, for which a diacritic is allowed to make a new
   // stand-alone blob.
-  double_VAR_H(noise_cert_punc, -2.5, "Threshold for new punc char certainty");
+  double_VAR_H(noise_cert_punc);
   // Factor of certainty margin for adding diacritics to not count as worse.
-  double_VAR_H(noise_cert_factor, 0.375, "Scaling on certainty diff from Hingepoint");
-  INT_VAR_H(noise_maxperblob, 8, "Max diacritics to apply to a blob");
-  INT_VAR_H(noise_maxperword, 16, "Max diacritics to apply to a word");
-  INT_VAR_H(debug_x_ht_level, 0, "Reestimate debug");
-  STRING_VAR_H(chs_leading_punct, "('`\"", "Leading punctuation");
-  STRING_VAR_H(chs_trailing_punct1, ").,;:?!", "1st Trailing punctuation");
-  STRING_VAR_H(chs_trailing_punct2, ")'`\"", "2nd Trailing punctuation");
-  double_VAR_H(quality_rej_pc, 0.08, "good_quality_doc lte rejection limit");
-  double_VAR_H(quality_blob_pc, 0.0, "good_quality_doc gte good blobs limit");
-  double_VAR_H(quality_outline_pc, 1.0, "good_quality_doc lte outline error limit");
-  double_VAR_H(quality_char_pc, 0.95, "good_quality_doc gte good char limit");
-  INT_VAR_H(quality_min_initial_alphas_reqd, 2, "alphas in a good word");
-  INT_VAR_H(tessedit_tess_adaption_mode, 0x27, "Adaptation decision algorithm for tess");
-  BOOL_VAR_H(tessedit_minimal_rej_pass1, false, "Do minimal rejection on pass 1 output");
-  BOOL_VAR_H(tessedit_test_adaption, false, "Test adaption criteria");
-  BOOL_VAR_H(test_pt, false, "Test for point");
-  double_VAR_H(test_pt_x, 99999.99, "xcoord");
-  double_VAR_H(test_pt_y, 99999.99, "ycoord");
-  INT_VAR_H(multilang_debug_level, 0, "Print multilang debug info.");
-  INT_VAR_H(paragraph_debug_level, 0, "Print paragraph debug info.");
-  BOOL_VAR_H(paragraph_text_based, true,
-             "Run paragraph detection on the post-text-recognition "
-             "(more accurate)");
-  BOOL_VAR_H(lstm_use_matrix, 1, "Use ratings matrix/beam searct with lstm");
-  STRING_VAR_H(outlines_odd, "%| ", "Non standard number of outlines");
-  STRING_VAR_H(outlines_2, "ij!?%\":;", "Non standard number of outlines");
-  BOOL_VAR_H(tessedit_good_quality_unrej, true, "Reduce rejection on good docs");
-  BOOL_VAR_H(tessedit_use_reject_spaces, true, "Reject spaces?");
-  double_VAR_H(tessedit_reject_doc_percent, 65.00, "%rej allowed before rej whole doc");
-  double_VAR_H(tessedit_reject_block_percent, 45.00, "%rej allowed before rej whole block");
-  double_VAR_H(tessedit_reject_row_percent, 40.00, "%rej allowed before rej whole row");
-  double_VAR_H(tessedit_whole_wd_rej_row_percent, 70.00,
-               "Number of row rejects in whole word rejects"
-               "which prevents whole row rejection");
-  BOOL_VAR_H(tessedit_preserve_blk_rej_perfect_wds, true,
-             "Only rej partially rejected words in block rejection");
-  BOOL_VAR_H(tessedit_preserve_row_rej_perfect_wds, true,
-             "Only rej partially rejected words in row rejection");
-  BOOL_VAR_H(tessedit_dont_blkrej_good_wds, false, "Use word segmentation quality metric");
-  BOOL_VAR_H(tessedit_dont_rowrej_good_wds, false, "Use word segmentation quality metric");
-  INT_VAR_H(tessedit_preserve_min_wd_len, 2, "Only preserve wds longer than this");
-  BOOL_VAR_H(tessedit_row_rej_good_docs, true, "Apply row rejection to good docs");
-  double_VAR_H(tessedit_good_doc_still_rowrej_wd, 1.1,
-               "rej good doc wd if more than this fraction rejected");
-  BOOL_VAR_H(tessedit_reject_bad_qual_wds, true, "Reject all bad quality wds");
-  BOOL_VAR_H(tessedit_debug_doc_rejection, false, "Page stats");
-  BOOL_VAR_H(tessedit_debug_quality_metrics, false, "Output data to debug file");
-  BOOL_VAR_H(bland_unrej, false, "unrej potential with no checks");
-  double_VAR_H(quality_rowrej_pc, 1.1, "good_quality_doc gte good char limit");
-  BOOL_VAR_H(unlv_tilde_crunching, false, "Mark v.bad words for tilde crunch");
-  BOOL_VAR_H(hocr_font_info, false, "Add font info to hocr output");
-  BOOL_VAR_H(hocr_char_boxes, false, "Add coordinates for each character to hocr output");
-  BOOL_VAR_H(crunch_early_merge_tess_fails, true, "Before word crunch?");
-  BOOL_VAR_H(crunch_early_convert_bad_unlv_chs, false, "Take out ~^ early?");
-  double_VAR_H(crunch_terrible_rating, 80.0, "crunch rating lt this");
-  BOOL_VAR_H(crunch_terrible_garbage, true, "As it says");
-  double_VAR_H(crunch_poor_garbage_cert, -9.0, "crunch garbage cert lt this");
-  double_VAR_H(crunch_poor_garbage_rate, 60, "crunch garbage rating lt this");
-  double_VAR_H(crunch_pot_poor_rate, 40, "POTENTIAL crunch rating lt this");
-  double_VAR_H(crunch_pot_poor_cert, -8.0, "POTENTIAL crunch cert lt this");
-  double_VAR_H(crunch_del_rating, 60, "POTENTIAL crunch rating lt this");
-  double_VAR_H(crunch_del_cert, -10.0, "POTENTIAL crunch cert lt this");
-  double_VAR_H(crunch_del_min_ht, 0.7, "Del if word ht lt xht x this");
-  double_VAR_H(crunch_del_max_ht, 3.0, "Del if word ht gt xht x this");
-  double_VAR_H(crunch_del_min_width, 3.0, "Del if word width lt xht x this");
-  double_VAR_H(crunch_del_high_word, 1.5, "Del if word gt xht x this above bl");
-  double_VAR_H(crunch_del_low_word, 0.5, "Del if word gt xht x this below bl");
-  double_VAR_H(crunch_small_outlines_size, 0.6, "Small if lt xht x this");
-  INT_VAR_H(crunch_rating_max, 10, "For adj length in rating per ch");
-  INT_VAR_H(crunch_pot_indicators, 1, "How many potential indicators needed");
-  BOOL_VAR_H(crunch_leave_ok_strings, true, "Don't touch sensible strings");
-  BOOL_VAR_H(crunch_accept_ok, true, "Use acceptability in okstring");
-  BOOL_VAR_H(crunch_leave_accept_strings, false, "Don't pot crunch sensible strings");
-  BOOL_VAR_H(crunch_include_numerals, false, "Fiddle alpha figures");
-  INT_VAR_H(crunch_leave_lc_strings, 4, "Don't crunch words with long lower case strings");
-  INT_VAR_H(crunch_leave_uc_strings, 4, "Don't crunch words with long lower case strings");
-  INT_VAR_H(crunch_long_repetitions, 3, "Crunch words with long repetitions");
-  INT_VAR_H(crunch_debug, 0, "As it says");
-  INT_VAR_H(fixsp_non_noise_limit, 1, "How many non-noise blbs either side?");
-  double_VAR_H(fixsp_small_outlines_size, 0.28, "Small if lt xht x this");
-  BOOL_VAR_H(tessedit_prefer_joined_punct, false, "Reward punctuation joins");
-  INT_VAR_H(fixsp_done_mode, 1, "What constitutes done for spacing");
-  INT_VAR_H(debug_fix_space_level, 0, "Contextual fixspace debug");
-  STRING_VAR_H(numeric_punctuation, ".,", "Punct. chs expected WITHIN numbers");
-  INT_VAR_H(x_ht_acceptance_tolerance, 8, "Max allowed deviation of blob top outside of font data");
-  INT_VAR_H(x_ht_min_change, 8, "Min change in xht before actually trying it");
-  INT_VAR_H(superscript_debug, 0, "Debug level for sub & superscript fixer");
-  double_VAR_H(superscript_worse_certainty, 2.0,
-               "How many times worse "
-               "certainty does a superscript position glyph need to be for us "
-               "to try classifying it as a char with a different baseline?");
-  double_VAR_H(superscript_bettered_certainty, 0.97,
-               "What reduction in "
-               "badness do we think sufficient to choose a superscript over "
-               "what we'd thought.  For example, a value of 0.6 means we want "
-               "to reduce badness of certainty by 40%");
-  double_VAR_H(superscript_scaledown_ratio, 0.4,
-               "A superscript scaled down more than this is unbelievably "
-               "small.  For example, 0.3 means we expect the font size to "
-               "be no smaller than 30% of the text line font size.");
-  double_VAR_H(subscript_max_y_top, 0.5,
-               "Maximum top of a character measured as a multiple of x-height "
-               "above the baseline for us to reconsider whether it's a "
-               "subscript.");
-  double_VAR_H(superscript_min_y_bottom, 0.3,
-               "Minimum bottom of a character measured as a multiple of "
-               "x-height above the baseline for us to reconsider whether it's "
-               "a superscript.");
-  BOOL_VAR_H(tessedit_write_block_separators, false, "Write block separators in output");
-  BOOL_VAR_H(tessedit_write_rep_codes, false, "Write repetition char code");
-  BOOL_VAR_H(tessedit_write_unlv, false, "Write .unlv output file");
-  BOOL_VAR_H(tessedit_create_txt, false, "Write .txt output file");
-  BOOL_VAR_H(tessedit_create_hocr, false, "Write .html hOCR output file");
-  BOOL_VAR_H(tessedit_create_alto, false, "Write .xml ALTO output file");
-  BOOL_VAR_H(tessedit_create_lstmbox, false, "Write .box file for LSTM training");
-  BOOL_VAR_H(tessedit_create_tsv, false, "Write .tsv output file");
-  BOOL_VAR_H(tessedit_create_wordstrbox, false, "Write WordStr format .box output file");
-  BOOL_VAR_H(tessedit_create_pdf, false, "Write .pdf output file");
-  BOOL_VAR_H(textonly_pdf, false, "Create PDF with only one invisible text layer");
-  INT_VAR_H(jpg_quality, 85, "Set JPEG quality level");
-  INT_VAR_H(user_defined_dpi, 0, "Specify DPI for input image");
-  INT_VAR_H(min_characters_to_try, 50, "Specify minimum characters to try during OSD");
-  STRING_VAR_H(unrecognised_char, "|", "Output char for unidentified blobs");
-  INT_VAR_H(suspect_level, 99, "Suspect marker level");
-  INT_VAR_H(suspect_short_words, 2, "Don't Suspect dict wds longer than this");
-  BOOL_VAR_H(suspect_constrain_1Il, false, "UNLV keep 1Il chars rejected");
-  double_VAR_H(suspect_rating_per_ch, 999.9, "Don't touch bad rating limit");
-  double_VAR_H(suspect_accept_rating, -999.9, "Accept good rating limit");
-  BOOL_VAR_H(tessedit_minimal_rejection, false, "Only reject tess failures");
-  BOOL_VAR_H(tessedit_zero_rejection, false, "Don't reject ANYTHING");
-  BOOL_VAR_H(tessedit_word_for_word, false, "Make output have exactly one word per WERD");
-  BOOL_VAR_H(tessedit_zero_kelvin_rejection, false, "Don't reject ANYTHING AT ALL");
-  INT_VAR_H(tessedit_reject_mode, 0, "Rejection algorithm");
-  BOOL_VAR_H(tessedit_rejection_debug, false, "Adaption debug");
-  BOOL_VAR_H(tessedit_flip_0O, true, "Contextual 0O O0 flips");
-  double_VAR_H(tessedit_lower_flip_hyphen, 1.5, "Aspect ratio dot/hyphen test");
-  double_VAR_H(tessedit_upper_flip_hyphen, 1.8, "Aspect ratio dot/hyphen test");
-  BOOL_VAR_H(rej_trust_doc_dawg, false, "Use DOC dawg in 11l conf. detector");
-  BOOL_VAR_H(rej_1Il_use_dict_word, false, "Use dictword test");
-  BOOL_VAR_H(rej_1Il_trust_permuter_type, true, "Don't double check");
-  BOOL_VAR_H(rej_use_tess_accepted, true, "Individual rejection control");
-  BOOL_VAR_H(rej_use_tess_blanks, true, "Individual rejection control");
-  BOOL_VAR_H(rej_use_good_perm, true, "Individual rejection control");
-  BOOL_VAR_H(rej_use_sensible_wd, false, "Extend permuter check");
-  BOOL_VAR_H(rej_alphas_in_number_perm, false, "Extend permuter check");
-  double_VAR_H(rej_whole_of_mostly_reject_word_fract, 0.85, "if >this fract");
-  INT_VAR_H(tessedit_image_border, 2, "Rej blbs near image edge limit");
-  STRING_VAR_H(ok_repeated_ch_non_alphanum_wds, "-?*\075", "Allow NN to unrej");
-  STRING_VAR_H(conflict_set_I_l_1, "Il1[]", "Il1 conflict set");
-  INT_VAR_H(min_sane_x_ht_pixels, 8, "Reject any x-ht lt or eq than this");
-  BOOL_VAR_H(tessedit_create_boxfile, false, "Output text with boxes");
-  INT_VAR_H(tessedit_page_number, -1, "-1 -> All pages, else specific page to process");
-  BOOL_VAR_H(tessedit_write_images, false, "Capture the image from the IPE");
-  BOOL_VAR_H(interactive_display_mode, false, "Run interactively?");
-  STRING_VAR_H(file_type, ".tif", "Filename extension");
-  BOOL_VAR_H(tessedit_override_permuter, true, "According to dict_word");
-  STRING_VAR_H(tessedit_load_sublangs, "", "List of languages to load with this one");
-  BOOL_VAR_H(tessedit_use_primary_params_model, false,
-             "In multilingual mode use params model of the primary language");
+  double_VAR_H(noise_cert_factor);
+  INT_VAR_H(noise_maxperblob);
+  INT_VAR_H(noise_maxperword);
+  INT_VAR_H(debug_x_ht_level);
+  STRING_VAR_H(chs_leading_punct);
+  STRING_VAR_H(chs_trailing_punct1);
+  STRING_VAR_H(chs_trailing_punct2);
+  double_VAR_H(quality_rej_pc);
+  double_VAR_H(quality_blob_pc);
+  double_VAR_H(quality_outline_pc);
+  double_VAR_H(quality_char_pc);
+  INT_VAR_H(quality_min_initial_alphas_reqd);
+  INT_VAR_H(tessedit_tess_adaption_mode);
+  BOOL_VAR_H(tessedit_minimal_rej_pass1);
+  BOOL_VAR_H(tessedit_test_adaption);
+  BOOL_VAR_H(test_pt);
+  double_VAR_H(test_pt_x);
+  double_VAR_H(test_pt_y);
+  INT_VAR_H(multilang_debug_level);
+  INT_VAR_H(paragraph_debug_level);
+  BOOL_VAR_H(paragraph_text_based);
+  BOOL_VAR_H(lstm_use_matrix);
+  STRING_VAR_H(outlines_odd);
+  STRING_VAR_H(outlines_2);
+  BOOL_VAR_H(tessedit_good_quality_unrej);
+  BOOL_VAR_H(tessedit_use_reject_spaces);
+  double_VAR_H(tessedit_reject_doc_percent);
+  double_VAR_H(tessedit_reject_block_percent);
+  double_VAR_H(tessedit_reject_row_percent);
+  double_VAR_H(tessedit_whole_wd_rej_row_percent);
+  BOOL_VAR_H(tessedit_preserve_blk_rej_perfect_wds);
+  BOOL_VAR_H(tessedit_preserve_row_rej_perfect_wds);
+  BOOL_VAR_H(tessedit_dont_blkrej_good_wds);
+  BOOL_VAR_H(tessedit_dont_rowrej_good_wds);
+  INT_VAR_H(tessedit_preserve_min_wd_len);
+  BOOL_VAR_H(tessedit_row_rej_good_docs);
+  double_VAR_H(tessedit_good_doc_still_rowrej_wd);
+  BOOL_VAR_H(tessedit_reject_bad_qual_wds);
+  BOOL_VAR_H(tessedit_debug_doc_rejection);
+  BOOL_VAR_H(tessedit_debug_quality_metrics);
+  BOOL_VAR_H(bland_unrej);
+  double_VAR_H(quality_rowrej_pc);
+  BOOL_VAR_H(unlv_tilde_crunching);
+  BOOL_VAR_H(hocr_font_info);
+  BOOL_VAR_H(hocr_char_boxes);
+  BOOL_VAR_H(crunch_early_merge_tess_fails);
+  BOOL_VAR_H(crunch_early_convert_bad_unlv_chs);
+  double_VAR_H(crunch_terrible_rating);
+  BOOL_VAR_H(crunch_terrible_garbage);
+  double_VAR_H(crunch_poor_garbage_cert);
+  double_VAR_H(crunch_poor_garbage_rate);
+  double_VAR_H(crunch_pot_poor_rate);
+  double_VAR_H(crunch_pot_poor_cert);
+  double_VAR_H(crunch_del_rating);
+  double_VAR_H(crunch_del_cert);
+  double_VAR_H(crunch_del_min_ht);
+  double_VAR_H(crunch_del_max_ht);
+  double_VAR_H(crunch_del_min_width);
+  double_VAR_H(crunch_del_high_word);
+  double_VAR_H(crunch_del_low_word);
+  double_VAR_H(crunch_small_outlines_size);
+  INT_VAR_H(crunch_rating_max);
+  INT_VAR_H(crunch_pot_indicators);
+  BOOL_VAR_H(crunch_leave_ok_strings);
+  BOOL_VAR_H(crunch_accept_ok);
+  BOOL_VAR_H(crunch_leave_accept_strings);
+  BOOL_VAR_H(crunch_include_numerals);
+  INT_VAR_H(crunch_leave_lc_strings);
+  INT_VAR_H(crunch_leave_uc_strings);
+  INT_VAR_H(crunch_long_repetitions);
+  INT_VAR_H(crunch_debug);
+  INT_VAR_H(fixsp_non_noise_limit);
+  double_VAR_H(fixsp_small_outlines_size);
+  BOOL_VAR_H(tessedit_prefer_joined_punct);
+  INT_VAR_H(fixsp_done_mode);
+  INT_VAR_H(debug_fix_space_level);
+  STRING_VAR_H(numeric_punctuation);
+  INT_VAR_H(x_ht_acceptance_tolerance);
+  INT_VAR_H(x_ht_min_change);
+  INT_VAR_H(superscript_debug);
+  double_VAR_H(superscript_worse_certainty);
+  double_VAR_H(superscript_bettered_certainty);
+  double_VAR_H(superscript_scaledown_ratio);
+  double_VAR_H(subscript_max_y_top);
+  double_VAR_H(superscript_min_y_bottom);
+  BOOL_VAR_H(tessedit_write_block_separators);
+  BOOL_VAR_H(tessedit_write_rep_codes);
+  BOOL_VAR_H(tessedit_write_unlv);
+  BOOL_VAR_H(tessedit_create_txt);
+  BOOL_VAR_H(tessedit_create_hocr);
+  BOOL_VAR_H(tessedit_create_alto);
+  BOOL_VAR_H(tessedit_create_lstmbox);
+  BOOL_VAR_H(tessedit_create_tsv);
+  BOOL_VAR_H(tessedit_create_wordstrbox);
+  BOOL_VAR_H(tessedit_create_pdf);
+  BOOL_VAR_H(textonly_pdf);
+  INT_VAR_H(jpg_quality);
+  INT_VAR_H(user_defined_dpi);
+  INT_VAR_H(min_characters_to_try);
+  STRING_VAR_H(unrecognised_char);
+  INT_VAR_H(suspect_level);
+  INT_VAR_H(suspect_short_words);
+  BOOL_VAR_H(suspect_constrain_1Il);
+  double_VAR_H(suspect_rating_per_ch);
+  double_VAR_H(suspect_accept_rating);
+  BOOL_VAR_H(tessedit_minimal_rejection);
+  BOOL_VAR_H(tessedit_zero_rejection);
+  BOOL_VAR_H(tessedit_word_for_word);
+  BOOL_VAR_H(tessedit_zero_kelvin_rejection);
+  INT_VAR_H(tessedit_reject_mode);
+  BOOL_VAR_H(tessedit_rejection_debug);
+  BOOL_VAR_H(tessedit_flip_0O);
+  double_VAR_H(tessedit_lower_flip_hyphen);
+  double_VAR_H(tessedit_upper_flip_hyphen);
+  BOOL_VAR_H(rej_trust_doc_dawg);
+  BOOL_VAR_H(rej_1Il_use_dict_word);
+  BOOL_VAR_H(rej_1Il_trust_permuter_type);
+  BOOL_VAR_H(rej_use_tess_accepted);
+  BOOL_VAR_H(rej_use_tess_blanks);
+  BOOL_VAR_H(rej_use_good_perm);
+  BOOL_VAR_H(rej_use_sensible_wd);
+  BOOL_VAR_H(rej_alphas_in_number_perm);
+  double_VAR_H(rej_whole_of_mostly_reject_word_fract);
+  INT_VAR_H(tessedit_image_border);
+  STRING_VAR_H(ok_repeated_ch_non_alphanum_wds);
+  STRING_VAR_H(conflict_set_I_l_1);
+  INT_VAR_H(min_sane_x_ht_pixels);
+  BOOL_VAR_H(tessedit_create_boxfile);
+  INT_VAR_H(tessedit_page_number);
+  BOOL_VAR_H(tessedit_write_images);
+  BOOL_VAR_H(interactive_display_mode);
+  STRING_VAR_H(file_type);
+  BOOL_VAR_H(tessedit_override_permuter);
+  STRING_VAR_H(tessedit_load_sublangs);
+  BOOL_VAR_H(tessedit_use_primary_params_model);
   // Min acceptable orientation margin (difference in scores between top and 2nd
   // choice in OSResults::orientations) to believe the page orientation.
-  double_VAR_H(min_orientation_margin, 7.0, "Min acceptable orientation margin");
-  BOOL_VAR_H(textord_tabfind_show_vlines, false, "Debug line finding");
-  BOOL_VAR_H(textord_use_cjk_fp_model, false, "Use CJK fixed pitch model");
-  BOOL_VAR_H(poly_allow_detailed_fx, false, "Allow feature extractors to see the original outline");
-  BOOL_VAR_H(tessedit_init_config_only, false,
-             "Only initialize with the config file. Useful if the instance is "
-             "not going to be used for OCR but say only for layout analysis.");
-  BOOL_VAR_H(textord_equation_detect, false, "Turn on equation detector");
-  BOOL_VAR_H(textord_tabfind_vertical_text, true, "Enable vertical detection");
-  BOOL_VAR_H(textord_tabfind_force_vertical_text, false, "Force using vertical text page mode");
-  double_VAR_H(textord_tabfind_vertical_text_ratio, 0.5,
-               "Fraction of textlines deemed vertical to use vertical page "
-               "mode");
-  double_VAR_H(textord_tabfind_aligned_gap_fraction, 0.75,
-               "Fraction of height used as a minimum gap for aligned blobs.");
-  INT_VAR_H(tessedit_parallelize, 0, "Run in parallel where possible");
-  BOOL_VAR_H(preserve_interword_spaces, false, "Preserve multiple interword spaces");
-  STRING_VAR_H(page_separator, "\f", "Page separator (default is form feed control character)");
-  INT_VAR_H(lstm_choice_mode, 0,
-            "Allows to include alternative symbols choices in the hOCR "
-            "output. "
-            "Valid input values are 0, 1 and 2. 0 is the default value. "
-            "With 1 the alternative symbol choices per timestep are included. "
-            "With 2 the alternative symbol choices are extracted from the CTC "
-            "process instead of the lattice. The choices are mapped per "
-            "character.");
-  INT_VAR_H(lstm_choice_iterations, 5,
-            "Sets the number of cascading iterations for the Beamsearch in "
-            "lstm_choice_mode. Note that lstm_choice_mode must be set to "
-            "a value greater than 0 to produce results.");
-  double_VAR_H(lstm_rating_coefficient, 5,
-               "Sets the rating coefficient for the lstm choices. The smaller "
-               "the coefficient, the better are the ratings for each choice "
-               "and less information is lost due to the cut off at 0. The "
-               "standard value is 5.");
-  BOOL_VAR_H(pageseg_apply_music_mask, true,
-             "Detect music staff and remove intersecting components");
+  double_VAR_H(min_orientation_margin);
+  BOOL_VAR_H(textord_tabfind_show_vlines);
+  BOOL_VAR_H(textord_use_cjk_fp_model);
+  BOOL_VAR_H(poly_allow_detailed_fx);
+  BOOL_VAR_H(tessedit_init_config_only);
+#ifndef DISABLED_LEGACY_ENGINE
+  BOOL_VAR_H(textord_equation_detect);
+#endif // ndef DISABLED_LEGACY_ENGINE
+  BOOL_VAR_H(textord_tabfind_vertical_text);
+  BOOL_VAR_H(textord_tabfind_force_vertical_text);
+  double_VAR_H(textord_tabfind_vertical_text_ratio);
+  double_VAR_H(textord_tabfind_aligned_gap_fraction);
+  INT_VAR_H(tessedit_parallelize);
+  BOOL_VAR_H(preserve_interword_spaces);
+  STRING_VAR_H(page_separator);
+  INT_VAR_H(lstm_choice_mode);
+  INT_VAR_H(lstm_choice_iterations);
+  double_VAR_H(lstm_rating_coefficient);
+  BOOL_VAR_H(pageseg_apply_music_mask);
 
   //// ambigsrecog.cpp /////////////////////////////////////////////////////////
   FILE *init_recog_training(const char *filename);
@@ -1071,8 +1007,10 @@ private:
   Tesseract *most_recently_used_;
   // The size of the font table, ie max possible font id + 1.
   int font_table_size_;
+#ifndef DISABLED_LEGACY_ENGINE
   // Equation detector. Note: this pointer is NOT owned by the class.
   EquationDetect *equ_detect_;
+#endif // ndef DISABLED_LEGACY_ENGINE
   // LSTM recognizer, if available.
   LSTMRecognizer *lstm_recognizer_;
   // Output "page" number (actually line number) using TrainLineRecognizer.

src/ccmain/tfacepp.cpp

@@ -47,14 +47,7 @@ void Tesseract::recog_word(WERD_RES *word) {
   ASSERT_HOST(!word->chopped_word->blobs.empty());
   recog_word_recursive(word);
   word->SetupBoxWord();
-  if (word->best_choice->length() != word->box_word->length()) {
-    tprintf(
-        "recog_word ASSERT FAIL String:\"%s\"; "
-        "Strlen=%d; #Blobs=%d\n",
-        word->best_choice->debug_string().c_str(), word->best_choice->length(),
-        word->box_word->length());
-  }
-  ASSERT_HOST(word->best_choice->length() == word->box_word->length());
+  ASSERT_HOST(static_cast<unsigned>(word->best_choice->length()) == word->box_word->length());
   // Check that the ratings matrix size matches the sum of all the
   // segmentation states.
   if (!word->StatesAllValid()) {
@@ -82,7 +75,7 @@ void Tesseract::recog_word(WERD_RES *word) {
   // Factored out from control.cpp
   ASSERT_HOST((word->best_choice == nullptr) == (word->raw_choice == nullptr));
   if (word->best_choice == nullptr || word->best_choice->empty() ||
-      static_cast<int>(strspn(word->best_choice->unichar_string().c_str(), " ")) ==
+      strspn(word->best_choice->unichar_string().c_str(), " ") ==
           word->best_choice->length()) {
     word->tess_failed = true;
     word->reject_map.initialise(word->box_word->length());
@@ -99,7 +92,7 @@ void Tesseract::recog_word(WERD_RES *word) {
  * Convert the output back to editor form.
  **********************************************************************/
 void Tesseract::recog_word_recursive(WERD_RES *word) {
-  int word_length = word->chopped_word->NumBlobs(); // no of blobs
+  auto word_length = word->chopped_word->NumBlobs(); // no of blobs
   if (word_length > MAX_UNDIVIDED_LENGTH) {
     return split_and_recog_word(word);
   }
@@ -134,7 +127,7 @@ void Tesseract::split_and_recog_word(WERD_RES *word) {
   // Find the biggest blob gap in the chopped_word.
   int bestgap = -INT32_MAX;
   int split_index = 0;
-  for (int b = 1; b < word->chopped_word->NumBlobs(); ++b) {
+  for (unsigned b = 1; b < word->chopped_word->NumBlobs(); ++b) {
     TBOX prev_box = word->chopped_word->blobs[b - 1]->bounding_box();
     TBOX blob_box = word->chopped_word->blobs[b]->bounding_box();
     int gap = blob_box.left() - prev_box.right();
@@ -167,7 +160,7 @@ void Tesseract::split_and_recog_word(WERD_RES *word) {
  * and will now be owned by the caller.  New blamer bundles are forged for the
  * two pieces.
  **********************************************************************/
-void Tesseract::split_word(WERD_RES *word, int split_pt, WERD_RES **right_piece,
+void Tesseract::split_word(WERD_RES *word, unsigned split_pt, WERD_RES **right_piece,
                            BlamerBundle **orig_blamer_bundle) const {
   ASSERT_HOST(split_pt > 0 && split_pt < word->chopped_word->NumBlobs());
 
@@ -181,7 +174,7 @@ void Tesseract::split_word(WERD_RES *word, int split_pt, WERD_RES **right_piece,
   TWERD *chopped = word->chopped_word;
   auto *chopped2 = new TWERD;
   chopped2->blobs.reserve(chopped->NumBlobs() - split_pt);
-  for (int i = split_pt; i < chopped->NumBlobs(); ++i) {
+  for (auto i = split_pt; i < chopped->NumBlobs(); ++i) {
     chopped2->blobs.push_back(chopped->blobs[i]);
   }
   chopped->blobs.resize(split_pt);

src/ccmain/thresholder.cpp

@@ -25,8 +25,10 @@
 #endif
 
 #include <allheaders.h>
+#include <tesseract/baseapi.h> // for api->GetIntVariable()
 
-#include <cstdint> // for uint32_t
+#include <algorithm> // for std::max, std::min
+#include <cstdint>   // for uint32_t
 #include <cstring>
 #include <tuple>
 
@@ -186,7 +188,8 @@ void ImageThresholder::SetImage(const Image pix) {
 }
 
 std::tuple<bool, Image, Image, Image> ImageThresholder::Threshold(
-                                                         ThresholdMethod method) {
+                                                      TessBaseAPI *api,
+                                                      ThresholdMethod method) {
   Image pix_binary = nullptr;
   Image pix_thresholds = nullptr;
 
@@ -196,19 +199,83 @@ std::tuple<bool, Image, Image, Image> ImageThresholder::Threshold(
     Image original = GetPixRect();
     pix_binary = original.copy();
     original.destroy();
-    return std::make_tuple(false, nullptr, pix_binary, nullptr);
+    return std::make_tuple(true, nullptr, pix_binary, nullptr);
   }
 
   auto pix_grey = GetPixRectGrey();
 
   int r;
+
+  l_int32 pix_w, pix_h;
+  pixGetDimensions(pix_grey, &pix_w, &pix_h, nullptr);
+
+  bool thresholding_debug;
+  api->GetBoolVariable("thresholding_debug", &thresholding_debug);
+  if (thresholding_debug) {
+    tprintf("\nimage width: %d  height: %d  ppi: %d\n", pix_w, pix_h, yres_);
+  }
+
   if (method == ThresholdMethod::Sauvola) {
-    r = pixSauvolaBinarizeTiled(pix_grey, 25, 0.40, 300, 300, pix_thresholds,
-                                pix_binary);
-  } else {
-    // AdaptiveOtsu.
-    r = pixOtsuAdaptiveThreshold(pix_grey, 300, 300, 0, 0, 0.1,
-                                 pix_thresholds, pix_binary);
+    int window_size;
+    double window_size_factor;
+    api->GetDoubleVariable("thresholding_window_size", &window_size_factor);
+    window_size = window_size_factor * yres_;
+    window_size = std::max(7, window_size);
+    window_size = std::min(pix_w < pix_h ? pix_w - 3 : pix_h - 3, window_size);
+    int half_window_size = window_size / 2;
+
+    // factor for image division into tiles; >= 1
+    l_int32 nx, ny;
+    // tiles size will be approx. 250 x 250 pixels
+    nx = std::max(1, (pix_w + 125) / 250);
+    ny = std::max(1, (pix_h + 125) / 250);
+    auto xrat = pix_w / nx;
+    auto yrat = pix_h / ny;
+    if (xrat < half_window_size + 2) {
+      nx = pix_w / (half_window_size + 2);
+    }
+    if (yrat < half_window_size + 2) {
+      ny = pix_h / (half_window_size + 2);
+    }
+
+    double kfactor;
+    api->GetDoubleVariable("thresholding_kfactor", &kfactor);
+    kfactor = std::max(0.0, kfactor);
+
+    if (thresholding_debug) {
+      tprintf("window size: %d  kfactor: %.3f  nx:%d  ny: %d\n", window_size, kfactor, nx, ny);
+    }
+
+    r = pixSauvolaBinarizeTiled(pix_grey, half_window_size, kfactor, nx, ny,
+                               (PIX**)pix_thresholds,
+                                (PIX**)pix_binary);
+  } else { // if (method == ThresholdMethod::LeptonicaOtsu)
+    int tile_size;
+    double tile_size_factor;
+    api->GetDoubleVariable("thresholding_tile_size", &tile_size_factor);
+    tile_size = tile_size_factor * yres_;
+    tile_size = std::max(16, tile_size);
+
+    int smooth_size;
+    double smooth_size_factor;
+    api->GetDoubleVariable("thresholding_smooth_kernel_size",
+                         &smooth_size_factor);
+    smooth_size_factor = std::max(0.0, smooth_size_factor);
+    smooth_size = smooth_size_factor * yres_;
+    int half_smooth_size = smooth_size / 2;
+
+    double score_fraction;
+    api->GetDoubleVariable("thresholding_score_fraction", &score_fraction);
+
+    if (thresholding_debug) {
+      tprintf("tile size: %d  smooth_size: %d  score_fraction: %.2f\n", tile_size, smooth_size, score_fraction);
+    }
+
+    r = pixOtsuAdaptiveThreshold(pix_grey, tile_size, tile_size,
+                                 half_smooth_size, half_smooth_size,
+                                 score_fraction,
+                                 (PIX**)pix_thresholds,
+                                 (PIX**)pix_binary);
   }
 
   bool ok = (r == 0);

src/ccmain/thresholder.h

@@ -20,7 +20,6 @@
 #define TESSERACT_CCMAIN_THRESHOLDER_H_
 
 #include <tesseract/export.h>
-#include <tesseract/publictypes.h>
 
 #include <vector> // for std::vector
 
@@ -28,6 +27,15 @@ struct Pix;
 
 namespace tesseract {
 
+enum class ThresholdMethod {
+  Otsu,          // Tesseract's legacy Otsu
+  LeptonicaOtsu, // Leptonica's Otsu
+  Sauvola,       // Leptonica's Sauvola
+  Max,           // Number of Thresholding methods
+};
+
+class TessBaseAPI;
+
 /// Base class for all tesseract image thresholding classes.
 /// Specific classes can add new thresholding methods by
 /// overriding ThresholdToPix.
@@ -121,7 +129,7 @@ public:
   /// Returns false on error.
   virtual bool ThresholdToPix(Image *pix);
 
-  virtual std::tuple<bool, Image, Image, Image> Threshold(
+  virtual std::tuple<bool, Image, Image, Image> Threshold(TessBaseAPI *api,
                                                           ThresholdMethod method);
 
   // Gets a pix that contains an 8 bit threshold value at each pixel. The

src/ccstruct/blamer.cpp

@@ -72,7 +72,7 @@ void BlamerBundle::SetWordTruth(const UNICHARSET &unicharset, const char *truth_
   std::vector<char> lengths;
   unicharset.encode_string(truth_str, false, &encoding, &lengths, nullptr);
   int total_length = 0;
-  for (int i = 0; i < encoding.size(); total_length += lengths[i++]) {
+  for (size_t i = 0; i < encoding.size(); total_length += lengths[i++]) {
     std::string uch(truth_str + total_length);
     uch.resize(lengths[i] - total_length);
     UNICHAR_ID id = encoding[i];
@@ -119,7 +119,7 @@ bool BlamerBundle::ChoiceIsCorrect(const WERD_CHOICE *word_choice) const {
   }
   const UNICHARSET *uni_set = word_choice->unicharset();
   std::string normed_choice_str;
-  for (int i = 0; i < word_choice->length(); ++i) {
+  for (unsigned i = 0; i < word_choice->length(); ++i) {
     normed_choice_str += uni_set->get_normed_unichar(word_choice->unichar_id(i));
   }
   std::string truth_str = TruthString();
@@ -155,7 +155,7 @@ void BlamerBundle::SetupNormTruthWord(const DENORM &denorm) {
   TPOINT botright;
   TPOINT norm_topleft;
   TPOINT norm_botright;
-  for (int b = 0; b < truth_word_.length(); ++b) {
+  for (unsigned b = 0; b < truth_word_.length(); ++b) {
     const TBOX &box = truth_word_.BlobBox(b);
     topleft.x = box.left();
     topleft.y = box.top();
@@ -175,8 +175,7 @@ void BlamerBundle::SplitBundle(int word1_right, int word2_left, bool debug, Blam
                                BlamerBundle *bundle2) const {
   std::string debug_str;
   // Find truth boxes that correspond to the split in the blobs.
-  int b;
-  int begin2_truth_index = -1;
+  unsigned begin2_truth_index = 0;
   if (incorrect_result_reason_ != IRR_NO_TRUTH && truth_has_char_boxes_) {
     debug_str = "Looking for truth split at";
     debug_str += " end1_x " + std::to_string(word1_right);
@@ -184,7 +183,7 @@ void BlamerBundle::SplitBundle(int word1_right, int word2_left, bool debug, Blam
     debug_str += "\nnorm_truth_word boxes:\n";
     if (norm_truth_word_.length() > 1) {
       norm_truth_word_.BlobBox(0).print_to_str(debug_str);
-      for (b = 1; b < norm_truth_word_.length(); ++b) {
+      for (unsigned b = 1; b < norm_truth_word_.length(); ++b) {
         norm_truth_word_.BlobBox(b).print_to_str(debug_str);
         if ((abs(word1_right - norm_truth_word_.BlobBox(b - 1).right()) < norm_box_tolerance_) &&
             (abs(word2_left - norm_truth_word_.BlobBox(b).left()) < norm_box_tolerance_)) {
@@ -204,7 +203,7 @@ void BlamerBundle::SplitBundle(int word1_right, int word2_left, bool debug, Blam
     bundle2->truth_has_char_boxes_ = true;
     bundle2->norm_box_tolerance_ = norm_box_tolerance_;
     BlamerBundle *curr_bb = bundle1;
-    for (b = 0; b < norm_truth_word_.length(); ++b) {
+    for (unsigned b = 0; b < norm_truth_word_.length(); ++b) {
       if (b == begin2_truth_index) {
         curr_bb = bundle2;
       }
@@ -264,7 +263,7 @@ void BlamerBundle::BlameClassifier(const UNICHARSET &unicharset, const TBOX &blo
     return; // Nothing to do here.
   }
 
-  for (int b = 0; b < norm_truth_word_.length(); ++b) {
+  for (unsigned b = 0; b < norm_truth_word_.length(); ++b) {
     const TBOX &truth_box = norm_truth_word_.BlobBox(b);
     // Note that we are more strict on the bounding box boundaries here
     // than in other places (chopper, segmentation search), since we do
@@ -311,10 +310,9 @@ void BlamerBundle::SetChopperBlame(const WERD_RES *word, bool debug) {
   if (NoTruth() || !truth_has_char_boxes_ || word->chopped_word->blobs.empty()) {
     return;
   }
-  std::string debug_str;
   bool missing_chop = false;
   int num_blobs = word->chopped_word->blobs.size();
-  int box_index = 0;
+  unsigned box_index = 0;
   int blob_index = 0;
   int16_t truth_x = -1;
   while (box_index < truth_word_.length() && blob_index < num_blobs) {
@@ -367,7 +365,7 @@ void BlamerBundle::BlameClassifierOrLangModel(const WERD_RES *word, const UNICHA
   if (valid_permuter) {
     // Find out whether best choice is a top choice.
     best_choice_is_dict_and_top_choice_ = true;
-    for (int i = 0; i < word->best_choice->length(); ++i) {
+    for (unsigned i = 0; i < word->best_choice->length(); ++i) {
       BLOB_CHOICE_IT blob_choice_it(word->GetBlobChoices(i));
       ASSERT_HOST(!blob_choice_it.empty());
       BLOB_CHOICE *first_choice = nullptr;
@@ -415,7 +413,7 @@ void BlamerBundle::SetupCorrectSegmentation(const TWERD *word, bool debug) {
   }
   int blob_index = 0;
   int16_t next_box_x = word->blobs[blob_index]->bounding_box().right();
-  for (int truth_idx = 0; blob_index < num_blobs && truth_idx < norm_truth_word_.length();
+  for (unsigned truth_idx = 0; blob_index < num_blobs && truth_idx < norm_truth_word_.length();
        ++blob_index) {
     ++next_box_col;
     int16_t curr_box_x = next_box_x;
@@ -442,7 +440,7 @@ void BlamerBundle::SetupCorrectSegmentation(const TWERD *word, bool debug) {
   }
   if (blob_index < num_blobs || // trailing blobs
       correct_segmentation_cols_.size() != norm_truth_word_.length()) {
-    debug_str += 
+    debug_str +=
         "Blamer failed to find correct segmentation"
         " (tolerance=" +
         std::to_string(norm_box_tolerance_);
@@ -478,7 +476,7 @@ void BlamerBundle::InitForSegSearch(const WERD_CHOICE *best_choice, MATRIX *rati
   // Fill pain points for any unclassifed blob corresponding to the
   // correct segmentation state.
   debug_str += "Correct segmentation:\n";
-  for (int idx = 0; idx < correct_segmentation_cols_.size(); ++idx) {
+  for (unsigned idx = 0; idx < correct_segmentation_cols_.size(); ++idx) {
     debug_str += "col=" + std::to_string(correct_segmentation_cols_[idx]);
     debug_str += " row=" + std::to_string(correct_segmentation_rows_[idx]);
     debug_str += "\n";

src/ccstruct/blobbox.cpp

@@ -33,6 +33,7 @@
 #include <allheaders.h> // for pixGetHeight, pixGetPixel
 
 #include <algorithm> // for max, min
+#include <cmath>
 #include <cstdint>   // for INT32_MAX, INT16_MAX
 
 #define PROJECTION_MARGIN 10 // arbitrary
@@ -133,7 +134,7 @@ void BLOBNBOX::chop(       // chop blobs
   BLOBNBOX_IT blob_it;        // blob iterator
 
   // get no of chops
-  blobcount = static_cast<int16_t>(floor(box.width() / xheight));
+  blobcount = static_cast<int16_t>(std::floor(box.width() / xheight));
   if (blobcount > 1 && cblob_ptr != nullptr) {
     // width of each
     blobwidth = static_cast<float>(box.width() + 1) / blobcount;
@@ -150,12 +151,12 @@ void BLOBNBOX::chop(       // chop blobs
         UpdateRange(test_ymin, test_ymax, &ymin, &ymax);
       } while (blob != end_it->data());
       if (ymin < ymax) {
-        leftx = static_cast<int16_t>(floor(rightx - blobwidth));
+        leftx = static_cast<int16_t>(std::floor(rightx - blobwidth));
         if (leftx < box.left()) {
           leftx = box.left(); // clip to real box
         }
-        bl = ICOORD(leftx, static_cast<int16_t>(floor(ymin)));
-        tr = ICOORD(static_cast<int16_t>(ceil(rightx)), static_cast<int16_t>(ceil(ymax)));
+        bl = ICOORD(leftx, static_cast<int16_t>(std::floor(ymin)));
+        tr = ICOORD(static_cast<int16_t>(std::ceil(rightx)), static_cast<int16_t>(std::ceil(ymax)));
         if (blobindex == 0) {
           box = TBOX(bl, tr); // change box
         } else {

src/ccstruct/blobbox.h

@@ -740,8 +740,11 @@ public:
     TO_ROW_IT row_it = &row_list;
     for (row_it.mark_cycle_pt(); !row_it.cycled_list(); row_it.forward()) {
       auto row = row_it.data();
-      tprintf("Row range (%g,%g), para_c=%g, blobcount=%" PRId32 "\n", row->min_y(), row->max_y(),
-              row->parallel_c(), row->blob_list()->length());
+      tprintf("Row range (%g,%g), para_c=%g, blobcount=%" PRId32 "\n",
+              static_cast<double>(row->min_y()),
+              static_cast<double>(row->max_y()),
+              static_cast<double>(row->parallel_c()),
+              row->blob_list()->length());
     }
   }
 
@@ -803,7 +806,6 @@ private:
 };
 
 ELISTIZEH(TO_BLOCK)
-extern double_VAR_H(textord_error_weight, 3, "Weighting for error in believability");
 void find_cblob_limits( // get y limits
     C_BLOB *blob,       // blob to search
     float leftx,        // x limits

src/ccstruct/blobs.cpp

@@ -558,7 +558,9 @@ void TBLOB::GetPreciseBoundingBox(TBOX *precise_box) const {
 // Eg x_coords[1] is a collection of the x-coords of edges at y=bottom + 1.
 void TBLOB::GetEdgeCoords(const TBOX &box, std::vector<std::vector<int>> &x_coords,
                           std::vector<std::vector<int>> &y_coords) const {
+  x_coords.clear();
   x_coords.resize(box.height());
+  y_coords.clear();
   y_coords.resize(box.width());
   CollectEdges(box, nullptr, nullptr, &x_coords, &y_coords);
   // Sort the output vectors.
@@ -869,12 +871,15 @@ TBOX TWERD::bounding_box() const {
 
 // Merges the blobs from start to end, not including end, and deletes
 // the blobs between start and end.
-void TWERD::MergeBlobs(int start, int end) {
-  if (start >= blobs.size() - 1) {
+void TWERD::MergeBlobs(unsigned start, unsigned end) {
+  if (end > blobs.size()) {
+    end = blobs.size();
+  }
+  if (start >= end) {
     return; // Nothing to do.
   }
   TESSLINE *outline = blobs[start]->outlines;
-  for (int i = start + 1; i < end && i < blobs.size(); ++i) {
+  for (auto i = start + 1; i < end; ++i) {
     TBLOB *next_blob = blobs[i];
     // Take the outlines from the next blob.
     if (outline == nullptr) {
@@ -893,7 +898,7 @@ void TWERD::MergeBlobs(int start, int end) {
   }
   // Remove dead blobs from the vector.
   // TODO: optimize.
-  for (int i = start + 1; i < end && start + 1 < blobs.size(); ++i) {
+  for (auto i = start + 1; i < end && start + 1 < blobs.size(); ++i) {
     blobs.erase(blobs.begin() + start + 1);
   }
 }
@@ -925,8 +930,8 @@ bool divisible_blob(TBLOB *blob, bool italic_blob, TPOINT *location) {
     if (outline1->is_hole) {
       continue; // Holes do not count as separable.
     }
-    TPOINT mid_pt1(static_cast<int16_t>((outline1->topleft.x + outline1->botright.x) / 2),
-                   static_cast<int16_t>((outline1->topleft.y + outline1->botright.y) / 2));
+    TPOINT mid_pt1((outline1->topleft.x + outline1->botright.x) / 2,
+                   (outline1->topleft.y + outline1->botright.y) / 2);
     int mid_prod1 = mid_pt1.cross(vertical);
     int min_prod1, max_prod1;
     outline1->MinMaxCrossProduct(vertical, &min_prod1, &max_prod1);
@@ -934,8 +939,8 @@ bool divisible_blob(TBLOB *blob, bool italic_blob, TPOINT *location) {
       if (outline2->is_hole) {
         continue; // Holes do not count as separable.
       }
-      TPOINT mid_pt2(static_cast<int16_t>((outline2->topleft.x + outline2->botright.x) / 2),
-                     static_cast<int16_t>((outline2->topleft.y + outline2->botright.y) / 2));
+      TPOINT mid_pt2((outline2->topleft.x + outline2->botright.x) / 2,
+                     (outline2->topleft.y + outline2->botright.y) / 2);
       int mid_prod2 = mid_pt2.cross(vertical);
       int min_prod2, max_prod2;
       outline2->MinMaxCrossProduct(vertical, &min_prod2, &max_prod2);
@@ -972,8 +977,8 @@ void divide_blobs(TBLOB *blob, TBLOB *other_blob, bool italic_blob, const TPOINT
   int location_prod = location.cross(vertical);
 
   while (outline != nullptr) {
-    TPOINT mid_pt(static_cast<int16_t>((outline->topleft.x + outline->botright.x) / 2),
-                  static_cast<int16_t>((outline->topleft.y + outline->botright.y) / 2));
+    TPOINT mid_pt((outline->topleft.x + outline->botright.x) / 2,
+                  (outline->topleft.y + outline->botright.y) / 2);
     int mid_prod = mid_pt.cross(vertical);
     if (mid_prod < location_prod) {
       // Outline is in left blob.

src/ccstruct/blobs.h

@@ -28,7 +28,7 @@
 
 #include <tesseract/publictypes.h> // for OcrEngineMode
 
-#include <cstdint> // for int16_t
+#include "tesstypes.h" // for TDimension
 
 struct Pix;
 
@@ -46,8 +46,8 @@ class WERD;
 ----------------------------------------------------------------------*/
 
 struct TPOINT {
-  TPOINT() : x(0), y(0) {}
-  TPOINT(int16_t vx, int16_t vy) : x(vx), y(vy) {}
+  TPOINT() = default;
+  TPOINT(TDimension vx, TDimension vy) : x(vx), y(vy) {}
   TPOINT(const ICOORD &ic) : x(ic.x()), y(ic.y()) {}
 
   void operator+=(const TPOINT &other) {
@@ -86,8 +86,8 @@ struct TPOINT {
     return x * x + y * y;
   }
 
-  int16_t x; // absolute x coord.
-  int16_t y; // absolute y coord.
+  TDimension x = 0; // absolute x coord.
+  TDimension y = 0; // absolute y coord.
 };
 
 using VECTOR = TPOINT; // structure for coordinates.
@@ -196,7 +196,7 @@ struct EDGEPT {
   bool is_hidden = false;
   uint8_t runlength = 0;
   int8_t dir = 0;
-  int8_t fixed = 0;
+  bool fixed = false;
   EDGEPT *next = nullptr;           // anticlockwise element
   EDGEPT *prev = nullptr;           // clockwise element
   C_OUTLINE *src_outline = nullptr; // Outline it came from.
@@ -446,14 +446,14 @@ struct TWERD {
   void ComputeBoundingBoxes();
 
   // Returns the number of blobs in the word.
-  int NumBlobs() const {
+  unsigned NumBlobs() const {
     return blobs.size();
   }
   TBOX bounding_box() const;
 
   // Merges the blobs from start to end, not including end, and deletes
   // the blobs between start and end.
-  void MergeBlobs(int start, int end);
+  void MergeBlobs(unsigned start, unsigned end);
 
 #ifndef GRAPHICS_DISABLED
   void plot(ScrollView *window);

src/ccstruct/boxword.cpp

@@ -46,7 +46,7 @@ void BoxWord::CopyFrom(const BoxWord &src) {
   length_ = src.length_;
   boxes_.clear();
   boxes_.reserve(length_);
-  for (int i = 0; i < length_; ++i) {
+  for (unsigned i = 0; i < length_; ++i) {
     boxes_.push_back(src.boxes_[i]);
   }
 }
@@ -60,10 +60,11 @@ BoxWord *BoxWord::CopyFromNormalized(TWERD *tessword) {
   // Allocate memory.
   boxword->boxes_.reserve(boxword->length_);
 
-  for (int b = 0; b < boxword->length_; ++b) {
+  for (unsigned b = 0; b < boxword->length_; ++b) {
     TBLOB *tblob = tessword->blobs[b];
     TBOX blob_box;
-    for (TESSLINE *outline = tblob->outlines; outline != nullptr; outline = outline->next) {
+    for (TESSLINE *outline = tblob->outlines; outline != nullptr;
+         outline = outline->next) {
       EDGEPT *edgept = outline->loop;
       // Iterate over the edges.
       do {
@@ -89,10 +90,11 @@ BoxWord *BoxWord::CopyFromNormalized(TWERD *tessword) {
 // expanding slightly, then clipping to the blobs from the original_word
 // that overlap. If not null, the block provides the inverse rotation.
 void BoxWord::ClipToOriginalWord(const BLOCK *block, WERD *original_word) {
-  for (int i = 0; i < length_; ++i) {
+  for (unsigned i = 0; i < length_; ++i) {
     TBOX box = boxes_[i];
     // Expand by a single pixel, as the poly approximation error is 1 pixel.
-    box = TBOX(box.left() - 1, box.bottom() - 1, box.right() + 1, box.top() + 1);
+    box =
+        TBOX(box.left() - 1, box.bottom() - 1, box.right() + 1, box.top() + 1);
     // Now find the original box that matches.
     TBOX original_box;
     C_BLOB_IT b_it(original_word->cblob_list());
@@ -106,16 +108,19 @@ void BoxWord::ClipToOriginalWord(const BLOCK *block, WERD *original_word) {
       }
     }
     if (!original_box.null_box()) {
-      if (NearlyEqual<int>(original_box.left(), box.left(), kBoxClipTolerance)) {
+      if (NearlyEqual<int>(original_box.left(), box.left(),
+                           kBoxClipTolerance)) {
         box.set_left(original_box.left());
       }
-      if (NearlyEqual<int>(original_box.right(), box.right(), kBoxClipTolerance)) {
+      if (NearlyEqual<int>(original_box.right(), box.right(),
+                           kBoxClipTolerance)) {
         box.set_right(original_box.right());
       }
       if (NearlyEqual<int>(original_box.top(), box.top(), kBoxClipTolerance)) {
         box.set_top(original_box.top());
       }
-      if (NearlyEqual<int>(original_box.bottom(), box.bottom(), kBoxClipTolerance)) {
+      if (NearlyEqual<int>(original_box.bottom(), box.bottom(),
+                           kBoxClipTolerance)) {
         box.set_bottom(original_box.bottom());
       }
     }
@@ -130,18 +135,18 @@ void BoxWord::ClipToOriginalWord(const BLOCK *block, WERD *original_word) {
 
 // Merges the boxes from start to end, not including end, and deletes
 // the boxes between start and end.
-void BoxWord::MergeBoxes(int start, int end) {
-  start = ClipToRange(start, 0, length_);
-  end = ClipToRange(end, 0, length_);
+void BoxWord::MergeBoxes(unsigned start, unsigned end) {
+  start = ClipToRange(start, 0U, length_);
+  end = ClipToRange(end, 0U, length_);
   if (end <= start + 1) {
     return;
   }
-  for (int i = start + 1; i < end; ++i) {
+  for (unsigned i = start + 1; i < end; ++i) {
     boxes_[start] += boxes_[i];
   }
   int shrinkage = end - 1 - start;
   length_ -= shrinkage;
-  for (int i = start + 1; i < length_; ++i) {
+  for (unsigned i = start + 1; i < length_; ++i) {
     boxes_[i] = boxes_[i + shrinkage];
   }
   boxes_.resize(length_);
@@ -149,7 +154,7 @@ void BoxWord::MergeBoxes(int start, int end) {
 
 // Inserts a new box before the given index.
 // Recomputes the bounding box.
-void BoxWord::InsertBox(int index, const TBOX &box) {
+void BoxWord::InsertBox(unsigned index, const TBOX &box) {
   if (index < length_) {
     boxes_.insert(boxes_.begin() + index, box);
   } else {
@@ -161,15 +166,15 @@ void BoxWord::InsertBox(int index, const TBOX &box) {
 
 // Changes the box at the given index to the new box.
 // Recomputes the bounding box.
-void BoxWord::ChangeBox(int index, const TBOX &box) {
+void BoxWord::ChangeBox(unsigned index, const TBOX &box) {
   boxes_[index] = box;
   ComputeBoundingBox();
 }
 
 // Deletes the box with the given index, and shuffles up the rest.
 // Recomputes the bounding box.
-void BoxWord::DeleteBox(int index) {
-  ASSERT_HOST(0 <= index && index < length_);
+void BoxWord::DeleteBox(unsigned index) {
+  ASSERT_HOST(index < length_);
   boxes_.erase(boxes_.begin() + index);
   --length_;
   ComputeBoundingBox();
@@ -185,7 +190,7 @@ void BoxWord::DeleteAllBoxes() {
 // Computes the bounding box of the word.
 void BoxWord::ComputeBoundingBox() {
   bbox_ = TBOX();
-  for (int i = 0; i < length_; ++i) {
+  for (unsigned i = 0; i < length_; ++i) {
     bbox_ += boxes_[i];
   }
 }
@@ -193,8 +198,9 @@ void BoxWord::ComputeBoundingBox() {
 // This and other putatively are the same, so call the (permanent) callback
 // for each blob index where the bounding boxes match.
 // The callback is deleted on completion.
-void BoxWord::ProcessMatchedBlobs(const TWERD &other, std::function<void(int)> cb) const {
-  for (int i = 0; i < length_ && i < other.NumBlobs(); ++i) {
+void BoxWord::ProcessMatchedBlobs(const TWERD &other,
+                                  const std::function<void(int)> &cb) const {
+  for (unsigned i = 0; i < length_ && i < other.NumBlobs(); ++i) {
     TBOX blob_box = other.blobs[i]->bounding_box();
     if (blob_box == boxes_[i]) {
       cb(i);

src/ccstruct/boxword.h

@@ -52,19 +52,19 @@ public:
 
   // Merges the boxes from start to end, not including end, and deletes
   // the boxes between start and end.
-  void MergeBoxes(int start, int end);
+  void MergeBoxes(unsigned start, unsigned end);
 
   // Inserts a new box before the given index.
   // Recomputes the bounding box.
-  void InsertBox(int index, const TBOX &box);
+  void InsertBox(unsigned index, const TBOX &box);
 
   // Changes the box at the given index to the new box.
   // Recomputes the bounding box.
-  void ChangeBox(int index, const TBOX &box);
+  void ChangeBox(unsigned index, const TBOX &box);
 
   // Deletes the box with the given index, and shuffles up the rest.
   // Recomputes the bounding box.
-  void DeleteBox(int index);
+  void DeleteBox(unsigned index);
 
   // Deletes all the boxes stored in BoxWord.
   void DeleteAllBoxes();
@@ -72,15 +72,16 @@ public:
   // This and other putatively are the same, so call the (permanent) callback
   // for each blob index where the bounding boxes match.
   // The callback is deleted on completion.
-  void ProcessMatchedBlobs(const TWERD &other, std::function<void(int)> cb) const;
+  void ProcessMatchedBlobs(const TWERD &other,
+                           const std::function<void(int)> &cb) const;
 
   const TBOX &bounding_box() const {
     return bbox_;
   }
-  int length() const {
+  unsigned length() const {
     return length_;
   }
-  const TBOX &BlobBox(int index) const {
+  const TBOX &BlobBox(unsigned index) const {
     return boxes_[index];
   }
 
@@ -88,7 +89,7 @@ private:
   void ComputeBoundingBox();
 
   TBOX bbox_;
-  int length_;
+  unsigned length_;
   std::vector<TBOX> boxes_;
 };
 

src/ccstruct/detlinefit.cpp

@@ -145,7 +145,7 @@ double DetLineFit::ConstrainedFit(const FCOORD &direction, double min_dist, doub
   if (debug) {
     tprintf("Constrained fit to dir %g, %g = %d, %d :%zu distances:\n", direction.x(), direction.y(),
             line_pt->x(), line_pt->y(), distances_.size());
-    for (int i = 0; i < distances_.size(); ++i) {
+    for (unsigned i = 0; i < distances_.size(); ++i) {
       tprintf("%d: %d, %d -> %g\n", i, distances_[i].data().x(), distances_[i].data().y(),
               distances_[i].key());
     }
@@ -260,7 +260,7 @@ void DetLineFit::ComputeDistances(const ICOORD &start, const ICOORD &end) {
   // Compute the distance of each point from the line.
   int prev_abs_dist = 0;
   int prev_dot = 0;
-  for (int i = 0; i < pts_.size(); ++i) {
+  for (unsigned i = 0; i < pts_.size(); ++i) {
     ICOORD pt_vector = pts_[i].pt;
     pt_vector -= start;
     int dot = line_vector % pt_vector;

src/ccstruct/fontinfo.cpp

@@ -83,7 +83,7 @@ bool FontInfoTable::SetContainsMultipleFontProperties(
   }
   int first_font = font_set[0].fontinfo_id;
   uint32_t properties = at(first_font).properties;
-  for (int f = 1; f < font_set.size(); ++f) {
+  for (unsigned f = 1; f < font_set.size(); ++f) {
     if (at(font_set[f].fontinfo_id).properties != properties) {
       return true;
     }
@@ -95,7 +95,7 @@ bool FontInfoTable::SetContainsMultipleFontProperties(
 void FontInfoTable::MoveSpacingInfoFrom(FontInfoTable *other) {
   using namespace std::placeholders; // for _1, _2
   set_clear_callback(std::bind(FontInfoDeleteCallback, _1));
-  for (int i = 0; i < other->size(); ++i) {
+  for (unsigned i = 0; i < other->size(); ++i) {
     std::vector<FontSpacingInfo *> *spacing_vec = other->at(i).spacing_vec;
     if (spacing_vec != nullptr) {
       int target_index = get_index(other->at(i));
@@ -117,7 +117,7 @@ void FontInfoTable::MoveTo(UnicityTable<FontInfo> *target) {
   target->clear();
   using namespace std::placeholders; // for _1, _2
   target->set_clear_callback(std::bind(FontInfoDeleteCallback, _1));
-  for (int i = 0; i < size(); ++i) {
+  for (unsigned i = 0; i < size(); ++i) {
     // Bit copy the FontInfo and steal all the pointers.
     target->push_back(at(i));
     at(i).name = nullptr;

src/ccstruct/fontinfo.h

@@ -77,8 +77,7 @@ struct FontInfo {
 
   // Reserves unicharset_size spots in spacing_vec.
   void init_spacing(int unicharset_size) {
-    spacing_vec = new std::vector<FontSpacingInfo *>();
-    spacing_vec->resize(unicharset_size);
+    spacing_vec = new std::vector<FontSpacingInfo *>(unicharset_size);
   }
   // Adds the given pointer to FontSpacingInfo to spacing_vec member
   // (FontInfo class takes ownership of the pointer).

src/ccstruct/image.cpp

@@ -22,7 +22,7 @@
 namespace tesseract {
 
 Image Image::clone() const {
-  return pixClone(pix_);
+  return pix_ ? pixClone(pix_) : nullptr;
 }
 
 Image Image::copy() const {

src/ccstruct/image.h

@@ -35,7 +35,7 @@ public:
   bool operator!=(decltype(nullptr)) const { return pix_ != nullptr; }
   explicit operator bool() const { return pix_ != nullptr; }
   operator Pix *() const { return pix_; }
-  operator Pix **() { return &pix_; }
+  explicit operator Pix **() { return &pix_; }
   Pix *operator->() const { return pix_; }
 
   // api

src/ccstruct/imagedata.cpp

@@ -43,7 +43,8 @@ const int kMaxReadAhead = 8;
 
 ImageData::ImageData() : page_number_(-1), vertical_text_(false) {}
 // Takes ownership of the pix and destroys it.
-ImageData::ImageData(bool vertical, Image pix) : page_number_(0), vertical_text_(vertical) {
+ImageData::ImageData(bool vertical, Image pix)
+    : page_number_(0), vertical_text_(vertical) {
   SetPix(pix);
 }
 ImageData::~ImageData() {
@@ -55,8 +56,8 @@ ImageData::~ImageData() {
 // Builds and returns an ImageData from the basic data. Note that imagedata,
 // truth_text, and box_text are all the actual file data, NOT filenames.
 ImageData *ImageData::Build(const char *name, int page_number, const char *lang,
-                            const char *imagedata, int imagedatasize, const char *truth_text,
-                            const char *box_text) {
+                            const char *imagedata, int imagedatasize,
+                            const char *truth_text, const char *box_text) {
   auto *image_data = new ImageData();
   image_data->imagefilename_ = name;
   image_data->page_number_ = page_number;
@@ -67,7 +68,8 @@ ImageData *ImageData::Build(const char *name, int page_number, const char *lang,
   memcpy(&image_data->image_data_[0], imagedata, imagedatasize);
   if (!image_data->AddBoxes(box_text)) {
     if (truth_text == nullptr || truth_text[0] == '\0') {
-      tprintf("Error: No text corresponding to page %d from image %s!\n", page_number, name);
+      tprintf("Error: No text corresponding to page %d from image %s!\n",
+              page_number, name);
       delete image_data;
       return nullptr;
     }
@@ -210,8 +212,9 @@ Image ImageData::GetPix() const {
 // The return value is the scaled Pix, which must be pixDestroyed after use,
 // and scale_factor (if not nullptr) is set to the scale factor that was applied
 // to the image to achieve the target_height.
-Image ImageData::PreScale(int target_height, int max_height, float *scale_factor, int *scaled_width,
-                         int *scaled_height, std::vector<TBOX> *boxes) const {
+Image ImageData::PreScale(int target_height, int max_height,
+                          float *scale_factor, int *scaled_width,
+                          int *scaled_height, std::vector<TBOX> *boxes) const {
   int input_width = 0;
   int input_height = 0;
   Image src_pix = GetPix();
@@ -231,8 +234,8 @@ Image ImageData::PreScale(int target_height, int max_height, float *scale_factor
   // Get the scaled image.
   Image pix = pixScale(src_pix, im_factor, im_factor);
   if (pix == nullptr) {
-    tprintf("Scaling pix of size %d, %d by factor %g made null pix!!\n", input_width, input_height,
-            im_factor);
+    tprintf("Scaling pix of size %d, %d by factor %g made null pix!!\n",
+            input_width, input_height, im_factor);
     src_pix.destroy();
     return nullptr;
   }
@@ -278,9 +281,9 @@ void ImageData::Display() const {
   }
   int width = pixGetWidth(pix);
   int height = pixGetHeight(pix);
-  auto *win =
-      new ScrollView("Imagedata", 100, 100, 2 * (width + 2 * kTextSize),
-                     2 * (height + 4 * kTextSize), width + 10, height + 3 * kTextSize, true);
+  auto *win = new ScrollView("Imagedata", 100, 100, 2 * (width + 2 * kTextSize),
+                             2 * (height + 4 * kTextSize), width + 10,
+                             height + 3 * kTextSize, true);
   win->Draw(pix, 0, height - 1);
   pix.destroy();
   // Draw the boxes.
@@ -292,7 +295,7 @@ void ImageData::Display() const {
   }
   win->TextAttributes("Arial", text_size, false, false, false);
   if (!boxes_.empty()) {
-    for (int b = 0; b < boxes_.size(); ++b) {
+    for (unsigned b = 0; b < boxes_.size(); ++b) {
       boxes_[b].plot(win);
       win->Text(boxes_[b].left(), height + kTextSize, box_texts_[b].c_str());
     }
@@ -309,10 +312,11 @@ void ImageData::Display() const {
 
 // Adds the supplied boxes and transcriptions that correspond to the correct
 // page number.
-void ImageData::AddBoxes(const std::vector<TBOX> &boxes, const std::vector<std::string> &texts,
+void ImageData::AddBoxes(const std::vector<TBOX> &boxes,
+                         const std::vector<std::string> &texts,
                          const std::vector<int> &box_pages) {
   // Copy the boxes and make the transcription.
-  for (int i = 0; i < box_pages.size(); ++i) {
+  for (unsigned i = 0; i < box_pages.size(); ++i) {
     if (page_number_ >= 0 && box_pages[i] != page_number_) {
       continue;
     }
@@ -346,7 +350,8 @@ Image ImageData::GetPixInternal(const std::vector<char> &image_data) {
   Image pix = nullptr;
   if (!image_data.empty()) {
     // Convert the array to an image.
-    const auto *u_data = reinterpret_cast<const unsigned char *>(&image_data[0]);
+    const auto *u_data =
+        reinterpret_cast<const unsigned char *>(&image_data[0]);
     pix = pixReadMem(u_data, image_data.size());
   }
   return pix;
@@ -361,23 +366,25 @@ bool ImageData::AddBoxes(const char *box_text) {
     std::vector<std::string> texts;
     std::vector<int> box_pages;
     if (ReadMemBoxes(page_number_, /*skip_blanks*/ false, box_text,
-                     /*continue_on_failure*/ true, &boxes, &texts, nullptr, &box_pages)) {
+                     /*continue_on_failure*/ true, &boxes, &texts, nullptr,
+                     &box_pages)) {
       AddBoxes(boxes, texts, box_pages);
       return true;
     } else {
-      tprintf("Error: No boxes for page %d from image %s!\n", page_number_, imagefilename_.c_str());
+      tprintf("Error: No boxes for page %d from image %s!\n", page_number_,
+              imagefilename_.c_str());
     }
   }
   return false;
 }
 
 DocumentData::DocumentData(const std::string &name)
-    : document_name_(name)
-    , pages_offset_(-1)
-    , total_pages_(-1)
-    , memory_used_(0)
-    , max_memory_(0)
-    , reader_(nullptr) {}
+    : document_name_(name),
+      pages_offset_(-1),
+      total_pages_(-1),
+      memory_used_(0),
+      max_memory_(0),
+      reader_(nullptr) {}
 
 DocumentData::~DocumentData() {
   if (thread.joinable()) {
@@ -392,15 +399,16 @@ DocumentData::~DocumentData() {
 
 // Reads all the pages in the given lstmf filename to the cache. The reader
 // is used to read the file.
-bool DocumentData::LoadDocument(const char *filename, int start_page, int64_t max_memory,
-                                FileReader reader) {
+bool DocumentData::LoadDocument(const char *filename, int start_page,
+                                int64_t max_memory, FileReader reader) {
   SetDocument(filename, max_memory, reader);
   pages_offset_ = start_page;
   return ReCachePages();
 }
 
 // Sets up the document, without actually loading it.
-void DocumentData::SetDocument(const char *filename, int64_t max_memory, FileReader reader) {
+void DocumentData::SetDocument(const char *filename, int64_t max_memory,
+                               FileReader reader) {
   std::lock_guard<std::mutex> lock_p(pages_mutex_);
   std::lock_guard<std::mutex> lock(general_mutex_);
   document_name_ = filename;
@@ -435,19 +443,23 @@ void DocumentData::LoadPageInBackground(int index) {
   if (IsPageAvailable(index, &page)) {
     return;
   }
-  std::lock_guard<std::mutex> lock(pages_mutex_);
-  if (pages_offset_ == index) {
-    return;
+  {
+    std::lock_guard<std::mutex> lock(pages_mutex_);
+    if (pages_offset_ == index) {
+      return;
+    }
+    pages_offset_ = index;
+    for (auto page : pages_) {
+      delete page;
+    }
+    pages_.clear();
   }
-  pages_offset_ = index;
-  for (auto page : pages_) {
-    delete page;
-  }
-  pages_.clear();
   if (thread.joinable()) {
     thread.join();
   }
-  thread = std::thread(&tesseract::DocumentData::ReCachePages, this);
+  // Don't run next statement asynchronously because that would
+  // create too many threads on Linux (see issue #3111).
+  ReCachePages();
 }
 
 // Returns a pointer to the page with the given index, modulo the total
@@ -481,7 +493,8 @@ bool DocumentData::IsPageAvailable(int index, ImageData **page) {
   }
   if (num_pages > 0) {
     index = Modulo(index, num_pages);
-    if (pages_offset_ <= index && index < pages_offset_ + pages_.size()) {
+    if (pages_offset_ <= index &&
+        static_cast<unsigned>(index) < pages_offset_ + pages_.size()) {
       *page = pages_[index - pages_offset_]; // Page is available already.
       return true;
     }
@@ -501,8 +514,8 @@ int64_t DocumentData::UnCache() {
   pages_offset_ = -1;
   set_total_pages(-1);
   set_memory_used(0);
-  tprintf("Unloaded document %s, saving %" PRId64 " memory\n", document_name_.c_str(),
-          memory_saved);
+  tprintf("Unloaded document %s, saving %" PRId64 " memory\n",
+          document_name_.c_str(), memory_saved);
   return memory_saved;
 }
 
@@ -534,8 +547,8 @@ bool DocumentData::ReCachePages() {
   }
   pages_.clear();
   TFile fp;
-  if (!fp.Open(document_name_.c_str(), reader_) || !fp.DeSerializeSize(&loaded_pages) ||
-      loaded_pages <= 0) {
+  if (!fp.Open(document_name_.c_str(), reader_) ||
+      !fp.DeSerializeSize(&loaded_pages) || loaded_pages <= 0) {
     tprintf("Deserialize header failed: %s\n", document_name_.c_str());
     return false;
   }
@@ -548,7 +561,8 @@ bool DocumentData::ReCachePages() {
     if (!fp.DeSerialize(&non_null)) {
       break;
     }
-    if (page < pages_offset_ || (max_memory_ > 0 && memory_used() > max_memory_)) {
+    if (page < pages_offset_ ||
+        (max_memory_ > 0 && memory_used() > max_memory_)) {
       if (non_null && !ImageData::SkipDeSerialize(&fp)) {
         break;
       }
@@ -570,16 +584,17 @@ bool DocumentData::ReCachePages() {
     }
   }
   if (page < loaded_pages) {
-    tprintf("Deserialize failed: %s read %d/%d lines\n", document_name_.c_str(), page,
-            loaded_pages);
+    tprintf("Deserialize failed: %s read %d/%d lines\n", document_name_.c_str(),
+            page, loaded_pages);
     for (auto page : pages_) {
       delete page;
     }
     pages_.clear();
   } else if (loaded_pages > 1) {
     // Avoid lots of messages for training with single line images.
-    tprintf("Loaded %zu/%d lines (%d-%zu) of document %s\n", pages_.size(), loaded_pages,
-            pages_offset_ + 1, pages_offset_ + pages_.size(), document_name_.c_str());
+    tprintf("Loaded %zu/%d lines (%d-%zu) of document %s\n", pages_.size(),
+            loaded_pages, pages_offset_ + 1, pages_offset_ + pages_.size(),
+            document_name_.c_str());
   }
   set_total_pages(loaded_pages);
   return !pages_.empty();
@@ -597,7 +612,8 @@ DocumentCache::~DocumentCache() {
 // Adds all the documents in the list of filenames, counting memory.
 // The reader is used to read the files.
 bool DocumentCache::LoadDocuments(const std::vector<std::string> &filenames,
-                                  CachingStrategy cache_strategy, FileReader reader) {
+                                  CachingStrategy cache_strategy,
+                                  FileReader reader) {
   cache_strategy_ = cache_strategy;
   int64_t fair_share_memory = 0;
   // In the round-robin case, each DocumentData handles restricting its content
@@ -606,7 +622,7 @@ bool DocumentCache::LoadDocuments(const std::vector<std::string> &filenames,
   if (cache_strategy_ == CS_ROUND_ROBIN) {
     fair_share_memory = max_memory_ / filenames.size();
   }
-  for (auto filename : filenames) {
+  for (const auto &filename : filenames) {
     auto *document = new DocumentData(filename);
     document->SetDocument(filename.c_str(), fair_share_memory, reader);
     AddToCache(document);
@@ -628,7 +644,8 @@ bool DocumentCache::AddToCache(DocumentData *data) {
 }
 
 // Finds and returns a document by name.
-DocumentData *DocumentCache::FindDocument(const std::string &document_name) const {
+DocumentData *DocumentCache::FindDocument(
+    const std::string &document_name) const {
   for (auto *document : documents_) {
     if (document->document_name() == document_name) {
       return document;
@@ -692,7 +709,8 @@ const ImageData *DocumentCache::GetPageSequential(int serial) {
     }
   }
   int doc_index = serial / num_pages_per_doc_ % num_docs;
-  const ImageData *doc = documents_[doc_index]->GetPage(serial % num_pages_per_doc_);
+  const ImageData *doc =
+      documents_[doc_index]->GetPage(serial % num_pages_per_doc_);
   // Count up total memory. Background loading makes it more complicated to
   // keep a running count.
   int64_t total_memory = 0;
@@ -706,7 +724,8 @@ const ImageData *DocumentCache::GetPageSequential(int serial) {
     // we create a hole between them and then un-caching the backmost occupied
     // will work for both.
     int num_in_front = CountNeighbourDocs(doc_index, 1);
-    for (int offset = num_in_front - 2; offset > 1 && total_memory >= max_memory_; --offset) {
+    for (int offset = num_in_front - 2;
+         offset > 1 && total_memory >= max_memory_; --offset) {
       int next_index = (doc_index + offset) % num_docs;
       total_memory -= documents_[next_index]->UnCache();
     }
@@ -714,7 +733,8 @@ const ImageData *DocumentCache::GetPageSequential(int serial) {
     // we take away the document that a 2nd reader is using, it will put it
     // back and make a hole between.
     int num_behind = CountNeighbourDocs(doc_index, -1);
-    for (int offset = num_behind; offset < 0 && total_memory >= max_memory_; ++offset) {
+    for (int offset = num_behind; offset < 0 && total_memory >= max_memory_;
+         ++offset) {
       int next_index = (doc_index + offset + num_docs) % num_docs;
       total_memory -= documents_[next_index]->UnCache();
     }