2009-07-11 10:50:24 +08:00
|
|
|
///////////////////////////////////////////////////////////////////////
|
|
|
|
// File: tessdatamanager.h
|
|
|
|
// Description: Functions to handle loading/combining tesseract data files.
|
|
|
|
// Author: Daria Antonova
|
|
|
|
// Created: Wed Jun 03 11:26:43 PST 2009
|
|
|
|
//
|
|
|
|
// (C) Copyright 2009, Google Inc.
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
//
|
|
|
|
///////////////////////////////////////////////////////////////////////
|
|
|
|
|
|
|
|
#ifndef TESSERACT_CCUTIL_TESSDATAMANAGER_H_
|
|
|
|
#define TESSERACT_CCUTIL_TESSDATAMANAGER_H_
|
|
|
|
|
|
|
|
#include <stdio.h>
|
2013-09-23 23:26:50 +08:00
|
|
|
|
2009-07-11 10:50:24 +08:00
|
|
|
#include "host.h"
|
2013-09-23 23:26:50 +08:00
|
|
|
#include "strngs.h"
|
2009-07-11 10:50:24 +08:00
|
|
|
#include "tprintf.h"
|
2017-07-15 02:14:23 +08:00
|
|
|
#include "version.h"
|
2009-07-11 10:50:24 +08:00
|
|
|
|
|
|
|
static const char kTrainedDataSuffix[] = "traineddata";
|
|
|
|
|
2010-05-21 07:07:24 +08:00
|
|
|
// When adding new tessdata types and file suffixes, please make sure to
|
|
|
|
// update TessdataType enum, kTessdataFileSuffixes and kTessdataFileIsText.
|
2009-07-11 10:50:24 +08:00
|
|
|
static const char kLangConfigFileSuffix[] = "config";
|
|
|
|
static const char kUnicharsetFileSuffix[] = "unicharset";
|
|
|
|
static const char kAmbigsFileSuffix[] = "unicharambigs";
|
|
|
|
static const char kBuiltInTemplatesFileSuffix[] = "inttemp";
|
|
|
|
static const char kBuiltInCutoffsFileSuffix[] = "pffmtable";
|
|
|
|
static const char kNormProtoFileSuffix[] = "normproto";
|
|
|
|
static const char kPuncDawgFileSuffix[] = "punc-dawg";
|
|
|
|
static const char kSystemDawgFileSuffix[] = "word-dawg";
|
|
|
|
static const char kNumberDawgFileSuffix[] = "number-dawg";
|
|
|
|
static const char kFreqDawgFileSuffix[] = "freq-dawg";
|
2010-11-24 02:34:14 +08:00
|
|
|
static const char kFixedLengthDawgsFileSuffix[] = "fixed-length-dawgs";
|
|
|
|
static const char kCubeUnicharsetFileSuffix[] = "cube-unicharset";
|
|
|
|
static const char kCubeSystemDawgFileSuffix[] = "cube-word-dawg";
|
2012-02-02 11:14:43 +08:00
|
|
|
static const char kShapeTableFileSuffix[] = "shapetable";
|
|
|
|
static const char kBigramDawgFileSuffix[] = "bigram-dawg";
|
|
|
|
static const char kUnambigDawgFileSuffix[] = "unambig-dawg";
|
2013-09-23 23:26:50 +08:00
|
|
|
static const char kParamsModelFileSuffix[] = "params-model";
|
2016-11-08 07:38:07 +08:00
|
|
|
static const char kLSTMModelFileSuffix[] = "lstm";
|
|
|
|
static const char kLSTMPuncDawgFileSuffix[] = "lstm-punc-dawg";
|
|
|
|
static const char kLSTMSystemDawgFileSuffix[] = "lstm-word-dawg";
|
|
|
|
static const char kLSTMNumberDawgFileSuffix[] = "lstm-number-dawg";
|
2017-07-15 02:14:23 +08:00
|
|
|
static const char kLSTMUnicharsetFileSuffix[] = "lstm-unicharset";
|
|
|
|
static const char kLSTMRecoderFileSuffix[] = "lstm-recoder";
|
|
|
|
static const char kVersionFileSuffix[] = "version";
|
2009-07-11 10:50:24 +08:00
|
|
|
|
|
|
|
namespace tesseract {
|
|
|
|
|
|
|
|
enum TessdataType {
|
2010-11-24 02:34:14 +08:00
|
|
|
TESSDATA_LANG_CONFIG, // 0
|
|
|
|
TESSDATA_UNICHARSET, // 1
|
|
|
|
TESSDATA_AMBIGS, // 2
|
|
|
|
TESSDATA_INTTEMP, // 3
|
|
|
|
TESSDATA_PFFMTABLE, // 4
|
|
|
|
TESSDATA_NORMPROTO, // 5
|
|
|
|
TESSDATA_PUNC_DAWG, // 6
|
|
|
|
TESSDATA_SYSTEM_DAWG, // 7
|
|
|
|
TESSDATA_NUMBER_DAWG, // 8
|
|
|
|
TESSDATA_FREQ_DAWG, // 9
|
2013-09-23 23:26:50 +08:00
|
|
|
TESSDATA_FIXED_LENGTH_DAWGS, // 10 // deprecated
|
2016-12-14 06:37:40 +08:00
|
|
|
TESSDATA_CUBE_UNICHARSET, // 11 // deprecated
|
|
|
|
TESSDATA_CUBE_SYSTEM_DAWG, // 12 // deprecated
|
2012-02-02 11:14:43 +08:00
|
|
|
TESSDATA_SHAPE_TABLE, // 13
|
|
|
|
TESSDATA_BIGRAM_DAWG, // 14
|
|
|
|
TESSDATA_UNAMBIG_DAWG, // 15
|
2013-09-23 23:26:50 +08:00
|
|
|
TESSDATA_PARAMS_MODEL, // 16
|
2016-11-08 07:38:07 +08:00
|
|
|
TESSDATA_LSTM, // 17
|
|
|
|
TESSDATA_LSTM_PUNC_DAWG, // 18
|
|
|
|
TESSDATA_LSTM_SYSTEM_DAWG, // 19
|
|
|
|
TESSDATA_LSTM_NUMBER_DAWG, // 20
|
2017-07-15 02:14:23 +08:00
|
|
|
TESSDATA_LSTM_UNICHARSET, // 21
|
|
|
|
TESSDATA_LSTM_RECODER, // 22
|
|
|
|
TESSDATA_VERSION, // 23
|
2009-07-11 10:50:24 +08:00
|
|
|
|
|
|
|
TESSDATA_NUM_ENTRIES
|
|
|
|
};
|
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* kTessdataFileSuffixes[i] indicates the file suffix for
|
|
|
|
* tessdata of type i (from TessdataType enum).
|
|
|
|
*/
|
2016-11-08 07:38:07 +08:00
|
|
|
static const char *const kTessdataFileSuffixes[] = {
|
|
|
|
kLangConfigFileSuffix, // 0
|
|
|
|
kUnicharsetFileSuffix, // 1
|
|
|
|
kAmbigsFileSuffix, // 2
|
|
|
|
kBuiltInTemplatesFileSuffix, // 3
|
|
|
|
kBuiltInCutoffsFileSuffix, // 4
|
|
|
|
kNormProtoFileSuffix, // 5
|
|
|
|
kPuncDawgFileSuffix, // 6
|
|
|
|
kSystemDawgFileSuffix, // 7
|
|
|
|
kNumberDawgFileSuffix, // 8
|
|
|
|
kFreqDawgFileSuffix, // 9
|
|
|
|
kFixedLengthDawgsFileSuffix, // 10 // deprecated
|
2016-12-14 06:37:40 +08:00
|
|
|
kCubeUnicharsetFileSuffix, // 11 // deprecated
|
|
|
|
kCubeSystemDawgFileSuffix, // 12 // deprecated
|
2016-11-08 07:38:07 +08:00
|
|
|
kShapeTableFileSuffix, // 13
|
|
|
|
kBigramDawgFileSuffix, // 14
|
|
|
|
kUnambigDawgFileSuffix, // 15
|
|
|
|
kParamsModelFileSuffix, // 16
|
|
|
|
kLSTMModelFileSuffix, // 17
|
|
|
|
kLSTMPuncDawgFileSuffix, // 18
|
|
|
|
kLSTMSystemDawgFileSuffix, // 19
|
|
|
|
kLSTMNumberDawgFileSuffix, // 20
|
2017-07-15 02:14:23 +08:00
|
|
|
kLSTMUnicharsetFileSuffix, // 21
|
|
|
|
kLSTMRecoderFileSuffix, // 22
|
|
|
|
kVersionFileSuffix, // 23
|
2010-05-21 07:07:24 +08:00
|
|
|
};
|
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* TessdataType could be updated to contain more entries, however
|
|
|
|
* we do not expect that number to be astronomically high.
|
|
|
|
* In order to automatically detect endianness TessdataManager will
|
|
|
|
* flip the bits if actual_tessdata_num_entries_ is larger than
|
|
|
|
* kMaxNumTessdataEntries.
|
|
|
|
*/
|
2009-07-11 10:50:24 +08:00
|
|
|
static const int kMaxNumTessdataEntries = 1000;
|
|
|
|
|
|
|
|
|
|
|
|
class TessdataManager {
|
|
|
|
public:
|
2017-07-15 02:14:23 +08:00
|
|
|
TessdataManager() : reader_(nullptr), is_loaded_(false), swap_(false) {
|
|
|
|
SetVersionString(TESSERACT_VERSION_STR);
|
|
|
|
}
|
2017-04-28 06:48:23 +08:00
|
|
|
explicit TessdataManager(FileReader reader)
|
2017-07-15 02:14:23 +08:00
|
|
|
: reader_(reader), is_loaded_(false), swap_(false) {
|
|
|
|
SetVersionString(TESSERACT_VERSION_STR);
|
|
|
|
}
|
2009-07-11 10:50:24 +08:00
|
|
|
~TessdataManager() {}
|
|
|
|
|
2017-04-28 06:48:23 +08:00
|
|
|
bool swap() const { return swap_; }
|
|
|
|
bool is_loaded() const { return is_loaded_; }
|
|
|
|
|
2017-07-15 00:30:14 +08:00
|
|
|
// Lazily loads from the the given filename. Won't actually read the file
|
|
|
|
// until it needs it.
|
|
|
|
void LoadFileLater(const char *data_file_name);
|
2010-11-30 08:53:31 +08:00
|
|
|
/**
|
2017-04-28 06:48:23 +08:00
|
|
|
* Opens and reads the given data file right now.
|
2014-09-13 04:41:19 +08:00
|
|
|
* @return true on success.
|
2010-11-30 08:53:31 +08:00
|
|
|
*/
|
2017-04-28 06:48:23 +08:00
|
|
|
bool Init(const char *data_file_name);
|
|
|
|
// Loads from the given memory buffer as if a file, remembering name as some
|
|
|
|
// arbitrary source id for caching.
|
|
|
|
bool LoadMemBuffer(const char *name, const char *data, int size);
|
2017-07-15 00:30:14 +08:00
|
|
|
// Overwrites a single entry of the given type.
|
|
|
|
void OverwriteEntry(TessdataType type, const char *data, int size);
|
2017-04-28 06:48:23 +08:00
|
|
|
|
|
|
|
// Saves to the given filename.
|
|
|
|
bool SaveFile(const STRING &filename, FileWriter writer) const;
|
|
|
|
// Serializes to the given vector.
|
|
|
|
void Serialize(GenericVector<char> *data) const;
|
|
|
|
// Resets to the initial state, keeping the reader.
|
|
|
|
void Clear();
|
|
|
|
|
|
|
|
// Prints a directory of contents.
|
|
|
|
void Directory() const;
|
|
|
|
|
2017-07-15 02:14:23 +08:00
|
|
|
// Returns true if the component requested is present.
|
|
|
|
bool IsComponentAvailable(TessdataType type) const {
|
|
|
|
return !entries_[type].empty();
|
|
|
|
}
|
2017-04-28 06:48:23 +08:00
|
|
|
// Opens the given TFile pointer to the given component type.
|
|
|
|
// Returns false in case of failure.
|
|
|
|
bool GetComponent(TessdataType type, TFile *fp);
|
2017-07-15 02:14:23 +08:00
|
|
|
// As non-const version except it can't load the component if not already
|
|
|
|
// loaded.
|
|
|
|
bool GetComponent(TessdataType type, TFile *fp) const;
|
|
|
|
|
|
|
|
// Returns the current version string.
|
|
|
|
string VersionString() const;
|
|
|
|
// Sets the version string to the given v_str.
|
|
|
|
void SetVersionString(const string &v_str);
|
2009-07-11 10:50:24 +08:00
|
|
|
|
2016-12-14 06:37:40 +08:00
|
|
|
// Returns true if the base Tesseract components are present.
|
2017-04-28 06:48:23 +08:00
|
|
|
bool IsBaseAvailable() const {
|
|
|
|
return !entries_[TESSDATA_UNICHARSET].empty() &&
|
|
|
|
!entries_[TESSDATA_INTTEMP].empty();
|
|
|
|
}
|
2016-12-14 06:37:40 +08:00
|
|
|
|
|
|
|
// Returns true if the LSTM components are present.
|
2017-04-28 06:48:23 +08:00
|
|
|
bool IsLSTMAvailable() const { return !entries_[TESSDATA_LSTM].empty(); }
|
2016-12-14 06:37:40 +08:00
|
|
|
|
2013-09-23 23:26:50 +08:00
|
|
|
// Return the name of the underlying data file.
|
|
|
|
const STRING &GetDataFileName() const { return data_file_name_; }
|
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* Reads all the standard tesseract config and data files for a language
|
|
|
|
* at the given path and bundles them up into one binary data file.
|
|
|
|
* Returns true if the combined traineddata file was successfully written.
|
|
|
|
*/
|
2017-04-28 06:48:23 +08:00
|
|
|
bool CombineDataFiles(const char *language_data_path_prefix,
|
|
|
|
const char *output_filename);
|
2009-07-11 10:50:24 +08:00
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* Gets the individual components from the data_file_ with which the class was
|
|
|
|
* initialized. Overwrites the components specified by component_filenames.
|
|
|
|
* Writes the updated traineddata file to new_traineddata_filename.
|
|
|
|
*/
|
2010-05-21 07:07:24 +08:00
|
|
|
bool OverwriteComponents(const char *new_traineddata_filename,
|
|
|
|
char **component_filenames,
|
|
|
|
int num_new_components);
|
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* Extracts tessdata component implied by the name of the input file from
|
|
|
|
* the combined traineddata loaded into TessdataManager.
|
|
|
|
* Writes the extracted component to the file indicated by the file name.
|
|
|
|
* E.g. if the filename given is somepath/somelang.unicharset, unicharset
|
|
|
|
* will be extracted from the data loaded into the TessdataManager and will
|
|
|
|
* be written to somepath/somelang.unicharset.
|
|
|
|
* @return true if the component was successfully extracted, false if the
|
|
|
|
* component was not present in the traineddata loaded into TessdataManager.
|
|
|
|
*/
|
2010-05-21 07:07:24 +08:00
|
|
|
bool ExtractToFile(const char *filename);
|
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* Fills type with TessdataType of the tessdata component represented by the
|
|
|
|
* given file name. E.g. tessdata/eng.unicharset -> TESSDATA_UNICHARSET.
|
|
|
|
* @return true if the tessdata component type could be determined
|
|
|
|
* from the given file name.
|
|
|
|
*/
|
2010-05-21 07:07:24 +08:00
|
|
|
static bool TessdataTypeFromFileSuffix(const char *suffix,
|
2017-04-28 06:48:23 +08:00
|
|
|
TessdataType *type);
|
2010-05-21 07:07:24 +08:00
|
|
|
|
2010-07-27 21:23:23 +08:00
|
|
|
/**
|
|
|
|
* Tries to determine tessdata component file suffix from filename,
|
|
|
|
* returns true on success.
|
|
|
|
*/
|
2010-05-21 07:07:24 +08:00
|
|
|
static bool TessdataTypeFromFileName(const char *filename,
|
2017-04-28 06:48:23 +08:00
|
|
|
TessdataType *type);
|
2009-07-11 10:50:24 +08:00
|
|
|
|
2010-05-21 07:07:24 +08:00
|
|
|
private:
|
2017-04-28 06:48:23 +08:00
|
|
|
// Name of file it came from.
|
|
|
|
STRING data_file_name_;
|
|
|
|
// Function to load the file when we need it.
|
|
|
|
FileReader reader_;
|
|
|
|
// True if the file has been loaded.
|
|
|
|
bool is_loaded_;
|
2012-02-02 11:14:43 +08:00
|
|
|
// True if the bytes need swapping.
|
|
|
|
bool swap_;
|
2017-04-28 06:48:23 +08:00
|
|
|
// Contents of each element of the traineddata file.
|
|
|
|
GenericVector<char> entries_[TESSDATA_NUM_ENTRIES];
|
2009-07-11 10:50:24 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace tesseract
|
|
|
|
|
|
|
|
#endif // TESSERACT_CCUTIL_TESSDATAMANAGER_H_
|