mirror of
https://github.com/tesseract-ocr/tesseract.git
synced 2024-11-24 19:19:05 +08:00
116 lines
3.9 KiB
C++
116 lines
3.9 KiB
C++
// (C) Copyright 2017, Google Inc.
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#if defined(_WIN32)
|
|
# include <io.h> // for _access
|
|
#else
|
|
# include <unistd.h> // for access
|
|
#endif
|
|
|
|
#include "dawg.h"
|
|
#include "include_gunit.h"
|
|
#include "trie.h"
|
|
#include "unicharset.h"
|
|
#ifdef INCLUDE_TENSORFLOW
|
|
# include "util/utf8/unicodetext.h" // for UnicodeText
|
|
#endif
|
|
|
|
namespace tesseract {
|
|
|
|
// Replacement for std::filesystem::exists (C++-17)
|
|
static bool file_exists(const char *filename) {
|
|
#if defined(_WIN32)
|
|
return _access(filename, 0) == 0;
|
|
#else
|
|
return access(filename, 0) == 0;
|
|
#endif
|
|
}
|
|
|
|
class TatweelTest : public ::testing::Test {
|
|
protected:
|
|
void SetUp() override {
|
|
static std::locale system_locale("");
|
|
std::locale::global(system_locale);
|
|
}
|
|
|
|
TatweelTest() {
|
|
#ifdef INCLUDE_TENSORFLOW
|
|
std::string filename = TestDataNameToPath("ara.wordlist");
|
|
if (file_exists(filename.c_str())) {
|
|
std::string wordlist(u8"\u0640");
|
|
CHECK_OK(file::GetContents(filename, &wordlist, file::Defaults()));
|
|
// Put all the unicodes in the unicharset_.
|
|
UnicodeText text;
|
|
text.PointToUTF8(wordlist.data(), wordlist.size());
|
|
int num_tatweel = 0;
|
|
for (auto it = text.begin(); it != text.end(); ++it) {
|
|
std::string utf8 = it.get_utf8_string();
|
|
if (utf8.find(u8"\u0640") != std::string::npos)
|
|
++num_tatweel;
|
|
unicharset_.unichar_insert(utf8.c_str());
|
|
}
|
|
LOG(INFO) << "Num tatweels in source data=" << num_tatweel;
|
|
EXPECT_GT(num_tatweel, 0);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
std::string TestDataNameToPath(const std::string &name) {
|
|
return file::JoinPath(TESTDATA_DIR, name);
|
|
}
|
|
UNICHARSET unicharset_;
|
|
};
|
|
|
|
TEST_F(TatweelTest, UnicharsetIgnoresTatweel) {
|
|
// This test verifies that the unicharset ignores the Tatweel character.
|
|
for (int i = 0; i < unicharset_.size(); ++i) {
|
|
const char *utf8 = unicharset_.id_to_unichar(i);
|
|
EXPECT_EQ(strstr(utf8, reinterpret_cast<const char *>(u8"\u0640")), nullptr);
|
|
}
|
|
}
|
|
|
|
TEST_F(TatweelTest, DictIgnoresTatweel) {
|
|
// This test verifies that the dictionary ignores the Tatweel character.
|
|
tesseract::Trie trie(tesseract::DAWG_TYPE_WORD, "ara", SYSTEM_DAWG_PERM, unicharset_.size(), 0);
|
|
std::string filename = TestDataNameToPath("ara.wordlist");
|
|
if (!file_exists(filename.c_str())) {
|
|
LOG(INFO) << "Skip test because of missing " << filename;
|
|
GTEST_SKIP();
|
|
} else {
|
|
EXPECT_TRUE(trie.read_and_add_word_list(filename.c_str(), unicharset_,
|
|
tesseract::Trie::RRP_REVERSE_IF_HAS_RTL));
|
|
EXPECT_EQ(0, trie.check_for_words(filename.c_str(), unicharset_, false));
|
|
}
|
|
}
|
|
|
|
TEST_F(TatweelTest, UnicharsetLoadKeepsTatweel) {
|
|
// This test verifies that a load of an existing unicharset keeps any
|
|
// existing tatweel for backwards compatibility.
|
|
std::string filename = TestDataNameToPath("ara.unicharset");
|
|
if (!file_exists(filename.c_str())) {
|
|
LOG(INFO) << "Skip test because of missing " << filename;
|
|
GTEST_SKIP();
|
|
} else {
|
|
EXPECT_TRUE(unicharset_.load_from_file(filename.c_str()));
|
|
int num_tatweel = 0;
|
|
for (int i = 0; i < unicharset_.size(); ++i) {
|
|
const char *utf8 = unicharset_.id_to_unichar(i);
|
|
if (strstr(utf8, reinterpret_cast<const char *>(u8"\u0640")) != nullptr) {
|
|
++num_tatweel;
|
|
}
|
|
}
|
|
LOG(INFO) << "Num tatweels in unicharset=" << num_tatweel;
|
|
EXPECT_EQ(num_tatweel, 4);
|
|
}
|
|
}
|
|
|
|
} // namespace tesseract
|