// (C) Copyright 2017, Google Inc. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // http://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #if defined(_WIN32) # include // for _access #else # include // for access #endif #include "dawg.h" #include "include_gunit.h" #include "trie.h" #include "unicharset.h" #ifdef INCLUDE_TENSORFLOW # include "util/utf8/unicodetext.h" // for UnicodeText #endif namespace tesseract { // Replacement for std::filesystem::exists (C++-17) static bool file_exists(const char *filename) { #if defined(_WIN32) return _access(filename, 0) == 0; #else return access(filename, 0) == 0; #endif } class TatweelTest : public ::testing::Test { protected: void SetUp() override { static std::locale system_locale(""); std::locale::global(system_locale); } TatweelTest() { #ifdef INCLUDE_TENSORFLOW std::string filename = TestDataNameToPath("ara.wordlist"); if (file_exists(filename.c_str())) { std::string wordlist(u8"\u0640"); CHECK_OK(file::GetContents(filename, &wordlist, file::Defaults())); // Put all the unicodes in the unicharset_. UnicodeText text; text.PointToUTF8(wordlist.data(), wordlist.size()); int num_tatweel = 0; for (auto it = text.begin(); it != text.end(); ++it) { std::string utf8 = it.get_utf8_string(); if (utf8.find(u8"\u0640") != std::string::npos) ++num_tatweel; unicharset_.unichar_insert(utf8.c_str()); } LOG(INFO) << "Num tatweels in source data=" << num_tatweel; EXPECT_GT(num_tatweel, 0); } #endif } std::string TestDataNameToPath(const std::string &name) { return file::JoinPath(TESTDATA_DIR, name); } UNICHARSET unicharset_; }; TEST_F(TatweelTest, UnicharsetIgnoresTatweel) { // This test verifies that the unicharset ignores the Tatweel character. for (int i = 0; i < unicharset_.size(); ++i) { const char *utf8 = unicharset_.id_to_unichar(i); EXPECT_EQ(strstr(utf8, reinterpret_cast(u8"\u0640")), nullptr); } } TEST_F(TatweelTest, DictIgnoresTatweel) { // This test verifies that the dictionary ignores the Tatweel character. tesseract::Trie trie(tesseract::DAWG_TYPE_WORD, "ara", SYSTEM_DAWG_PERM, unicharset_.size(), 0); std::string filename = TestDataNameToPath("ara.wordlist"); if (!file_exists(filename.c_str())) { LOG(INFO) << "Skip test because of missing " << filename; GTEST_SKIP(); } else { EXPECT_TRUE(trie.read_and_add_word_list(filename.c_str(), unicharset_, tesseract::Trie::RRP_REVERSE_IF_HAS_RTL)); EXPECT_EQ(0, trie.check_for_words(filename.c_str(), unicharset_, false)); } } TEST_F(TatweelTest, UnicharsetLoadKeepsTatweel) { // This test verifies that a load of an existing unicharset keeps any // existing tatweel for backwards compatibility. std::string filename = TestDataNameToPath("ara.unicharset"); if (!file_exists(filename.c_str())) { LOG(INFO) << "Skip test because of missing " << filename; GTEST_SKIP(); } else { EXPECT_TRUE(unicharset_.load_from_file(filename.c_str())); int num_tatweel = 0; for (int i = 0; i < unicharset_.size(); ++i) { const char *utf8 = unicharset_.id_to_unichar(i); if (strstr(utf8, reinterpret_cast(u8"\u0640")) != nullptr) { ++num_tatweel; } } LOG(INFO) << "Num tatweels in unicharset=" << num_tatweel; EXPECT_EQ(num_tatweel, 4); } } } // namespace tesseract