mirror/tesseract
1
0
Fork 0
mirror of https://github.com/tesseract-ocr/tesseract.git synced 2024-11-30 23:49:05 +08:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity
81fbd878dd
tesseract/unittest/paragraphs_test.cc
Stefan Weil de6a759744 unittest: Add paragraphs_test 
Signed-off-by: Stefan Weil <sw@weilnetz.de>
2018-10-12 16:23:10 +02:00

705 lines
38 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// (C) Copyright 2017, Google Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string> // for std::string
#include "absl/strings/str_cat.h" // for absl::StrCat
#include "absl/strings/str_join.h" // for absl::StrJoin
#include "absl/strings/str_split.h" // for absl::StrSplit
#include "include_gunit.h" // for TEST
#include "log.h" // for LOG
#include "genericvector.h"
// ccmain
#include "paragraphs.h"
#include "paragraphs_internal.h"
// ccstruct
#include "ocrpara.h"
namespace { // anonymous namespace
// Functions for making monospace ASCII trial text for the paragraph detector.
const tesseract::ParagraphJustification kLeft = tesseract::JUSTIFICATION_LEFT;
const tesseract::ParagraphJustification kCenter =
tesseract::JUSTIFICATION_CENTER;
const tesseract::ParagraphJustification kRight = tesseract::JUSTIFICATION_RIGHT;
const tesseract::ParagraphJustification kUnknown =
tesseract::JUSTIFICATION_UNKNOWN;
enum TextModelInputType {
PCONT = 0, // Continuation line of a paragraph (default).
PSTART = 1, // First line of a paragraph.
PNONE = 2, // Not a paragraph line.
};
struct TextAndModel {
const char* ascii;
TextModelInputType model_type;
// fields corresponding to PARA (see ccstruct/ocrpara.h)
ParagraphModel model;
bool is_very_first_or_continuation;
bool is_list_item;
};
// Imagine that the given text is typewriter ASCII with each character ten
// pixels wide and twenty pixels high and return an appropriate row_info.
void AsciiToRowInfo(const char* text, int row_number,
tesseract::RowInfo* info) {
const int kCharWidth = 10;
const int kLineSpace = 30;
info->text = text;
info->has_leaders =
strstr(text, "...") != nullptr || strstr(text, ". . .") != nullptr;
info->has_drop_cap = false;
info->pix_ldistance = info->pix_rdistance = 0;
info->average_interword_space = kCharWidth;
info->pix_xheight = kCharWidth;
info->lword_text = info->rword_text = "";
info->ltr = true;
std::vector<std::string> words = absl::StrSplit(text, ' ', absl::SkipEmpty());
info->num_words = words.size();
if (info->num_words < 1) return;
info->lword_text = words[0].c_str();
info->rword_text = words[words.size() - 1].c_str();
int lspace = 0;
while (lspace < info->text.size() && text[lspace] == ' ') {
lspace++;
}
int rspace = 0;
while (rspace < info->text.size() &&
text[info->text.size() - rspace - 1] == ' ') {
rspace++;
}
int top = -kLineSpace * row_number;
int bottom = top - kLineSpace;
int row_right = kCharWidth * info->text.size();
int lword_width = kCharWidth * info->lword_text.size();
int rword_width = kCharWidth * info->rword_text.size();
info->pix_ldistance = lspace * kCharWidth;
info->pix_rdistance = rspace * kCharWidth;
info->lword_box =
TBOX(info->pix_ldistance, bottom, info->pix_ldistance + lword_width, top);
info->rword_box = TBOX(row_right - info->pix_rdistance - rword_width, bottom,
row_right - info->pix_rdistance, top);
tesseract::LeftWordAttributes(
nullptr, nullptr, info->lword_text, &info->lword_indicates_list_item,
&info->lword_likely_starts_idea, &info->lword_likely_ends_idea);
tesseract::RightWordAttributes(
nullptr, nullptr, info->rword_text, &info->rword_indicates_list_item,
&info->rword_likely_starts_idea, &info->rword_likely_ends_idea);
}
void MakeAsciiRowInfos(const TextAndModel* row_infos, int n,
GenericVector<tesseract::RowInfo>* output) {
output->clear();
tesseract::RowInfo info;
for (int i = 0; i < n; i++) {
AsciiToRowInfo(row_infos[i].ascii, i, &info);
output->push_back(info);
}
}
// Given n rows of reference ground truth, evaluate whether the n rows
// of PARA * pointers yield the same paragraph breakpoints.
void EvaluateParagraphDetection(const TextAndModel* correct, int n,
const GenericVector<PARA*>& detector_output) {
int incorrect_breaks = 0;
int missed_breaks = 0;
int poorly_matched_models = 0;
int bad_crowns = 0;
int bad_list_items = 0;
ASSERT_EQ(detector_output.size(), n);
for (int i = 1; i < n; i++) {
bool has_break = correct[i].model_type != PCONT;
bool detected_break = (detector_output[i - 1] != detector_output[i]);
if (has_break && !detected_break) missed_breaks++;
if (detected_break && !has_break) incorrect_breaks++;
if (has_break) {
if (correct[i].model_type == PNONE) {
if (detector_output[i]->model != nullptr) {
poorly_matched_models++;
}
} else {
if (correct[i].model.justification() != kUnknown &&
(detector_output[i]->model == nullptr ||
!correct[i].model.Comparable(*detector_output[i]->model))) {
poorly_matched_models++;
}
}
if (correct[i].is_very_first_or_continuation ^
detector_output[i]->is_very_first_or_continuation) {
bad_crowns++;
}
if (correct[i].is_list_item ^ detector_output[i]->is_list_item) {
bad_list_items++;
}
}
}
EXPECT_EQ(incorrect_breaks, 0);
EXPECT_EQ(missed_breaks, 0);
EXPECT_EQ(poorly_matched_models, 0);
EXPECT_EQ(bad_list_items, 0);
EXPECT_EQ(bad_crowns, 0);
if (incorrect_breaks || missed_breaks || poorly_matched_models ||
bad_list_items || bad_crowns) {
std::vector<std::string> dbg_lines;
dbg_lines.push_back("# ==========================");
dbg_lines.push_back("# Correct paragraph breaks:");
dbg_lines.push_back("# ==========================");
for (int i = 0; i < n; i++) {
if (correct[i].model_type != PCONT) {
dbg_lines.push_back(absl::StrCat(
correct[i].ascii, " # ", correct[i].model.ToString().string(),
correct[i].is_very_first_or_continuation ? " crown" : "",
correct[i].is_list_item ? " li" : ""));
} else {
dbg_lines.push_back(correct[i].ascii);
}
}
dbg_lines.push_back("");
dbg_lines.push_back("# ==========================");
dbg_lines.push_back("# Paragraph detector output:");
dbg_lines.push_back("# ==========================");
for (int i = 0; i < n; i++) {
std::string annotation;
if (i == 0 || (detector_output[i - 1] != detector_output[i])) {
if (detector_output[i] && detector_output[i]->model) {
annotation += absl::StrCat(
" # ", detector_output[i]->model->ToString().string(),
detector_output[i]->is_very_first_or_continuation ? " crown" : "",
detector_output[i]->is_list_item ? " li" : "");
} else {
annotation = " # Unmodeled paragraph.";
}
}
dbg_lines.push_back(absl::StrCat(correct[i].ascii, annotation));
}
LOG(INFO) << "Discrepency!\n" << absl::StrJoin(dbg_lines, "\n");
}
}
void TestParagraphDetection(const TextAndModel* correct, int num_rows) {
GenericVector<tesseract::RowInfo> row_infos;
GenericVector<PARA*> row_owners;
PARA_LIST paragraphs;
GenericVector<ParagraphModel*> models;
MakeAsciiRowInfos(correct, num_rows, &row_infos);
int debug_level(3);
tesseract::DetectParagraphs(debug_level, &row_infos, &row_owners, &paragraphs,
&models);
EvaluateParagraphDetection(correct, num_rows, row_owners);
models.delete_data_pointers();
}
TEST(ParagraphsTest, ListItemsIdentified) {
EXPECT_TRUE(tesseract::AsciiLikelyListItem("iii"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("A."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("B."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("C."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("1."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("2."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("3."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("1"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("2"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("3"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("[[1]]"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("A-1."));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("A-2"));
EXPECT_TRUE(tesseract::AsciiLikelyListItem("(A)(i)"));
EXPECT_FALSE(tesseract::AsciiLikelyListItem("The"));
EXPECT_FALSE(tesseract::AsciiLikelyListItem("first"));
EXPECT_FALSE(tesseract::AsciiLikelyListItem("house"));
EXPECT_FALSE(tesseract::AsciiLikelyListItem("Oregonian."));
EXPECT_FALSE(tesseract::AsciiLikelyListItem("on."));
}
typedef ParagraphModel PModel;
const TextAndModel kTwoSimpleParagraphs[] = {
{" Look here, I have a paragraph.", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"This paragraph starts at the top", PCONT, PModel(), false, false},
{"of the page and takes 3 lines. ", PCONT, PModel(), false, false},
{" Here I have a second paragraph", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"which indicates that the first ", PCONT, PModel(), false, false},
{"paragraph is not a continuation ", PCONT, PModel(), false, false},
{"from a previous page, as it is ", PCONT, PModel(), false, false},
{"indented just like this second ", PCONT, PModel(), false, false},
{"paragraph. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestSimpleParagraphDetection) {
TestParagraphDetection(kTwoSimpleParagraphs,
ABSL_ARRAYSIZE(kTwoSimpleParagraphs));
}
const TextAndModel kFewCluesWithCrown[] = {
{"This paragraph starts at the top", PSTART, PModel(kLeft, 0, 20, 0, 0),
true, false},
{"of the page and takes two lines.", PCONT, PModel(), false, false},
{" Here I have a second paragraph", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"which indicates that the first ", PCONT, PModel(), false, false},
{"paragraph is a continuation from", PCONT, PModel(), false, false},
{"a previous page, as it is ", PCONT, PModel(), false, false},
{"indented just like this second ", PCONT, PModel(), false, false},
{"paragraph. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestFewCluesWithCrown) {
TestParagraphDetection(kFewCluesWithCrown,
ABSL_ARRAYSIZE(kFewCluesWithCrown));
}
const TextAndModel kCrownedParagraph[] = {
{"The first paragraph on a page is", PSTART, PModel(kLeft, 0, 20, 0, 0),
true, false},
{"often not indented as the rest ", PCONT, PModel(), false, false},
{"of the paragraphs are. Nonethe-", PCONT, PModel(), false, false},
{"less it should be counted as the", PCONT, PModel(), false, false},
{"same type of paragraph. ", PCONT, PModel(), false, false},
{" The second and third para- ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"graphs are both indented two ", PCONT, PModel(), false, false},
{"spaces. ", PCONT, PModel(), false, false},
{" The first paragraph has what ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"fmt refers to as a 'crown.' ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestCrownParagraphDetection) {
TestParagraphDetection(kCrownedParagraph, ABSL_ARRAYSIZE(kCrownedParagraph));
}
const TextAndModel kFlushLeftParagraphs[] = {
{"It is sometimes the case that", PSTART, PModel(kLeft, 0, 0, 0, 0), false, false},
{"flush left paragraphs (those", PCONT, PModel(), false, false},
{"with no body indent) are not", PCONT, PModel(), false, false},
{"actually crowns. ", PCONT, PModel(), false, false},
{"Instead, further paragraphs are", PSTART, PModel(kLeft, 0, 0, 0, 0), false, false},
{"also flush left aligned. Usual-", PCONT, PModel(), false, false},
{"ly, these paragraphs are set", PCONT, PModel(), false, false},
{"apart vertically by some white-", PCONT, PModel(), false, false},
{"space, but you can also detect", PCONT, PModel(), false, false},
{"them by observing the big empty", PCONT, PModel(), false, false},
{"space at the ends of the para-", PCONT, PModel(), false, false},
{"graphs. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsText, TestRealFlushLeftParagraphs) {
TestParagraphDetection(kFlushLeftParagraphs,
ABSL_ARRAYSIZE(kFlushLeftParagraphs));
}
const TextAndModel kSingleFullPageContinuation[] = {
{"sometimes a page is one giant", PSTART, PModel(kLeft, 0, 20, 0, 0), true, false},
{"continuation. It flows from", PCONT, PModel(), false, false},
{"line to line, using the full", PCONT, PModel(), false, false},
{"column width with no clear", PCONT, PModel(), false, false},
{"paragraph break, because it", PCONT, PModel(), false, false},
{"actually doesn't have one. It", PCONT, PModel(), false, false},
{"is the middle of one monster", PCONT, PModel(), false, false},
{"paragraph continued from the", PCONT, PModel(), false, false},
{"previous page and continuing", PCONT, PModel(), false, false},
{"onto the next page. There-", PCONT, PModel(), false, false},
{"fore, it ends up getting", PCONT, PModel(), false, false},
{"marked as a crown and then", PCONT, PModel(), false, false},
{"getting re-marked as any ex-", PCONT, PModel(), false, false},
{"isting model. Not great, but", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestSingleFullPageContinuation) {
const TextAndModel* correct = kSingleFullPageContinuation;
int num_rows = ABSL_ARRAYSIZE(kSingleFullPageContinuation);
GenericVector<tesseract::RowInfo> row_infos;
GenericVector<PARA*> row_owners;
PARA_LIST paragraphs;
GenericVector<ParagraphModel*> models;
models.push_back(new ParagraphModel(kLeft, 0, 20, 0, 10));
MakeAsciiRowInfos(correct, num_rows, &row_infos);
tesseract::DetectParagraphs(3, &row_infos, &row_owners, &paragraphs, &models);
EvaluateParagraphDetection(correct, num_rows, row_owners);
models.delete_data_pointers();
}
const TextAndModel kRightAligned[] = {
{"Right-aligned paragraphs are", PSTART, PModel(kRight, 0, 0, 0, 0), false, false},
{" uncommon in Left-to-Right", PCONT, PModel(), false, false},
{" languages, but they do", PCONT, PModel(), false, false},
{" exist.", PCONT, PModel(), false, false},
{" Mostly, however, they're", PSTART, PModel(kRight, 0, 0, 0, 0), false, false},
{" horribly tiny paragraphs in", PCONT, PModel(), false, false},
{" tables on which we have no", PCONT, PModel(), false, false},
{" chance anyways.", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestRightAlignedParagraph) {
TestParagraphDetection(kRightAligned, ABSL_ARRAYSIZE(kRightAligned));
}
const TextAndModel kTinyParagraphs[] = {
{" Occasionally, interspersed with", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"obvious paragraph text, you might", PCONT, PModel(), false, false},
{"find short exchanges of dialogue ", PCONT, PModel(), false, false},
{"between characters. ", PCONT, PModel(), false, false},
{" 'Oh?' ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{" 'Don't be confused!' ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{" 'Not me!' ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{" One naive approach would be to ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"mark a new paragraph whenever one", PCONT, PModel(), false, false},
{"of the statistics (left, right or", PCONT, PModel(), false, false},
{"center) changes from one text-", PCONT, PModel(), false, false},
{"line to the next. Such an", PCONT, PModel(), false, false},
{"approach would misclassify the", PCONT, PModel(), false, false},
{"tiny paragraphs above as a single", PCONT, PModel(), false, false},
{"paragraph. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestTinyParagraphs) {
TestParagraphDetection(kTinyParagraphs, ABSL_ARRAYSIZE(kTinyParagraphs));
}
const TextAndModel kComplexPage1[] = {
{" Awesome ", PSTART, PModel(kCenter, 0, 0, 0, 0), false, false},
{" Centered Title ", PCONT, PModel(), false, false},
{" Paragraph Detection ", PCONT, PModel(), false, false},
{" OCR TEAM ", PCONT, PModel(), false, false},
{" 10 November 2010 ", PCONT, PModel(), false, false},
{" ", PNONE, PModel(), false, false},
{" Look here, I have a paragraph.", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"This paragraph starts at the top", PCONT, PModel(), false, false},
{"of the page and takes 3 lines. ", PCONT, PModel(), false, false},
{" Here I have a second paragraph", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"which indicates that the first ", PCONT, PModel(), false, false},
{"paragraph is not a continuation ", PCONT, PModel(), false, false},
{"from a previous page, as it is ", PCONT, PModel(), false, false},
{"indented just like this second ", PCONT, PModel(), false, false},
{"paragraph. ", PCONT, PModel(), false, false},
{" Here is a block quote. It ", PSTART, PModel(kLeft, 30, 0, 0, 0),
true, false},
{" looks like the prior text ", PCONT, PModel(), false, false},
{" but it is indented more ", PCONT, PModel(), false, false},
{" and is fully justified. ", PCONT, PModel(), false, false},
{" So how does one deal with ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"centered text, block quotes, ", PCONT, PModel(), false, false},
{"normal paragraphs, and lists ", PCONT, PModel(), false, false},
{"like what follows? ", PCONT, PModel(), false, false},
{"1. Make a plan. ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{"2. Use a heuristic, for example,", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" looking for lines where the ", PCONT, PModel(), false, false},
{" first word of the next line ", PCONT, PModel(), false, false},
{" would fit on the previous ", PCONT, PModel(), false, false},
{" line. ", PCONT, PModel(), false, false},
{"8. Try to implement the plan in ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" Python and try it out. ", PCONT, PModel(), false, false},
{"4. Determine how to fix the ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" mistakes. ", PCONT, PModel(), false, false},
{"5. Repeat. ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" For extra painful penalty work", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"you can try to identify source ", PCONT, PModel(), false, false},
{"code. Ouch! ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestComplexPage1) {
TestParagraphDetection(kComplexPage1, ABSL_ARRAYSIZE(kComplexPage1));
}
// The same as above, but wider.
const TextAndModel kComplexPage2[] = {
{" Awesome ", PSTART,
PModel(kCenter, 0, 0, 0, 0), false, false},
{" Centered Title ", PCONT, PModel(), false, false},
{" Paragraph Detection ", PCONT, PModel(), false, false},
{" OCR TEAM ", PCONT, PModel(), false, false},
{" 10 November 2010 ", PCONT, PModel(), false, false},
{" ", PNONE, PModel(), false, false},
{" Look here, I have a paragraph. ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"This paragraph starts at the top of", PCONT, PModel(), false, false},
{"the page and takes 3 lines. ", PCONT, PModel(), false, false},
{" Here I have a second paragraph ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"which indicates that the first ", PCONT, PModel(), false, false},
{"paragraph is not a continuation ", PCONT, PModel(), false, false},
{"from a previous page, as it is in- ", PCONT, PModel(), false, false},
{"dented just like this second para- ", PCONT, PModel(), false, false},
{"graph. ", PCONT, PModel(), false, false},
{" Here is a block quote. It ", PSTART, PModel(kLeft, 30, 0, 0, 0),
true, false},
{" looks like the prior text ", PCONT, PModel(), false, false},
{" but it is indented more ", PCONT, PModel(), false, false},
{" and is fully justified. ", PCONT, PModel(), false, false},
{" So how does one deal with center-", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"ed text, block quotes, normal para-", PCONT, PModel(), false, false},
{"graphs, and lists like what follow?", PCONT, PModel(), false, false},
{"1. Make a plan. ", PCONT, PModel(), false, false}, // BUG!!
{"2. Use a heuristic, for example, ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" looking for lines where the ", PCONT, PModel(), false, false},
{" first word of the next line ", PCONT, PModel(), false, false},
{" would fit on the previous line. ", PCONT, PModel(), false, false},
{"8. Try to implement the plan in ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" Python and try it out. ", PCONT, PModel(), false, false},
{"4. Determine how to fix the ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" mistakes. ", PCONT, PModel(), false, false},
{"5. Repeat. ", PSTART, PModel(kLeft, 0, 0, 30, 0),
false, true},
{" For extra painful penalty work ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"you can try to identify source ", PCONT, PModel(), false, false},
{"code. Ouch! ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestComplexPage2) {
TestParagraphDetection(kComplexPage2, ABSL_ARRAYSIZE(kComplexPage2));
}
const TextAndModel kSubtleCrown[] = {
{"The first paragraph on a page is", PSTART, PModel(kLeft, 0, 20, 0, 0),
true, false},
{"often not indented as the rest ", PCONT, PModel(), false, false},
{"of the paragraphs are. Nonethe-", PCONT, PModel(), false, false},
{"less it should be counted as the", PCONT, PModel(), false, false},
{"same type of paragraph. ", PCONT, PModel(), false, false},
{" Even a short second paragraph ", PSTART, PModel(kLeft, 0, 20, 0, 0), false, false},
{"should suffice. ", PCONT, PModel(), false, false},
{" 1235 ", PNONE, PModel(), false, false},
};
TEST(ParagraphsTest, TestSubtleCrown) {
TestParagraphDetection(kSubtleCrown, ABSL_ARRAYSIZE(kSubtleCrown) - 1);
}
TEST(ParagraphsTest, TestStrayLineInBlock) {
TestParagraphDetection(kSubtleCrown, ABSL_ARRAYSIZE(kSubtleCrown));
}
const TextAndModel kUnlvRep3AO[] = {
{" Defined contribution plans cover employees in Australia, New", PSTART,
PModel(kLeft, 0, 50, 0, 0), false, false},
{"Zealand, Spain, the United Kingdom and some U.S. subsidiaries. ", PCONT, PModel(), false, false},
{"In addition, employees in the U.S. are eligible to participate in ", PCONT, PModel(), false, false},
{"defined contribution plans (Employee Savings Plans) by contribut-", PCONT, PModel(), false, false},
{"ing a portion of their compensation. The Company matches com- ", PCONT, PModel(), false, false},
{"pensation, depending on Company profit levels. Contributions ", PCONT, PModel(), false, false},
{"charged to income for defined contribution plans were $92 in ", PCONT, PModel(), false, false},
{"1993, $98 in 1992 and $89 in 1991. ", PCONT, PModel(), false, false},
{" In addition to providing pension benefits, the Company pro- ", PSTART,
PModel(kLeft, 0, 50, 0, 0), false, false},
{"vides certain health care and life insurance benefits to retired ", PCONT, PModel(), false, false},
{"employees. As discussed in Note A, the Company adopted FASB ", PCONT, PModel(), false, false},
{"Statement No. 106 effective January 1, 1992. Previously, the ", PCONT, PModel(), false, false},
{"Company recognized the cost of providing these benefits as the ", PCONT, PModel(), false, false},
{"benefits were paid. These pretax costs amounted to $53 in 1991. ", PCONT, PModel(), false, false},
{"The Company continues to fund most of the cost of these medical ", PCONT, PModel(), false, false},
{"and life insurance benefits in the year incurred. ", PCONT, PModel(), false, false},
{" The U.S. plan covering the parent company is the largest plan.",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"It provides medical and life insurance benefits including hospital, ", PCONT, PModel(), false, false},
{"physicians’ services and major medical expense benefits and life ", PCONT, PModel(), false, false},
{"insurance benefits. The plan provides benefits supplemental to ", PCONT, PModel(), false, false},
{"Medicare after retirees are eligible for these benefits. The cost of ", PCONT, PModel(), false, false},
{"these benefits are shared by the Company and the retiree, with the ", PCONT, PModel(), false, false},
{"Company portion increasing as the retiree has increased years of ", PCONT, PModel(), false, false},
{"credited service. The Company has the ability to change these ", PCONT, PModel(), false, false},
{"benefits at any time. ", PCONT, PModel(), false, false},
{" Effective October 1993, the Company amended its health ", PSTART,
PModel(kLeft, 0, 50, 0, 0), false, false},
{"benefits plan in the U.S. to cap the cost absorbed by the Company ", PCONT, PModel(), false, false},
{"at approximately twice the 1993 cost per person for employees who", PCONT, PModel(), false, false},
{"retire after December 31, 1993. The effect of this amendment was ", PCONT, PModel(), false, false},
{"to reduce the December 31, 1993 accumulated postretirement ", PCONT, PModel(), false, false},
{"benefit obligation by $327. It also reduced the net periodic postre- ", PCONT, PModel(), false, false},
{"tirement cost by $21 for 1993 and is estimated to reduce this cost ", PCONT, PModel(), false, false},
{"for 1994 by approximately $83. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, TestUnlvInsurance) {
TestParagraphDetection(kUnlvRep3AO, ABSL_ARRAYSIZE(kUnlvRep3AO));
}
// The basic outcome we want for something with a bunch of leader dots is that
// we group each logical entry as a separate item. Without knowledge of
// leaders, we would most likely mark the text below as a simple right aligned
// paragraph or two.
// This example comes from Volume 9886293, Page 5
const TextAndModel kTableOfContents[] = {
{"1 Hmong People ........... 1", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Hmong Origins . . . . . 1", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Language . . . . . . . 1", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Proverbs . . . . . . 2", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Discussion . . . . 2", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Riddles . . . . . . . 2", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Discussion . . . . 3", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Appearance . . . . . 3", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Hmong History . . . . . 4", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Hmong in SE Asia . . . 4", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Hmong in the West . . .5", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Hmong in the USA . . . 5", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
{" Discussion . . . . 6", PSTART, PModel(kUnknown, 0, 0, 0, 0), false, false},
};
TEST(ParagraphsTest, TestSplitsOutLeaderLines) {
TestParagraphDetection(kTableOfContents, ABSL_ARRAYSIZE(kTableOfContents));
}
const TextAndModel kTextWithSourceCode[] = {
{" A typical page of a programming book may contain", PSTART,
PModel(kLeft, 0, 20, 0, 0), false, false},
{"examples of source code to exemplify an algorithm ", PCONT, PModel(), false, false},
{"being described in prose. Such examples should be", PCONT, PModel(), false, false},
{"rendered as lineated text, meaning text with ", PCONT, PModel(), false, false},
{"explicit line breaks but without extra inter-line ", PCONT, PModel(), false, false},
{"spacing. Accidentally finding stray paragraphs in", PCONT, PModel(), false, false},
{"source code would lead to a bad reading experience", PCONT, PModel(), false, false},
{"when the text is re-flowed. ", PCONT, PModel(), false, false},
{" Let's show this by describing the function fact-", PSTART,
PModel(kLeft, 0, 20, 0, 0), false, false},
{"orial. Factorial is a simple recursive function ", PCONT, PModel(), false, false},
{"which grows very quickly. So quickly, in fact, ", PCONT, PModel(), false, false},
{"that the typical C implementation will only work ", PCONT, PModel(), false, false},
{"for values less than about 12: ", PCONT, PModel(), false, false},
{" ", PNONE, PModel(), false, false},
{" # Naive implementation in C ", PCONT, PModel(), false, false},
{" int factorial(int n) { ", PCONT, PModel(), false, false},
{" if (n < 2) ", PCONT, PModel(), false, false},
{" return 1; ", PCONT, PModel(), false, false},
{" return n * factorial(n - 1); ", PCONT, PModel(), false, false},
{" } ", PCONT, PModel(), false, false},
{" ", PCONT, PModel(), false, false},
{" The C programming language does not have built- ", PSTART,
PModel(kLeft, 0, 20, 0, 0), false, false},
{"in support for detecting integer overflow, so this", PCONT, PModel(), false, false},
{"naive implementation simply returns random values ", PCONT, PModel(), false, false},
{"if even a moderate sized n is provided. ", PCONT, PModel(), false, false},
};
TEST(ParagraphsTest, NotDistractedBySourceCode) {
TestParagraphDetection(kTextWithSourceCode,
ABSL_ARRAYSIZE(kTextWithSourceCode));
}
const TextAndModel kOldManAndSea[] = {
{"royal palm which are called guano and in it there was a bed, a",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"table, one chair, and a place on the dirt floor to cook with charcoal.", PCONT, PModel(), false, false},
{"On the brown walls of the flattened, overlapping leaves of the", PCONT, PModel(), false, false},
{"sturdy fibered guano there was a picture in color of the Sacred", PCONT, PModel(), false, false},
{"Heart of Jesus and another of the Virgin of Cobre. These were", PCONT, PModel(), false, false},
{"relics of his wife. Once there had been a tinted photograph of his", PCONT, PModel(), false, false},
{"wife on the wall but he had taken it down because it made him too", PCONT, PModel(), false, false},
{"lonely to see it and it was on the shelf in the corner under his clean", PCONT, PModel(), false, false},
{"shirt. ", PCONT, PModel(), false, false},
{" \"What do you have to eat?\" the boy asked. ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"A pot of yellow rice with fish. Do you want some?\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"No. I will eat at home. Do you want me to make the fire?\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"No. I will make it later on. Or I may eat the rice cold.\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"May I take the cast net?\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"Of course.\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" There was no cast net and the boy remembered when they had",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"sold it. But they went through this fiction every day. There was no", PCONT, PModel(), false, false},
{"pot of yellow rice and fish and the boy knew this too. "
" ", PCONT, PModel(), false, false},
{" \"Eighty-five is a lucky number,\" the old man said. \"How",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"would you like to see me bring one in that dressed out over a "
"thou-", PCONT, PModel(), false, false},
{"sand pounds? "
" ", PCONT, PModel(), false, false},
{" \"I'll get the cast net and go for sardines. Will you sit in the "
"sun",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"in the doorway?\" "
" ", PCONT, PModel(), false, false},
{" \"Yes. I have yesterday's paper and I will read the baseball.\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" The boy did not know whether yesterday's paper was a fiction",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"too. But the old man brought it out from under the bed. ", PCONT, PModel(), false, false},
{" \"Pedrico gave it to me at the bodega,\" he explained. "
" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"I'll be back when I have the sardines. I'll keep yours and mine",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"together on ice and we can share them in the morning. When I", PCONT, PModel(), false, false},
{"come back you can tell me about the baseball.\" ", PCONT, PModel(), false, false},
{" \"The Yankees cannot lose.\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"But I fear the Indians of Cleveland.\" ",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{" \"Have faith in the Yankees my son. Think of the great Di-",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"Maggio.\" ", PCONT, PModel(), false, false},
{" \"I fear both the Tigers of Detroit and the Indians of Cleve-",
PSTART, PModel(kLeft, 0, 50, 0, 0), false, false},
{"land.\" ", PCONT, PModel(), false, false}
};
TEST(ParagraphsTest, NotOverlyAggressiveWithBlockQuotes) {
TestParagraphDetection(kOldManAndSea, ABSL_ARRAYSIZE(kOldManAndSea));
}
const TextAndModel kNewZealandIndex[] = {
{"Oats, 51 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"O'Brien, Gregory, 175 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Occupational composition, 110,", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{" 138 ", PCONT, PModel(), false, false},
{"OECD rankings, 155, 172 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Okiato (original capital), 47 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Oil shock: 1974, xxx, 143; 1979,", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{" 145 ", PCONT, PModel(), false, false},
{"Old Age Pensions, xxii, 89-90 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Old World evils, 77 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Oliver, W. H., 39, 77, 89 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Olssen, Erik, 45, 64, 84 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Olympic Games, 1924, 111, 144 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Once on Chunuk Bair, 149 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Once Were Warriors, xxxiii, 170", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"On—shore whaling, xvi ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Opotiki, xix ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Orakau battle of, xviii, 57 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"O’Regan, Tipene, 170, 198-99 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Organic agriculture, 177 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Orwell, George, 151 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago, xvii, 45, 49-50, 70 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago block, xvii ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago Daily Times, 67 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago Girls’ High School, xix, 61,", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{" 85 ", PCONT, PModel(), false, false},
{"Otago gold rushes, 61-63 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago Peninsula, xx ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otago Provincial Council, 68 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Otaki, 33 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false},
{"Owls Do Cry, 139 ", PSTART, PModel(kLeft, 0, 0, 30, 0), false, false}
};
TEST(ParagraphsTest, IndexPageTest) {
TestParagraphDetection(kNewZealandIndex, ABSL_ARRAYSIZE(kNewZealandIndex));
}
// TOOO(eger): Add some right-to-left examples, and fix the algorithm as needed.
} // namespace
Reference in New Issue View Git Blame Copy Permalink