21 #pragma warning(disable:4244) // Conversion warnings
30 #include "allheaders.h"
75 static void clear_any_old_text(BLOCK_LIST *block_list) {
76 BLOCK_IT block_it(block_list);
77 for (block_it.mark_cycle_pt();
78 !block_it.cycled_list(); block_it.forward()) {
79 ROW_IT row_it(block_it.data()->row_list());
80 for (row_it.mark_cycle_pt(); !row_it.cycled_list(); row_it.forward()) {
81 WERD_IT word_it(row_it.data()->word_list());
82 for (word_it.mark_cycle_pt();
83 !word_it.cycled_list(); word_it.forward()) {
84 word_it.data()->set_text(
"");
111 bool find_segmentation,
112 BLOCK_LIST *block_list) {
120 int box_count = boxes.
size();
121 int box_failures = 0;
129 PAGE_RES* page_res = find_segmentation ?
131 clear_any_old_text(block_list);
133 for (
int i = 0; i < boxes.
size() - 1; i++) {
134 bool foundit =
false;
135 if (page_res !=
NULL) {
138 full_texts[i].
string());
141 boxes[i + 1], full_texts[i].
string());
150 "FAILURE! Couldn't find a matching blob");
154 if (page_res ==
NULL) {
162 tprintf(
" Boxes read from boxfile: %6d\n", box_count);
163 if (box_failures > 0)
164 tprintf(
" Boxes failed resegmentation: %6d\n", box_failures);
171 static double MedianXHeight(BLOCK_LIST *block_list) {
172 BLOCK_IT block_it(block_list);
173 STATS xheights(0, block_it.data()->bounding_box().height());
174 for (block_it.mark_cycle_pt();
175 !block_it.cycled_list(); block_it.forward()) {
176 ROW_IT row_it(block_it.data()->row_list());
177 for (row_it.mark_cycle_pt(); !row_it.cycled_list(); row_it.forward()) {
181 return xheights.median();
187 double median_xheight = MedianXHeight(block_list);
190 BLOCK_IT b_it(block_list);
191 for (b_it.mark_cycle_pt(); !b_it.cycled_list(); b_it.forward()) {
192 BLOCK* block = b_it.data();
194 for (r_it.mark_cycle_pt(); !r_it.cycled_list(); r_it.forward ()) {
195 ROW* row = r_it.data();
196 float diff = fabs(row->
x_height() - median_xheight);
197 if (diff > max_deviation) {
199 tprintf(
"row xheight=%g, but median xheight = %g\n",
211 BLOCK_LIST *block_list) {
214 BLOCK_IT b_it(block_list);
215 for (b_it.mark_cycle_pt(); !b_it.cycled_list(); b_it.forward()) {
216 BLOCK* block = b_it.data();
218 for (r_it.mark_cycle_pt(); !r_it.cycled_list(); r_it.forward ()) {
219 ROW* row = r_it.data();
221 for (w_it.mark_cycle_pt(); !w_it.cycled_list(); w_it.forward()) {
222 WERD* word = w_it.data();
224 delete w_it.extract();
235 while ((word_res = pr_it.
word()) !=
NULL) {
237 pr_it.
row()->
row, word_res);
259 tprintf(
"Maximally chopping word at:");
264 float rating =
static_cast<float>(
MAX_INT8);
279 const double e = exp(1.0);
281 int right_chop_index = 0;
286 &blob_number)) !=
NULL) {
288 BLOB_CHOICE* left_choice = blob_choices[blob_number];
289 rating = left_choice->
rating() / e;
294 rating - 0.125f, -rating, -1,
296 blob_choices.
insert(right_choice, blob_number + 1);
314 static double BoxMissMetric(
const TBOX& box1,
const TBOX& box2) {
316 double miss_metric = box1.
area()- overlap_area;
317 miss_metric /= box1.
area();
318 miss_metric *= box2.
area() - overlap_area;
319 miss_metric /= box2.
area();
332 const TBOX& box,
const TBOX& next_box,
333 const char* correct_text) {
335 tprintf(
"\nAPPLY_BOX: in ResegmentCharBox() for %s\n", correct_text);
339 for (word_res = page_res_it.
word(); word_res !=
NULL;
340 word_res = page_res_it.
forward()) {
348 for (
int i = 0; i < word_len; ++i) {
351 for (blob_count = 0; i + blob_count < word_len; ++blob_count) {
357 double current_box_miss_metric = BoxMissMetric(blob_box, box);
358 double next_box_miss_metric = BoxMissMetric(blob_box, next_box);
362 tprintf(
"Current miss metric = %g, next = %g\n",
363 current_box_miss_metric, next_box_miss_metric);
365 if (current_box_miss_metric > next_box_miss_metric)
367 char_box += blob_box;
369 if (blob_count > 0) {
371 tprintf(
"Index [%d, %d) seem good.\n", i, i + blob_count);
374 (box.
x_gap(next_box) < -3 ||
375 (prev_box !=
NULL && prev_box->
x_gap(box) < -3))) {
386 tprintf(
"%d Blobs match: blob box:", blob_count);
395 for (
int j = 1; j < blob_count; ++j) {
430 const TBOX& box,
const TBOX& next_box,
431 const char* correct_text) {
433 tprintf(
"\nAPPLY_BOX: in ResegmentWordBox() for %s\n", correct_text);
436 BLOCK_IT b_it(block_list);
437 for (b_it.mark_cycle_pt(); !b_it.cycled_list(); b_it.forward()) {
438 BLOCK* block = b_it.data();
442 for (r_it.mark_cycle_pt(); !r_it.cycled_list(); r_it.forward()) {
443 ROW* row = r_it.data();
447 for (w_it.mark_cycle_pt(); !w_it.cycled_list(); w_it.forward()) {
448 WERD* word = w_it.data();
458 for (blob_it.mark_cycle_pt(); !blob_it.cycled_list();
460 C_BLOB* blob = blob_it.data();
464 double current_box_miss_metric = BoxMissMetric(blob_box, box);
465 double next_box_miss_metric = BoxMissMetric(blob_box, next_box);
469 tprintf(
"Current miss metric = %g, next = %g\n",
470 current_box_miss_metric, next_box_miss_metric);
472 if (current_box_miss_metric > next_box_miss_metric)
482 if (new_word ==
NULL) {
486 w_it.add_to_end(new_word);
488 C_BLOB_IT new_blob_it(new_word->
cblob_list());
489 new_blob_it.add_to_end(blob_it.extract());
495 return new_word !=
NULL;
510 tprintf(
"APPLY_BOX: FAILURE: can't find class_id for '%s'\n",
516 tprintf(
"APPLY_BOX: FAILURE: can't find segmentation for '%s'\n",
528 for (
int step = 0; *utf8 !=
'\0'; utf8 += step) {
529 const char* next_space = strchr(utf8,
' ');
530 if (next_space ==
NULL)
531 next_space = utf8 + strlen(utf8);
532 step = next_space - utf8;
534 if (class_id == INVALID_UNICHAR_ID) {
537 while (utf8[step] ==
' ')
556 for (
int i = 0; i < word_length; ++i) {
557 for (
int j = 1; j <=
kMaxGroupSize && i + j <= word_length; ++j) {
559 word_res->
seam_array, i, i + j - 1,
"Applybox",
573 float best_rating = 0.0f;
575 &search_segmentation, &best_rating, &word_res->
best_state);
576 for (
int i = 0; i < word_length; ++i)
577 choices[i].delete_data_pointers();
599 for (
int i = 0; i < target_text.
size(); ++i) {
613 int choices_pos,
int choices_length,
620 for (
int length = 1; length <= choices[choices_pos].
size(); ++length) {
622 float choice_rating = 0.0f;
624 BLOB_CHOICE_IT choice_it(choices[choices_pos][length - 1]);
625 for (choice_it.mark_cycle_pt(); !choice_it.cycled_list();
626 choice_it.forward()) {
628 choice_rating = choice->
rating();
630 if (class_id == target_text[text_index]) {
634 if (class_id < table.
size() && table[class_id] !=
NULL) {
635 AmbigSpec_IT spec_it(table[class_id]);
636 for (spec_it.mark_cycle_pt(); !spec_it.cycled_list();
638 const AmbigSpec *ambig_spec = spec_it.data();
640 if (ambig_spec->
wrong_ngram[1] == INVALID_UNICHAR_ID &&
644 if (!spec_it.cycled_list())
648 if (choice_it.cycled_list())
651 if (choices_pos + length == choices_length &&
652 text_index + 1 == target_text.
size()) {
655 tprintf(
"Complete match, rating = %g, best=%g, seglength=%d, best=%d\n",
656 rating + choice_rating, *best_rating, segmentation->
size(),
657 best_segmentation->
size());
659 if (best_segmentation->
empty() || rating + choice_rating < *best_rating) {
660 *best_segmentation = *segmentation;
661 *best_rating = rating + choice_rating;
663 }
else if (choices_pos + length < choices_length &&
664 text_index + 1 < target_text.
size()) {
666 tprintf(
"Match found for %d=%s:%s, at %d+%d, recursing...\n",
667 target_text[text_index],
669 choice_it.data()->unichar_id() == target_text[text_index]
671 choices_pos, length);
673 SearchForText(choices, choices_pos + length, choices_length, target_text,
674 text_index + 1, rating + choice_rating, segmentation,
675 best_rating, best_segmentation);
677 tprintf(
"End recursion for %d=%s\n", target_text[text_index],
690 int ok_blob_count = 0;
691 int bad_blob_count = 0;
692 int ok_word_count = 0;
693 int unlabelled_words = 0;
701 for (
int c = 0; c < blob_count; ++c) {
710 INVALID_UNICHAR_ID, word_res->
best_state[c], 1.0f, -1.0f);
712 if (ok_in_word > 0) {
713 ok_blob_count += ok_in_word;
720 tprintf(
"APPLY_BOXES: Unlabelled word at :");
736 tprintf(
" Found %d good blobs.\n", ok_blob_count);
737 if (bad_blob_count > 0) {
738 tprintf(
" Leaving %d unlabelled blobs in %d words.\n",
739 bad_blob_count, ok_word_count);
741 if (unlabelled_words > 0)
742 tprintf(
" %d remaining unlabelled words deleted.\n", unlabelled_words);
748 const char *box_ch,
const char *err_msg) {
749 tprintf(
"APPLY_BOXES: boxfile line %d/%s ((%d,%d),(%d,%d)): %s\n",
750 boxfile_lineno + 1, box_ch,
760 word_res->correct_text.size());
761 for (
int i = 0; i < word_res->correct_text.size(); ++i) {
765 word_res->correct_text[i].split(
' ', &tokens);
768 word_res->best_state[i],
771 word_res->ClearWordChoices();
772 word_res->LogNewRawChoice(choice);
773 word_res->LogNewCookedChoice(1,
false, choice);
787 tprintf(
"Generated training data for %d words\n", word_count);
bool ResegmentCharBox(PAGE_RES *page_res, const TBOX *prev_box, const TBOX &box, const TBOX &next_box, const char *correct_text)
bool LogNewRawChoice(WERD_CHOICE *word_choice)
void CorrectClassifyWords(PAGE_RES *page_res)
tesseract::BoxWord * box_word
bool classify_bln_numeric_mode
const UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
const double kMaxXHeightDeviationFraction
bool HasAnySplits() const
bool assume_fixed_pitch_char_segment
void set_text(const char *new_text)
ROW_RES * next_row() const
bool LogNewCookedChoice(int max_num_choices, bool debug, WERD_CHOICE *word_choice)
void append_unichar_id_space_allocated(UNICHAR_ID unichar_id, int blob_count, float rating, float certainty)
bool almost_equal(const TBOX &box, int tolerance) const
const TBOX & BlobBox(int index) const
void set_x_height(float new_xheight)
void MergeBoxes(int start, int end)
void set_permuter(uinT8 perm)
void add(inT32 value, inT32 count)
TBOX bounding_box() const
GenericVector< STRING > correct_text
UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE+1]
PAGE_RES * ApplyBoxes(const STRING &fname, bool find_segmentation, BLOCK_LIST *block_list)
void print_ratings_list(const char *msg, BLOB_CHOICE_LIST *ratings, const UNICHARSET ¤t_unicharset)
BLOCK_RES * block() const
bool FindSegmentation(const GenericVector< UNICHAR_ID > &target_text, WERD_RES *word_res)
void TidyUp(PAGE_RES *page_res)
WERD_RES * restart_page()
void insert(T t, int index)
void FakeClassifyWord(int blob_count, BLOB_CHOICE **choices)
const UnicharAmbigs & getUnicharAmbigs() const
void CloneChoppedToRebuild()
void ReportFailedBox(int boxfile_lineno, TBOX box, const char *box_ch, const char *err_msg)
void set_certainty(float newrat)
const char *const id_to_unichar(UNICHAR_ID id) const
const UNICHARSET * uch_set
int tessedit_ocr_engine_mode
bool poly_allow_detailed_fx
TBOX bounding_box() const
virtual BLOB_CHOICE_LIST * classify_piece(const GenericVector< SEAM * > &seams, inT16 start, inT16 end, const char *description, TWERD *word, BlamerBundle *blamer_bundle)
void bounding_box(ICOORD &bottom_left, ICOORD &top_right) const
get box
GenericVector< SEAM * > seam_array
void SearchForText(const GenericVector< BLOB_CHOICE_LIST * > *choices, int choices_pos, int choices_length, const GenericVector< UNICHAR_ID > &target_text, int text_index, float rating, GenericVector< int > *segmentation, float *best_rating, GenericVector< int > *best_segmentation)
const UnicharAmbigsVector & dang_ambigs() const
void InsertSeam(int blob_number, SEAM *seam)
const char * text() const
bool major_overlap(const TBOX &box) const
bool ResegmentWordBox(BLOCK_LIST *block_list, const TBOX &box, const TBOX &next_box, const char *correct_text)
TBOX intersection(const TBOX &box) const
void ApplyBoxTraining(const STRING &fontname, PAGE_RES *page_res)
UNICHAR_ID correct_ngram_id
GenericVector< TBLOB * > blobs
int x_gap(const TBOX &box) const
int IntCastRounded(double x)
void MaximallyChopWord(const GenericVector< TBOX > &boxes, BLOCK *block, ROW *row, WERD_RES *word_res)
const TBOX & bounding_box() const
bool SetupForRecognition(const UNICHARSET &unicharset_in, tesseract::Tesseract *tesseract, Pix *pix, int norm_mode, const TBOX *norm_box, bool numeric_mode, bool use_body_size, bool allow_detailed_fx, ROW *row, const BLOCK *block)
TBOX bounding_box() const
GenericVector< int > best_state
void set_rating(float newrat)
void LearnWord(const char *fontname, WERD_RES *word)
PAGE_RES * SetupApplyBoxes(const GenericVector< TBOX > &boxes, BLOCK_LIST *block_list)
ROW_LIST * row_list()
get rows
void PreenXHeights(BLOCK_LIST *block_list)
bool ConvertStringToUnichars(const char *utf8, GenericVector< UNICHAR_ID > *class_ids)
const char * string() const
ROW_RES * prev_row() const
void set_flag(WERD_FLAGS mask, BOOL8 value)
UNICHAR_ID unichar_id() const
bool textord_use_cjk_fp_model
BlamerBundle * blamer_bundle
SEAM * chop_one_blob(const GenericVector< TBOX > &boxes, const GenericVector< BLOB_CHOICE * > &blob_choices, WERD_RES *word_res, int *blob_number)
bool ReadAllBoxes(int target_page, bool skip_blanks, const STRING &filename, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
C_BLOB_LIST * cblob_list()
void ReSegmentByClassification(PAGE_RES *page_res)