21 #pragma warning(disable:4244) // Conversion warnings 129 int expected_outline_count;
131 if (
STRING (outlines_odd).contains (c))
133 else if (
STRING (outlines_2).contains (c))
134 expected_outline_count = 2;
136 expected_outline_count = 1;
137 return abs (outline_count - expected_outline_count);
141 BOOL8 good_quality_doc) {
142 if ((tessedit_good_quality_unrej && good_quality_doc))
143 unrej_good_quality_words(page_res_it);
144 doc_and_block_rejection(page_res_it, good_quality_doc);
145 if (unlv_tilde_crunching) {
146 tilde_crunch(page_res_it);
147 tilde_delete(page_res_it);
171 while (page_res_it.
word () !=
NULL) {
172 check_debug_pt (page_res_it.
word (), 100);
174 word = page_res_it.
word ();
176 if (word->
reject_map[i].accept_if_good_quality ())
184 quality_rowrej_pc)) {
185 word = page_res_it.
word ();
187 (tessedit_unrej_any_wd ||
188 acceptable_word_string(*word->
uch_set,
192 unrej_good_chs(word, page_res_it.
row ()->
row);
198 current_row = page_res_it.
row ();
199 while ((page_res_it.
word () !=
NULL) &&
200 (page_res_it.
row () == current_row))
203 check_debug_pt (page_res_it.
word (), 110);
208 current_block =
NULL;
210 while (page_res_it.
word () !=
NULL) {
211 if (current_block != page_res_it.
block ()) {
212 current_block = page_res_it.
block ();
216 if (current_row != page_res_it.
row ()) {
217 current_row = page_res_it.
row ();
237 BOOL8 good_quality_doc) {
244 BOOL8 prev_word_rejected;
245 inT16 char_quality = 0;
246 inT16 accepted_char_quality;
251 if (tessedit_debug_doc_rejection) {
252 tprintf(
"REJECT ALL #chars: %d #Rejects: %d; \n",
257 if (tessedit_debug_doc_rejection) {
258 tprintf(
"NO PAGE REJECTION #chars: %d # Rejects: %d; \n",
267 while ((word = page_res_it.
word()) !=
NULL) {
268 current_block = page_res_it.
block();
272 tessedit_reject_block_percent) {
273 if (tessedit_debug_block_rejection) {
274 tprintf(
"REJECTING BLOCK %d #chars: %d; #Rejects: %d\n",
278 prev_word_rejected =
FALSE;
279 while ((word = page_res_it.
word()) !=
NULL &&
280 (page_res_it.
block() == current_block)) {
281 if (tessedit_preserve_blk_rej_perfect_wds) {
284 if (rej_word && tessedit_dont_blkrej_good_wds &&
286 acceptable_word_string(
291 word_char_quality(word, page_res_it.
row()->
row,
293 &accepted_char_quality);
305 if (tessedit_use_reject_spaces &&
306 prev_word_rejected &&
312 prev_word_rejected = rej_word;
316 if (tessedit_debug_block_rejection) {
317 tprintf(
"NOT REJECTING BLOCK %d #chars: %d # Rejects: %d; \n",
324 while (page_res_it.
word() !=
NULL &&
325 page_res_it.
block() == current_block) {
326 current_row = page_res_it.
row();
335 tessedit_reject_row_percent &&
338 tessedit_whole_wd_rej_row_percent) {
339 if (tessedit_debug_block_rejection) {
340 tprintf(
"REJECTING ROW %d #chars: %d; #Rejects: %d\n",
344 prev_word_rejected =
FALSE;
345 while ((word = page_res_it.
word()) !=
NULL &&
346 page_res_it.
row () == current_row) {
348 if (!tessedit_row_rej_good_docs && good_quality_doc) {
351 tessedit_good_doc_still_rowrej_wd;
352 }
else if (tessedit_preserve_row_rej_perfect_wds) {
356 if (rej_word && tessedit_dont_rowrej_good_wds &&
358 acceptable_word_string(*word->
uch_set,
362 word_char_quality(word, page_res_it.
row()->
row,
364 &accepted_char_quality);
376 if (tessedit_use_reject_spaces &&
377 prev_word_rejected &&
383 prev_word_rejected = rej_word;
387 if (tessedit_debug_block_rejection) {
388 tprintf(
"NOT REJECTING ROW %d #chars: %d # Rejects: %d; \n",
391 while (page_res_it.
word() !=
NULL &&
392 page_res_it.
row() == current_row)
412 while (page_res_it.
word () !=
NULL) {
430 while (page_res_it.
word() !=
NULL) {
436 word = page_res_it.
word();
438 if (crunch_early_convert_bad_unlv_chs)
439 convert_bad_unlv_chs(word);
441 if (crunch_early_merge_tess_fails)
445 found_terrible_word =
FALSE;
447 prev_potential_marked =
FALSE;
450 ok_dict_word = safe_dict_word(word);
451 garbage_level = garbage_word (word, ok_dict_word);
454 (terrible_word_crunch (word, garbage_level))) {
455 if (crunch_debug > 0) {
456 tprintf (
"T CRUNCHING: \"%s\"\n",
460 if (prev_potential_marked) {
462 if (crunch_debug > 0) {
463 tprintf (
"P1 CRUNCHING: \"%s\"\n",
469 prev_potential_marked =
FALSE;
471 found_terrible_word =
TRUE;
474 (potential_word_crunch (word,
475 garbage_level, ok_dict_word))) {
476 if (found_terrible_word) {
477 if (crunch_debug > 0) {
478 tprintf (
"P2 CRUNCHING: \"%s\"\n",
483 else if (!prev_potential_marked) {
484 copy_it = page_res_it;
485 prev_potential_marked =
TRUE;
486 if (crunch_debug > 1) {
487 tprintf (
"P3 CRUNCHING: \"%s\"\n",
493 found_terrible_word =
FALSE;
495 prev_potential_marked =
FALSE;
496 if (crunch_debug > 2) {
497 tprintf (
"NO CRUNCH: \"%s\"\n",
519 if (adjusted_len > crunch_rating_max)
520 adjusted_len = crunch_rating_max;
523 if (rating_per_ch > crunch_terrible_rating)
525 else if (crunch_terrible_garbage && (garbage_level ==
G_TERRIBLE))
528 (garbage_level !=
G_OK))
530 else if ((rating_per_ch > crunch_poor_garbage_rate) &&
531 (garbage_level !=
G_OK))
534 if (crunch_mode > 0) {
535 if (crunch_debug > 2) {
536 tprintf (
"Terrible_word_crunch (%d) on \"%s\"\n",
547 BOOL8 ok_dict_word) {
552 BOOL8 word_crunchable;
553 int poor_indicator_count = 0;
555 word_crunchable = !crunch_leave_accept_strings ||
557 (acceptable_word_string(*word->
uch_set,
562 if (adjusted_len > 10)
566 if (rating_per_ch > crunch_pot_poor_rate) {
567 if (crunch_debug > 2) {
568 tprintf(
"Potential poor rating on \"%s\"\n",
571 poor_indicator_count++;
574 if (word_crunchable &&
576 if (crunch_debug > 2) {
577 tprintf(
"Potential poor cert on \"%s\"\n",
580 poor_indicator_count++;
583 if (garbage_level !=
G_OK) {
584 if (crunch_debug > 2) {
585 tprintf(
"Potential garbage on \"%s\"\n",
588 poor_indicator_count++;
590 return poor_indicator_count >= crunch_pot_indicators;
598 inT16 debug_delete_mode;
600 inT16 x_debug_delete_mode;
604 while (page_res_it.
word() !=
NULL) {
605 word = page_res_it.
word();
607 delete_mode = word_deletable (word, debug_delete_mode);
610 if (crunch_debug > 0) {
611 tprintf (
"BOL CRUNCH DELETING(%d): \"%s\"\n",
616 deleting_from_bol =
TRUE;
618 if (marked_delete_point) {
620 x_delete_mode = word_deletable (copy_it.
word (),
621 x_debug_delete_mode);
622 if (crunch_debug > 0) {
623 tprintf (
"EOL CRUNCH DELETING(%d): \"%s\"\n",
631 if (crunch_debug > 0) {
632 tprintf (
"EOL CRUNCH DELETING(%d): \"%s\"\n",
637 deleting_from_bol =
FALSE;
638 marked_delete_point =
FALSE;
641 if (!marked_delete_point) {
642 copy_it = page_res_it;
643 marked_delete_point =
TRUE;
648 deleting_from_bol =
FALSE;
650 marked_delete_point =
FALSE;
656 if (!crunch_early_merge_tess_fails)
698 int isolated_digits = 0;
699 int isolated_alphas = 0;
700 int bad_char_count = 0;
705 int alpha_repetition_count = 0;
706 int longest_alpha_repetition_count = 0;
707 int longest_lower_run_len = 0;
708 int lower_string_count = 0;
709 int longest_upper_run_len = 0;
710 int upper_string_count = 0;
711 int total_alpha_count = 0;
712 int total_digit_count = 0;
714 for (; *str !=
'\0'; str += *(lengths++)) {
719 case SUBSEQUENT_UPPER:
721 state = SUBSEQUENT_UPPER;
722 upper_string_count++;
723 if (longest_upper_run_len < upper_string_count)
724 longest_upper_run_len = upper_string_count;
726 alpha_repetition_count++;
727 if (longest_alpha_repetition_count < alpha_repetition_count) {
728 longest_alpha_repetition_count = alpha_repetition_count;
733 alpha_repetition_count = 1;
741 alpha_repetition_count = 1;
742 upper_string_count = 1;
749 case SUBSEQUENT_LOWER:
751 state = SUBSEQUENT_LOWER;
752 lower_string_count++;
753 if (longest_lower_run_len < lower_string_count)
754 longest_lower_run_len = lower_string_count;
756 alpha_repetition_count++;
757 if (longest_alpha_repetition_count < alpha_repetition_count) {
758 longest_alpha_repetition_count = alpha_repetition_count;
763 alpha_repetition_count = 1;
771 alpha_repetition_count = 1;
772 lower_string_count = 1;
780 state = SUBSEQUENT_NUM;
792 if (*lengths == 1 && *str ==
' ')
821 if (crunch_include_numerals) {
822 total_alpha_count += total_digit_count - isolated_digits;
825 if (crunch_leave_ok_strings && len >= 4 &&
826 2 * (total_alpha_count - isolated_alphas) > len &&
827 longest_alpha_repetition_count < crunch_long_repetitions) {
828 if ((crunch_accept_ok &&
829 acceptable_word_string(*word->
uch_set, str, lengths) !=
831 longest_lower_run_len > crunch_leave_lc_strings ||
832 longest_upper_run_len > crunch_leave_uc_strings)
836 strpbrk(str,
" ") ==
NULL &&
841 acceptable_word_string(*word->
uch_set, str, lengths) !=
845 ok_chars = len - bad_char_count - isolated_digits -
846 isolated_alphas - tess_rejs;
848 if (crunch_debug > 3) {
849 tprintf(
"garbage_word: \"%s\"\n",
851 tprintf(
"LEN: %d bad: %d iso_N: %d iso_A: %d rej: %d\n",
853 bad_char_count, isolated_digits, isolated_alphas, tess_rejs);
855 if (bad_char_count == 0 &&
857 (len > isolated_digits + isolated_alphas || len <= 2))
860 if (tess_rejs > ok_chars ||
861 (tess_rejs > 0 && (bad_char_count + tess_rejs) * 2 > len))
865 dodgy_chars = 2 * tess_rejs + bad_char_count + isolated_digits +
867 if (dodgy_chars > 5 || (dodgy_chars / (
float) len) > 0.5)
872 dodgy_chars = 2 * tess_rejs + bad_char_count;
873 if ((len == 4 && dodgy_chars > 2) ||
874 (len == 3 && dodgy_chars > 2) || dodgy_chars >= len)
927 if ((failure_count (word) * 1.5) > word_len) {
939 if (rating_per_ch > crunch_del_rating) {
973 for (; *str !=
'\0'; str++) {
983 inT16 outline_count = 0;
984 inT16 small_outline_count = 0;
986 float small_limit =
kBlnXHeight * crunch_small_outlines_size;
988 for (
int b = 0; b < word->
NumBlobs(); ++b) {
992 box = ol->bounding_box();
994 max_dimension = box.
height();
996 max_dimension = box.
width();
997 if (max_dimension < small_limit)
998 small_outline_count++;
1001 return small_outline_count >= outline_count;
BOOL8 flag(WERD_FLAGS mask) const
void tilde_crunch(PAGE_RES_IT &page_res_it)
inT16 word_blob_quality(WERD_RES *word, ROW *row)
inT16 failure_count(WERD_RES *word)
const int kBlnBaselineOffset
BOOL8 noise_outlines(TWERD *word)
void reject_whole_page(PAGE_RES_IT &page_res_it)
WERD_RES * restart_page()
void unrej_good_quality_words(PAGE_RES_IT &page_res_it)
void CountAcceptedBlobs(int index)
void word_char_quality(WERD_RES *word, ROW *row, inT16 *match_count, inT16 *accepted_match_count)
TBOX bounding_box() const
DocQualCallbacks(WERD_RES *word0)
const UNICHARSET * uch_set
_ConstTessMemberResultCallback_0_0< false, R, T1 >::base * NewPermanentTessCallback(const T1 *obj, R(T2::*member)() const)
CRUNCH_MODE word_deletable(WERD_RES *word, inT16 &delete_mode)
const UNICHAR_ID unichar_id(int index) const
inT32 whole_word_rej_count
BOOL8 quality_recoverable_rejects()
inT16 count_outline_errs(char c, inT16 outline_count)
const char * string() const
const STRING & unichar_string() const
GenericVector< TBLOB * > blobs
void AcceptIfGoodQuality(int index)
void ProcessMatchedBlobs(const TWERD &other, TessCallback1< int > *cb) const
BOOL8 potential_word_crunch(WERD_RES *word, GARBAGE_LEVEL garbage_level, BOOL8 ok_dict_word)
void CountMatchingBlobs(int index)
void doc_and_block_rejection(PAGE_RES_IT &page_res_it, BOOL8 good_quality_doc)
GARBAGE_LEVEL garbage_word(WERD_RES *word, BOOL8 ok_dict_word)
bool get_islower(UNICHAR_ID unichar_id) const
CRUNCH_MODE unlv_crunch_mode
BLOCK_RES * block() const
void unrej_good_chs(WERD_RES *word, ROW *row)
bool get_isdigit(UNICHAR_ID unichar_id) const
ROW_RES * prev_row() const
tesseract::BoxWord * bln_boxes
BOOL8 terrible_word_crunch(WERD_RES *word, GARBAGE_LEVEL garbage_level)
void rej_word_block_rej()
void tilde_delete(PAGE_RES_IT &page_res_it)
inT16 word_outline_errs(WERD_RES *word)
POLY_BLOCK * poly_block() const
void quality_based_rejection(PAGE_RES_IT &page_res_it, BOOL8 good_quality_doc)
const STRING & unichar_lengths() const
const UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
inT16 accepted_match_count
WERD_CHOICE * best_choice
bool get_isupper(UNICHAR_ID unichar_id) const
void convert_bad_unlv_chs(WERD_RES *word_res)
void set_unichar_id(UNICHAR_ID unichar_id, int index)