35 #if defined(_MSC_VER) || defined(ANDROID) 36 double log2(
double n) {
37 return log(n) / log(2.0);
47 :
INT_MEMBER(language_model_debug_level, 0,
"Language model debug level",
48 dict->getCCUtil()->params()),
50 "Turn on/off the use of character ngram model",
51 dict->getCCUtil()->params()),
53 "Maximum order of the character ngram model",
54 dict->getCCUtil()->params()),
55 INT_MEMBER(language_model_viterbi_list_max_num_prunable, 10,
56 "Maximum number of prunable (those for which" 57 " PrunablePath() is true) entries in each viterbi list" 58 " recorded in BLOB_CHOICEs",
59 dict->getCCUtil()->params()),
60 INT_MEMBER(language_model_viterbi_list_max_size, 500,
61 "Maximum size of viterbi lists recorded in BLOB_CHOICEs",
62 dict->getCCUtil()->params()),
64 "To avoid overly small denominators use this as the " 65 "floor of the probability returned by the ngram model.",
66 dict->getCCUtil()->params()),
68 "Average classifier score of a non-matching unichar.",
69 dict->getCCUtil()->params()),
70 BOOL_MEMBER(language_model_ngram_use_only_first_uft8_step, false,
71 "Use only the first UTF8 step of the given string" 72 " when computing log probabilities.",
73 dict->getCCUtil()->params()),
75 "Strength of the character ngram model relative to the" 76 " character classifier ",
77 dict->getCCUtil()->params()),
79 "Factor to bring log-probs into the same range as ratings" 80 " when multiplied by outline length ",
81 dict->getCCUtil()->params()),
82 BOOL_MEMBER(language_model_ngram_space_delimited_language, true,
83 "Words are delimited by space",
84 dict->getCCUtil()->params()),
85 INT_MEMBER(language_model_min_compound_length, 3,
86 "Minimum length of compound words",
87 dict->getCCUtil()->params()),
89 "Penalty for words not in the frequent word dictionary",
90 dict->getCCUtil()->params()),
92 "Penalty for non-dictionary words",
93 dict->getCCUtil()->params()),
95 "Penalty for inconsistent punctuation",
96 dict->getCCUtil()->params()),
98 "Penalty for inconsistent case",
99 dict->getCCUtil()->params()),
101 "Penalty for inconsistent script",
102 dict->getCCUtil()->params()),
104 "Penalty for inconsistent character type",
105 dict->getCCUtil()->params()),
109 "Penalty for inconsistent font",
110 dict->getCCUtil()->params()),
112 "Penalty for inconsistent spacing",
113 dict->getCCUtil()->params()),
116 dict->getCCUtil()->params()),
117 INT_MEMBER(wordrec_display_segmentations, 0,
"Display Segmentations",
118 dict->getCCUtil()->params()),
120 "Use sigmoidal score for certainty",
121 dict->getCCUtil()->params()),
122 fontinfo_table_(fontinfo_table), dict_(dict),
123 fixed_pitch_(false), max_char_wh_ratio_(0.0),
124 acceptable_choice_found_(false) {
139 bool fixed_pitch,
float max_char_wh_ratio,
140 float rating_cert_scale) {
176 static void ScanParentsForCaseMix(
const UNICHARSET& unicharset,
178 if (parent_node ==
NULL)
return;
180 for (vit.mark_cycle_pt(); !vit.cycled_list(); vit.forward()) {
187 if (other_case == unichar_id)
continue;
193 for (vit2.mark_cycle_pt(); !vit2.cycled_list() &&
194 vit2.data()->curr_b->unichar_id() != other_case;
196 if (!vit2.cycled_list()) {
205 static bool HasBetterCaseVariant(
const UNICHARSET& unicharset,
207 BLOB_CHOICE_LIST* choices) {
210 if (other_case == choice_id || other_case == INVALID_UNICHAR_ID)
214 BLOB_CHOICE_IT bc_it(choices);
215 for (bc_it.mark_cycle_pt(); !bc_it.cycled_list(); bc_it.forward()) {
217 if (better_choice->
unichar_id() == other_case)
219 else if (better_choice == choice)
247 bool just_classified,
248 int curr_col,
int curr_row,
249 BLOB_CHOICE_LIST *curr_list,
256 tprintf(
"\nUpdateState: col=%d row=%d %s",
257 curr_col, curr_row, just_classified ?
"just_classified" :
"");
259 tprintf(
"(parent=%p)\n", parent_node);
265 bool new_changed =
false;
271 bool has_alnum_mix =
false;
272 if (parent_node !=
NULL) {
276 tprintf(
"No parents found to process\n");
280 has_alnum_mix =
true;
284 has_alnum_mix =
false;;
285 ScanParentsForCaseMix(unicharset, parent_node);
287 parent_node->
Print(
"Parent viterbi list");
292 ViterbiStateEntry_IT vit;
293 BLOB_CHOICE_IT c_it(curr_list);
294 for (c_it.mark_cycle_pt(); !c_it.cycled_list(); c_it.forward()) {
306 if (c_it.at_first() || !new_changed)
310 if (first_digit == choice) blob_choice_flags |=
kDigitFlag;
312 if (parent_node ==
NULL) {
324 if (HasBetterCaseVariant(unicharset, choice, curr_list))
330 blob_choice_flags, denom, word_end, curr_col, curr_row,
331 choice, curr_state,
NULL, pain_points,
332 word_res, best_choice_bundle, blamer_bundle);
341 c_it.data(), blob_choice_flags,
342 unicharset, word_res, &vit,
343 &top_choice_flags)) !=
NULL) {
356 HasBetterCaseVariant(unicharset, choice, curr_list))
361 top_choice_flags, denom, word_end, curr_col, curr_row,
362 c_it.data(), curr_state, parent_vse, pain_points,
363 word_res, best_choice_bundle, blamer_bundle);
378 BLOB_CHOICE_IT c_it(curr_list);
381 for (c_it.mark_cycle_pt(); !c_it.cycled_list(); c_it.forward()) {
382 UNICHAR_ID unichar_id = c_it.data()->unichar_id();
384 if (first_unichar ==
NULL) first_unichar = c_it.data();
386 *first_lower = c_it.data();
390 *first_upper = c_it.data();
393 *first_digit = c_it.data();
397 bool mixed = (*first_lower !=
NULL || *first_upper !=
NULL) &&
398 *first_digit !=
NULL;
399 if (*first_lower ==
NULL) *first_lower = first_unichar;
400 if (*first_upper ==
NULL) *first_upper = first_unichar;
401 if (*first_digit ==
NULL) *first_digit = first_unichar;
414 if (parent_node ==
NULL)
return -1;
420 float lower_rating = 0.0f;
421 float upper_rating = 0.0f;
422 float digit_rating = 0.0f;
423 float top_rating = 0.0f;
426 for (vit.mark_cycle_pt(); !vit.cycled_list(); vit.forward()) {
433 while (unichar_id == INVALID_UNICHAR_ID &&
439 if (unichar_id != INVALID_UNICHAR_ID) {
441 if (top_lower ==
NULL || lower_rating > rating) {
443 lower_rating = rating;
446 if (top_upper ==
NULL || upper_rating > rating) {
448 upper_rating = rating;
451 if (top_digit ==
NULL || digit_rating > rating) {
453 digit_rating = rating;
457 if (top_choice ==
NULL || top_rating > rating) {
463 if (top_choice ==
NULL)
return -1;
466 if (top_lower ==
NULL) top_lower = top_choice;
468 if (top_upper ==
NULL) top_upper = top_choice;
470 if (top_digit ==
NULL) top_digit = top_choice;
481 return mixed ? 1 : 0;
488 bool just_classified,
bool mixed_alnum,
const BLOB_CHOICE* bc,
490 WERD_RES* word_res, ViterbiStateEntry_IT* vse_it,
492 for (; !vse_it->cycled_list(); vse_it->forward()) {
496 if (!just_classified && !parent_vse->
updated)
continue;
498 parent_vse->
Print(
"Considering");
500 *top_choice_flags = blob_choice_flags;
513 (mixed_alnum || *top_choice_flags == 0))
519 (mixed_alnum || *top_choice_flags == 0))
528 tprintf(
"Parent %s has competition %s\n",
552 int curr_col,
int curr_row,
560 ViterbiStateEntry_IT vit;
562 tprintf(
"AddViterbiStateEntry for unichar %s rating=%.4f" 563 " certainty=%.4f top_choice_flags=0x%x",
567 tprintf(
" parent_vse=%p\n", parent_vse);
572 if (curr_state !=
NULL &&
576 tprintf(
"AddViterbiStateEntry: viterbi list is full!\n");
585 float outline_length =
592 denom, curr_col, curr_row, outline_length, parent_vse);
595 bool liked_by_language_model = dawg_info !=
NULL ||
599 if (!liked_by_language_model && top_choice_flags == 0) {
601 tprintf(
"Language model components very early pruned this entry\n");
622 if (!liked_by_language_model && top_choice_flags == 0) {
624 tprintf(
"Language model components early pruned this entry\n");
633 word_res, &consistency_info);
634 if (dawg_info != NULL && consistency_info.
invalid_punc) {
641 parent_vse, word_res, &associate_stats);
642 if (parent_vse != NULL) {
649 parent_vse, b, 0.0, outline_length,
650 consistency_info, associate_stats, top_choice_flags, dawg_info,
671 tprintf(
"Language model components did not like this entry\n");
685 tprintf(
"Discarded ViterbiEntry with high cost %g max cost %g\n",
696 best_choice_bundle, blamer_bundle);
699 new_vse != best_choice_bundle->
best_vse) {
701 tprintf(
"Discarded ViterbiEntry with high cost %g\n", new_vse->
cost);
724 for (vit.mark_cycle_pt(); !vit.cycled_list(); vit.forward()) {
733 if (prunable_counter > 0 &&
PrunablePath(*curr_vse)) --prunable_counter;
735 if (prunable_counter == 0) {
738 tprintf(
"Set viterbi_state_entries_prunable_max_cost to %g\n",
741 prunable_counter = -1;
748 new_vse->
Print(
"New");
750 curr_state->
Print(
"Updated viterbi list");
760 for (vit.mark_cycle_pt(); !vit.cycled_list() && new_vse->
top_choice_flags &&
761 new_vse->
cost >= vit.data()->cost; vit.forward()) {
767 tprintf(
"GenerateTopChoiceInfo: top_choice_flags=0x%x\n",
774 int curr_col,
int curr_row,
779 if (parent_vse ==
NULL) {
803 if (parent_vse ==
NULL || word_end ||
809 bool has_word_ending =
false;
817 has_word_ending =
true;
821 if (!has_word_ending)
return NULL;
835 for (
int i = 0; i < normed_ids.
size(); ++i) {
837 tprintf(
"Test Letter OK for unichar %d, normed %d\n",
840 word_end && i == normed_ids.
size() - 1);
843 }
else if (i < normed_ids.
size() - 1) {
848 tprintf(
"Letter was OK for unichar %d, normed %d\n",
864 const char *unichar,
float certainty,
float denom,
865 int curr_col,
int curr_row,
float outline_length,
868 const char *pcontext_ptr =
"";
869 int pcontext_unichar_step_len = 0;
870 if (parent_vse ==
NULL) {
875 pcontext_unichar_step_len =
879 int unichar_step_len = 0;
882 float ngram_and_classifier_cost =
884 pcontext_ptr, &unichar_step_len,
885 &pruned, &ngram_cost);
889 ngram_and_classifier_cost *=
892 if (parent_vse !=
NULL) {
893 ngram_and_classifier_cost +=
899 int num_remove = (unichar_step_len + pcontext_unichar_step_len -
901 if (num_remove > 0) pcontext_unichar_step_len -= num_remove;
902 while (num_remove > 0 && *pcontext_ptr !=
'\0') {
912 pcontext_ptr, pcontext_unichar_step_len, pruned, ngram_cost,
913 ngram_and_classifier_cost);
914 ngram_info->
context += unichar;
924 int *unichar_step_len,
925 bool *found_small_prob,
927 const char *context_ptr = context;
928 char *modified_context =
NULL;
929 char *modified_context_end =
NULL;
930 const char *unichar_ptr = unichar;
931 const char *unichar_end = unichar_ptr + strlen(unichar_ptr);
934 while (unichar_ptr < unichar_end &&
937 tprintf(
"prob(%s | %s)=%g\n", unichar_ptr, context_ptr,
941 ++(*unichar_step_len);
947 if (unichar_ptr < unichar_end) {
948 if (modified_context ==
NULL) {
949 int context_len = strlen(context);
951 new char[context_len + strlen(unichar_ptr) + step + 1];
952 strncpy(modified_context, context, context_len);
953 modified_context_end = modified_context + context_len;
954 context_ptr = modified_context;
956 strncpy(modified_context_end, unichar_ptr - step, step);
957 modified_context_end += step;
958 *modified_context_end =
'\0';
961 prob /=
static_cast<float>(*unichar_step_len);
964 *found_small_prob =
true;
967 *ngram_cost = -1.0*log2(prob);
968 float ngram_and_classifier_cost =
972 tprintf(
"-log [ p(%s) * p(%s | %s) ] = -log2(%g*%g) = %g\n", unichar,
974 ngram_and_classifier_cost);
976 if (modified_context !=
NULL)
delete[] modified_context;
977 return ngram_and_classifier_cost;
981 if (curr_list->empty())
return 1.0f;
984 BLOB_CHOICE_IT c_it(curr_list);
985 for (c_it.mark_cycle_pt(); !c_it.cycled_list(); c_it.forward()) {
1019 consistency_info->
punc_ref = NO_EDGE;
1022 bool prev_is_numalpha = (parent_b !=
NULL &&
1028 (is_apos && prev_is_numalpha)) ?
1030 if (consistency_info->
punc_ref == NO_EDGE ||
1038 node, pattern_unichar_id, word_end) : NO_EDGE;
1039 if (consistency_info->
punc_ref == NO_EDGE) {
1054 }
else if ((parent_b !=
NULL) && unicharset.
get_isupper(unichar_id)) {
1077 if (parent_vse !=
NULL &&
1083 consistency_info->
script_id = parent_script_id;
1085 if (consistency_info->
script_id != parent_script_id) {
1101 int fontinfo_id = -1;
1110 tprintf(
"pfont %s pfont %s font %s font2 %s common %s(%d)\n",
1122 bool expected_gap_found =
false;
1125 if (fontinfo_id >= 0) {
1126 ASSERT_HOST(fontinfo_id < fontinfo_table_->size());
1128 parent_b->
unichar_id(), unichar_id, &temp_gap)) {
1129 expected_gap = temp_gap;
1130 expected_gap_found =
true;
1135 int num_addends = 0;
1138 for (
int i = 0; i < 4; ++i) {
1141 }
else if (i == 1) {
1143 }
else if (i == 2) {
1148 ASSERT_HOST(temp_fid < 0 || fontinfo_table_->size());
1150 parent_b->
unichar_id(), unichar_id, &temp_gap)) {
1151 expected_gap += temp_gap;
1155 expected_gap_found = (num_addends > 0);
1156 if (num_addends > 0) {
1157 expected_gap /=
static_cast<float>(num_addends);
1160 if (expected_gap_found) {
1162 static_cast<float>(word_res->
GetBlobsGap(curr_col-1));
1163 float gap_ratio = expected_gap / actual_gap;
1169 if (gap_ratio < 0.0f || gap_ratio > 2.0f) {
1173 tprintf(
"spacing for %s(%d) %s(%d) col %d: expected %g actual %g\n",
1176 unichar_id, curr_col, expected_gap, actual_gap);
1190 tprintf(
"ComputeAdjustedPathCost %g ParamsModel features:\n", cost);
1193 tprintf(
"%s=%g\n", kParamsTrainingFeatureTypeName[f], features[f]);
1199 float adjustment = 1.0f;
1212 static_cast<float>(vse->
length);
1233 blamer_bundle, &truth_path);
1241 word->
print(
"UpdateBestChoice() constructed word");
1245 if (blamer_bundle !=
NULL) {
1252 tprintf(
"Raw features extracted from %s (cost=%g) [ ",
1276 tprintf(
"Updated raw choice\n");
1300 best_choice_bundle->
updated =
true;
1301 best_choice_bundle->
best_vse = vse;
1303 tprintf(
"Updated best choice\n");
1315 if (blamer_bundle !=
NULL) {
1329 int len = vse.
length <= kMaxSmallWordUnichars ? 0 :
1330 vse.
length <= kMaxMediumWordUnichars ? 1 : 2;
1380 if (truth_path !=
NULL) {
1382 (blamer_bundle !=
NULL &&
1393 float full_wh_ratio_mean = 0.0f;
1397 static_cast<float>(vse->
length));
1404 int total_blobs = 0;
1405 for (i = (vse->
length-1); i >= 0; --i) {
1406 if (blamer_bundle !=
NULL && truth_path !=
NULL && *truth_path &&
1408 *truth_path =
false;
1412 total_blobs += num_blobs;
1417 if ((full_wh_ratio_mean != 0.0f &&
1423 tprintf(
"full_wh_ratio_var += (%g-%g)^2\n",
1436 if (curr_vse ==
NULL)
break;
1437 curr_b = curr_vse->
curr_b;
1442 if (full_wh_ratio_mean != 0.0f) {
bool SizesDistinct(UNICHAR_ID id1, UNICHAR_ID id2) const
virtual UNICHAR_ID edge_letter(EDGE_REF edge_ref) const =0
Returns UNICHAR_ID stored in the edge indicated by the given EDGE_REF.
void InitForWord(const WERD_CHOICE *prev_word, bool fixed_pitch, float max_char_wh_ratio, float rating_cert_scale)
bool AcceptablePath(const ViterbiStateEntry &vse)
bool UpdateState(bool just_classified, int curr_col, int curr_row, BLOB_CHOICE_LIST *curr_list, LanguageModelState *parent_node, LMPainPoints *pain_points, WERD_RES *word_res, BestChoiceBundle *best_choice_bundle, BlamerBundle *blamer_bundle)
int NumInconsistentSpaces() const
inT16 fontinfo_id() const
bool language_model_ngram_on
const Dawg * GetPuncDawg() const
Return the points to the punctuation dawg.
ViterbiStateEntry * GetNextParentVSE(bool just_classified, bool mixed_alnum, const BLOB_CHOICE *bc, LanguageModelFlagsType blob_choice_flags, const UNICHARSET &unicharset, WERD_RES *word_res, ViterbiStateEntry_IT *vse_it, LanguageModelFlagsType *top_choice_flags) const
DawgPositionVector * beginning_active_dawgs_
int tessedit_truncate_wordchoice_log
GenericVector< int > blob_widths
LanguageModelDawgInfo * GenerateDawgInfo(bool word_end, int curr_col, int curr_row, const BLOB_CHOICE &b, const ViterbiStateEntry *parent_vse)
ViterbiStateEntry_LIST viterbi_state_entries
float CertaintyScore(float cert)
void print_state(const char *msg) const
ViterbiStateEntry * best_vse
int LetterIsOkay(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
Calls letter_is_okay_ member function.
float ngram_and_classifier_cost
void set_x_heights(float min_height, float max_height)
LanguageModelDawgInfo * dawg_info
float full_wh_ratio_total
const MATRIX_COORD & matrix_cell()
LanguageModelFlagsType top_choice_flags
bool acceptable_choice_found_
int GetBlobsGap(int blob_index)
static int utf8_step(const char *utf8_str)
AssociateStats associate_stats
#define BOOL_MEMBER(name, val, comment, vec)
static const UNICHAR_ID kPatternUnicharID
void ComputeXheightConsistency(const BLOB_CHOICE *b, bool is_punc)
DawgPositionVector * active_dawgs
static const LanguageModelFlagsType kXhtConsistentFlag
void reset_hyphen_vars(bool last_word_on_line)
LanguageModel(const UnicityTable< FontInfo > *fontinfo_table, Dict *dict)
int get_script(UNICHAR_ID unichar_id) const
bool LogNewRawChoice(WERD_CHOICE *word_choice)
const UNICHARSET * uch_set
static void ExtractFeaturesFromPath(const ViterbiStateEntry &vse, float features[])
bool AddViterbiStateEntry(LanguageModelFlagsType top_choice_flags, float denom, bool word_end, int curr_col, int curr_row, BLOB_CHOICE *b, LanguageModelState *curr_state, ViterbiStateEntry *parent_vse, LMPainPoints *pain_points, WERD_RES *word_res, BestChoiceBundle *best_choice_bundle, BlamerBundle *blamer_bundle)
bool GuidedSegsearchStillGoing() const
bool PrunablePath(const ViterbiStateEntry &vse)
bool hyphenated() const
Returns true if we've recorded the beginning of a hyphenated word.
int NumInconsistentCase() const
WERD_CHOICE * ConstructWord(ViterbiStateEntry *vse, WERD_RES *word_res, DANGERR *fixpt, BlamerBundle *blamer_bundle, bool *truth_path)
DawgPositionVector * active_dawgs
void set_rating(float new_val)
int language_model_viterbi_list_max_size
void GenerateTopChoiceInfo(ViterbiStateEntry *new_vse, const ViterbiStateEntry *parent_vse, LanguageModelState *lms)
ViterbiStateEntry * competing_vse
bool has_hyphen_end(UNICHAR_ID unichar_id, bool first_pos) const
Check whether the word has a hyphen at the end.
ParamsModel params_model_
#define BOOL_INIT_MEMBER(name, val, comment, vec)
double language_model_ngram_small_prob
bool is_apostrophe(UNICHAR_ID unichar_id)
bool language_model_ngram_use_only_first_uft8_step
static int Compare(const void *e1, const void *e2)
int SetTopParentLowerUpperDigit(LanguageModelState *parent_node) const
DawgPositionVector * updated_dawgs
int context_unichar_step_len
virtual bool end_of_word(EDGE_REF edge_ref) const =0
int wordrec_display_segmentations
static const float kBadRating
int language_model_viterbi_list_max_num_prunable
float viterbi_state_entries_prunable_max_cost
void UpdateBestChoice(ViterbiStateEntry *vse, LMPainPoints *pain_points, WERD_RES *word_res, BestChoiceBundle *best_choice_bundle, BlamerBundle *blamer_bundle)
int viterbi_state_entries_length
void Print(const char *msg)
float ComputeCost(const float features[]) const
bool compound_marker(UNICHAR_ID unichar_id)
bool NoDangerousAmbig(WERD_CHOICE *BestChoice, DANGERR *fixpt, bool fix_replaceable, MATRIX *ratings)
void SetScriptPositions(bool small_caps, TWERD *word)
int language_model_debug_level
const char * string() const
const STRING & unichar_string() const
const GenericVector< UNICHAR_ID > & normed_ids(UNICHAR_ID unichar_id) const
GenericVector< TBLOB * > blobs
const UNICHARSET & getUnicharset() const
#define INT_MEMBER(name, val, comment, vec)
DawgPositionVector * very_beginning_active_dawgs_
bool get_isalpha(UNICHAR_ID unichar_id) const
float ComputeNgramCost(const char *unichar, float certainty, float denom, const char *context, int *unichar_step_len, bool *found_small_prob, float *ngram_prob)
bool GetTopLowerUpperDigit(BLOB_CHOICE_LIST *curr_list, BLOB_CHOICE **first_lower, BLOB_CHOICE **first_upper, BLOB_CHOICE **first_digit) const
int num_inconsistent_spaces
float BodyMinXHeight() const
bool language_model_ngram_space_delimited_language
int InconsistentXHeight() const
double language_model_ngram_nonmatch_score
bool get_islower(UNICHAR_ID unichar_id) const
#define double_MEMBER(name, val, comment, vec)
void Print(const char *msg) const
double language_model_ngram_rating_factor
static NODE_REF GetStartingNode(const Dawg *dawg, EDGE_REF edge_ref)
Returns the appropriate next node given the EDGE_REF.
void adjust_word(WERD_CHOICE *word, bool nonword, XHeightConsistencyEnum xheight_consistency, float additional_adjust, bool modify_rating, bool debug)
Adjusts the rating of the given word.
LanguageModelNgramInfo * ngram_info
double ProbabilityInContext(const char *context, int context_bytes, const char *character, int character_bytes)
Calls probability_in_context_ member function.
ViterbiStateEntry * parent_vse
static const LanguageModelFlagsType kDigitFlag
const CHAR_FRAGMENT * get_fragment(UNICHAR_ID unichar_id) const
PointerVector< LanguageModelState > beam
const UnicityTable< FontInfo > * fontinfo_table_
bool get_isdigit(UNICHAR_ID unichar_id) const
bool HasAlnumChoice(const UNICHARSET &unicharset)
double language_model_penalty_increment
float ComputeDenom(BLOB_CHOICE_LIST *curr_list)
LanguageModelNgramInfo * GenerateNgramInfo(const char *unichar, float certainty, float denom, int curr_col, int curr_row, float outline_length, const ViterbiStateEntry *parent_vse)
unsigned char LanguageModelFlagsType
static const LanguageModelFlagsType kUpperCaseFlag
void set_blob_choice(int index, int blob_count, const BLOB_CHOICE *blob_choice)
bool get_ispunctuation(UNICHAR_ID unichar_id) const
static const LanguageModelFlagsType kSmallestRatingFlag
bool AcceptableChoice(const WERD_CHOICE &best_choice, XHeightConsistencyEnum xheight_consistency)
Returns true if the given best_choice is good enough to stop.
static const LanguageModelFlagsType kLowerCaseFlag
float features[PTRAIN_NUM_FEATURE_TYPES]
void FillConsistencyInfo(int curr_col, bool word_end, BLOB_CHOICE *b, ViterbiStateEntry *parent_vse, WERD_RES *word_res, LMConsistencyInfo *consistency_info)
const char *const id_to_unichar(UNICHAR_ID id) const
double language_model_penalty_non_freq_dict_word
const Dawg * GetDawg(int index) const
Return i-th dawg pointer recorded in the dawgs_ vector.
void default_dawgs(DawgPositionVector *anylength_dawgs, bool suppress_patterns) const
void string_and_lengths(STRING *word_str, STRING *word_lengths_str) const
UNICHAR_ID unichar_id() const
void ComputeAssociateStats(int col, int row, float max_char_wh_ratio, ViterbiStateEntry *parent_vse, WERD_RES *word_res, AssociateStats *associate_stats)
void set_hyphen_word(const WERD_CHOICE &word, const DawgPositionVector &active_dawgs)
float BodyMaxXHeight() const
int viterbi_state_entries_prunable_length
void set_dangerous_ambig_found_(bool value)
virtual EDGE_REF edge_char_of(NODE_REF node, UNICHAR_ID unichar_id, bool word_end) const =0
Returns the edge that corresponds to the letter out of this node.
inT16 fontinfo_id2() const
void set_best_choice_is_dict_and_top_choice(bool value)
float ComputeAdjustedPathCost(ViterbiStateEntry *vse)
UNICHAR_ID get_other_case(UNICHAR_ID unichar_id) const
double language_model_ngram_scale_factor
bool MatrixPositionCorrect(int index, const MATRIX_COORD &coord)
WERD_CHOICE * best_choice
int prev_word_unichar_step_len_
void DisplaySegmentation(TWERD *word)
int language_model_ngram_order
bool LogNewCookedChoice(int max_num_choices, bool debug, WERD_CHOICE *word_choice)
int NumInconsistentChartype() const
void init_active_dawgs(DawgPositionVector *active_dawgs, bool ambigs_mode) const
static const float kMaxAvgNgramCost
double language_model_penalty_non_dict_word
bool get_isupper(UNICHAR_ID unichar_id) const
static float ComputeOutlineLength(float rating_cert_scale, const BLOB_CHOICE &b)
void set_certainty(float new_val)
int language_model_min_compound_length
int correct_segmentation_length() const
void AddHypothesis(const tesseract::ParamsTrainingHypothesis &hypo)
XHeightConsistencyEnum xht_decision
bool correct_segmentation_explored_
LMConsistencyInfo consistency_info
void UpdateBestRating(float rating)
void set_permuter(uinT8 perm)
bool PosAndSizeAgree(const BLOB_CHOICE &other, float x_height, bool debug) const
float ComputeConsistencyAdjustment(const LanguageModelDawgInfo *dawg_info, const LMConsistencyInfo &consistency_info)