19 #ifndef TESSERACT_DICT_DICT_H_ 20 #define TESSERACT_DICT_DICT_H_ 36 #define MAX_WERD_LENGTH (inT64) 128 55 static const int kRatingPad = 4;
56 static const char kDictWildcard[] =
"\u2606";
57 static const int kDictMaxWildcards = 2;
60 static const char kHyphenSymbol[] =
"-";
61 static const char kSlashSymbol[] =
"/";
62 static const char kQuestionSymbol[] =
"?";
63 static const char kApostropheSymbol[] =
"'";
64 static const float kSimCertaintyScale = -10.0;
65 static const float kSimCertaintyOffset = -10.0;
66 static const float kSimilarityFloor = 100.0;
67 static const int kDocDictMaxRepChars = 4;
79 : active_dawgs(d), updated_dawgs(up), permuter(p) {}
97 return getCCUtil()->unicharset;
100 return getCCUtil()->unicharset;
103 return getCCUtil()->unichar_ambigs;
109 getUnicharset().normed_ids(unichar_id);
110 return normed_ids.
size() == 1 &&
111 (normed_ids[0] == hyphen_unichar_id_ ||
112 normed_ids[0] == slash_unichar_id_);
118 getUnicharset().normed_ids(unichar_id);
119 return normed_ids.
size() == 1 && normed_ids[0] == apostrophe_unichar_id_;
126 !last_word_on_line_ && hyphen_word_;
130 return this->hyphenated() ? hyphen_word_->length() : 0;
136 if (this->hyphenated()) {
137 *word = *hyphen_word_;
138 if (hyphen_debug_level) word->
print(
"copy_hyphen_info: ");
143 if (!last_word_on_line_ || first_pos)
146 getUnicharset().normed_ids(unichar_id);
147 return normed_ids.
size() == 1 && normed_ids[0] == hyphen_unichar_id_;
151 int word_index = word.
length() - 1;
152 return has_hyphen_end(word.
unichar_id(word_index), word_index == 0);
157 void reset_hyphen_vars(
bool last_word_on_line);
179 bool ambigs_mode)
const;
182 bool suppress_patterns)
const;
195 void go_deeper_dawg_fxn(
198 bool word_ending,
WERD_CHOICE *word,
float certainties[],
199 float *limit,
WERD_CHOICE *best_choice,
int *attempts_left,
200 void *void_more_args);
203 void (
Dict::*go_deeper_fxn_)(
const char *debug,
205 int char_choice_index,
208 float certainties[],
float *limit,
210 void *void_more_args);
214 void permute_choices(
217 int char_choice_index,
230 int char_choice_index,
239 bool fragment_state_okay(
UNICHAR_ID curr_unichar_id,
240 float curr_rating,
float curr_certainty,
242 const char *debug,
int word_ending,
248 bool fix_replaceable,
257 void ReplaceAmbig(
int wrong_ngram_begin_index,
int wrong_ngram_size,
262 int LengthOfShortestAlphaRun(
const WERD_CHOICE &WordChoice);
272 bool AcceptableChoice(
const WERD_CHOICE& best_choice,
277 bool AcceptableResult(
WERD_RES* word);
278 void EndDangerousAmbigs();
280 void DebugWordChoices();
282 void SettupStopperPass1();
284 void SettupStopperPass2();
302 if (pending_words_ !=
NULL)
303 pending_words_->clear();
304 if (document_words_ !=
NULL)
305 document_words_->clear();
344 int def_letter_is_okay(
void* void_dawg_args,
347 int (
Dict::*letter_is_okay_)(
void* void_dawg_args,
352 return (this->*letter_is_okay_)(void_dawg_args,
unichar_id, word_end);
357 double (
Dict::*probability_in_context_)(
const char*
lang,
361 int character_bytes);
365 const char* character,
366 int character_bytes) {
367 return (this->*probability_in_context_)(
368 getCCUtil()->lang.string(),
369 context, context_bytes,
375 const char* lang,
const char* context,
int context_bytes,
376 const char* character,
int character_bytes) {
378 (void) context_bytes;
380 (void) character_bytes;
383 double ngram_probability_in_context(
const char* lang,
386 const char* character,
387 int character_bytes);
390 float (
Dict::*params_model_classify_)(
const char *
lang,
void *path);
391 float ParamsModelClassify(
const char *lang,
void *path);
395 return (this->*params_model_classify_)(
396 getCCUtil()->lang.string(), path);
401 return wildcard_unichar_id_;
404 inline const int NumDawgs()
const {
return dawgs_.size(); }
406 inline const Dawg *
GetDawg(
int index)
const {
return dawgs_[index]; }
413 if (edge_ref == NO_EDGE)
return 0;
415 if (node == 0) node = NO_EDGE;
423 if (!dawg)
return ch;
424 switch (dawg->
type()) {
426 return getUnicharset().get_isdigit(ch) ? Dawg::kPatternUnicharID : ch;
453 int valid_word(
const WERD_CHOICE &word,
bool numbers_ok)
const;
455 return valid_word(word,
false);
458 return valid_word(word,
true);
463 return valid_word(word);
475 void add_document_word(
const WERD_CHOICE &best_choice);
479 float additional_adjust,
484 wordseg_rating_adjust_factor_ = f;
510 bool last_word_on_line_;
518 bool dawg_cache_is_ours_;
522 Trie *pending_words_;
536 Trie *document_words_;
539 float wordseg_rating_adjust_factor_;
541 FILE *output_ambig_words_file_;
547 STRING_VAR_H(user_words_file,
"",
"A filename of user-provided words.");
549 "A suffix of user-provided words located in tessdata.");
551 "A filename of user-provided patterns.");
553 "A suffix of user-provided patterns located in tessdata.");
554 BOOL_VAR_H(load_system_dawg,
true,
"Load system word dawg.");
555 BOOL_VAR_H(load_freq_dawg,
true,
"Load frequent word dawg.");
556 BOOL_VAR_H(load_unambig_dawg,
true,
"Load unambiguous word dawg.");
558 "Load dawg with punctuation patterns.");
559 BOOL_VAR_H(load_number_dawg,
true,
"Load dawg with number patterns.");
561 "Load dawg with special word bigrams.");
563 "Score penalty (0.1 = 10%) added if there are subscripts " 564 "or superscripts in a word, but it is otherwise OK.");
566 "Score penalty (0.1 = 10%) added if an xheight is " 569 "Score multiplier for word matches which have good case and" 570 "are frequent in the given language (lower is better).");
573 "Score multiplier for word matches that have good case " 574 "(lower is better).");
577 "Default score multiplier for word matches, which may have " 578 "case issues (lower is better).");
582 "Multipler to for the best choice from the ngram model.");
585 "Score multiplier for glyph fragment segmentations which " 586 "do not match a dictionary word (lower is better).");
589 "Score multiplier for poorly cased strings that are not in" 590 " the dictionary and generally look like garbage (lower is" 593 "Output file for ambiguities found in the dictionary");
594 INT_VAR_H(dawg_debug_level, 0,
"Set to 1 for general debug info" 595 ", to 2 for more details, to 3 to see all the debug messages");
596 INT_VAR_H(hyphen_debug_level, 0,
"Debug level for hyphenated words.");
597 INT_VAR_H(max_viterbi_list_size, 10,
"Maximum size of viterbi list.");
599 "Use only the first UTF8 step of the given string" 600 " when computing log probabilities.");
603 "Certainty threshold for non-dict words");
604 double_VAR_H(stopper_phase2_certainty_rejection_offset, 1.0,
605 "Reject certainty offset");
607 "Size of dict word to be treated as non-dict word");
609 "Certainty to add for each dict char above small word size.");
611 "Max certaintly variation allowed in a word (in sigma)");
612 INT_VAR_H(stopper_debug_level, 0,
"Stopper debug level");
613 BOOL_VAR_H(stopper_no_acceptable_choices,
false,
614 "Make AcceptableChoice() always return false. Useful" 615 " when there is a need to explore all segmentations");
617 "Deprecated- backward compatability only");
618 INT_VAR_H(tessedit_truncate_wordchoice_log, 10,
"Max words to keep in list");
619 STRING_VAR_H(word_to_debug,
"",
"Word for which stopper debug information" 620 " should be printed to stdout");
622 "Lengths of unichars in word_to_debug");
623 INT_VAR_H(fragments_debug, 0,
"Debug character fragments");
624 BOOL_VAR_H(segment_nonalphabetic_script,
false,
625 "Don't use any alphabetic-specific tricks." 626 "Set to true in the traineddata config file for" 627 " scripts that are cursive or inherently fixed-pitch");
630 "Worst certainty for using pending dictionary");
631 double_VAR_H(doc_dict_certainty_threshold, -2.25,
"Worst certainty" 632 " for words that can be inserted into the document dictionary");
633 INT_VAR_H(max_permuter_attempts, 10000,
"Maximum number of different" 634 " character choices to consider during permutation." 635 " This limit is especially useful when user patterns" 636 " are specified, since overly generic patterns can result in" 637 " dawg search exploring an overly large number of options.");
641 #endif // THIRD_PARTY_TESSERACT_DICT_DICT_H_ int valid_word_or_number(const WERD_CHOICE &word) const
double def_probability_in_context(const char *lang, const char *context, int context_bytes, const char *character, int character_bytes)
Default (no-op) implementation of probability in context function.
const Dawg * GetPuncDawg() const
Return the points to the punctuation dawg.
int LetterIsOkay(void *void_dawg_args, UNICHAR_ID unichar_id, bool word_end) const
Calls letter_is_okay_ member function.
void copy_hyphen_info(WERD_CHOICE *word) const
int hyphen_base_size() const
Size of the base word (the part on the line before) of a hyphenated word.
const int NumDawgs() const
Return the number of dawgs in the dawgs_ vector.
GenericVector< Dawg * > DawgVector
static bool valid_word_permuter(uinT8 perm, bool numbers_ok)
Check all the DAWGs to see if this word is in any of them.
DawgPositionVector * active_dawgs
float CallParamsModelClassify(void *path)
void ResetDocumentDictionary()
void SetWordsegRatingAdjustFactor(float f)
Set wordseg_rating_adjust_factor_ to the given value.
const CCUtil * getCCUtil() const
void update_best_choice(const WERD_CHOICE &word, WERD_CHOICE *best_choice)
bool hyphenated() const
Returns true if we've recorded the beginning of a hyphenated word.
const UNICHAR_ID unichar_id(int index) const
const CHAR_FRAGMENT * fragment
#define STRING_VAR_H(name, val, comment)
bool has_hyphen_end(UNICHAR_ID unichar_id, bool first_pos) const
Check whether the word has a hyphen at the end.
bool is_apostrophe(UNICHAR_ID unichar_id)
DawgPositionVector * updated_dawgs
const Dawg * GetUnambigDawg() const
Return the points to the unambiguous words dawg.
bool compound_marker(UNICHAR_ID unichar_id)
DawgArgs(DawgPositionVector *d, DawgPositionVector *up, PermuterType p)
const UNICHARSET & getUnicharset() const
bool has_hyphen_end(const WERD_CHOICE &word) const
Same as above, but check the unichar at the end of the word.
int valid_word(const WERD_CHOICE &word) const
static NODE_REF GetStartingNode(const Dawg *dawg, EDGE_REF edge_ref)
Returns the appropriate next node given the EDGE_REF.
double ProbabilityInContext(const char *context, int context_bytes, const char *character, int character_bytes)
Calls probability_in_context_ member function.
#define double_VAR_H(name, val, comment)
const UnicharAmbigs & getUnicharAmbigs() const
const UNICHAR_ID WildcardID() const
void SetWildcardID(UNICHAR_ID id)
const Dawg * GetDawg(int index) const
Return i-th dawg pointer recorded in the dawgs_ vector.
UNICHARSET & getUnicharset()
int valid_word(const char *string) const
This function is used by api/tesseract_cube_combiner.cpp.
UNICHAR_ID char_for_dawg(UNICHAR_ID ch, const Dawg *dawg) const
#define BOOL_VAR_H(name, val, comment)
virtual NODE_REF next_node(EDGE_REF edge_ref) const =0
#define INT_VAR_H(name, val, comment)