16 #include "allheaders.h" 67 "Min number of samples per proto as % of total");
69 "Max percentage of samples in a cluster which have more" 70 " than 1 feature in that cluster");
72 "Desired independence between dimensions");
74 "Desired confidence in prototypes created");
94 usage +=
" [.tr files ...]";
101 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_min_samples_fraction)));
103 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_max_illegal)));
105 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_independence)));
107 MAX(0.0,
MIN(1.0,
double(FLAGS_clusterconfig_confidence)));
109 if (!FLAGS_configfile.empty()) {
111 FLAGS_configfile.c_str(),
121 STRING shape_table_file = file_prefix;
122 shape_table_file += kShapeTableFileSuffix;
123 FILE* shape_fp = fopen(shape_table_file.
string(),
"rb");
124 if (shape_fp !=
NULL) {
129 tprintf(
"Error: Failed to read shape table %s\n",
130 shape_table_file.
string());
132 int num_shapes = shape_table->
NumShapes();
133 tprintf(
"Read shape table %s of %d shapes\n",
134 shape_table_file.
string(), num_shapes);
138 tprintf(
"Warning: No shape table file present: %s\n",
139 shape_table_file.
string());
146 STRING shape_table_file = file_prefix;
147 shape_table_file += kShapeTableFileSuffix;
148 FILE* fp = fopen(shape_table_file.
string(),
"wb");
151 fprintf(stderr,
"Error writing shape table: %s\n",
152 shape_table_file.
string());
156 fprintf(stderr,
"Error creating shape table: %s\n",
157 shape_table_file.
string());
181 if (!FLAGS_D.empty()) {
182 *file_prefix += FLAGS_D.
c_str();
189 bool shape_analysis =
false;
190 if (shape_table !=
NULL) {
192 if (*shape_table !=
NULL)
193 shape_analysis =
true;
195 shape_analysis =
true;
203 if (FLAGS_T.empty()) {
206 if (!FLAGS_F.empty()) {
212 if (!FLAGS_X.empty()) {
219 const char* page_name;
222 tprintf(
"Reading %s ...\n", page_name);
227 int pagename_len = strlen(page_name);
228 char *fontinfo_file_name =
new char[pagename_len + 7];
229 strncpy(fontinfo_file_name, page_name, pagename_len - 2);
230 strcpy(fontinfo_file_name + pagename_len - 2,
"fontinfo");
232 delete[] fontinfo_file_name;
235 if (FLAGS_load_images) {
236 STRING image_name = page_name;
245 if (!FLAGS_output_trainer.empty()) {
246 FILE* fp = fopen(FLAGS_output_trainer.c_str(),
"wb");
248 tprintf(
"Can't create saved trainer data!\n");
255 bool success =
false;
256 tprintf(
"Loading master trainer from file:%s\n",
258 FILE* fp = fopen(FLAGS_T.c_str(),
"rb");
260 tprintf(
"Can't read file %s to initialize master trainer\n",
267 tprintf(
"Deserialize of master trainer failed!\n");
274 if (!FLAGS_O.empty() &&
276 fprintf(stderr,
"Failed to save unicharset to file %s\n", FLAGS_O.c_str());
280 if (shape_table !=
NULL) {
283 if (*shape_table ==
NULL) {
286 tprintf(
"Flat shape table summary: %s\n",
287 (*shape_table)->SummaryStr().string());
289 (*shape_table)->set_unicharset(trainer->
unicharset());
344 if (strcmp (LabeledList->
Label, Label) == 0)
345 return (LabeledList);
372 strcpy (LabeledList->
Label, Label);
376 return (LabeledList);
384 const char *feature_name,
int max_samples,
386 FILE* file,
LIST* training_samples) {
410 LIST it = *training_samples;
416 while (fgets(buffer, 2048, file) !=
NULL) {
417 if (buffer[0] ==
'\n')
420 sscanf(buffer,
"%*s %s", unichar);
424 tprintf(
"Error: Size of unicharset in training is " 425 "greater than MAX_NUM_CLASSES\n");
429 char_sample =
FindList(*training_samples, unichar);
430 if (char_sample ==
NULL) {
432 *training_samples =
push(*training_samples, char_sample);
435 feature_samples = char_desc->
FeatureSets[feature_type];
437 char_sample->
List =
push(char_sample->
List, feature_samples);
444 if (feature_type != i)
472 FeatureList = char_sample->
List;
497 free(LabeledList->
Label);
504 const char* program_feature_type) {
534 FeatureList = char_sample->
List;
541 for (j = 0; j < N; j++)
547 if ( Sample !=
NULL ) free( Sample );
556 bool debug = strcmp(FLAGS_test_ch.c_str(), label) == 0;
558 LIST pProtoList = ProtoList;
566 LIST list_it = ProtoList;
569 if (test_p != Prototype && !test_p->
Merged) {
573 if (dist < best_dist) {
581 tprintf(
"Merging red clusters (%d+%d) at %g,%g and %g,%g\n",
583 best_match->
Mean[0], best_match->
Mean[1],
584 Prototype->
Mean[0], Prototype->
Mean[1]);
593 }
else if (best_match !=
NULL) {
595 tprintf(
"Red proto at %g,%g matched a green one at %g,%g\n",
596 Prototype->
Mean[0], Prototype->
Mean[1],
597 best_match->
Mean[0], best_match->
Mean[1]);
603 pProtoList = ProtoList;
610 tprintf(
"Red proto at %g,%g becoming green\n",
611 Prototype->
Mean[0], Prototype->
Mean[1]);
648 BOOL8 KeepInsigProtos,
658 pProtoList = ProtoList;
674 for (i=0; i < N; i++)
679 for (i=0; i < N; i++)
688 for (i=0; i < N; i++)
697 for (i=0; i < N; i++)
705 NewProtoList =
push_last(NewProtoList, NewProto);
709 return (NewProtoList);
722 if (strcmp (MergeClass->
Label, Label) == 0)
737 strcpy (MergeClass->
Label, Label);
765 free (MergeClass->
Label);
775 LIST LabeledClassList) {
802 for(i=0; i < NumProtos; i++)
806 Values[0] = OldProto->
X;
807 Values[1] = OldProto->
Y;
808 Values[2] = OldProto->
Angle;
810 NewProto->
X = OldProto->
X;
811 NewProto->
Y = OldProto->
Y;
814 NewProto->
A = Values[0];
815 NewProto->
B = Values[1];
816 NewProto->
C = Values[2];
824 for(i=0; i < NumConfigs; i++)
828 for(j=0; j < NumWords; j++)
829 NewConfig[j] = OldConfig[j];
833 return float_classes;
840 register float Slope;
841 register float Intercept;
842 register float Normalizer;
844 Slope = tan (Values [2] * 2 *
PI);
845 Intercept = Values [1] - Slope * Values [0];
846 Normalizer = 1 / sqrt (Slope * Slope + 1.0);
848 Values [0] = Slope * Normalizer;
849 Values [1] = - Normalizer;
850 Values [2] = Intercept * Normalizer;
882 LabeledProtoList->
List =
push(LabeledProtoList->
List, Proto);
884 *NormProtoList =
push(*NormProtoList, LabeledProtoList);
890 BOOL8 CountSigProtos,
891 BOOL8 CountInsigProtos)
void InitFeatureDefs(FEATURE_DEFS_STRUCT *featuredefs)
CLUSTERER * MakeClusterer(inT16 SampleSize, const PARAM_DESC ParamDesc[])
void move(UnicityTable< T > *from)
void Normalize(float *Values)
LIST RemoveInsignificantProtos(LIST ProtoList, BOOL8 KeepSigProtos, BOOL8 KeepInsigProtos, int N)
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
CLUSTERER * SetUpForClustering(const FEATURE_DEFS_STRUCT &FeatureDefs, LABELEDLIST char_sample, const char *program_feature_type)
void ReadTrainingSamples(const FEATURE_DEFS_STRUCT &feature_defs, const char *feature_name, int max_samples, UNICHARSET *unicharset, FILE *file, LIST *training_samples)
MERGE_CLASS_NODE * MERGE_CLASS
bool save_to_file(const char *const filename) const
int ShortNameToFeatureType(const FEATURE_DEFS_STRUCT &FeatureDefs, const char *ShortName)
void FreeTrainingSamples(LIST CharList)
void Init(uinT8 xbuckets, uinT8 ybuckets, uinT8 thetabuckets)
BIT_VECTOR NewBitVector(int NumBits)
bool DeSerialize(bool swap, FILE *fp)
#define ProtoIn(Class, Pid)
void FreeLabeledClassList(LIST ClassList)
struct LABELEDLISTNODE * LABELEDLIST
const UNICHARSET & unicharset() const
bool AddSpacingInfo(const char *filename)
const char * c_str() const
const char * GetNextFilename(int argc, const char *const *argv)
LIST push(LIST list, void *element)
const int kBoostXYBuckets
const FEATURE_DESC_STRUCT * FeatureDesc[NUM_FEATURE_TYPES]
DOUBLE_PARAM_FLAG(clusterconfig_min_samples_fraction, Config.MinSamples,"Min number of samples per proto as % of total")
void FreeNormProtoList(LIST CharList)
void FreeClass(CLASS_TYPE Class)
void SetFeatureSpace(const IntFeatureSpace &fs)
MasterTrainer * LoadTrainingData(int argc, const char *const *argv, bool replication, ShapeTable **shape_table, STRING *file_prefix)
void MergeInsignificantProtos(LIST ProtoList, const char *label, CLUSTERER *Clusterer, CLUSTERCONFIG *Config)
inT32 MergeClusters(inT16 N, register PARAM_DESC ParamDesc[], register inT32 n1, register inT32 n2, register FLOAT32 m[], register FLOAT32 m1[], register FLOAT32 m2[])
UnicityTableEqEq< int > font_set
INT_PARAM_FLAG(debug_level, 0,"Level of Trainer debugging")
ShapeTable * LoadShapeTable(const STRING &file_prefix)
void ParseArguments(int *argc, char ***argv)
void ReadTrainingSamples(const char *page_name, const FEATURE_DEFS_STRUCT &feature_defs, bool verification)
void SetupFlatShapeTable(ShapeTable *shape_table)
bool Serialize(FILE *fp) const
SAMPLE * MakeSample(CLUSTERER *Clusterer, const FLOAT32 *Feature, inT32 CharID)
const char * string() const
LABELEDLIST NewLabeledList(const char *Label)
bool Serialize(FILE *fp) const
MERGE_CLASS NewLabeledClass(const char *Label)
CLASS_TYPE NewClass(int NumProtos, int NumConfigs)
void FreeLabeledList(LABELEDLIST LabeledList)
static bool ReadParamsFile(const char *file, SetParamConstraint constraint, ParamsVectors *member_params)
#define WordsInVectorOfSize(NumBits)
CLASS_STRUCT * SetUpForFloat2Int(const UNICHARSET &unicharset, LIST LabeledClassList)
void WriteShapeTable(const STRING &file_prefix, const ShapeTable &shape_table)
FEATURE_SET_STRUCT * FEATURE_SET
void LoadPageImages(const char *filename)
FEATURE_DEFS_STRUCT feature_defs
int NumberOfProtos(LIST ProtoList, BOOL8 CountSigProtos, BOOL8 CountInsigProtos)
STRING_PARAM_FLAG(configfile,"","File to load more configs from")
LABELEDLIST FindList(LIST List, char *Label)
CHAR_DESC ReadCharDescription(const FEATURE_DEFS_STRUCT &FeatureDefs, FILE *File)
FLOAT32 ComputeDistance(int k, PARAM_DESC *dim, FLOAT32 p1[], FLOAT32 p2[])
FEATURE_SET FeatureSets[NUM_FEATURE_TYPES]
void truncate_at(inT32 index)
bool contains_unichar(const char *const unichar_repr) const
bool LoadXHeights(const char *filename)
void AddToNormProtosList(LIST *NormProtoList, LIST ProtoList, char *CharName)
void memfree(void *element)
bool LoadFontInfo(const char *filename)
MERGE_CLASS FindClass(LIST List, const char *Label)
const UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
const PARAM_DESC * ParamDesc
void unichar_insert(const char *const unichar_repr)
bool DeSerialize(bool swap, FILE *fp)
void LoadUnicharset(const char *filename)
void FreeFeatureSet(FEATURE_SET FeatureSet)
void FreeProtoList(LIST *ProtoList)
void CleanUpUnusedData(LIST ProtoList)
const int kBoostDirBuckets
LIST push_last(LIST list, void *item)