Index: app/src/lem/Makefile =================================================================== --- app/src/lem/Makefile (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/Makefile (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,56 @@ +PAR=-Wno-deprecated -m32 -O3 -fpermissive +#-static +PAR2=-c -Wno-deprecated -m32 -O3 -fpermissive +LIB_PATH=../lib +COMMON_PATH=../common +CMDLINE_FILE='"../lem/cmdline.h"' + + +lem: main.cc lem.o $(LIB_PATH)/auttools.o $(LIB_PATH)/word.o \ + cmdline.c common_lem.o common.o symtab.o + g++ $(PAR) main.cc lem.o $(LIB_PATH)/auttools.o \ + $(LIB_PATH)/word.o cmdline.c common.o common_lem.o \ + symtab.o -o lem + +lem.o: lem.h lem.cc + g++ $(PAR2) lem.cc + +# alphabet.o: $(LIB_PATH)/alphabet.h $(LIB_PATH)/alphabet.cc +# g++ $(PAR2) $(LIB_PATH)/alphabet.cc + +# auttools.o: $(LIB_PATH)/auttools.h $(LIB_PATH)/auttools.cc +# g++ $(PAR2) $(LIB_PATH)/auttools.cc + +# word.o: $(LIB_PATH)/word.h $(LIB_PATH)/word.cc +# g++ $(PAR2) $(LIB_PATH)/word.cc + +# erro.o: $(LIB_PATH)/erro.h $(LIB_PATH)/erro.cc +# g++ $(PAR2) $(LIB_PATH)/erro.cc + +common.o: $(COMMON_PATH)/cmdline_common.ggo $(COMMON_PATH)/common.cc \ + $(COMMON_PATH)/common.h + g++ $(PAR2) -D _CMDLINE_FILE=$(CMDLINE_FILE) $(COMMON_PATH)/common.cc + +common_lem.o: cmdline.h common_lem.h common_lem.cc + g++ $(PAR2) common_lem.cc + +cmdline.c cmdline.h: cmdline.ggo + gengetopt -i cmdline.ggo --conf-parser + +cmdline.ggo: cmdline_lem.ggo ../common/cmdline_common.ggo + cat cmdline_lem.ggo ../common/cmdline_common.ggo > cmdline.ggo + +symtab.o: $(LIB_PATH)/symtab.h $(LIB_PATH)/symtab.cc + g++ $(PAR2) $(LIB_PATH)/symtab.cc + +clean: clean.cmdline + rm *.o || true + rm lem || true + +clean.cmdline: + rm cmdline.* || true + +copy: +ifdef UTT_BIN_DIR + cp lem $(UTT_BIN_DIR) +endif Index: app/src/lem/cmdline_lem.ggo =================================================================== --- app/src/lem/cmdline_lem.ggo (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/cmdline_lem.ggo (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,5 @@ +package "lem" +version "0.1" + +option "dictionary-home" - "D.h." string typestr="FILENAME" hidden no +option "dictionary" d "Dictionary" string typestr="FILENAME" default="lem.bin" no Index: app/src/lem/common_lem.cc =================================================================== --- app/src/lem/common_lem.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/common_lem.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,41 @@ +#include +#include +#include "common_lem.h" + +char dictionary[255]; + +void process_lem_options(gengetopt_args_info* args) +{ + + if(args->dictionary_given) + { + expand_path(args->dictionary_arg,dictionary); + } + else if (args->dictionary_home_given && args->language_given) + { + char buf[255]; + expand_path(args->dictionary_home_arg, buf); + sprintf(dictionary,"%s/%s/lem.bin",buf,args->language_arg); + } +} + + +// STARE +// if(args.dictionary_given) +// strcpy(dictionary, args.dictionary_arg); +// else { +// char path[256]; +// //sprintf(path, "/etc/utt/data/%s/%s", args.locale_arg, DICT_FILE); +// //if (file_accessible(path) == 0) +// // strcpy(dictionary, path); +// //else { +// sprintf(path, "%s/%s", utt_dir, DICT_FILE); +// if (file_accessible(path) == 0) +// strcpy(dictionary, path); +// else { +// fprintf(stderr, "Cannot find dictionary!\n"); +// exit(1); +// } +// //} +// } + Index: app/src/lem/common_lem.h =================================================================== --- app/src/lem/common_lem.h (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/common_lem.h (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,16 @@ +#ifndef __COMMON_LEM__H +#define __COMMON_LEM__H + +#include +#define _CMDLINE_FILE "../lem/cmdline.h" +#include "../common/common.h" + +#include "cmdline.h" + +#define DICT_FILE "lem.bin" + +extern char dictionary[]; + +extern void process_lem_options(gengetopt_args_info* args); + +#endif Index: app/src/lem/lem.cc =================================================================== --- app/src/lem/lem.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/lem.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,152 @@ +#include "lem.h" + +#include + + +/* Znajduje opisy slownikowe dla wyrazu. + * Parametry: + * form - wyraz, + * tab - referencja do tablicy Words (miejsce na wyniki) + * Wartosc: + * liczba dodanych opisow + */ +int Lem::ana(const char* form, Words& tab) { + + // sprawdzamy czy parametry wywolania sa poprawne + assert(form && &tab); + int count0 = tab.count(); + long l; + if ((l=_dict.next(_dict.gtra(0, form, FT::ftMAXPATH), ';'))>=0) + add_to_table(tab, form, l); + return tab.count()-count0; +} + + +/* Szukamy opisu slownikowego nastepnego wyrazu w buforze. + * Parametry: + * buf - bufor + * tab - miejsce na wyniki + * Wartosc: + * ilosc dodanych opisow + */ +int Lem::pref(char* buf, Words& tab) { + + // sprawdzamy czy parametry wywolania sa poprawne + assert(buf && &tab); + + int count0 = tab.count(); + long l; + char* buf0 = buf; + + if((l=_dict.pref(buf, ';'))>=0) { + char form[MAX_FORM]; + int len=buf-buf0; + form[len]='\0'; + add_to_table(tab,form,l); + } + return tab.count() - count0; +} + +/* Dodaje kolejne opisy do tablicy wynikow. + * Parametry: + * tab - tablica wynikow, + * f - wyraz, + * s - stan, na ktorym zaczyna sie pierwszy opis + */ +void Lem::add_to_table(Words& tab, const char* f, long s) { + + // sprawdzenie parametrow + assert(&tab); + assert(f); + + char des[FT::ftMAXPATH]; + + while (_dict.cont(s, des)) { + char* des1; + if ((des1=strtok(des, ";")) != NULL) + do { + if (tab.count() >= MAX_ALT) break; + tab.add(f, des1); + des1=strtok(NULL, ";"); + } while (des1!=NULL); + s=-1; + } +} + +void Lem::prn_dict() +{ + + char des[FT::ftMAXPATH]; + + long s=0; + + while (_dict.cont(s, des)) + { + printf("%s\n",des); + s=-1; + } +} + + +AuxLem::AuxLem(const char* filename) + : Lem(), _dict(SIZE) +{ + FILE* f; + char buf[MAX_LINE+2]; + f=fopen(filename,"r"); + for(long i=0; i=MAX_LINE-1) continue; // BEZ isalpha! + buf[l-1]='\0'; + char* sep=strchr(buf,';'); + if(sep==NULL) continue; + *sep='\0'; + long formind=_dict.add(buf); + if(formind>=0) + { + char* desc=strdup(sep+1); + info[formind]=desc; + } + else + fprintf(stderr,"AuxLem: Form not added: %s;%s.\n", buf,sep+1); + } + fclose(f); +}; + +//--------------------------------------------------------------------------- + +AuxLem::~AuxLem() +{ +// for(long i=0; i<_dict.count(); ++i) +// free(info[_dict.hashindex(i)]); + for(long i=0; i=0) + { + strcpy(des,info[ind]); + char* des1; + if((des1=strtok(des,";"))!=NULL) + do + { + if(tab.cnt>=MAXALT) break; + tab.add(form,des1); + des1=strtok(NULL,";"); + } while(des1!=NULL); + } + return tab.count()-count0; +} + +//--------------------------------------------------------------------------- + Index: app/src/lem/lem.h =================================================================== --- app/src/lem/lem.h (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/lem.h (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,50 @@ +#include "../lib/tfti.h" +#include "../lib/word.h" +#include "../lib/symtab.h" +#include "../lib/const.h" + +class Lem { + + protected: + // Alphabet& _alpha; + + // slownik + TFTiv _dict; + + void add_to_table(Words& tab, const char* f, long s); + + public: + + Lem() {}; + Lem(const char* d) + : _dict(d) {}; + virtual int ana(const char* form, Words& tab); + int pref(char* form, Words& tab); + void prn_dict(); + +}; + + +class AuxLem : public Lem { +public: + + static const int SIZE=1500000; + // static const int MAXLINE=1000; + static const int MAXALT=256; + + AuxLem(const char* filename); + ~AuxLem(); + +// int ana(const char* form, Grams& tab); + int ana(const char* form, Words& tab); + +// operator bool() { return _dict && info; } + +private: + SymbolTable _dict; + char* info[SIZE]; + +}; + + + Index: app/src/lem/main.cc =================================================================== --- app/src/lem/main.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) +++ app/src/lem/main.cc (revision 25ae32e4c2354e0ed6756bbe1de83f39cd814652) @@ -0,0 +1,132 @@ +#include "../lib/iotools.h" +#define _CMDLINE_FILE "../lem/cmdline.h" +#include "../common/common.h" +#include "common_lem.h" +#include "lem.h" +#include "cmdline.h" +#include + +int main(int argc, char** argv) { + +// setlocale(LC_CTYPE,""); //PO CO TO? +// setlocale(LC_COLLATE,""); // + + gengetopt_args_info args; + + if(cmdline_parser(argc, argv, &args) != 0) + exit(1); + + process_config_files(&args,argv[0]); + process_common_options(&args,argv[0]); + process_lem_options(&args); + + char line[MAX_LINE+1]; + char outline[MAX_LINE+1]; + char parms[MAX_LINE+1], desc[MAX_LINE+1], lemma[MAX_LINE+1]; + long line_count = 0; + + Lem* lem; + + if(strcmp(dictionary+strlen(dictionary)-4,".bin")==0) + lem = new Lem(dictionary); + else if(strcmp(dictionary+strlen(dictionary)-4,".dic")==0) + lem = new AuxLem(dictionary); + else + fprintf(stderr,"lem: Invalid dictionary file extension.\n"); + + Words tab; +// Segment seg; + + while (fgets(line, MAX_LINE, inputf)) + { + // strcpy(outline,line); + ++line_count; + + int start, len; + + if (!process_seg(line, args)) // TO POWINNO BYC WCZESNIEJ ZABEZPIECZONE + fputs(line, outputf); + else + { + char form[MAX_FORM]; + + tab.clear(); + getfield(line,input_field_prefix,form); + if (form==NULL) continue; + + lem->ana(form, tab); + if(tab.count()==0) + { + char form1[MAX_FORM]; // tymczasowo tak, trzeba zmienic ana + char* p; + strcpy(form1,form); + for(p=form1;*p;++p) *p=tolower(*p); + p=form1; + lem->ana(p,tab); + } + + if (tab.count() == 0) + fputs(line, failedf); + else + { // mamy jakies opisy w slowniku + + if(one_line) + { + char* descp=desc; + for (int i=0; i< tab.count(); ++i) + { + descp += sprintf(descp," %s%s,%s", output_field_prefix, tab[i].lemma(), tab[i].descr()); + } + strcpy(outline,line); + outline[strlen(outline)-1]='\0'; + strcat(outline,desc); + strcat(outline,"\n"); + fputs(outline, outputf); + if (copy_processed) + fputs(line,outputf); + } + else if(one_field) + { + char* descp=desc; + for (int i=0; i< tab.count(); ++i) + if(i==0) + descp += sprintf(descp," %s%s,%s", output_field_prefix, tab[i].lemma(), tab[i].descr()); + else + { + if(strcmp(tab[i].lemma(),tab[i-1].lemma())==0) + descp += sprintf(descp,",%s",tab[i].descr()); + else + descp += sprintf(descp,";%s,%s",tab[i].lemma(),tab[i].descr()); + } + + strcpy(outline,line); + outline[strlen(outline)-1]='\0'; + strcat(outline,desc); + strcat(outline,"\n"); + fputs(outline, outputf); + if (copy_processed) + fputs(line,outputf); + } + else + { + for (int i=0; i< tab.count(); ++i) + { + // kolejne opisy - kolejne linie. + sprintf(desc, " %s%s,%s\n", output_field_prefix, tab[i].lemma(), tab[i].descr()); + strcpy(outline,line); + outline[strlen(outline)-1]='\0'; + strcat(outline,desc); + fputs(outline, outputf); + } + if (copy_processed) + fputs(line,outputf); + } + } + } + + if(args.interactive_flag) + fflush(outputf), fflush(failedf); + + } + cmdline_parser_free(&args); +}