1 | #! /usr/bin/env perl |
---|
2 | |
---|
3 | #package: UAM Text Tools |
---|
4 | #component: compiledic |
---|
5 | #version: 1.3 |
---|
6 | #author: Tomasz Obrebski |
---|
7 | #author: Krzysztof Szarzyñski (2012 migration to OpenFST format) |
---|
8 | |
---|
9 | use utf8; |
---|
10 | |
---|
11 | use strict; |
---|
12 | use locale; |
---|
13 | use File::HomeDir; |
---|
14 | use File::Basename; |
---|
15 | use File::Temp; |
---|
16 | use File::Copy; |
---|
17 | use Getopt::Long; |
---|
18 | |
---|
19 | |
---|
20 | |
---|
21 | my $linesPerFile = 20000; |
---|
22 | |
---|
23 | |
---|
24 | |
---|
25 | Getopt::Long::Configure('no_ignore_case_always'); |
---|
26 | my $help=0; |
---|
27 | GetOptions("help|h" => \$help); |
---|
28 | |
---|
29 | if($help) |
---|
30 | { |
---|
31 | print <<'END' |
---|
32 | Usage: compiledic dictionaryfile.dic |
---|
33 | |
---|
34 | The dictionary file must be UTF8 without Byte Order Mark (BOM). |
---|
35 | To remove BOM see removeBom.sh |
---|
36 | |
---|
37 | Options: |
---|
38 | --help -h Help. |
---|
39 | END |
---|
40 | ; |
---|
41 | exit 0; |
---|
42 | } |
---|
43 | |
---|
44 | ################################################## |
---|
45 | |
---|
46 | @ARGV > 0 or die("Source dictionary not given.\n"); |
---|
47 | |
---|
48 | my $file = shift; |
---|
49 | |
---|
50 | -f $file or die("Source dictionary not found.\n"); |
---|
51 | |
---|
52 | $file =~ /(.*)\.dic/ or die("The input file must have .dic extension.\n"); |
---|
53 | |
---|
54 | my $filenameprefix = $1; |
---|
55 | |
---|
56 | ################################################## |
---|
57 | |
---|
58 | # Tworzymy katalog tymczasowy, gdzie wszystko bedzie umieszczane. |
---|
59 | my $tmp_root = File::Temp::tempdir( CLEANUP => 1 ); |
---|
60 | print "Using temp dir: $tmp_root\n"; |
---|
61 | |
---|
62 | |
---|
63 | ################################################## |
---|
64 | # Tworzymy tabele symboli: |
---|
65 | print "Generating the symbols table\t\t"; |
---|
66 | `python ./symbols.py > $tmp_root/symbols`;# or die "Failed!\n"; |
---|
67 | print "OK\n"; |
---|
68 | |
---|
69 | |
---|
70 | ################################################## |
---|
71 | # Dzielenie pliku slownika: |
---|
72 | print "Dividing the dictionary file\t\t"; |
---|
73 | |
---|
74 | open(IN, $file); |
---|
75 | |
---|
76 | my $lineCount = 0; |
---|
77 | my $fileCount = 0; |
---|
78 | |
---|
79 | open(FILE, ">$tmp_root/slo_$fileCount"); |
---|
80 | |
---|
81 | while (<IN>) { |
---|
82 | if (++$lineCount >= $linesPerFile) { |
---|
83 | $fileCount++; |
---|
84 | $lineCount = 0; |
---|
85 | close(FILE); |
---|
86 | open(FILE, ">$tmp_root/slo_".$fileCount); |
---|
87 | } |
---|
88 | print(FILE $_); |
---|
89 | } |
---|
90 | print "OK\n"; |
---|
91 | |
---|
92 | ################################################## |
---|
93 | # Budujemy male automaty: |
---|
94 | print "Building partial automata"; |
---|
95 | |
---|
96 | #32 kropki, fileCount plikow |
---|
97 | my $filesPerDot = $fileCount/32; |
---|
98 | my $files=$filesPerDot; |
---|
99 | my $dots=0; |
---|
100 | |
---|
101 | for (my $i=0; $i<=$fileCount; $i++) { |
---|
102 | |
---|
103 | if ($files >= $filesPerDot) { |
---|
104 | $files = 0; |
---|
105 | print "."; |
---|
106 | $dots++; |
---|
107 | } |
---|
108 | $files++; |
---|
109 | `python text2fst.py < $tmp_root/slo_$i > $tmp_root/slownik_$i.fst`; |
---|
110 | #`fstcompile --acceptor $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`; |
---|
111 | `fstcompile --acceptor --isymbols=$tmp_root/symbols $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`; |
---|
112 | move("$tmp_root/slownikC_$i.fst", "$tmp_root/slownik_$i.bin") or die "Cant create slownik_$i.bin\n"; |
---|
113 | } |
---|
114 | if ($dots < 32) { |
---|
115 | for (my $i=0; $i<32 - $dots; $i++) { |
---|
116 | print "."; |
---|
117 | } |
---|
118 | } |
---|
119 | |
---|
120 | print "OK\n"; |
---|
121 | |
---|
122 | ################################################## |
---|
123 | # Usuwamy czesci slownika: |
---|
124 | print "Deleteing $tmp_root/slo_ text files\t\t"; |
---|
125 | unlink <$tmp_root/slo_*> or die "Faiiled\n"; |
---|
126 | print "OK\n"; |
---|
127 | |
---|
128 | |
---|
129 | ################################################## |
---|
130 | # Budowanie koncowego automatu: |
---|
131 | print "Building final automaton"; |
---|
132 | |
---|
133 | #35 kropek... |
---|
134 | my $ndots=33; |
---|
135 | $filesPerDot = $fileCount/$ndots; |
---|
136 | $files=$filesPerDot; |
---|
137 | $dots=0; |
---|
138 | |
---|
139 | |
---|
140 | my $out_fst = "slownik.bin"; |
---|
141 | my $tmp_fst = "slownik_T.bin"; |
---|
142 | |
---|
143 | |
---|
144 | ###################################################################### |
---|
145 | # Budowanie jednego automatu |
---|
146 | ###################################################################### |
---|
147 | move("$tmp_root/slownik_0.bin", "$tmp_root/$out_fst") or die "Failed to move slownik_0.bin -> $out_fst\n"; |
---|
148 | for (my $i=1; $i<=$fileCount; $i++) { |
---|
149 | |
---|
150 | if ($files >= $filesPerDot) { |
---|
151 | $files = 0; |
---|
152 | print "."; |
---|
153 | $dots++; |
---|
154 | } |
---|
155 | $files++; |
---|
156 | `fstunion $tmp_root/$out_fst $tmp_root/slownik_$i.bin $tmp_root/$tmp_fst`; |
---|
157 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at union: slownik_$i\n"; |
---|
158 | `fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
159 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at rmepsilon: slownik_$i\n"; |
---|
160 | `fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
161 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at minimization: slownik_$i\n"; |
---|
162 | `fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
163 | |
---|
164 | |
---|
165 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") || die "Unable to move $tmp_root/$tmp_fst -> $out_fst!\n"; |
---|
166 | |
---|
167 | } |
---|
168 | |
---|
169 | if ($dots < $ndots) { |
---|
170 | for (my $i=0; $i<$ndots - $dots; $i++) { |
---|
171 | print "."; |
---|
172 | } |
---|
173 | } |
---|
174 | |
---|
175 | print "OK\n"; |
---|
176 | |
---|
177 | |
---|
178 | |
---|
179 | ###################################################################### |
---|
180 | # Minimalizacja automatu: |
---|
181 | ###################################################################### |
---|
182 | print "removing epsilon-transitions\t\t"; |
---|
183 | `fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
184 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n"; |
---|
185 | print "OK\n"; |
---|
186 | |
---|
187 | print "determinizing automaton\t\t"; |
---|
188 | `fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
189 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n"; |
---|
190 | print "OK\n"; |
---|
191 | |
---|
192 | print "minimizing automaton\t\t"; |
---|
193 | `fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`; |
---|
194 | move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n"; |
---|
195 | print "OK\n"; |
---|
196 | |
---|
197 | |
---|
198 | |
---|
199 | |
---|
200 | print "moving the FST to compiledic directory\t\t"; |
---|
201 | use Cwd; |
---|
202 | my $workdir = getcwd($0); |
---|
203 | move("$tmp_root/$out_fst", "$workdir/dictionary.bin") or die "Failed\n"; |
---|
204 | print "OK\n"; |
---|
205 | |
---|
206 | ######################################################## |
---|
207 | # Sprzatanie: |
---|
208 | print "removing temporary files\t\t"; |
---|
209 | |
---|
210 | unlink <$tmp_root/*> or die "Failed\nCan't delete contents of $tmp_root \n"; |
---|
211 | unlink ($tmp_root); |
---|
212 | print "OK\n"; |
---|
213 | |
---|
214 | print "Finished!\n"; |
---|