Index: src/compdic_utf8/compdic
===================================================================
--- src/compdic_utf8/compdic	(revision e7de6cc88c605c4f810cbc852e843294b4b0e8ac)
+++ src/compdic_utf8/compdic	(revision e7de6cc88c605c4f810cbc852e843294b4b0e8ac)
@@ -0,0 +1,214 @@
+#! /usr/bin/env perl
+
+#package:	UAM Text Tools
+#component:	compiledic
+#version:	1.3
+#author:	Tomasz Obrebski
+#author:    	Krzysztof Szarzyñski (2012 migration to OpenFST format)
+
+use utf8;
+
+use strict;
+use locale;
+use File::HomeDir;
+use File::Basename;
+use File::Temp;
+use File::Copy;
+use Getopt::Long;
+
+
+
+my $linesPerFile = 20000;
+
+
+
+Getopt::Long::Configure('no_ignore_case_always');
+my $help=0;
+GetOptions("help|h" => \$help);
+
+if($help)
+{
+    print <<'END'
+Usage: compiledic dictionaryfile.dic
+
+The dictionary file must be UTF8 without Byte Order Mark (BOM).
+To remove BOM see removeBom.sh
+
+Options:
+   --help -h                      Help.
+END
+;
+    exit 0;
+}
+
+##################################################
+
+@ARGV > 0   or die("Source dictionary not given.\n");
+
+my $file = shift;
+
+-f $file or die("Source dictionary not found.\n");
+
+$file =~ /(.*)\.dic/ or die("The input file must have .dic extension.\n");
+
+my $filenameprefix = $1;
+
+##################################################
+
+# Tworzymy katalog tymczasowy, gdzie wszystko bedzie umieszczane.
+my $tmp_root = File::Temp::tempdir( CLEANUP => 1 );
+print "Using temp dir: $tmp_root\n";
+
+
+##################################################
+# Tworzymy tabele symboli:
+print "Generating the symbols table\t\t";
+`python ./symbols.py > $tmp_root/symbols`;# or die "Failed!\n";
+print "OK\n";
+
+
+##################################################
+# Dzielenie pliku slownika:
+print "Dividing the dictionary file\t\t";
+
+open(IN, $file);
+
+my $lineCount = 0;
+my $fileCount = 0;
+
+open(FILE, ">$tmp_root/slo_$fileCount");
+
+while (<IN>) {
+    if (++$lineCount >= $linesPerFile) {
+	$fileCount++;
+	$lineCount = 0;
+	close(FILE);
+	open(FILE, ">$tmp_root/slo_".$fileCount);
+    }
+    print(FILE $_);
+}
+print "OK\n";
+
+##################################################
+# Budujemy male automaty:
+print "Building partial automata";
+
+#32 kropki, fileCount plikow
+my $filesPerDot = $fileCount/32;
+my $files=$filesPerDot;
+my $dots=0;
+
+for (my $i=0; $i<=$fileCount; $i++) {
+
+    if ($files >= $filesPerDot) {
+	$files = 0;
+	print ".";
+	$dots++;
+    }
+    $files++;
+    `python text2fst.py < $tmp_root/slo_$i > $tmp_root/slownik_$i.fst`;
+     #`fstcompile --acceptor $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`;
+     `fstcompile --acceptor --isymbols=$tmp_root/symbols $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`;
+     move("$tmp_root/slownikC_$i.fst", "$tmp_root/slownik_$i.bin") or die "Cant create slownik_$i.bin\n";
+}
+if ($dots < 32) {
+    for (my $i=0; $i<32 - $dots; $i++) {
+	print ".";
+    }
+}
+
+print "OK\n";
+
+##################################################
+# Usuwamy czesci slownika:
+print "Deleteing $tmp_root/slo_ text files\t\t";
+unlink <$tmp_root/slo_*> or die "Faiiled\n";
+print "OK\n";
+
+
+##################################################
+# Budowanie koncowego automatu:
+print "Building final automaton";
+
+#35 kropek...
+my $ndots=33;
+$filesPerDot = $fileCount/$ndots;
+$files=$filesPerDot;
+$dots=0;
+
+
+my $out_fst = "slownik.bin";
+my $tmp_fst = "slownik_T.bin";
+
+
+######################################################################
+# Budowanie jednego automatu
+######################################################################
+move("$tmp_root/slownik_0.bin", "$tmp_root/$out_fst") or die "Failed to move slownik_0.bin -> $out_fst\n";
+for (my $i=1; $i<=$fileCount; $i++) {
+
+    if ($files >= $filesPerDot) {
+	$files = 0;
+	print ".";
+	$dots++;
+    }
+    $files++;
+    `fstunion $tmp_root/$out_fst $tmp_root/slownik_$i.bin $tmp_root/$tmp_fst`;
+    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at union: slownik_$i\n";
+    `fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at rmepsilon: slownik_$i\n";
+    `fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at minimization: slownik_$i\n";
+    `fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+    
+
+    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") || die "Unable to move $tmp_root/$tmp_fst -> $out_fst!\n";
+ 
+}
+
+if ($dots < $ndots) {
+    for (my $i=0; $i<$ndots - $dots; $i++) {
+	print ".";
+    }
+}
+
+print "OK\n";
+
+
+
+######################################################################
+# Minimalizacja automatu:
+######################################################################
+print "removing epsilon-transitions\t\t";
+`fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
+print "OK\n";
+
+print "determinizing automaton\t\t";
+`fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
+print "OK\n";
+
+print "minimizing automaton\t\t";
+`fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
+move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
+print "OK\n";
+
+
+
+
+print "moving the FST to compiledic directory\t\t";
+use Cwd;
+my $workdir = getcwd($0);
+move("$tmp_root/$out_fst", "$workdir/dictionary.bin") or die "Failed\n";
+print "OK\n";
+
+########################################################
+# Sprzatanie:
+print "removing temporary files\t\t";
+
+unlink <$tmp_root/*> or die "Failed\nCan't delete contents of $tmp_root \n";
+unlink ($tmp_root);
+print "OK\n";
+
+print "Finished!\n";