source: src/compdic_utf8/compdic @ f924e4b

#! /usr/bin/env perl

#package:       UAM Text Tools
#component:     compiledic
#version:       1.3
#author:        Tomasz Obrebski
#author:        Krzysztof Szarzyñski (2012 migration to OpenFST format)

use utf8;

use strict;
use locale;
use File::HomeDir;
use File::Basename;
use File::Temp;
use File::Copy;
use Getopt::Long;



my $linesPerFile = 20000;



Getopt::Long::Configure('no_ignore_case_always');
my $help=0;
GetOptions("help|h" => \$help);

if($help)
{
    print <<'END'
Usage: compiledic dictionaryfile.dic

The dictionary file must be UTF8 without Byte Order Mark (BOM).
To remove BOM see removeBom.sh

Options:
   --help -h                      Help.
END
;
    exit 0;
}

##################################################

@ARGV > 0   or die("Source dictionary not given.\n");

my $file = shift;

-f $file or die("Source dictionary not found.\n");

$file =~ /(.*)\.dic/ or die("The input file must have .dic extension.\n");

my $filenameprefix = $1;

##################################################

# Tworzymy katalog tymczasowy, gdzie wszystko bedzie umieszczane.
my $tmp_root = File::Temp::tempdir( CLEANUP => 1 );
print "Using temp dir: $tmp_root\n";


##################################################
# Tworzymy tabele symboli:
print "Generating the symbols table\t\t";
`python ./symbols.py > $tmp_root/symbols`;# or die "Failed!\n";
print "OK\n";


##################################################
# Dzielenie pliku slownika:
print "Dividing the dictionary file\t\t";

open(IN, $file);

my $lineCount = 0;
my $fileCount = 0;

open(FILE, ">$tmp_root/slo_$fileCount");

while (<IN>) {
    if (++$lineCount >= $linesPerFile) {
        $fileCount++;
        $lineCount = 0;
        close(FILE);
        open(FILE, ">$tmp_root/slo_".$fileCount);
    }
    print(FILE $_);
}
print "OK\n";

##################################################
# Budujemy male automaty:
print "Building partial automata";

#32 kropki, fileCount plikow
my $filesPerDot = $fileCount/32;
my $files=$filesPerDot;
my $dots=0;

for (my $i=0; $i<=$fileCount; $i++) {

    if ($files >= $filesPerDot) {
        $files = 0;
        print ".";
        $dots++;
    }
    $files++;
    `python text2fst.py < $tmp_root/slo_$i > $tmp_root/slownik_$i.fst`;
     #`fstcompile --acceptor $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`;
     `fstcompile --acceptor --isymbols=$tmp_root/symbols $tmp_root/slownik_$i.fst $tmp_root/slownikC_$i.fst`;
     move("$tmp_root/slownikC_$i.fst", "$tmp_root/slownik_$i.bin") or die "Cant create slownik_$i.bin\n";
}
if ($dots < 32) {
    for (my $i=0; $i<32 - $dots; $i++) {
        print ".";
    }
}

print "OK\n";

##################################################
# Usuwamy czesci slownika:
print "Deleteing $tmp_root/slo_ text files\t\t";
unlink <$tmp_root/slo_*> or die "Faiiled\n";
print "OK\n";


##################################################
# Budowanie koncowego automatu:
print "Building final automaton";

#35 kropek...
my $ndots=33;
$filesPerDot = $fileCount/$ndots;
$files=$filesPerDot;
$dots=0;


my $out_fst = "slownik.bin";
my $tmp_fst = "slownik_T.bin";


######################################################################
# Budowanie jednego automatu
######################################################################
move("$tmp_root/slownik_0.bin", "$tmp_root/$out_fst") or die "Failed to move slownik_0.bin -> $out_fst\n";
for (my $i=1; $i<=$fileCount; $i++) {

    if ($files >= $filesPerDot) {
        $files = 0;
        print ".";
        $dots++;
    }
    $files++;
    `fstunion $tmp_root/$out_fst $tmp_root/slownik_$i.bin $tmp_root/$tmp_fst`;
    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at union: slownik_$i\n";
    `fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`;
    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at rmepsilon: slownik_$i\n";
    `fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed at minimization: slownik_$i\n";
    `fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
    

    move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") || die "Unable to move $tmp_root/$tmp_fst -> $out_fst!\n";
 
}

if ($dots < $ndots) {
    for (my $i=0; $i<$ndots - $dots; $i++) {
        print ".";
    }
}

print "OK\n";



######################################################################
# Minimalizacja automatu:
######################################################################
print "removing epsilon-transitions\t\t";
`fstrmepsilon $tmp_root/$out_fst $tmp_root/$tmp_fst`;
move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
print "OK\n";

print "determinizing automaton\t\t";
`fstdeterminize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
print "OK\n";

print "minimizing automaton\t\t";
`fstminimize $tmp_root/$out_fst $tmp_root/$tmp_fst`;
move("$tmp_root/$tmp_fst", "$tmp_root/$out_fst") or die "Failed\n";
print "OK\n";




print "moving the FST to compiledic directory\t\t";
use Cwd;
my $workdir = getcwd($0);
move("$tmp_root/$out_fst", "$workdir/dictionary.bin") or die "Failed\n";
print "OK\n";

########################################################
# Sprzatanie:
print "removing temporary files\t\t";

unlink <$tmp_root/*> or die "Failed\nCan't delete contents of $tmp_root \n";
unlink ($tmp_root);
print "OK\n";

print "Finished!\n";