source: app/src/compiledic/compiledic @ 8d3e6ab

#! /usr/bin/env perl

$symfile='~/.utt/pl/pl_PL.iso-8859-2.sym';
$symfilenoext = $symfile;
$symfilenoext =~ s/\.sym$//;
$labfile = $symfilenoext . '.lab';
$sclfile = $symfilenoext . '.scl';

use locale;
#use strict;

##################################################
$linesPerFile = 20000;

if (@ARGV < 1) {
    print "usage: prep_user_dict.pl dictionary_file\n";
    exit;
}

my $file = shift; # @ARGV;
my $filenameprefix;

if ($file =~ /(.*)\.dic/)
{
    $filenameprefix = $1;
}
else
{
    print "The input file must have .dic extension.";
    exit(1);
}

# Przygotowanie etykiet

#`makeLabels.pl > labels.sym`;

`lexmakelab $symfilenoext`;

# Analiza pliku s³ownika

print "preparing file...........................................";

`sed -r "s/([[:punct:]])/\[\\1\]/g" < $file > temp1`;

`cp temp1 temp2`;

print "OK\n";

#dzielimy plik na wiele czê¶ci, uruchamiamy lexcomplex dla ka¿dej
#czê¶ci osobno, nastêpnie ³±czymy to za pomoc± programu fsmunion

#print "Dzielê s³ownik na mniejsze czê¶ci...";

open(IN, "./temp2");

$lineCount = 0;
$fileCount = 0;

`mkdir LemTEMP`;

open(FILE, ">LemTEMP/slo_0");

while (<IN>) {

    if (++$lineCount >= $linesPerFile) {
        $fileCount++;
        $lineCount = 0;

        close(FILE);
#       print "Tworzê nowy plik tymczasowy: slo_".$fileCount."\n";
        open(FILE, ">LemTEMP/slo_".$fileCount);
    }

    print(FILE $_);
}

#print "OK\n";

print "building partial automata";

#32 kropki, fileCount plikow
$filesPerDot = $fileCount/32;
$files=$filesPerDot;
$dots=0;

for ($i=0; $i<=$fileCount; $i++) {

    if ($files >= $filesPerDot) {
        $files = 0;
        print ".";
        $dots++;
    }
    $files++;

    $command = "lexcomplex -l $labfile -S $sclfile < LemTEMP/slo_".$i." > LemTEMP/slownik_".$i.".fsm";

    `$command`;

}
if ($dots < 32) {
    for ($i=0; $i<32 - $dots; $i++) {
        print ".";
    }
}

print "OK\n";

`rm LemTEMP/slo_*`;

print "building final automaton";

#35 kropek...
$ndots=33;
$filesPerDot = $fileCount/$ndots;
$files=$filesPerDot;
$dots=0;

`cp LemTEMP/slownik_0.fsm slownik1.fsm`;

for ($i=1; $i<=$filecount; $i++) {

    if ($files >= $filesPerDot) {
        $files = 0;
        print ".";
        $dots++;
    }
    $files++;

    $command = "fsmunion LemTEMP/slownik_".$i." slownik1.fsm > slownik2.fsm";

    `$command`;

    `mv slownik2.fsm slownik1.fsm`;
}

if ($dots < $ndots) {
    for ($i=0; $i<$ndots - $dots; $i++) {
        print ".";
    }
}

`fsmunion LemTEMP/* > slownik1.fsm`;

print "OK\n";

print "removing epsilon-transitions.............................";

`fsmrmepsilon slownik1.fsm > slownik2.fsm`;

`rm slownik1.fsm`;

print "OK\n";

print "determinizing automaton..................................";

`fsmdeterminize slownik2.fsm > slownik1.fsm`;

`rm slownik2.fsm`;

print "OK\n";

print "minimizing automaton.....................................";

`fsmminimize slownik1.fsm > slownik.fsm`;

#`rm slownik1.fsm`;

print "OK\n";

print "converting fsm format to bin.............................";

`fsmprint -i $labfile slownik.fsm > slownik.txt`;

`fsm2aut slownik.txt > slownik.aut`;

`aut2fsa < slownik.aut > $filenameprefix.bin`;

print "OK\n";

print "removing temporary files.................................";

`rm LemTEMP/*`;
`rmdir LemTEMP`;
`rm temp2`;
`rm slownik.fsm`;
`rm slownik.txt`;
`rm slownik.aut`;
`rm labels.*`;

print "OK\n";