Index: src/compdic/Makefile =================================================================== --- src/compdic/Makefile (revision f4bf33ed04eb342d17a377035e58f078b78159c3) +++ src/compdic/Makefile (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -16,9 +16,9 @@ ifdef BIN_DIR install -m 0755 compdic $(BIN_DIR) - install -m 0755 compdic-create-fst $(BIN_DIR) - install -m 0755 compdic-fst-add $(BIN_DIR) - install -m 0755 compdic-fst-minus $(BIN_DIR) + install -m 0755 compdic-update $(BIN_DIR) + install -m 0755 compdic-update-fst $(BIN_DIR) + install -m 0755 compdic-update-cats $(BIN_DIR) + install -m 0755 compdic-dic-to-fst $(BIN_DIR) install -m 0755 compdic-fst-to-bin $(BIN_DIR) - install -m 0755 compdic-fst-update $(BIN_DIR) install -m 0755 fsm2aut $(BIN_DIR) @@ -31,9 +31,10 @@ ifdef BIN_DIR rm $(BIN_DIR)/compdic - rm $(BIN_DIR)/compdic-create-fst - rm $(BIN_DIR)/compdic-fst-add - rm $(BIN_DIR)/compdic-fst-minus + rm $(BIN_DIR)/compdic-update + rm $(BIN_DIR)/compdic-update-fst + rm $(BIN_DIR)/compdic-update-cats + rm $(BIN_DIR)/compdic-dic-to-fst rm $(BIN_DIR)/compdic-fst-to-bin - rm $(BIN_DIR)/compdic-fst-update + rm $(BIN_DIR)/fsm2aut rm $(BIN_DIR)/aut2fsa Index: src/compdic/compdic =================================================================== --- src/compdic/compdic (revision abd28d16d662215e73293143d3773397f219ea93) +++ src/compdic/compdic (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -1,2 +1,3 @@ +#!/bin/bash no_of_parts=0 Index: src/compdic/compdic-dic-to-cats =================================================================== --- src/compdic/compdic-dic-to-cats (revision 555c7f814bf87fa10240e98295372095bc422115) +++ src/compdic/compdic-dic-to-cats (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -0,0 +1,18 @@ + +if [ $# -lt 2 ] +then + echo "Usage:" + echo " compdic-dic-to-cats " + echo "where" + echo " - file containig a list of words, one per line, iso-8859-2 encoded" + echo " - a file to which the compiled automaton in openfst format shoul be written" + exit 0 +fi + +source=$1 +cats=$2 + +echo generating cats file ... + +cat $source | cut -d ',' -f 2 | sort -u > $cats + Index: src/compdic/compdic-dic-to-fst =================================================================== --- src/compdic/compdic-dic-to-fst (revision 555c7f814bf87fa10240e98295372095bc422115) +++ src/compdic/compdic-dic-to-fst (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -0,0 +1,187 @@ +#! /bin/bash + +no_of_parts=0 + +while [ $# -gt 2 ] +do + case $1 + in + -p) + no_of_parts=$2 + shift 2 + ;; + + *) + echo "The arguments to use are" + echo "-p: number of parts" + shift 1 + ;; + esac +done + +if [ $# -lt 2 ] +then + echo "Usage:" + echo " compdic [-p ] " + echo "where" + echo " - file containig a list of words, one per line, iso-8859-2 encoded" + echo " - file to which the compiled automaton in openfst format will be written" + exit 0 +fi + + +source=$1 +fst=$2 + + +if [ $no_of_parts -eq 0 ] +then + no_of_parts=$(( `cat $1 | wc -l` / 75000 + 1 )) +fi + + +echo number of parts: $no_of_parts + + +tempdir=`mktemp -d /tmp/compdic.XXXXXX` + +alphabet=`tempfile -d $tempdir` + +cat < $alphabet + 0 +a 1 +A 2 +ä 3 +± 4 +¡ 5 +b 6 +B 7 +c 8 +C 9 +æ 10 +Æ 11 +d 12 +D 13 +e 14 +E 15 +é 16 +ê 17 +Ê 18 +f 19 +F 20 +g 21 +G 22 +h 23 +H 24 +i 25 +I 26 +j 27 +J 28 +k 29 +K 30 +l 31 +L 32 +³ 33 +£ 34 +m 35 +M 36 +n 37 +N 38 +ñ 39 +Ñ 40 +o 41 +O 42 +ö 43 +ó 44 +Ó 45 +p 46 +P 47 +q 48 +Q 49 +r 50 +R 51 +s 52 +S 53 +¶ 54 +Š 55 +t 56 +T 57 +u 58 +U 59 +ü 60 +v 61 +V 62 +w 63 +W 64 +x 65 +X 66 +y 67 +Y 68 +z 69 +Z 70 +Œ 71 +¬ 72 +¿ 73 +¯ 74 +0 75 +1 76 +2 77 +3 78 +4 79 +5 80 +6 81 +7 82 +8 83 +9 84 +_ 85 +- 86 +? 87 +! 88 +~ 89 +; 90 +, 91 +/ 92 +* 93 ++ 94 +EOF + + +no_of_lines=$(( (`cat $source | wc -l` / $no_of_parts) + 1 )) + +split -l $no_of_lines $source $tempdir/part. + +automaton=$tempdir/output.fst + +cat < $automaton +EOF + +n=0 + +for f in $tempdir/part.* +do + temp1=`tempfile -d $tempdir` + temp2=`tempfile -d $tempdir` + temp3=`tempfile -d $tempdir` + + n=$(( $n + 1 )) + echo processing part $n + + cat $f |\ + lst2fstext |\ + fstcompile --acceptor --isymbols=$alphabet |\ + fstrmepsilon |\ + fstdeterminize > $temp1 + fstminimize $temp1 $temp2 + + fstunion $automaton $temp2 | fstrmepsilon | fstdeterminize > $temp3 + fstminimize $temp3 $automaton +done + +echo generating binary automaton file ... + +cat $automaton | fsttopsort > $fst +rm -r $tempdir + +#echo generating cats file ... + +#cat $1 | cut -d ',' -f 2 | sort -u > $1.cats Index: src/compdic/compdic-fst-to-bin =================================================================== --- src/compdic/compdic-fst-to-bin (revision f4bf33ed04eb342d17a377035e58f078b78159c3) +++ src/compdic/compdic-fst-to-bin (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -1,3 +1,3 @@ - +#!/bin/bash if [ $# -lt 2 ] Index: src/compdic/compdic-update =================================================================== --- src/compdic/compdic-update (revision 555c7f814bf87fa10240e98295372095bc422115) +++ src/compdic/compdic-update (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -0,0 +1,48 @@ +#! /bin/bash + +. /etc/utt/compdic.conf + + +while [ $# -gt 2 ] +do + case $1 + in + -d) + dictionary_home=$2 + shift 2 + ;; + + *) + echo "The arguments to use are" + echo "-d " + shift 1 + ;; + esac +done + + +if [ $# -lt 2 ] +then + echo "Usage:" + echo " compdic-update [-d ] ..." + echo "where" + echo " - dictionary home directory" + echo " - dictionary name" + echo " - diff format file containing paths to be added/removed from dictionary" + exit 0 +fi + +fst=$dictionary_home/$1.fst +cats=$dictionary_home/$1.cats +bin=$dictionary_home/$1.bin +shift 1 +diffs=$* + +echo running "compdic-update-fst $fst $diffs" +compdic-update-fst $fst $diffs + +echo running "compdic-fst-to-bin $fst $bin" +compdic-fst-to-bin $fst $bin + +echo running "compdic-update-cats $cats $diffs" +compdic-update-cats $cats $diffs Index: src/compdic/compdic-update-cats =================================================================== --- src/compdic/compdic-update-cats (revision 555c7f814bf87fa10240e98295372095bc422115) +++ src/compdic/compdic-update-cats (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -0,0 +1,44 @@ +#! /bin/bash + +if [ $# -lt 2 ] +then + echo "Usage:" + echo " compdic-update-cats ..." + echo "where" + echo " - file containing the list morphosyntactic categories" + echo " - diff-format file containing lines to be added to/removed from dictionary" + exit 0 +fi + +tempdir=`mktemp -d /tmp/compdic.XXXXXX` + +cats=$1 +shift + +catplus=$tempdir/plus.cat +catminus=$tempdir/minus.cat +cattmp=$tempdir/tmp.cat + +touch $catplus +touch $catminus + +while (($#)) +do + echo processing $1 ... + cat $1 | egrep '^>' | sed -r 's/^> *//;s/[[:space:]].*$//' | cut -d ',' -f 2 | sort -u | canonize >> $catplus + cat $1 | egrep '^<' | sed -r 's/^< *//;s/[[:space:]].*$//' | cut -d ',' -f 2 | sort -u | canonize >> $catminus + shift +done + +echo updating $cats ... +# comm -23 $cats $catminus > $cattmp +# mv ${cats} ${cats}~ +# cat $cattmp $catplus | sort -u > $cats + +cat $cats $catplus | sort -u > $cattmp + +mv ${cats} ${cats}~ +mv ${cattmp} ${cats} +chmod a+r ${cats} + +rm -r $tempdir Index: src/compdic/compdic-update-fst =================================================================== --- src/compdic/compdic-update-fst (revision 555c7f814bf87fa10240e98295372095bc422115) +++ src/compdic/compdic-update-fst (revision 555c7f814bf87fa10240e98295372095bc422115) @@ -0,0 +1,67 @@ +#! /bin/bash + +if [ $# -lt 2 ] +then + echo "Usage:" + echo " compdic-update-fst ..." + echo "where" + echo " - fst format automaton" + echo " - diff format file containing paths to be added/removed from dictionary" + exit 0 +fi + +tempdir=`mktemp -d /tmp/compdic.XXXXXX` + +dict=$1 +shift + +dicplus=$tempdir/plus.dic +fstplus=$tempdir/plus.fst +dicminus=$tempdir/minus.dic +fstminus=$tempdir/minus.fst + +touch $dicplus +touch $dicminus + +while (($#)) +do + echo processing $1 ... + cat $1 | egrep '^>' | sed -r 's/^> *//;s/[[:space:]].*$//' | canonize >> $dicplus + cat $1 | egrep '^<' | sed -r 's/^< *//;s/[[:space:]].*$//' | canonize >> $dicminus + shift +done + +echo updating $dict ... + +if (( `cat $dicminus | wc -l` )) +then + tmpfst1=$tempdir/tmp1.fst + echo "running compdic-dic-to-fst $dicminus $fstminus" + compdic-dic-to-fst $dicminus $fstminus + echo "running fstdifference $dict $fstminus | fstdeterminize > $tmpfst1" + fstdifference $dict $fstminus > $tmpfst1 +else + tmpfst1=$dict +fi + +if (( `cat $dicplus | wc -l` )) +then + tmpfst2=$tempdir/tmp2.fst + echo "running compdic-dic-to-fst $dicplus $fstplus" + compdic-dic-to-fst $dicplus $fstplus + echo "running fstunion $tmpfst1 $fstplus | fstdeterminize | fstminimize > $tmpfst2" + fstunion $tmpfst1 $fstplus > $tmpfst2 +else + tmpfst2=$tmpfst1 +fi + + + +echo "running mv ${dict} ${dict}~" +mv ${dict} ${dict}~ + +echo "cat ${tmpfst2} | fstrmepsilon | fstdeterminize | fstminimize > ${dict}" +cat ${tmpfst2} | fstrmepsilon | fstdeterminize | fstminimize > ${dict} +chmod a+r ${dict} + +rm -r $tempdir