diff --git a/speechx/examples/ds2_ol/aishell/run_build_tlg.sh b/speechx/examples/ds2_ol/aishell/run_build_tlg.sh index 68a31de4fd411cd6614d45649ad77d57df541010..4394ac5a084c5f296e6861357da0a857cc17ea1e 100755 --- a/speechx/examples/ds2_ol/aishell/run_build_tlg.sh +++ b/speechx/examples/ds2_ol/aishell/run_build_tlg.sh @@ -55,7 +55,7 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then cp $unit data/local/dict/units.txt if [ ! -f $lexicon ];then - local/text_to_lexicon.py --has_key true --text $text --lexicon $lexicon + utils/text_to_lexicon.py --has_key true --text $text --lexicon $lexicon echo "Generate $lexicon from $text" fi diff --git a/utils/text_to_lexicon.py b/utils/text_to_lexicon.py new file mode 100755 index 0000000000000000000000000000000000000000..ba5ab60ac70335c8f7d199ed4d3f6ac942dd3106 --- /dev/null +++ b/utils/text_to_lexicon.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import argparse +from collections import Counter + + +def main(args): + counter = Counter() + with open(args.text, 'r') as fin, open(args.lexicon, 'w') as fout: + for line in fin: + line = line.strip() + if args.has_key: + utt, text = line.split(maxsplit=1) + words = text.split() + else: + words = line.split() + + counter.update(words) + + for word in counter: + val = " ".join(list(word)) + fout.write(f"{word}\t{val}\n") + fout.flush() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='text(line:utt1 中国 人) to lexicon(line:中国 中 国).') + parser.add_argument( + '--has_key', default=True, help='text path, with utt or not') + parser.add_argument( + '--text', required=True, help='text path. line: utt1 中国 人 or 中国 人') + parser.add_argument( + '--lexicon', required=True, help='lexicon path. line:中国 中 国') + args = parser.parse_args() + print(args) + + main(args)