add tarball utils (#626)

deepspeech/exps/u2/bin/alignment.py

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Alignment for U2 model."""
+from deepspeech.exps.u2.config import get_cfg_defaults
+from deepspeech.exps.u2.model import U2Tester as Tester
+from deepspeech.training.cli import default_argument_parser
+from deepspeech.utils.utility import print_arguments
+
+
+def main_sp(config, args):
+    exp = Tester(config, args)
+    exp.setup()
+    exp.run_align()
+
+
+def main(config, args):
+    main_sp(config, args)
+
+
+if __name__ == "__main__":
+    parser = default_argument_parser()
+    args = parser.parse_args()
+    print_arguments(args, globals())
+
+    # https://yaml.org/type/float.html
+    config = get_cfg_defaults()
+    if args.config:
+        config.merge_from_file(args.config)
+    if args.opts:
+        config.merge_from_list(args.opts)
+    config.freeze()
+    print(config)
+    if args.dump_config:
+        with open(args.dump_config, 'w') as f:
+            print(config, file=f)
+
+    main(config, args)

deepspeech/frontend/normalizer.py

@@ -179,8 +179,7 @@ class FeatureNormalizer(object):
                 wav_number += batch_size
 
                 if wav_number % 1000 == 0:
-                    logger.info('process {} wavs,{} frames'.format(wav_number,
-                                                                   all_number))
+                    logger.info(f'process {wav_number} wavs,{all_number} frames.')
 
         self.cmvn_info = {
             'mean_stat': list(all_mean_stat.tolist()),

env.sh

 export MAIN_ROOT=${PWD}
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/aishell/s0/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
-export PYTHONIOENCODING=UTF-8 
+export PYTHONIOENCODING=UTF-8
 export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
 
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/

examples/aishell/s1/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/librispeech/s0/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/librispeech/s1/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/spm/path.sh

 export MAIN_ROOT=${PWD}/../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/tiny/s0/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

examples/tiny/s1/path.sh

 export MAIN_ROOT=${PWD}/../../../
 
-export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH}
+export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
 export LC_ALL=C
 
 # Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C

requirements.txt

@@ -10,3 +10,4 @@ tensorboardX
 typeguard
 yacs
 pybind11
+textgrid

utils/tarball.sh

+#!/bin/bash
+
+if [ $# != 4 ];then
+    echo "usage: $0 ckpt_prefix model_config mean_std vocab"
+    exit -1
+fi
+
+ckpt_prefix=$1
+model_config=$2
+mean_std=$3
+vocab=$4
+
+output=release
+
+mkdir -p ${output}
+function clean() {
+    rm -rf ${output}
+}
+trap clean EXIT
+
+cp ${ckpt_prefix}.* ${output}
+cp ${model_config} ${mean_std} ${vocab} ${output}
+
+tar zcvf release.tar.gz ${output}
+
+echo "tarball done!"