fix set_device; more utils; args.opts support multi same name

deepspeech/exps/deepspeech2/model.py

@@ -403,7 +403,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()
@@ -635,7 +635,7 @@ class DeepSpeech2ExportTester(DeepSpeech2Tester):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
 

deepspeech/exps/u2/model.py

@@ -653,7 +653,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()

deepspeech/exps/u2_kaldi/model.py

@@ -637,7 +637,7 @@ class U2Tester(U2Trainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()

deepspeech/exps/u2_st/model.py

@@ -661,7 +661,7 @@ class U2STTester(U2STTrainer):
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
 
         self.setup_output_dir()
         self.setup_checkpointer()

deepspeech/training/cli.py

@@ -14,6 +14,20 @@
 import argparse
 
 
+class ExtendAction(argparse.Action):
+    """
+    [Since Python 3.8, the "extend" is available directly in stdlib]
+    (https://docs.python.org/3.8/library/argparse.html#action).
+    If you only have to support 3.8+ then defining it yourself is no longer required. 
+    Usage of stdlib "extend" action is exactly the same way as this answer originally described:
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, self.dest) or []
+        items.extend(values)
+        setattr(namespace, self.dest, items)
+
+
 def default_argument_parser():
     r"""A simple yet genral argument parser for experiments with parakeet.
 
@@ -42,6 +56,7 @@ def default_argument_parser():
         the parser
     """
     parser = argparse.ArgumentParser()
+    parser.register('action', 'extend', ExtendAction)
 
     train_group = parser.add_argument_group(
         title='Train Options', description=None)
@@ -64,10 +79,10 @@ def default_argument_parser():
         "--checkpoint_path", type=str, help="path to load checkpoint")
     train_group.add_argument(
         "--opts",
-        type=str,
-        default=[],
-        nargs='+',
-        help="overwrite --config file, passing in LIST[KEY VALUE] pairs")
+        action='extend',
+        nargs=2,
+        metavar=('key', 'val'),
+        help="overwrite --config field, passing (KEY VALUE) pairs")
     train_group.add_argument(
         "--dump-config", metavar="FILE", help="dump config to `this` file.")
 

deepspeech/training/trainer.py

@@ -119,7 +119,7 @@ class Trainer():
     def setup(self):
         """Setup the experiment.
         """
-        paddle.set_device('gpu' self.args.nprocs > 0 else 'cpu')
+        paddle.set_device('gpu' if self.args.nprocs > 0 else 'cpu')
         if self.parallel:
             self.init_parallel()
 
@@ -139,7 +139,7 @@ class Trainer():
         """A flag indicating whether the experiment should run with
         multiprocessing.
         """
-        return elf.args.nprocs > 0
+        return self.args.nprocs > 0
 
     def init_parallel(self):
         """Init environment for multiprocess training.

examples/librispeech/s1/local/test.sh

@@ -37,7 +37,8 @@ for type in attention ctc_greedy_search; do
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+    --opts decoding.decoding_method ${type} \
+    --opts decoding.batch_size ${batch_size}
 
     if [ $? -ne 0 ]; then
         echo "Failed in evaluation!"
@@ -53,7 +54,8 @@ for type in ctc_prefix_beam_search attention_rescoring; do
     --config ${config_path} \
     --result_file ${ckpt_prefix}.${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+    --opts decoding.decoding_method ${type} \
+    --opts decoding.batch_size ${batch_size}
 
     if [ $? -ne 0 ]; then
         echo "Failed in evaluation!"

examples/librispeech/s2/cmd.sh

+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentation for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# "qsub" (SGE, Torque, PBS, etc.)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partion" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by default for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/gpu.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi

examples/librispeech/s2/local/test.sh

 #!/bin/bash
 
+expdir=exp
+datadir=data
+
+lmtag=
+
+recog_set="test-clean test-other dev-clean dev-other"
+recog_set="test-clean"
+
+# bpemode (unigram or bpe)
+nbpe=5000
+bpemode=unigram
+bpeprefix="data/bpe_${bpemode}_${nbpe}"
+bpemodel=${bpeprefix}.model
+
 if [ $# != 3 ];then
     echo "usage: ${0} config_path dict_path ckpt_path_prefix"
     exit -1
@@ -8,13 +22,8 @@ fi
 ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
-device=gpu
-if [ ${ngpu} == 0 ];then
-    device=cpu
-fi
-
 config_path=$1
-dict_path=$2
+dict=$2
 ckpt_prefix=$3
 
 chunk_mode=false
@@ -30,50 +39,49 @@ echo "chunk mode ${chunk_mode}"
 #    exit 1
 #fi
 
-for type in attention ctc_greedy_search; do
-    echo "decoding ${type}"
-    if [ ${chunk_mode} == true ];then
-        # stream decoding only support batchsize=1
-        batch_size=1
-    else
-        batch_size=1
-    fi
-    python3 -u ${BIN_DIR}/test.py \
-    --model-name u2_kaldi \
-    --run-mode test \
-    --dict-path ${dict_path} \
-    --device ${device} \
-    --nproc 1 \
-    --config ${config_path} \
-    --result-file ${ckpt_prefix}.${type}.rsl \
-    --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+pids=() # initialize pids
 
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
-done
+for dmethd in attention ctc_greedy_search ctc_prefix_beam_search attention_rescoring; do
+(
+    for rtask in ${recog_set}; do
+    (
+        decode_dir=decode_${rtask}_${dmethd}_$(basename ${config_path%.*})_${lmtag}
+        feat_recog_dir=${datadir}
+        mkdir -p ${expdir}/${decode_dir}
+        mkdir -p ${feat_recog_dir}
+
+        # split data
+        split_json.sh ${feat_recog_dir}/manifest.${rtask} ${nj}
 
-for type in ctc_prefix_beam_search attention_rescoring; do
-    echo "decoding ${type}"
+        #### use CPU for decoding
+        ngpu=0
+
+        # set batchsize 0 to disable batch decoding
         batch_size=1
+        ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
             python3 -u ${BIN_DIR}/test.py \
             --model-name u2_kaldi \
             --run-mode test \
-    --dict-path ${dict_path} \
-    --device ${device} \
-    --nproc 1 \
+            --nproc ${ngpu} \
+            --dict-path ${dict} \
             --config ${config_path} \
-    --result-file ${ckpt_prefix}.${type}.rsl \
             --checkpoint_path ${ckpt_prefix} \
-    --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
+            --result-file ${expdir}/${decode_dir}/data.JOB.json \
+            --opts decoding.decoding_method ${dmethd} \
+            --opts decoding.batch_size ${batch_size} \
+            --opts data.test_manifest ${feat_recog_dir}/split${nj}/JOB/manifest.${rtask}
+
+        score_sclite.sh --bpe ${nbpe} --bpemodel ${bpemodel}.model --wer true ${expdir}/${decode_dir} ${dict}
 
-    if [ $? -ne 0 ]; then
-        echo "Failed in evaluation!"
-        exit 1
-    fi
+    ) &
+    pids+=($!) # store background pids
+    done
+) &
+pids+=($!) # store background pids
 done
 
+i=0; for pid in "${pids[@]}"; do wait ${pid} || ((++i)); done
+[ ${i} -gt 0 ] && echo "$0: ${i} background jobs are failed." && false
+echo "Finished"
 
 exit 0

examples/librispeech/s2/run.sh

 #!/bin/bash
 set -e
-source path.sh
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
 
 stage=0
 stop_stage=100

utils/README.md

+# Utils
+
+* [kaldi utils](https://github.com/kaldi-asr/kaldi/blob/cbed4ff688/egs/wsj/s5/utils)
\ No newline at end of file

utils/build_kenlm_model_from_arpa.sh

+#!/usr/bin/env bash
+# 2020 author Jiayu DU
+# Apache 2.0
+
+# This script reads in an Arpa format language model, and converts it into the
+# KenLM format language model.
+
+[ -f path.sh ] && . ./path.sh;
+
+# begin configuration section
+kenlm_opts="" # e.g. "-q 8 -b 8" for 8bits quantization
+model_type="trie" # "trie" or "probing". trie is smaller, probing is faster.
+# end configuration section
+
+. utils/parse_options.sh
+
+if [ $# != 2 ]; then
+  echo "Usage: "
+  echo "  $0 [options] <arpa-lm-path> <kenlm-path>"
+  echo "e.g.:"
+  echo "  $0 data/local/lm/4gram.arpa data/lang_test/G.trie"
+  echo "Options:"
+  echo "  --model-type can be either \"trie\" or \"probing\""
+  echo "  --kenlm-opts directly pass through to kenlm"
+  echo "    e.g. for 8bits quantization, feed \"-q 8 -b 8\""
+  exit 1;
+fi
+
+export LC_ALL=C
+
+arpa_lm=$1
+kenlm=$2
+
+if ! which build_binary >& /dev/null ; then
+  echo "$0: cannot find KenLM's build_binary tool,"
+  echo "check kenlm installation (tools/extras/install_kenlm_query_only.sh)."
+  exit 1
+fi
+
+mkdir -p $(dirname $kenlm)
+build_binary  $kenlm_opts  $model_type  $arpa_lm  $kenlm
+
+echo "$0: Successfully built arpa into kenlm format: $kenlm"
+exit 0
\ No newline at end of file

utils/parallel/run.pl

+#!/usr/bin/env perl
+use warnings; #sed replacement for -w perl parameter
+# In general, doing
+#  run.pl some.log a b c is like running the command a b c in
+# the bash shell, and putting the standard error and output into some.log.
+# To run parallel jobs (backgrounded on the host machine), you can do (e.g.)
+#  run.pl JOB=1:4 some.JOB.log a b c JOB is like running the command a b c JOB
+# and putting it in some.JOB.log, for each one. [Note: JOB can be any identifier].
+# If any of the jobs fails, this script will fail.
+
+# A typical example is:
+#  run.pl some.log my-prog "--opt=foo bar" foo \|  other-prog baz
+# and run.pl will run something like:
+# ( my-prog '--opt=foo bar' foo |  other-prog baz ) >& some.log
+#
+# Basically it takes the command-line arguments, quotes them
+# as necessary to preserve spaces, and evaluates them with bash.
+# In addition it puts the command line at the top of the log, and
+# the start and end times of the command at the beginning and end.
+# The reason why this is useful is so that we can create a different
+# version of this program that uses a queueing system instead.
+
+#use Data::Dumper;
+
+@ARGV < 2 && die "usage: run.pl log-file command-line arguments...";
+
+#print STDERR "COMMAND-LINE: " .  Dumper(\@ARGV) . "\n";
+$job_pick = 'all';
+$max_jobs_run = -1;
+$jobstart = 1;
+$jobend = 1;
+$ignored_opts = ""; # These will be ignored.
+
+# First parse an option like JOB=1:4, and any
+# options that would normally be given to
+# queue.pl, which we will just discard.
+
+for (my $x = 1; $x <= 2; $x++) { # This for-loop is to
+  # allow the JOB=1:n option to be interleaved with the
+  # options to qsub.
+  while (@ARGV >= 2 && $ARGV[0] =~ m:^-:) {
+    # parse any options that would normally go to qsub, but which will be ignored here.
+    my $switch = shift @ARGV;
+    if ($switch eq "-V") {
+      $ignored_opts .= "-V ";
+    } elsif ($switch eq "--max-jobs-run" || $switch eq "-tc") {
+      # we do support the option --max-jobs-run n, and its GridEngine form -tc n.
+      # if the command appears multiple times uses the smallest option.
+      if ( $max_jobs_run <= 0 ) {
+          $max_jobs_run =  shift @ARGV;
+      } else {
+        my $new_constraint = shift @ARGV;
+        if ( ($new_constraint < $max_jobs_run) ) {
+          $max_jobs_run = $new_constraint;
+        }
+      }
+      
+      if (! ($max_jobs_run > 0)) {
+        die "run.pl: invalid option --max-jobs-run $max_jobs_run";
+      }
+    } else {
+      my $argument = shift @ARGV;
+      if ($argument =~ m/^--/) {
+        print STDERR "run.pl: WARNING: suspicious argument '$argument' to $switch; starts with '-'\n";
+      }
+      if ($switch eq "-sync" && $argument =~ m/^[yY]/) {
+        $ignored_opts .= "-sync "; # Note: in the
+        # corresponding code in queue.pl it says instead, just "$sync = 1;".
+      } elsif ($switch eq "-pe") { # e.g. -pe smp 5
+        my $argument2 = shift @ARGV;
+        $ignored_opts .= "$switch $argument $argument2 ";
+      } elsif ($switch eq "--gpu") {
+        $using_gpu = $argument;
+      } elsif ($switch eq "--pick") {
+        if($argument =~ m/^(all|failed|incomplete)$/) {
+          $job_pick = $argument;
+        } else {
+          print STDERR "run.pl: ERROR: --pick argument must be one of 'all', 'failed' or 'incomplete'"
+        }
+      } else {
+        # Ignore option.
+        $ignored_opts .= "$switch $argument ";
+      }
+    }
+  }
+  if ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+):(\d+)$/) { # e.g. JOB=1:20
+    $jobname = $1;
+    $jobstart = $2;
+    $jobend = $3;
+    if ($jobstart > $jobend) {
+      die "run.pl: invalid job range $ARGV[0]";
+    }
+    if ($jobstart <= 0) {
+      die "run.pl: invalid job range $ARGV[0], start must be strictly positive (this is required for GridEngine compatibility).";
+    }
+    shift;
+  } elsif ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+)$/) { # e.g. JOB=1.
+    $jobname = $1;
+    $jobstart = $2;
+    $jobend = $2;
+    shift;
+  } elsif ($ARGV[0] =~ m/.+\=.*\:.*$/) {
+    print STDERR "run.pl: Warning: suspicious first argument to run.pl: $ARGV[0]\n";
+  }
+}
+
+# Users found this message confusing so we are removing it.
+# if ($ignored_opts ne "") {
+#   print STDERR "run.pl: Warning: ignoring options \"$ignored_opts\"\n";
+# }
+
+if ($max_jobs_run == -1) { # If --max-jobs-run option not set,
+                           # then work out the number of processors if possible,
+                           # and set it based on that.
+  $max_jobs_run = 0;
+  if ($using_gpu) {
+    if (open(P, "nvidia-smi -L |")) {
+      $max_jobs_run++ while (<P>);
+      close(P);
+    }
+    if ($max_jobs_run == 0) {
+      $max_jobs_run = 1;
+      print STDERR "run.pl: Warning: failed to detect number of GPUs from nvidia-smi, using ${max_jobs_run}\n";
+    }
+  } elsif (open(P, "</proc/cpuinfo")) {  # Linux
+    while (<P>) { if (m/^processor/) { $max_jobs_run++; } }
+    if ($max_jobs_run == 0) {
+      print STDERR "run.pl: Warning: failed to detect any processors from /proc/cpuinfo\n";
+      $max_jobs_run = 10;  # reasonable default.
+    }
+    close(P);
+  } elsif (open(P, "sysctl -a |")) {  # BSD/Darwin
+    while (<P>) {
+      if (m/hw\.ncpu\s*[:=]\s*(\d+)/) { # hw.ncpu = 4, or hw.ncpu: 4
+        $max_jobs_run = $1;
+        last;
+      }
+    }
+    close(P);
+    if ($max_jobs_run == 0) {
+      print STDERR "run.pl: Warning: failed to detect any processors from sysctl -a\n";
+      $max_jobs_run = 10;  # reasonable default.
+    }
+  } else {
+    # allow at most 32 jobs at once, on non-UNIX systems; change this code
+    # if you need to change this default.
+    $max_jobs_run = 32;
+  }
+  # The just-computed value of $max_jobs_run is just the number of processors
+  # (or our best guess); and if it happens that the number of jobs we need to
+  # run is just slightly above $max_jobs_run, it will make sense to increase
+  # $max_jobs_run to equal the number of jobs, so we don't have a small number
+  # of leftover jobs.
+  $num_jobs = $jobend - $jobstart + 1;
+  if (!$using_gpu &&
+      $num_jobs > $max_jobs_run && $num_jobs < 1.4 * $max_jobs_run) {
+    $max_jobs_run = $num_jobs;
+  }
+}
+
+sub pick_or_exit {
+  # pick_or_exit ( $logfile ) 
+  # Invoked before each job is started helps to run jobs selectively.
+  #
+  # Given the name of the output logfile decides whether the job must be 
+  # executed (by returning from the subroutine) or not (by terminating the
+  # process calling exit)
+  # 
+  # PRE: $job_pick is a global variable set by command line switch --pick
+  #      and indicates which class of jobs must be executed.
+  #
+  # 1) If a failed job is not executed the process exit code will indicate 
+  #    failure, just as if the task was just executed  and failed.
+  #
+  # 2) If a task is incomplete it will be executed. Incomplete may be either
+  #    a job whose log file does not contain the accounting notes in the end,
+  #    or a job whose log file does not exist.
+  #
+  # 3) If the $job_pick is set to 'all' (default behavior) a task will be
+  #    executed regardless of the result of previous attempts.
+  #
+  # This logic could have been implemented in the main execution loop
+  # but a subroutine to preserve the current level of readability of
+  # that part of the code.
+  #
+  # Alexandre Felipe, (o.alexandre.felipe@gmail.com) 14th of August of 2020
+  #
+  if($job_pick eq 'all'){
+    return; # no need to bother with the previous log
+  }
+  open my $fh, "<", $_[0] or return; # job not executed yet
+  my $log_line;
+  my $cur_line;
+  while ($cur_line = <$fh>) {
+    if( $cur_line =~ m/# Ended \(code .*/ ) {
+      $log_line = $cur_line;
+    }
+  }
+  close $fh;
+  if (! defined($log_line)){
+    return; # incomplete
+  }
+  if ( $log_line =~ m/# Ended \(code 0\).*/ ) {
+    exit(0); # complete
+  } elsif ( $log_line =~ m/# Ended \(code \d+(; signal \d+)?\).*/ ){
+    if ($job_pick !~ m/^(failed|all)$/) {
+      exit(1); # failed but not going to run
+    } else {
+      return; # failed
+    }
+  } elsif ( $log_line =~ m/.*\S.*/ ) {
+    return; # incomplete jobs are always run
+  }
+}
+
+
+$logfile = shift @ARGV;
+
+if (defined $jobname && $logfile !~ m/$jobname/ &&
+    $jobend > $jobstart) {
+  print STDERR "run.pl: you are trying to run a parallel job but "
+    . "you are putting the output into just one log file ($logfile)\n";
+  exit(1);
+}
+
+$cmd = "";
+
+foreach $x (@ARGV) {
+    if ($x =~ m/^\S+$/) { $cmd .=  $x . " "; }
+    elsif ($x =~ m:\":) { $cmd .= "'$x' "; }
+    else { $cmd .= "\"$x\" "; }
+}
+
+#$Data::Dumper::Indent=0;
+$ret = 0;
+$numfail = 0;
+%active_pids=();
+
+use POSIX ":sys_wait_h";
+for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
+  if (scalar(keys %active_pids) >= $max_jobs_run) {
+
+    # Lets wait for a change in any child's status
+    # Then we have to work out which child finished
+    $r = waitpid(-1, 0);
+    $code = $?;
+    if ($r < 0 ) { die "run.pl: Error waiting for child process"; } # should never happen.
+    if ( defined $active_pids{$r} ) {
+        $jid=$active_pids{$r};
+        $fail[$jid]=$code;
+        if ($code !=0) { $numfail++;}
+        delete $active_pids{$r};
+        # print STDERR "Finished: $r/$jid " .  Dumper(\%active_pids) . "\n";
+    } else {
+        die "run.pl: Cannot find the PID of the child process that just finished.";
+    }
+
+    # In theory we could do a non-blocking waitpid over all jobs running just
+    # to find out if only one or more jobs finished during the previous waitpid()
+    # However, we just omit this and will reap the next one in the next pass
+    # through the for(;;) cycle
+  }
+  $childpid = fork();
+  if (!defined $childpid) { die "run.pl: Error forking in run.pl (writing to $logfile)"; }
+  if ($childpid == 0) { # We're in the child... this branch
+    # executes the job and returns (possibly with an error status).
+    if (defined $jobname) {
+      $cmd =~ s/$jobname/$jobid/g;
+      $logfile =~ s/$jobname/$jobid/g;
+    }
+    # exit if the job does not need to be executed
+    pick_or_exit( $logfile );
+
+    system("mkdir -p `dirname $logfile` 2>/dev/null");
+    open(F, ">$logfile") || die "run.pl: Error opening log file $logfile";
+    print F "# " . $cmd . "\n";
+    print F "# Started at " . `date`;
+    $starttime = `date +'%s'`;
+    print F "#\n";
+    close(F);
+
+    # Pipe into bash.. make sure we're not using any other shell.
+    open(B, "|bash") || die "run.pl: Error opening shell command";
+    print B "( " . $cmd . ") 2>>$logfile >> $logfile";
+    close(B);                   # If there was an error, exit status is in $?
+    $ret = $?;
+
+    $lowbits = $ret & 127;
+    $highbits = $ret >> 8;
+    if ($lowbits != 0) { $return_str = "code $highbits; signal $lowbits" }
+    else { $return_str = "code $highbits"; }
+
+    $endtime = `date +'%s'`;
+    open(F, ">>$logfile") || die "run.pl: Error opening log file $logfile (again)";
+    $enddate = `date`;
+    chop $enddate;
+    print F "# Accounting: time=" . ($endtime - $starttime) . " threads=1\n";
+    print F "# Ended ($return_str) at " . $enddate . ", elapsed time " . ($endtime-$starttime) . " seconds\n";
+    close(F);
+    exit($ret == 0 ? 0 : 1);
+  } else {
+    $pid[$jobid] = $childpid;
+    $active_pids{$childpid} = $jobid;
+    # print STDERR "Queued: " .  Dumper(\%active_pids) . "\n";
+  }
+}
+
+# Now we have submitted all the jobs, lets wait until all the jobs finish
+foreach $child (keys %active_pids) {
+    $jobid=$active_pids{$child};
+    $r = waitpid($pid[$jobid], 0);
+    $code = $?;
+    if ($r == -1) { die "run.pl: Error waiting for child process"; } # should never happen.
+    if ($r != 0) { $fail[$jobid]=$code; $numfail++ if $code!=0; } # Completed successfully
+}
+
+# Some sanity checks:
+# The $fail array should not contain undefined codes
+# The number of non-zeros in that array  should be equal to $numfail
+# We cannot do foreach() here, as the JOB ids do not start at zero
+$failed_jids=0;
+for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) {
+  $job_return = $fail[$jobid];
+  if (not defined $job_return ) {
+    # print Dumper(\@fail);
+
+    die "run.pl: Sanity check failed: we have indication that some jobs are running " .
+      "even after we waited for all jobs to finish" ;
+  }
+  if ($job_return != 0 ){ $failed_jids++;}
+}
+if ($failed_jids != $numfail) {
+  die "run.pl: Sanity check failed: cannot find out how many jobs failed ($failed_jids x $numfail)."
+}
+if ($numfail > 0) { $ret = 1; }
+
+if ($ret != 0) {
+  $njobs = $jobend - $jobstart + 1;
+  if ($njobs == 1) {
+    if (defined $jobname) {
+      $logfile =~ s/$jobname/$jobstart/; # only one numbered job, so replace name with
+                                         # that job.
+    }
+    print STDERR "run.pl: job failed, log is in $logfile\n";
+    if ($logfile =~ m/JOB/) {
+      print STDERR "run.pl: probably you forgot to put JOB=1:\$nj in your script.";
+    }
+  }
+  else {
+    $logfile =~ s/$jobname/*/g;
+    print STDERR "run.pl: $numfail / $njobs failed, log is in $logfile\n";
+  }
+}
+
+
+exit ($ret);
\ No newline at end of file

utils/run.pl

+parallel/run.pl
\ No newline at end of file

utils/score_sclite.sh

+#!/usr/bin/env bash
+
+# Copyright 2017 Johns Hopkins University (Shinji Watanabe)
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+[ -f ./path.sh ] && . ./path.sh
+
+nlsyms=""
+wer=false
+bpe=""
+bpemodel=""
+remove_blank=true
+filter=""
+num_spkrs=1
+help_message="Usage: $0 <data-dir> <dict>"
+
+. utils/parse_options.sh
+
+if [ $# != 2 ]; then
+    echo "${help_message}"
+    exit 1;
+fi
+
+dir=$1
+dic=$2
+
+concatjson.py ${dir}/data.*.json > ${dir}/data.json
+
+if [ $num_spkrs -eq 1 ]; then
+  json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${dir}/ref.trn --hyps ${dir}/hyp.trn
+
+  if ${remove_blank}; then
+      sed -i.bak2 -r 's/<blank> //g' ${dir}/hyp.trn
+  fi
+  if [ -n "${nlsyms}" ]; then
+      cp ${dir}/ref.trn ${dir}/ref.trn.org
+      cp ${dir}/hyp.trn ${dir}/hyp.trn.org
+      filt.py -v ${nlsyms} ${dir}/ref.trn.org > ${dir}/ref.trn
+      filt.py -v ${nlsyms} ${dir}/hyp.trn.org > ${dir}/hyp.trn
+  fi
+  if [ -n "${filter}" ]; then
+      sed -i.bak3 -f ${filter} ${dir}/hyp.trn
+      sed -i.bak3 -f ${filter} ${dir}/ref.trn
+  fi
+
+  sclite -r ${dir}/ref.trn trn -h ${dir}/hyp.trn trn -i rm -o all stdout > ${dir}/result.txt
+
+  echo "write a CER (or TER) result in ${dir}/result.txt"
+  grep -e Avg -e SPKR -m 2 ${dir}/result.txt
+
+  if ${wer}; then
+      if [ -n "$bpe" ]; then
+  	    spm_decode --model=${bpemodel} --input_format=piece < ${dir}/ref.trn | sed -e "s/▁/ /g" > ${dir}/ref.wrd.trn
+  	    spm_decode --model=${bpemodel} --input_format=piece < ${dir}/hyp.trn | sed -e "s/▁/ /g" > ${dir}/hyp.wrd.trn
+      else
+  	    sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/ref.trn > ${dir}/ref.wrd.trn
+  	    sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/hyp.trn > ${dir}/hyp.wrd.trn
+      fi
+      sclite -r ${dir}/ref.wrd.trn trn -h ${dir}/hyp.wrd.trn trn -i rm -o all stdout > ${dir}/result.wrd.txt
+
+      echo "write a WER result in ${dir}/result.wrd.txt"
+      grep -e Avg -e SPKR -m 2 ${dir}/result.wrd.txt
+  fi
+elif [ ${num_spkrs} -lt 4 ]; then
+  ref_trns=""
+  hyp_trns=""
+  for i in $(seq ${num_spkrs}); do
+      ref_trns=${ref_trns}"${dir}/ref${i}.trn "
+      hyp_trns=${hyp_trns}"${dir}/hyp${i}.trn "
+  done
+  json2trn.py ${dir}/data.json ${dic} --num-spkrs ${num_spkrs} --refs ${ref_trns} --hyps ${hyp_trns}
+
+  for n in $(seq ${num_spkrs}); do
+      if ${remove_blank}; then
+          sed -i.bak2 -r 's/<blank> //g' ${dir}/hyp${n}.trn
+      fi
+      if [ -n "${nlsyms}" ]; then
+          cp ${dir}/ref${n}.trn ${dir}/ref${n}.trn.org
+          cp ${dir}/hyp${n}.trn ${dir}/hyp${n}.trn.org
+          filt.py -v ${nlsyms} ${dir}/ref${n}.trn.org > ${dir}/ref${n}.trn
+          filt.py -v ${nlsyms} ${dir}/hyp${n}.trn.org > ${dir}/hyp${n}.trn
+      fi
+      if [ -n "${filter}" ]; then
+          sed -i.bak3 -f ${filter} ${dir}/hyp${n}.trn
+          sed -i.bak3 -f ${filter} ${dir}/ref${n}.trn
+      fi
+  done
+
+  results_str=""
+  for (( i=0; i<$((num_spkrs * num_spkrs)); i++ )); do
+      ind_r=$((i / num_spkrs + 1))
+      ind_h=$((i % num_spkrs + 1))
+      results_str=${results_str}"${dir}/result_r${ind_r}h${ind_h}.txt "
+      sclite -r ${dir}/ref${ind_r}.trn trn -h ${dir}/hyp${ind_h}.trn trn -i rm -o all stdout > ${dir}/result_r${ind_r}h${ind_h}.txt
+  done
+
+  echo "write CER (or TER) results in ${dir}/result_r*h*.txt"
+  eval_perm_free_error.py --num-spkrs ${num_spkrs} \
+      ${results_str} > ${dir}/min_perm_result.json
+  sed -n '2,4p' ${dir}/min_perm_result.json
+
+  if ${wer}; then
+      for n in $(seq ${num_spkrs}); do
+          if [ -n "$bpe" ]; then
+              spm_decode --model=${bpemodel} --input_format=piece < ${dir}/ref${n}.trn | sed -e "s/▁/ /g" > ${dir}/ref${n}.wrd.trn
+              spm_decode --model=${bpemodel} --input_format=piece < ${dir}/hyp${n}.trn | sed -e "s/▁/ /g" > ${dir}/hyp${n}.wrd.trn
+          else
+              sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/ref${n}.trn > ${dir}/ref${n}.wrd.trn
+              sed -e "s/ //g" -e "s/(/ (/" -e "s/<space>/ /g" ${dir}/hyp${n}.trn > ${dir}/hyp${n}.wrd.trn
+          fi
+      done
+      results_str=""
+      for (( i=0; i<$((num_spkrs * num_spkrs)); i++ )); do
+          ind_r=$((i / num_spkrs + 1))
+          ind_h=$((i % num_spkrs + 1))
+          results_str=${results_str}"${dir}/result_r${ind_r}h${ind_h}.wrd.txt "
+          sclite -r ${dir}/ref${ind_r}.wrd.trn trn -h ${dir}/hyp${ind_h}.wrd.trn trn -i rm -o all stdout > ${dir}/result_r${ind_r}h${ind_h}.wrd.txt
+      done
+
+      echo "write WER results in ${dir}/result_r*h*.wrd.txt"
+      eval_perm_free_error.py --num-spkrs ${num_spkrs} \
+          ${results_str} > ${dir}/min_perm_result.wrd.json
+      sed -n '2,4p' ${dir}/min_perm_result.wrd.json
+  fi
+fi

utils/spk2utt_to_utt2spk.pl

+#!/usr/bin/env perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+while(<>){ 
+    @A = split(" ", $_);
+    @A > 1 || die "Invalid line in spk2utt file: $_";
+    $s = shift @A;
+    foreach $u ( @A ) {
+        print "$u $s\n";
+    }
+}

utils/split_data.sh

+#!/usr/bin/env bash
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+
+if [ $# != 2 ]; then
+  echo "Usage: split_data.sh data-dir num-to-split"
+  exit 1
+fi
+
+data=$1
+numsplit=$2
+
+if [ $numsplit -le 0 ]; then
+  echo "Invalid num-split argument $numsplit";
+  exit 1;
+fi
+
+n=0;
+feats=""
+wavs=""
+utt2spks=""
+texts=""
+
+nu=`cat $data/utt2spk | wc -l`
+nf=`cat $data/feats.scp | wc -l`
+nt=`cat $data/text | wc -l`
+if [ $nu -ne $nf ]; then
+  echo "split_data.sh: warning, #lines is (utt2spk,feats.scp) is ($nu,$nf);"
+  echo "this script may produce incorrectly split data."
+  echo "use utils/fix_data_dir.sh to fix this."
+fi
+if [ $nt -ne 0 -a $nu -ne $nt ]; then
+  echo "split_data.sh: warning, #lines is (utt2spk,text) is ($nu,$nt);"
+  echo "this script may produce incorrectly split data."
+  echo "use utils/fix_data_dir.sh to fix this."
+fi
+
+# utilsscripts/get_split.pl returns "0 1 2 3" or "00 01 .. 18 19" or whatever.
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  mkdir -p $data/split$numsplit/$n
+  feats="$feats $data/split$numsplit/$n/feats.scp"
+  wavs="$wavs $data/split$numsplit/$n/wav.scp"
+  texts="$texts $data/split$numsplit/$n/text"
+  utt2spks="$utt2spks $data/split$numsplit/$n/utt2spk"
+done
+
+split_scp.pl --utt2spk=$data/utt2spk $data/utt2spk $utt2spks
+split_scp.pl --utt2spk=$data/utt2spk $data/feats.scp $feats
+[ -f $data/wav.scp ] && \
+  split_scp.pl --utt2spk=$data/utt2spk $data/wav.scp $wavs
+[ -f $data/text ] && \
+  split_scp.pl --utt2spk=$data/utt2spk $data/text $texts
+
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  utt2spk_to_spk2utt.pl $data/split$numsplit/$n/utt2spk \
+    > $data/split$numsplit/$n/spk2utt
+  # for completeness, also split the spk2gender file
+  [ -f $data/spk2gender ] && \
+    filter_scp.pl $data/split$numsplit/$n/spk2utt $data/spk2gender \
+    > $data/split$numsplit/$n/spk2gender 
+done
+
+exit 0
\ No newline at end of file

utils/split_json.sh

+#!/usr/bin/env bash
+set -o errexit
+
+if [ $# != 2 ]; then
+  echo "Usage: split_data.sh manifest num-to-split"
+  exit 1
+fi
+
+data=data
+
+jsonfile=$1
+numsplit=$2
+
+if [ $numsplit -le 0 ]; then
+  echo "Invalid num-split argument $numsplit";
+  exit 1;
+fi
+
+n=0;
+jsons=""
+
+# utilsscripts/get_split.pl returns "0 1 2 3" or "00 01 .. 18 19" or whatever.
+# for n in `get_splits.pl $numsplit`; do
+for n in `seq 1 $numsplit`; do  # Changed this to usual number sequence -Arnab
+  mkdir -p $data/split$numsplit/$n
+  jsons="$jsons $data/split$numsplit/$n/${jsonfile}"
+done
+
+split_scp.pl $data/${jsonfile} $jsons
+
+exit 0
\ No newline at end of file

utils/split_scp.pl

+#!/usr/bin/env perl
+use warnings; #sed replacement for -w perl parameter
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+# This program splits up any kind of .scp or archive-type file.
+# If there is no utt2spk option it will work on any text  file and
+# will split it up with an approximately equal number of lines in
+# each but.
+# With the --utt2spk option it will work on anything that has the 
+# utterance-id as the first entry on each line; the utt2spk file is
+# of the form "utterance speaker" (on each line).
+# It splits it into equal size chunks as far as it can.  If you use
+# the utt2spk option it will make sure these chunks coincide with
+# speaker boundaries.  In this case, if there are more chunks
+# than speakers (and in some other circumstances), some of the 
+# resulting  chunks will be empty and it
+# will print a warning.
+# You will normally call this like:
+# split_scp.pl scp scp.1 scp.2 scp.3 ...
+# or
+# split_scp.pl --utt2spk=utt2spk scp scp.1 scp.2 scp.3 ...
+# Note that you can use this script to split the utt2spk file itself,
+# e.g. split_scp.pl --utt2spk=utt2spk utt2spk utt2spk.1 utt2spk.2 ...
+
+# You can also call the scripts like:
+# split_scp.pl -j 3 0 scp scp.0
+# [note: with this option, it assumes zero-based indexing of the split parts,
+# i.e. the second number must be 0 <= n < num-jobs.]
+
+$num_jobs = 0;
+$job_id = 0;
+$utt2spk_file = "";
+
+for ($x = 1; $x <= 2; $x++) {
+    if ($ARGV[0] eq "-j") {
+        shift @ARGV;
+        $num_jobs = shift @ARGV;
+        $job_id = shift @ARGV;
+        if ($num_jobs <= 0 || $job_id < 0 || $job_id >= $num_jobs) {
+            die "Invalid num-jobs and job-id: $num_jobs and $job_id";
+        }
+    }
+    if ($ARGV[0] =~ "--utt2spk=(.+)") {
+        $utt2spk_file=$1;
+        shift;
+    }
+}
+
+if(($num_jobs == 0 && @ARGV < 2) || ($num_jobs > 0 && (@ARGV < 1 || @ARGV > 2))) {
+    die "Usage: split_scp.pl [--utt2spk=<utt2spk_file>] in.scp out1.scp out2.scp ... \n" .
+        " or: split_scp.pl -j num-jobs job-id [--utt2spk=<utt2spk_file>] in.scp [out.scp]\n" .
+        " ... where 0 <= job-id < num-jobs.";
+}
+   
+$inscp = shift @ARGV;
+if ($num_jobs == 0) { # without -j option
+    @OUTPUTS = @ARGV;
+} else {
+    for ($j = 0; $j < $num_jobs; $j++) {
+        if ($j == $job_id) { 
+            if (@ARGV > 0) { push @OUTPUTS, $ARGV[0]; }
+            else { push @OUTPUTS, "-"; }
+        } else {
+            push @OUTPUTS, "/dev/null";
+        }
+    }
+} 
+
+if ($utt2spk_file ne "") {  # We have the --utt2spk option...
+    open(U, "<$utt2spk_file") || die "Failed to open utt2spk file $utt2spk_file";
+    while(<U>) {
+        @A = split;
+        @A == 2 || die "Bad line $_ in utt2spk file $utt2spk_file";
+        ($u,$s) = @A;
+        $utt2spk{$u} = $s;
+    }
+    open(I, "<$inscp") || die "Opening input scp file $inscp";
+    @spkrs = ();
+    while(<I>) {
+        @A = split;
+        if(@A == 0) { die "Empty or space-only line in scp file $inscp"; }
+        $u = $A[0];
+        $s = $utt2spk{$u};
+        if(!defined $s) { die "No such utterance $u in utt2spk file $utt2spk_file"; }
+        if(!defined $spk_count{$s}) { 
+            push @spkrs, $s; 
+            $spk_count{$s} = 0;
+            $spk_data{$s} = "";
+        }
+        $spk_count{$s}++;
+        $spk_data{$s} = $spk_data{$s} . $_;
+    }
+    # Now split as equally as possible ..
+    # First allocate spks to files by allocating an approximately
+    # equal number of speakers.
+    $numspks = @spkrs;  # number of speakers.
+    $numscps = @OUTPUTS; # number of output files.
+    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+        $scparray[$scpidx] = []; # [] is array reference.
+    }
+    for ($spkidx = 0; $spkidx < $numspks; $spkidx++) {
+        $scpidx = int(($spkidx*$numscps) / $numspks);
+        $spk = $spkrs[$spkidx];
+        push @{$scparray[$scpidx]}, $spk;
+        $scpcount[$scpidx] += $spk_count{$spk};
+    }
+
+    # Now will try to reassign beginning + ending speakers
+    # to different scp's and see if it gets more balanced.
+    # Suppose objf we're minimizing is sum_i (num utts in scp[i] - average)^2.
+    # We can show that if considering changing just 2 scp's, we minimize
+    # this by minimizing the squared difference in sizes.  This is
+    # equivalent to minimizing the absolute difference in sizes.  This
+    # shows this method is bound to converge.
+
+    $changed = 1;
+    while($changed) {
+        $changed = 0;
+        for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+            # First try to reassign ending spk of this scp.
+            if($scpidx < $numscps-1) {
+                $sz = @{$scparray[$scpidx]};
+                if($sz > 0) {
+                    $spk = $scparray[$scpidx]->[$sz-1];
+                    $count = $spk_count{$spk};
+                    $nutt1 = $scpcount[$scpidx];
+                    $nutt2 = $scpcount[$scpidx+1];
+                    if( abs( ($nutt2+$count) - ($nutt1-$count))
+                        < abs($nutt2 - $nutt1))  { # Would decrease
+                        # size-diff by reassigning spk...
+                        $scpcount[$scpidx+1] += $count;
+                        $scpcount[$scpidx] -= $count;
+                        pop @{$scparray[$scpidx]};
+                        unshift @{$scparray[$scpidx+1]}, $spk;
+                        $changed = 1;
+                    }
+                }
+            }
+            if($scpidx > 0 && @{$scparray[$scpidx]} > 0) {
+                $spk = $scparray[$scpidx]->[0];
+                $count = $spk_count{$spk};
+                $nutt1 = $scpcount[$scpidx-1];
+                $nutt2 = $scpcount[$scpidx];
+                if( abs( ($nutt2-$count) - ($nutt1+$count))
+                    < abs($nutt2 - $nutt1))  { # Would decrease
+                    # size-diff by reassigning spk...
+                    $scpcount[$scpidx-1] += $count;
+                    $scpcount[$scpidx] -= $count;
+                    shift @{$scparray[$scpidx]};
+                    push @{$scparray[$scpidx-1]}, $spk;
+                    $changed = 1;
+                }
+            }
+        }
+    }
+    # Now print out the files...
+    for($scpidx = 0; $scpidx < $numscps; $scpidx++) {
+        $scpfn = $OUTPUTS[$scpidx];
+        open(F, ">$scpfn") || die "Could not open scp file $scpfn for writing.";
+        $count = 0;
+        if(@{$scparray[$scpidx]} == 0) {
+            print STDERR "Warning: split_scp.pl producing empty .scp file $scpfn (too many splits and too few speakers?)\n";
+        } else {
+            foreach $spk ( @{$scparray[$scpidx]} ) {
+                print F $spk_data{$spk};
+                $count += $spk_count{$spk};
+            }
+            if($count != $scpcount[$scpidx]) { die "Count mismatch [code error]"; }
+        }
+        close(F);
+    }
+} else { 
+   # This block is the "normal" case where there is no --utt2spk 
+   # option and we just break into equal size chunks.
+
+    open(I, "<$inscp") || die "Opening input scp file $inscp";
+
+    $numscps = @OUTPUTS;  # size of array.
+    @F = ();
+    while(<I>) {
+        push @F, $_;
+    }
+    $numlines = @F;
+    if($numlines == 0) {
+        print STDERR "split_scp.pl: warning: empty input scp file $inscp";
+    }
+    $linesperscp = int( ($numlines+($numscps-1)) / $numscps); # the +$(numscps-1) forces rounding up.
+# [just doing int() rounds down].
+    for($scpidx = 0; $scpidx < @OUTPUTS; $scpidx++) {
+        $scpfile = $OUTPUTS[$scpidx];
+        open(O, ">$scpfile") || die "Opening output scp file $scpfile";
+        for($n = $linesperscp * $scpidx; $n < $numlines && $n < $linesperscp*($scpidx+1); $n++) {
+            print O $F[$n];
+        }
+        close(O) || die "Closing scp file $scpfile";
+    }
+}
\ No newline at end of file

utils/train_arpa_with_kenlm.sh

+#!/usr/bin/env bash
+
+# 2020 Author Jiayu DU
+# Apache 2.0
+
+# This script uses kenlm to estimate an arpa model from plain text,
+# it is a resort when you hit memory limit dealing with large corpus
+# kenlm estimates arpa using on-disk structure,
+# as long as you have big enough hard disk, memory shouldn't be a problem.
+# by default, kenlm use up to 50% of your local memory,
+# you can control this through -S option
+
+[ -f path.sh ] && . ./path.sh;
+
+kenlm_opts="" # e.g. "-o 4 -S 50% --prune 0 5 7 7"
+
+if [ $# != 4 ]; then
+  echo "$0 <text> <kaldi_symbol_table> <working_dir> <arpa_name>"
+  echo "e.g. $0 train.txt words.txt wdir 4gram"
+  exit 1
+fi
+
+text=$1
+symbol_table=$2
+dir=$3
+arpa_name=$4
+
+if ! which lmplz >& /dev/null ; then
+  echo "$0: cannot find training tool *lmplz*."
+  echo "tools/extras/install_kenlm_query_only.sh installs kenlm at tools/kenlm"
+  echo "it only supports runtime mode, to actually train an arpa using KenLM,"
+  echo "you need a complete KenLM installation(depends on EIGEN and BOOST),"
+  echo "follow KenLM's building instructions at (https://github.com/kpu/kenlm)"
+  exit 1
+fi
+
+# the text should be properly pre-processed, e.g:
+#   cleand, normalized and possibly word-segmented
+
+# get rid off irrelavent symbols
+grep -v '<eps>' $symbol_table \
+  | grep -v '#0' \
+  | grep -v '<unk>' | grep -v '<UNK>' \
+  | grep -v '<s>' | grep -v '</s>' \
+  | awk '{print $1}' \
+  > $dir/ngram.vocab
+
+# To make sure that kenlm & kaldi have strictly the same vocabulary:
+# 1. feed vocabulary into kenlm via --limit_vocab_file
+# 2. cat vocabulary to training text, so each word at least appear once
+# 
+# TL;DR reason:
+# Unlike SRILM's -limit-vocab, kenlm's --limit_vocab_file option 
+# spcifies a *valid* set of vocabulary, whereas *valid but unseen* 
+# words are discarded in final arpa.
+# So the trick is, 
+# we explicitly add kaldi's vocab(one word per line) to training text, 
+# making each word appear at least once.
+# kenlm never prunes unigram, 
+# so this always generates consistent kenlm vocabuary as kaldi has.
+# The effect of this is like add-one smoothing to unigram counts,
+# shouldn't have significant impacts in practice.
+cat $dir/ngram.vocab $text \
+  | lmplz $kenlm_opts --limit_vocab_file $dir/ngram.vocab \
+  > $dir/${arpa_name}.arpa
+
+echo "$0: Done training arpa to: $dir/${arpa_name}.arpa"
\ No newline at end of file

utils/utt2spk_to_spk2utt.pl

+#!/usr/bin/env perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# converts an utt2spk file to a spk2utt file.
+# Takes input from the stdin or from a file argument;
+# output goes to the standard out.
+
+if ( @ARGV > 1 ) {
+    die "Usage: utt2spk_to_spk2utt.pl [ utt2spk ] > spk2utt";
+}
+
+while(<>){ 
+    @A = split(" ", $_);
+    @A == 2 || die "Invalid line in utt2spk file: $_";
+    ($u,$s) = @A;
+    if(!$seen_spk{$s}) {
+        $seen_spk{$s} = 1;
+        push @spklist, $s;
+    }
+    push (@{$spk_hash{$s}}, "$u");
+}
+foreach $s (@spklist) {
+    $l = join(' ',@{$spk_hash{$s}});
+    print "$s $l\n";
+}
\ No newline at end of file