parallel data scripts; more mask test; need pybind11 repo

.gitignore

 .DS_Store
 *.pyc
 .vscode
-*.log
+*log
 *.pdmodel
 *.pdiparams*
 *.zip

deepspeech/exps/deepspeech2/model.py

@@ -170,7 +170,7 @@ class DeepSpeech2Trainer(Trainer):
             train_dataset,
             batch_sampler=batch_sampler,
             collate_fn=collate_fn,
-            num_workers=config.data.num_workers, )
+            num_workers=config.data.num_workers)
         self.valid_loader = DataLoader(
             dev_dataset,
             batch_size=config.data.batch_size,

examples/aishell/s0/local/data.sh

@@ -66,6 +66,7 @@ fi
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # format manifest with tokenids, vocab size
     for dataset in train dev test; do
+    {
         python3 ${MAIN_ROOT}/utils/format_data.py \
                 --feat_type "raw" \
                 --cmvn_path "data/mean_std.json" \
@@ -73,12 +74,14 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
                 --vocab_path="data/vocab.txt" \
                 --manifest_path="data/manifest.${dataset}.raw" \
                 --output_path="data/manifest.${dataset}"
-    done
 
         if [ $? -ne 0 ]; then
                 echo "Formt mnaifest failed. Terminated."
                 exit 1
         fi
+    } &
+    done
+    wait
 fi
 
 echo "Aishell data preparation done."

examples/aishell/s1/local/data.sh

@@ -67,6 +67,7 @@ fi
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # format manifest with tokenids, vocab size
     for dataset in train dev test; do
+    {
         python3 ${MAIN_ROOT}/utils/format_data.py \
             --feat_type "raw" \
             --cmvn_path "data/mean_std.json" \
@@ -74,12 +75,14 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
             --vocab_path="data/vocab.txt" \
             --manifest_path="data/manifest.${dataset}.raw" \
             --output_path="data/manifest.${dataset}"
-    done
 
         if [ $? -ne 0 ]; then
             echo "Formt mnaifest failed. Terminated."
             exit 1
         fi
+    } &
+    done
+    wait
 fi
 
 echo "Aishell data preparation done."

requirements.txt

@@ -9,3 +9,4 @@ sox
 tensorboardX
 typeguard
 yacs
+pybind11

tests/mask_test.py

@@ -48,7 +48,9 @@ class TestU2Model(unittest.TestCase):
 
     def test_make_pad_mask(self):
         res = make_pad_mask(self.lengths)
+        res1 = make_non_pad_mask(self.lengths).logical_not()
         self.assertSequenceEqual(res.numpy().tolist(), self.pad_masks.tolist())
+        self.assertSequenceEqual(res.numpy().tolist(), res1.tolist())
 
 
 if __name__ == '__main__':