Merge branch 'develop' of upstream into ctc_decoder_deploy

README.md

@@ -419,13 +419,56 @@ python deploy/demo_server.py --help
 python deploy/demo_client.py --help
 ```
 
+## Released Models
+
+#### Speech Model Released
+
+Language  | Model Name | Training Data | Training Hours
+:-----------: | :------------: | :----------: |  -------:
+English  | [LibriSpeech Model](http://cloud.dlnel.org/filepub/?uuid=17404caf-cf19-492f-9707-1fad07c19aae) | [LibriSpeech Dataset](http://www.openslr.org/12/) | 960 h
+English  | [Internal English Model](to-be-added) | Baidu English Dataset | 8000 h
+Mandarin | [Aishell Model](http://cloud.dlnel.org/filepub/?uuid=6c83b9d8-3255-4adf-9726-0fe0be3d0274) | [Aishell Dataset](http://www.openslr.org/33/) | 151 h
+Mandarin | [Internal Mandarin Model](to-be-added) | Baidu Mandarin Dataset | 2917 h
+
+#### Language Model Released
+
+Language Model | Training Data | Token-based | Size | Filter Configuraiton
+:-------------:| :------------:| :-----: | -----: | -----------------:
+[English LM (Median)](http://paddlepaddle.bj.bcebos.com/model_zoo/speech/common_crawl_00.prune01111.trie.klm) |  To Be Added | Word-based | 8.3 GB | To Be Added
+[English LM (Big)](to-be-added) |  To Be Added | Word-based | X.X GB | To Be Added
+[Mandarin LM (Median)](http://cloud.dlnel.org/filepub/?uuid=d21861e4-4ed6-45bb-ad8e-ae417a43195e) |  To Be Added | Character-based | 2.8 GB | To Be Added
+[Mandarin LM (Big)](to-be-added) |  To Be Added | Character-based | X.X GB | To Be Added
+
 ## Experiments and Benchmarks
 
-TODO: to be added
+#### English Model Evaluation (Word Error Rate)
 
-## Released Models
+Test Set                | LibriSpeech Model | Internal English Model
+:---------------------: | :---------------: | :-------------------:
+LibriSpeech-Test-Clean  |   7.9             |   X.X
+LibriSpeech-Test-Other  |   X.X             |   X.X
+VoxForge-Test           |   X.X             |   X.X
+Baidu-English-Test      |   X.X             |   X.X
 
-TODO: to be added
+#### English Model Evaluation (Character Error Rate)
+
+Test Set                | LibriSpeech Model | Internal English Model
+:---------------------: | :---------------: | :-------------------:
+LibriSpeech-Test-Clean  |   X.X             |   X.X
+LibriSpeech-Test-Other  |   X.X             |   X.X
+VoxForge-Test           |   X.X             |   X.X
+Baidu-English-Test      |   X.X             |   X.X
+
+#### Mandarin Model Evaluation (Character Error Rate)
+
+Test Set                | Aishell Model     | Internal Mandarin Model
+:---------------------: | :---------------: | :-------------------:
+Aishell-Test            |   X.X             |   X.X
+Baidu-Mandarin-Test     |   X.X             |   X.X
+
+#### Multiple GPU Efficiency
+
+TODO: To Be Added
 
 ## Questions and Help
 

cloud/pcloud_submit.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 TRAIN_MANIFEST="cloud/cloud_manifests/cloud.manifest.train"
 DEV_MANIFEST="cloud/cloud_manifests/cloud.manifest.dev"

cloud/pcloud_train.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 TRAIN_MANIFEST=$1
 DEV_MANIFEST=$2

cloud/pcloud_upload_data.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 mkdir cloud_manifests
 

deploy/demo_server.py

@@ -100,7 +100,7 @@ class AsrRequestHandler(SocketServer.BaseRequestHandler):
         finish_time = time.time()
         print("Response Time: %f, Transcript: %s" %
               (finish_time - start_time, transcript))
-        self.request.sendall(transcript)
+        self.request.sendall(transcript.encode('utf-8'))
 
     def _write_to_file(self, data):
         # prepare save dir and filename

examples/librispeech/run_data.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_infer.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_infer_golden.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_test.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_test_golden.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_train.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/librispeech/run_tune.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/mandarin/run_demo_client.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/mandarin/run_demo_server.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 # TODO: replace the model with a mandarin model
 
 pushd ../.. > /dev/null

examples/tiny/run_data.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_infer.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_infer_golden.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_test.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_test_golden.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_train.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

examples/tiny/run_tune.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 pushd ../.. > /dev/null
 

model_utils/model.py

@@ -8,6 +8,7 @@ import os
 import time
 import logging
 import gzip
+from distutils.dir_util import mkpath
 import paddle.v2 as paddle
 from decoders.swig_wrapper import Scorer
 from decoders.swig_wrapper import ctc_greedy_decoder
@@ -85,7 +86,7 @@ class DeepSpeech2Model(object):
         """
         # prepare model output directory
         if not os.path.exists(output_model_dir):
-            os.mkdir(output_model_dir)
+            mkpath(output_model_dir)
 
         # prepare optimizer and trainer
         optimizer = paddle.optimizer.Adam(

models/aishell/download_model.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 source ../../utils/utility.sh
 

models/librispeech/download_model.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 source ../../utils/utility.sh
 

models/lm/download_lm_ch.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 source ../../utils/utility.sh
 

models/lm/download_lm_en.sh

-#! /usr/bin/bash
+#! /usr/bin/env bash
 
 source ../../utils/utility.sh
 

setup.sh

-#!/bin/bash
+#! /usr/bin/env bash
 
 # install python dependencies
 if [ -f "requirements.txt" ]; then