Some ch2 and 8 changes.

.dockerignore

 Dockerfile
+.git/
+.gitignore

.gitignore

@@ -6,3 +6,4 @@ pandoc.template
 py_env*
 *.ipynb
 build
+Dockerfile

.pre-commit-config.yaml

@@ -35,8 +35,8 @@
     hooks:
     -   id: convert-markdown-into-html
         name: convert-markdown-into-html
-        description: Convert README.md into index.html and README.en.md into index.en.html
+        description: Convert README.md into index.html and README.cn.md into index.cn.html
         entry: python .pre-commit-hooks/convert_markdown_into_html.py
         language: system
-        files: .+README(\.en)?\.md$
+        files: .+README(\.cn)?\.md$
 

.tools/build_docker.sh

 #!/bin/bash
+set -xe
+
 cur_path="$(cd "$(dirname "$0")" && pwd -P)"
-cd $cur_path/../
+cd "$cur_path"/../
 
-#convert md to ipynb
-.tools/convert-markdown-into-ipynb-and-test.sh
+#paddle production image name
+if [ ! -n "$1" ]; then
+  paddle_image=paddlepaddle/paddle
+else
+  paddle_image=$1
+fi
+
+#paddle production image tag
+if [ ! -n "$2" ]; then
+  paddle_tag=0.10.0
+else
+  paddle_tag=$2
+fi
+
+#paddle book image name
+if [ ! -n "$3" ]; then
+  book_image=paddlepaddle/book
+else
+  book_image=$3
+fi
 
-paddle_tag=0.10.0rc2
-book_tag=latest
+#paddle book image tag
+if [ ! -n "$4" ]; then
+  book_tag=latest
+else
+  book_tag=$4
+fi
 
 #generate docker file
 if [ ${USE_UBUNTU_REPO_MIRROR} ]; then
@@ -20,24 +44,44 @@ echo "paddle_tag:"$paddle_tag
 echo "book_tag:"$book_tag
 
 cat > Dockerfile <<EOF
-FROM paddlepaddle/paddle:${paddle_tag}
+FROM ${paddle_image}:${paddle_tag}
 MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
 
 COPY . /book
+EOF
+
+if [ -n "${http_proxy}" ]; then
+cat >> Dockerfile <<EOF
+ENV http_proxy ${http_proxy}
+ENV https_proxy ${http_proxy}
+EOF
+fi
 
+cat >> Dockerfile <<EOF
 RUN pip install -U nltk \
     && python /book/.tools/cache_dataset.py
 
 RUN ${update_mirror_cmd}
     apt-get update && \
-    apt-get install -y locales && \
-    apt-get -y install gcc && \
+    apt-get install -y locales patch && \
+    apt-get -y install gcc curl git && \
     apt-get -y clean && \
     localedef -f UTF-8 -i en_US en_US.UTF-8 && \
-    pip install -U matplotlib jupyter numpy requests scipy
+    pip install --upgrade pip && \
+    pip install -U notedown pillow matplotlib jupyter numpy requests scipy
+
+RUN curl https://storage.googleapis.com/golang/go1.8.linux-amd64.tar.gz -o go1.8.linux-amd64.tar.gz && \
+    tar -zxvf go1.8.linux-amd64.tar.gz -C /usr/local/ && \
+    rm go1.8.linux-amd64.tar.gz
+
+ENV GOROOT /usr/local/go
+ENV PATH \${GOROOT}/bin:\${PATH}
+
+#convert md to ipynb
+RUN /bin/bash /book/.tools/convert-markdown-into-ipynb-and-test.sh
 
 EXPOSE 8888
-CMD ["sh", "-c", "jupyter notebook --ip=0.0.0.0 --no-browser --NotebookApp.token='' --NotebookApp.disable_check_xsrf=True /book/"]
+CMD ["sh", "-c", "jupyter notebook --ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.disable_check_xsrf=True /book/"]
 EOF
 
-docker build --no-cache  -t paddlepaddle/book:${paddle_tag}  -t paddlepaddle/book:${book_tag} .
+docker build --no-cache -t ${book_image}:${paddle_tag} -t ${book_image}:${book_tag} .

.tools/convert-markdown-into-ipynb-and-test.sh

@@ -11,7 +11,7 @@ cur_path="$(cd "$(dirname "$0")" && pwd -P)"
 cd $cur_path/../
 
 #convert md to ipynb
-for file in */{README,README\.en}.md ; do
+for file in */{README,README\.cn}.md ; do
     ~/go/bin/markdown-to-ipynb < $file > ${file%.*}".ipynb"
     if [ $? -ne 0 ]; then
         echo >&2 "markdown-to-ipynb $file error"
@@ -24,7 +24,7 @@ if [[ -z $TEST_EMBEDDED_PYTHON_SCRIPTS ]]; then
 fi
 
 #exec ipynb's py file
-for file in */{README,README\.en}.ipynb ; do
+for file in */{README,README\.cn}.ipynb ; do
     pushd $PWD > /dev/null
     cd $(dirname $file) > /dev/null
 

.tools/notedown.sh

+#!/bin/bash
+set -xe
+
+cd /book
+
+#convert md to ipynb
+for file in */{README,README\.cn}.md ; do
+    notedown $file > ${file%.*}.ipynb
+done

.tools/templates/index.cn.html.json

+{
+  "is_en": false,
+  "chapters": [
+    {
+      "name": "新手入门",
+      "link": "./01.fit_a_line/index.cn.html"
+    },
+    {
+      "name": "识别数字",
+      "link": "./02.recognize_digits/index.cn.html"
+    },
+    {
+      "name": "图像分类",
+      "link": "./03.image_classification/index.cn.html"
+    },
+    {
+      "name": "词向量",
+      "link": "./04.word2vec/index.cn.html"
+    },
+    {
+      "name": "个性化推荐",
+      "link": "./05.recommender_system/index.cn.html"
+    },
+    {
+      "name": "情感分析",
+      "link": "./06.understand_sentiment/index.cn.html"
+    },
+    {
+      "name": "语义角色标注",
+      "link": "./07.label_semantic_roles/index.cn.html"
+    },
+    {
+      "name": "机器翻译",
+      "link": "./08.machine_translation/index.cn.html"
+    }
+  ]
+}

.tools/templates/index.en.html.json

-{
-  "is_en": true,
-  "chapters": [
-    {
-      "name": "Linear Regression",
-      "link": "./01.fit_a_line/index.en.html"
-    },
-    {
-      "name": "Recognize Digits",
-      "link": "./02.recognize_digits/index.en.html"
-    },
-    {
-      "name": "Image Classification",
-      "link": "./03.image_classification/index.en.html"
-    },
-    {
-      "name": "Word2Vec",
-      "link": "./04.word2vec/index.en.html"
-    },
-    {
-      "name": "Sentiment Analysis",
-      "link": "./05.understand_sentiment/index.en.html"
-    },
-    {
-      "name": "Semantic Role Labeling",
-      "link": "./06.label_semantic_roles/index.en.html"
-    },
-    {
-      "name": "Machine Translation",
-      "link": "./07.machine_translation/index.en.html"
-    },
-    {
-      "name": "Personalized Recommendation",
-      "link": "./08.recommender_system/index.en.html"
-    }
-  ]
-}

.tools/templates/index.html.json

 {
-  "is_en": false,
+  "is_en": true,
   "chapters": [
     {
-      "name": "新手入门",
+      "name": "Linear Regression",
       "link": "./01.fit_a_line/index.html"
     },
     {
-      "name": "识别数字",
+      "name": "Recognize Digits",
       "link": "./02.recognize_digits/index.html"
     },
     {
-      "name": "图像分类",
+      "name": "Image Classification",
       "link": "./03.image_classification/index.html"
     },
     {
-      "name": "词向量",
+      "name": "Word2Vec",
       "link": "./04.word2vec/index.html"
     },
     {
-      "name": "情感分析",
-      "link": "./05.understand_sentiment/index.html"
+      "name": "Personalized Recommendation",
+      "link": "./05.recommender_system/index.html"
     },
     {
-      "name": "语义角色标注",
-      "link": "./06.label_semantic_roles/index.html"
+      "name": "Sentiment Analysis",
+      "link": "./06.understand_sentiment/index.html"
     },
     {
-      "name": "机器翻译",
-      "link": "./07.machine_translation/index.html"
+      "name": "Semantic Role Labeling",
+      "link": "./07.label_semantic_roles/index.html"
     },
     {
-      "name": "个性化推荐",
-      "link": "./08.recommender_system/index.html"
+      "name": "Machine Translation",
+      "link": "./08.machine_translation/index.html"
     }
   ]
 }

.tools/templates/index.html.tmpl

@@ -78,12 +78,12 @@
             </a>
           </li>
           <li class="nav-item {% if not is_en %}active{% endif %}">
-            <a class="nav-link" href="{% if is_en %}./index.html{% else %}#{%endif%}">
+            <a class="nav-link" href="{% if is_en %}./index.cn.html{% else %}#{%endif%}">
                 {% if is_en %}中文{% else %}深度学习入门{% endif %}
             </a>
           </li>
           <li class="nav-item {% if is_en %}active{% endif %}">
-            <a class="nav-link" href="{% if is_en %}#{% else %}./index.en.html{% endif %}">
+            <a class="nav-link" href="{% if is_en %}#{% else %}./index.html{% endif %}">
                 {% if is_en %}Deep Learning 101{% else %}English{% endif %}
             </a>
           </li>
@@ -119,7 +119,7 @@
                 </div>
             </div>
             <div class="col">
-                <iframe src="./01.fit_a_line/index{% if is_en %}.en{% endif %}.html" style="border: none; overflow-y : hidden" width="100%" height="100%" name="content_iframe" id="content_iframe">
+                <iframe src="./01.fit_a_line/index{% if not is_en %}.cn{% endif %}.html" style="border: none; overflow-y : hidden" width="100%" height="100%" name="content_iframe" id="content_iframe">
                 </iframe>
             </div>
         </div>

.travis.yml

@@ -16,7 +16,7 @@ addons:
       - python2.7-dev
       - golang
 before_install:
-  -  pip install -U virtualenv pre-commit pip
+  -  sudo pip install -U virtualenv pre-commit pip
   -  GOPATH=/tmp/go go get -u github.com/wangkuiyi/ipynb/markdown-to-ipynb
 script:
   -  PATH=/tmp/go/bin:$PATH .travis/precommit.sh

01.fit_a_line/README.cn.md

+# 线性回归
+让我们从经典的线性回归（Linear Regression \[[1](#参考文献)\]）模型开始这份教程。在这一章里，你将使用真实的数据集建立起一个房价预测模型，并且了解到机器学习中的若干重要概念。
+
+本教程源代码目录在[book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/137.html)。
+
+## 背景介绍
+给定一个大小为$n$的数据集  ${\{y_{i}, x_{i1}, ..., x_{id}\}}_{i=1}^{n}$，其中$x_{i1}, \ldots, x_{id}$是第$i$个样本$d$个属性上的取值，$y_i$是该样本待预测的目标。线性回归模型假设目标$y_i$可以被属性间的线性组合描述，即
+
+$$y_i = \omega_1x_{i1} + \omega_2x_{i2} + \ldots + \omega_dx_{id} + b,  i=1,\ldots,n$$
+
+例如，在我们将要建模的房价预测问题里，$x_{ij}$是描述房子$i$的各种属性（比如房间的个数、周围学校和医院的个数、交通状况等），而 $y_i$是房屋的价格。
+
+初看起来，这个假设实在过于简单了，变量间的真实关系很难是线性的。但由于线性回归模型有形式简单和易于建模分析的优点，它在实际问题中得到了大量的应用。很多经典的统计学习、机器学习书籍\[[2,3,4](#参考文献)\]也选择对线性模型独立成章重点讲解。
+
+## 效果展示
+我们使用从[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)获得的波士顿房价数据集进行模型的训练和预测。下面的散点图展示了使用模型对部分房屋价格进行的预测。其中，每个点的横坐标表示同一类房屋真实价格的中位数，纵坐标表示线性回归模型根据特征预测的结果，当二者值完全相等的时候就会落在虚线上。所以模型预测得越准确，则点离虚线越近。
+<p align="center">
+    <img src = "image/predictions.png" width=400><br/>
+    图1. 预测值 V.S. 真实值
+</p>
+
+## 模型概览
+
+### 模型定义
+
+在波士顿房价数据集中，和房屋相关的值共有14个：前13个用来描述房屋相关的各种信息，即模型中的 $x_i$；最后一个值为我们要预测的该类房屋价格的中位数，即模型中的 $y_i$。因此，我们的模型就可以表示成：
+
+$$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
+
+$\hat{Y}$ 表示模型的预测结果，用来和真实值$Y$区分。模型要学习的参数即：$\omega_1, \ldots, \omega_{13}, b$。
+
+建立模型后，我们需要给模型一个优化目标，使得学到的参数能够让预测值$\hat{Y}$尽可能地接近真实值$Y$。这里我们引入损失函数（[Loss Function](https://en.wikipedia.org/wiki/Loss_function)，或Cost Function）这个概念。 输入任意一个数据样本的目标值$y_{i}$和模型给出的预测值$\hat{y_{i}}$，损失函数输出一个非负的实值。这个实值通常用来反映模型误差的大小。
+
+对于线性回归模型来讲，最常见的损失函数就是均方误差（Mean Squared Error， [MSE](https://en.wikipedia.org/wiki/Mean_squared_error)）了，它的形式是：
+
+$$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
+
+即对于一个大小为$n$的测试集，$MSE$是$n$个数据预测结果误差平方的均值。
+
+### 训练过程
+
+定义好模型结构之后，我们要通过以下几个步骤进行模型训练
+ 1. 初始化参数，其中包括权重$\omega_i$和偏置$b$，对其进行初始化（如0均值，1方差）。
+ 2. 网络正向传播计算网络输出和损失函数。
+ 3. 根据损失函数进行反向误差传播 （[backpropagation](https://en.wikipedia.org/wiki/Backpropagation)），将网络误差从输出层依次向前传递, 并更新网络中的参数。
+ 4. 重复2~3步骤，直至网络训练误差达到规定的程度或训练轮次达到设定值。
+
+## 数据集
+
+### 数据集接口的封装
+首先加载需要的包
+
+```python
+import paddle.v2 as paddle
+import paddle.v2.dataset.uci_housing as uci_housing
+```
+
+我们通过uci_housing模块引入了数据集合[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)
+
+其中，在uci_housing模块中封装了：
+
+1. 数据下载的过程。下载数据保存在~/.cache/paddle/dataset/uci_housing/housing.data。
+2. [数据预处理](#数据预处理)的过程。
+
+
+### 数据集介绍
+这份数据集共506行，每行包含了波士顿郊区的一类房屋的相关信息及该类房屋价格的中位数。其各维属性的意义如下：
+
+| 属性名 | 解释 | 类型 |
+| ------| ------ | ------ |
+| CRIM | 该镇的人均犯罪率 | 连续值 |
+| ZN | 占地面积超过25,000平方呎的住宅用地比例 | 连续值 |
+| INDUS | 非零售商业用地比例 | 连续值 |
+| CHAS | 是否邻近 Charles River  | 离散值，1=邻近；0=不邻近 |
+| NOX | 一氧化氮浓度 | 连续值 |
+| RM | 每栋房屋的平均客房数 | 连续值 |
+| AGE | 1940年之前建成的自用单位比例 | 连续值 |
+| DIS | 到波士顿5个就业中心的加权距离 | 连续值 |
+| RAD | 到径向公路的可达性指数 | 连续值 |
+| TAX | 全值财产税率 | 连续值 |
+| PTRATIO | 学生与教师的比例 | 连续值 |
+| B | 1000(BK - 0.63)^2，其中BK为黑人占比 | 连续值 |
+| LSTAT | 低收入人群占比 | 连续值 |
+| MEDV | 同类房屋价格的中位数 | 连续值 |
+
+### 数据预处理
+#### 连续值与离散值
+观察一下数据，我们的第一个发现是：所有的13维属性中，有12维的连续值和1维的离散值（CHAS）。离散值虽然也常使用类似0、1、2这样的数字表示，但是其含义与连续值是不同的，因为这里的差值没有实际意义。例如，我们用0、1、2来分别表示红色、绿色和蓝色的话，我们并不能因此说“蓝色和红色”比“绿色和红色”的距离更远。所以通常对一个有$d$个可能取值的离散属性，我们会将它们转为$d$个取值为0或1的二值属性或者将每个可能取值映射为一个多维向量。不过就这里而言，因为CHAS本身就是一个二值属性，就省去了这个麻烦。
+
+#### 属性的归一化
+另外一个稍加观察即可发现的事实是，各维属性的取值范围差别很大（如图2所示）。例如，属性B的取值范围是[0.32, 396.90]，而属性NOX的取值范围是[0.3850, 0.8170]。这里就要用到一个常见的操作-归一化（normalization）了。归一化的目标是把各位属性的取值范围放缩到差不多的区间，例如[-0.5,0.5]。这里我们使用一种很常见的操作方法：减掉均值，然后除以原取值范围。
+
+做归一化（或 [Feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)）至少有以下3个理由：
+- 过大或过小的数值范围会导致计算时的浮点上溢或下溢。
+- 不同的数值范围会导致不同属性对模型的重要性不同（至少在训练的初始阶段如此），而这个隐含的假设常常是不合理的。这会对优化的过程造成困难，使训练时间大大的加长。
+- 很多的机器学习技巧/模型（例如L1，L2正则项，向量空间模型-Vector Space Model）都基于这样的假设：所有的属性取值都差不多是以0为均值且取值范围相近的。
+
+<p align="center">
+    <img src = "image/ranges.png" width=550><br/>
+    图2. 各维属性的取值范围
+</p>
+
+#### 整理训练集与测试集
+我们将数据集分割为两份：一份用于调整模型的参数，即进行模型的训练，模型在这份数据集上的误差被称为**训练误差**；另外一份被用来测试，模型在这份数据集上的误差被称为**测试误差**。我们训练模型的目的是为了通过从训练数据中找到规律来预测未知的新数据，所以测试误差是更能反映模型表现的指标。分割数据的比例要考虑到两个因素：更多的训练数据会降低参数估计的方差，从而得到更可信的模型；而更多的测试数据会降低测试误差的方差，从而得到更可信的测试误差。我们这个例子中设置的分割比例为$8:2$
+
+
+在更复杂的模型训练过程中，我们往往还会多使用一种数据集：验证集。因为复杂的模型中常常还有一些超参数（[Hyperparameter](https://en.wikipedia.org/wiki/Hyperparameter_optimization)）需要调节，所以我们会尝试多种超参数的组合来分别训练多个模型，然后对比它们在验证集上的表现选择相对最好的一组超参数，最后才使用这组参数下训练的模型在测试集上评估测试误差。由于本章训练的模型比较简单，我们暂且忽略掉这个过程。
+
+## 训练
+
+`fit_a_line/trainer.py`演示了训练的整体过程。
+
+### 初始化PaddlePaddle
+
+```python
+paddle.init(use_gpu=False, trainer_count=1)
+```
+
+### 模型配置
+
+线性回归的模型其实就是一个采用线性激活函数（linear activation，`LinearActivation`）的全连接层（fully-connected layer，`fc_layer`）：
+
+```python
+x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
+y_predict = paddle.layer.fc(input=x,
+                                size=1,
+                                act=paddle.activation.Linear())
+y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
+cost = paddle.layer.mse_cost(input=y_predict, label=y)
+```
+### 创建参数
+
+```python
+parameters = paddle.parameters.create(cost)
+```
+
+### 创建Trainer
+
+```python
+optimizer = paddle.optimizer.Momentum(momentum=0)
+
+trainer = paddle.trainer.SGD(cost=cost,
+                             parameters=parameters,
+                             update_equation=optimizer)
+```
+
+### 读取数据且打印训练的中间信息
+
+PaddlePaddle提供一个
+[reader机制](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
+来读取数据。 Reader返回的数据可以包括多列，我们需要一个Python dict把列
+序号映射到网络里的数据层。
+
+```python
+feeding={'x': 0, 'y': 1}
+```
+
+此外，我们还可以提供一个 event handler，来打印训练的进度：
+
+```python
+# event_handler to print training and testing info
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f" % (
+                event.pass_id, event.batch_id, event.cost)
+
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                uci_housing.test(), batch_size=2),
+            feeding=feeding)
+        print "Test %d, Cost %f" % (event.pass_id, result.cost)
+```
+
+```python
+# event_handler to print training and testing info
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 10 == 0:  # every 10 batches, record a train cost
+            cost_ploter.append(train_title, step, event.cost)
+
+        if step % 100 == 0: # every 100 batches, record a test cost
+            result = trainer.test(
+                reader=paddle.batch(
+                    uci_housing.test(), batch_size=2),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
+
+        if step % 100 == 0: # every 100 batches, update cost plot
+            cost_ploter.plot()
+
+        step += 1
+
+    if isinstance(event, paddle.event.EndPass):
+        if event.pass_id % 10 == 0:
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
+```
+
+### 开始训练
+
+```python
+trainer.train(
+    reader=paddle.batch(
+        paddle.reader.shuffle(
+            uci_housing.train(), buf_size=500),
+        batch_size=2),
+    feeding=feeding,
+    event_handler=event_handler_plot,
+    num_passes=30)
+```
+
+![png](./image/train_and_test.png)
+
+### 应用模型
+
+#### 1. 生成测试数据
+
+```python
+test_data_creator = paddle.dataset.uci_housing.test()
+test_data = []
+test_label = []
+
+for item in test_data_creator():
+    test_data.append((item[0],))
+    test_label.append(item[1])
+    if len(test_data) == 5:
+        break
+```
+
+#### 2. 推测 inference
+
+```python
+# load parameters from tar file.
+# users can remove the comments and change the model name
+# with open('params_pass_20.tar', 'r') as f:
+#     parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=y_predict, parameters=parameters, input=test_data)
+
+for i in xrange(len(probs)):
+    print "label=" + str(test_label[i][0]) + ", predict=" + str(probs[i][0])
+```
+
+## 总结
+在这章里，我们借助波士顿房价这一数据集，介绍了线性回归模型的基本概念，以及如何使用PaddlePaddle实现训练和测试的过程。很多的模型和技巧都是从简单的线性回归模型演化而来，因此弄清楚线性模型的原理和局限非常重要。
+
+
+## 参考文献
+1. https://en.wikipedia.org/wiki/Linear_regression
+2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
+3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
+4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
+
+<br/>
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

01.fit_a_line/README.en.ipynb

-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# Linear Regression\n",
-        "Let us begin the tutorial with a classical problem called Linear Regression \\[[1](#References)\\]. In this chapter, we will train a model from a realistic dataset to predict home prices. Some important concepts in Machine Learning will be covered through this example.\n",
-        "\n",
-        "The source code for this tutorial lives on [book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line). For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).\n",
-        "\n",
-        "## Problem Setup\n",
-        "Suppose we have a dataset of $n$ real estate properties. These real estate properties will be referred to as *homes* in this chapter for clarity.\n",
-        "\n",
-        "Each home is associated with $d$ attributes. The attributes describe characteristics such the number of rooms in the home, the number of schools or hospitals in the neighborhood, and the traffic condition nearby.\n",
-        "\n",
-        "In our problem setup, the attribute $x_{i,j}$ denotes the $j$th characteristic of the $i$th home. In addition, $y_i$ denotes the price of the $i$th home. Our task is to predict $y_i$ given a set of attributes $\\{x_{i,1}, ..., x_{i,d}\\}$. We assume that the price of a home is a linear combination of all of its attributes, namely,\n",
-        "\n",
-        "$$y_i = \\omega_1x_{i,1} + \\omega_2x_{i,2} + \\ldots + \\omega_dx_{i,d} + b,  i=1,\\ldots,n$$\n",
-        "\n",
-        "where $\\vec{\\omega}$ and $b$ are the model parameters we want to estimate. Once they are learned, we will be able to predict the price of a home, given the attributes associated with it. We call this model **Linear Regression**. In other words, we want to regress a value against several values linearly. In practice, a linear model is often too simplistic to capture the real relationships between the variables. Yet, because Linear Regression is easy to train and analyze, it has been applied to a large number of real problems. As a result, it is an important topic in many classic Statistical Learning and Machine Learning textbooks \\[[2,3,4](#References)\\].\n",
-        "\n",
-        "## Results Demonstration\n",
-        "We first show the result of our model. The dataset [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) is used to train a linear model to predict the home prices in Boston. The figure below shows the predictions the model makes for some home prices. The $X$-axis represents the median value of the prices of simlilar homes within a bin, while the $Y$-axis represents the home value our linear model predicts. The dotted line represents points where $X=Y$. When reading the diagram, the more precise the model predicts, the closer the point is to the dotted line.\n",
-        "\u003cp align=\"center\"\u003e\n",
-        "    \u003cimg src = \"image/predictions_en.png\" width=400\u003e\u003cbr/\u003e\n",
-        "    Figure 1. Predicted Value V.S. Actual Value\n",
-        "\u003c/p\u003e\n",
-        "\n",
-        "## Model Overview\n",
-        "\n",
-        "### Model Definition\n",
-        "\n",
-        "In the UCI Housing Data Set, there are 13 home attributes $\\{x_{i,j}\\}$ that are related to the median home price $y_i$, which we aim to predict. Thus, our model can be written as:\n",
-        "\n",
-        "$$\\hat{Y} = \\omega_1X_{1} + \\omega_2X_{2} + \\ldots + \\omega_{13}X_{13} + b$$\n",
-        "\n",
-        "where $\\hat{Y}$ is the predicted value used to differentiate from actual value $Y$. The model learns parameters $\\omega_1, \\ldots, \\omega_{13}, b$, where the entries of $\\vec{\\omega}$ are **weights** and $b$ is **bias**.\n",
-        "\n",
-        "Now we need an objective to optimize, so that the learned parameters can make $\\hat{Y}$ as close to $Y$ as possible. Let's refer to the concept of [Loss Function (Cost Function)](https://en.wikipedia.org/wiki/Loss_function). A loss function must output a non-negative value, given any pair of the actual value $y_i$ and the predicted value $\\hat{y_i}$. This value reflects the magnitutude of the model error.\n",
-        "\n",
-        "For Linear Regression, the most common loss function is [Mean Square Error (MSE)](https://en.wikipedia.org/wiki/Mean_squared_error) which has the following form:\n",
-        "\n",
-        "$$MSE=\\frac{1}{n}\\sum_{i=1}^{n}{(\\hat{Y_i}-Y_i)}^2$$\n",
-        "\n",
-        "That is, for a dataset of size $n$, MSE is the average value of the the prediction sqaure errors.\n",
-        "\n",
-        "### Training\n",
-        "\n",
-        "After setting up our model, there are several major steps to go through to train it:\n",
-        "1. Initialize the parameters including the weights $\\vec{\\omega}$ and the bias $b$. For example, we can set their mean values as $0$s, and their standard deviations as $1$s.\n",
-        "2. Feedforward. Evaluate the network output and compute the corresponding loss.\n",
-        "3. [Backpropagate](https://en.wikipedia.org/wiki/Backpropagation) the errors. The errors will be propagated from the output layer back to the input layer, during which the model parameters will be updated with the corresponding errors.\n",
-        "4. Repeat steps 2~3, until the loss is below a predefined threshold or the maximum number of repeats is reached.\n",
-        "\n",
-        "## Dataset\n",
-        "\n",
-        "### Python Dataset Modules\n",
-        "\n",
-        "Our program starts with importing necessary packages:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "import paddle.v2 as paddle\n",
-        "import paddle.v2.dataset.uci_housing as uci_housing\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "We encapsulated the [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) in our Python module `uci_housing`.  This module can\n",
-        "\n",
-        "1. download the dataset to `~/.cache/paddle/dataset/uci_housing/housing.data`, if not yet, and\n",
-        "2.  [preprocesses](#preprocessing) the dataset.\n",
-        "\n",
-        "### An Introduction of the Dataset\n",
-        "\n",
-        "The UCI housing dataset has 506 instances. Each instance describes the attributes of a house in surburban Boston.  The attributes are explained below:\n",
-        "\n",
-        "| Attribute Name | Characteristic | Data Type |\n",
-        "| ------| ------ | ------ |\n",
-        "| CRIM | per capita crime rate by town | Continuous|\n",
-        "| ZN | proportion of residential land zoned for lots over 25,000 sq.ft. | Continuous |\n",
-        "| INDUS | proportion of non-retail business acres per town | Continuous |\n",
-        "| CHAS | Charles River dummy variable | Discrete, 1 if tract bounds river; 0 otherwise|\n",
-        "| NOX | nitric oxides concentration (parts per 10 million) | Continuous |\n",
-        "| RM | average number of rooms per dwelling | Continuous |\n",
-        "| AGE | proportion of owner-occupied units built prior to 1940 | Continuous |\n",
-        "| DIS | weighted distances to five Boston employment centres | Continuous |\n",
-        "| RAD | index of accessibility to radial highways | Continuous |\n",
-        "| TAX | full-value property-tax rate per $10,000 | Continuous |\n",
-        "| PTRATIO | pupil-teacher ratio by town | Continuous |\n",
-        "| B | 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town | Continuous |\n",
-        "| LSTAT | % lower status of the population | Continuous |\n",
-        "| MEDV | Median value of owner-occupied homes in $1000's | Continuous |\n",
-        "\n",
-        "The last entry is the median home price.\n",
-        "\n",
-        "### Preprocessing\n",
-        "#### Continuous and Discrete Data\n",
-        "We define a feature vector of length 13 for each home, where each entry corresponds to an attribute. Our first observation is that, among the 13 dimensions, there are 12 continuous dimensions and 1 discrete dimension.\n",
-        "\n",
-        "Note that although a discrete value is also written as numeric values such as 0, 1, or 2, its meaning differs from a continuous value drastically.  The linear difference between two discrete values has no meaning. For example, suppose $0$, $1$, and $2$ are used to represent colors *Red*, *Green*, and *Blue* respectively. Judging from the numeric representation of these colors, *Red* differs more from *Blue* than it does from *Green*. Yet in actuality, it is not true that extent to which the color *Blue* is different from *Red* is greater than the extent to which *Green* is different from *Red*. Therefore, when handling a discrete feature that has $d$ possible values, we usually convert it to $d$ new features where each feature takes a binary value, $0$ or $1$, indicating whether the original value is absent or present. Alternatively, the discrete features can be mapped onto a continuous multi-dimensional vector through an embedding table. For our problem here, because CHAS itself is a binary discrete value, we do not need to do any preprocessing.\n",
-        "\n",
-        "#### Feature Normalization\n",
-        "We also observe a huge difference among the value ranges of the 13 features (Figure 2). For instance, the values of feature *B* fall in $[0.32, 396.90]$, whereas those of feature *NOX* has a range of $[0.3850, 0.8170]$. An effective optimization would require data normalization. The goal of data normalization is to scale te values of each feature into roughly the same range, perhaps $[-0.5, 0.5]$. Here, we adopt a popular normalization technique where we substract the mean value from the feature value and divide the result by the width of the original range.\n",
-        "\n",
-        "There are at least three reasons for [Feature Normalization](https://en.wikipedia.org/wiki/Feature_scaling) (Feature Scaling):\n",
-        "- A value range that is too large or too small might cause floating number overflow or underflow during computation.\n",
-        "- Different value ranges might result in varying *importances* of different features to the model (at least in the beginning of the training process). This assumption about the data is often unreasonable, making the optimization difficult, which in turn results in increased training time.\n",
-        "- Many machine learning techniques or models (e.g., *L1/L2 regularization* and *Vector Space Model*) assumes that all the features have roughly zero means and their value ranges are similar.\n",
-        "\n",
-        "\u003cp align=\"center\"\u003e\n",
-        "    \u003cimg src = \"image/ranges_en.png\" width=550\u003e\u003cbr/\u003e\n",
-        "    Figure 2. The value ranges of the features\n",
-        "\u003c/p\u003e\n",
-        "\n",
-        "#### Prepare Training and Test Sets\n",
-        "We split the dataset in two, one for adjusting the model parameters, namely, for model training, and the other for model testing. The model error on the former is called the **training error**, and the error on the latter is called the **test error**. Our goal in training a model is to find the statistical dependency between the outputs and the inputs, so that we can predict new outputs given new inputs. As a result, the test error reflects the performance of the model better than the training error does. We consider two things when deciding the ratio of the training set to the test set: 1) More training data will decrease the variance of the parameter estimation, yielding more reliable models; 2) More test data will decrease the variance of the test error, yielding more reliable test errors. One standard split ratio is $8:2$.\n",
-        "\n",
-        "\n",
-        "When training complex models, we usually have one more split: the validation set. Complex models usually have [Hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_optimization) that need to be set before the training process, such as the number of layers in the network. Because hyperparameters are not part of the model parameters, they cannot be trained using the same loss function. Thus we will try several sets of hyperparameters to train several models and cross-validate them on the validation set to pick the best one; finally, the selected trained model is tested on the test set. Because our model is relatively simple, we will omit this validation process.\n",
-        "\n",
-        "\n",
-        "## Training\n",
-        "\n",
-        "`fit_a_line/trainer.py` demonstrates the training using [PaddlePaddle](http://paddlepaddle.org).\n",
-        "\n",
-        "### Initialize PaddlePaddle\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "paddle.init(use_gpu=False, trainer_count=1)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### Model Configuration\n",
-        "\n",
-        "Logistic regression is essentially a fully-connected layer with linear activation:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))\n",
-        "y_predict = paddle.layer.fc(input=x,\n",
-        "                                size=1,\n",
-        "                                act=paddle.activation.Linear())\n",
-        "y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))\n",
-        "cost = paddle.layer.mse_cost(input=y_predict, label=y)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### Create Parameters\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "parameters = paddle.parameters.create(cost)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### Create Trainer\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "optimizer = paddle.optimizer.Momentum(momentum=0)\n",
-        "\n",
-        "trainer = paddle.trainer.SGD(cost=cost,\n",
-        "                             parameters=parameters,\n",
-        "                             update_equation=optimizer)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### Feeding Data\n",
-        "\n",
-        "PaddlePaddle provides the\n",
-        "[reader mechanism](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)\n",
-        "for loadinng training data. A reader may return multiple columns, and we need a Python dictionary to specify the mapping from column index to data layers.\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "feeding={'x': 0, 'y': 1}\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "Moreover, an event handler is provided to print the training progress:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "import matplotlib.pyplot as plt\n",
-        "from IPython import display\n",
-        "import cPickle\n",
-        "\n",
-        "step=0\n",
-        "\n",
-        "train_costs=[],[]\n",
-        "test_costs=[],[]\n",
-        "\n",
-        "def event_handler(event):\n",
-        "    global step\n",
-        "    global train_costs\n",
-        "    global test_costs\n",
-        "    if isinstance(event, paddle.event.EndIteration):\n",
-        "        need_plot = False\n",
-        "        if step % 10 == 0:  # every 10 batches, record a train cost\n",
-        "            train_costs[0].append(step)\n",
-        "            train_costs[1].append(event.cost)\n",
-        "\n",
-        "        if step % 1000 == 0: # every 1000 batches, record a test cost\n",
-        "            result = trainer.test(\n",
-        "                reader=paddle.batch(\n",
-        "                    uci_housing.test(), batch_size=2),\n",
-        "                feeding=feeding)\n",
-        "            test_costs[0].append(step)\n",
-        "            test_costs[1].append(result.cost)\n",
-        "\n",
-        "        if step % 100 == 0: # every 100 batches, update cost plot\n",
-        "            plt.plot(*train_costs)\n",
-        "            plt.plot(*test_costs)\n",
-        "            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')\n",
-        "            display.clear_output(wait=True)\n",
-        "            display.display(plt.gcf())\n",
-        "            plt.gcf().clear()\n",
-        "        step += 1\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### Start Training\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "trainer.train(\n",
-        "    reader=paddle.batch(\n",
-        "        paddle.reader.shuffle(\n",
-        "            uci_housing.train(), buf_size=500),\n",
-        "        batch_size=2),\n",
-        "    feeding=feeding,\n",
-        "    event_handler=event_handler,\n",
-        "    num_passes=30)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "![png](./image/train-and-test.png)\n",
-        "\n",
-        "## Summary\n",
-        "This chapter introduces *Linear Regression* and how to train and test this model with PaddlePaddle, using the UCI Housing Data Set. Because a large number of more complex models and techniques are derived from linear regression, it is important to understand its underlying theory and limitation.\n",
-        "\n",
-        "\n",
-        "## References\n",
-        "1. https://en.wikipedia.org/wiki/Linear_regression\n",
-        "2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.\n",
-        "3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.\n",
-        "4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.\n",
-        "\n",
-        "\u003cbr/\u003e\n",
-        "This tutorial is contributed by \u003ca xmlns:cc=\"http://creativecommons.org/ns#\" href=\"http://book.paddlepaddle.org\" property=\"cc:attributionName\" rel=\"cc:attributionURL\"\u003ePaddlePaddle\u003c/a\u003e, and licensed under a \u003ca rel=\"license\" href=\"http://creativecommons.org/licenses/by-nc-sa/4.0/\"\u003eCreative Commons Attribution-NonCommercial-ShareAlike 4.0 International License\u003c/a\u003e.\n"
-      ]
-    }
-  ],
-  "metadata": {
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.0"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

01.fit_a_line/README.en.md

-# Linear Regression
-Let us begin the tutorial with a classical problem called Linear Regression \[[1](#References)\]. In this chapter, we will train a model from a realistic dataset to predict home prices. Some important concepts in Machine Learning will be covered through this example.
-
-The source code for this tutorial lives on [book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line). For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.en.md).
-
-## Problem Setup
-Suppose we have a dataset of $n$ real estate properties. These real estate properties will be referred to as *homes* in this chapter for clarity.
-
-Each home is associated with $d$ attributes. The attributes describe characteristics such the number of rooms in the home, the number of schools or hospitals in the neighborhood, and the traffic condition nearby.
-
-In our problem setup, the attribute $x_{i,j}$ denotes the $j$th characteristic of the $i$th home. In addition, $y_i$ denotes the price of the $i$th home. Our task is to predict $y_i$ given a set of attributes $\{x_{i,1}, ..., x_{i,d}\}$. We assume that the price of a home is a linear combination of all of its attributes, namely,
-
-$$y_i = \omega_1x_{i,1} + \omega_2x_{i,2} + \ldots + \omega_dx_{i,d} + b,  i=1,\ldots,n$$
-
-where $\vec{\omega}$ and $b$ are the model parameters we want to estimate. Once they are learned, we will be able to predict the price of a home, given the attributes associated with it. We call this model **Linear Regression**. In other words, we want to regress a value against several values linearly. In practice, a linear model is often too simplistic to capture the real relationships between the variables. Yet, because Linear Regression is easy to train and analyze, it has been applied to a large number of real problems. As a result, it is an important topic in many classic Statistical Learning and Machine Learning textbooks \[[2,3,4](#References)\].
-
-## Results Demonstration
-We first show the result of our model. The dataset [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) is used to train a linear model to predict the home prices in Boston. The figure below shows the predictions the model makes for some home prices. The $X$-axis represents the median value of the prices of simlilar homes within a bin, while the $Y$-axis represents the home value our linear model predicts. The dotted line represents points where $X=Y$. When reading the diagram, the more precise the model predicts, the closer the point is to the dotted line.
-<p align="center">
-    <img src = "image/predictions_en.png" width=400><br/>
-    Figure 1. Predicted Value V.S. Actual Value
-</p>
-
-## Model Overview
-
-### Model Definition
-
-In the UCI Housing Data Set, there are 13 home attributes $\{x_{i,j}\}$ that are related to the median home price $y_i$, which we aim to predict. Thus, our model can be written as:
-
-$$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
-
-where $\hat{Y}$ is the predicted value used to differentiate from actual value $Y$. The model learns parameters $\omega_1, \ldots, \omega_{13}, b$, where the entries of $\vec{\omega}$ are **weights** and $b$ is **bias**.
-
-Now we need an objective to optimize, so that the learned parameters can make $\hat{Y}$ as close to $Y$ as possible. Let's refer to the concept of [Loss Function (Cost Function)](https://en.wikipedia.org/wiki/Loss_function). A loss function must output a non-negative value, given any pair of the actual value $y_i$ and the predicted value $\hat{y_i}$. This value reflects the magnitutude of the model error.
-
-For Linear Regression, the most common loss function is [Mean Square Error (MSE)](https://en.wikipedia.org/wiki/Mean_squared_error) which has the following form:
-
-$$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
-
-That is, for a dataset of size $n$, MSE is the average value of the the prediction sqaure errors.
-
-### Training
-
-After setting up our model, there are several major steps to go through to train it:
-1. Initialize the parameters including the weights $\vec{\omega}$ and the bias $b$. For example, we can set their mean values as $0$s, and their standard deviations as $1$s.
-2. Feedforward. Evaluate the network output and compute the corresponding loss.
-3. [Backpropagate](https://en.wikipedia.org/wiki/Backpropagation) the errors. The errors will be propagated from the output layer back to the input layer, during which the model parameters will be updated with the corresponding errors.
-4. Repeat steps 2~3, until the loss is below a predefined threshold or the maximum number of repeats is reached.
-
-## Dataset
-
-### Python Dataset Modules
-
-Our program starts with importing necessary packages:
-
-```python
-import paddle.v2 as paddle
-import paddle.v2.dataset.uci_housing as uci_housing
-```
-
-We encapsulated the [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) in our Python module `uci_housing`.  This module can
-
-1. download the dataset to `~/.cache/paddle/dataset/uci_housing/housing.data`, if not yet, and
-2.  [preprocesses](#preprocessing) the dataset.
-
-### An Introduction of the Dataset
-
-The UCI housing dataset has 506 instances. Each instance describes the attributes of a house in surburban Boston.  The attributes are explained below:
-
-| Attribute Name | Characteristic | Data Type |
-| ------| ------ | ------ |
-| CRIM | per capita crime rate by town | Continuous|
-| ZN | proportion of residential land zoned for lots over 25,000 sq.ft. | Continuous |
-| INDUS | proportion of non-retail business acres per town | Continuous |
-| CHAS | Charles River dummy variable | Discrete, 1 if tract bounds river; 0 otherwise|
-| NOX | nitric oxides concentration (parts per 10 million) | Continuous |
-| RM | average number of rooms per dwelling | Continuous |
-| AGE | proportion of owner-occupied units built prior to 1940 | Continuous |
-| DIS | weighted distances to five Boston employment centres | Continuous |
-| RAD | index of accessibility to radial highways | Continuous |
-| TAX | full-value property-tax rate per $10,000 | Continuous |
-| PTRATIO | pupil-teacher ratio by town | Continuous |
-| B | 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town | Continuous |
-| LSTAT | % lower status of the population | Continuous |
-| MEDV | Median value of owner-occupied homes in $1000's | Continuous |
-
-The last entry is the median home price.
-
-### Preprocessing
-#### Continuous and Discrete Data
-We define a feature vector of length 13 for each home, where each entry corresponds to an attribute. Our first observation is that, among the 13 dimensions, there are 12 continuous dimensions and 1 discrete dimension.
-
-Note that although a discrete value is also written as numeric values such as 0, 1, or 2, its meaning differs from a continuous value drastically.  The linear difference between two discrete values has no meaning. For example, suppose $0$, $1$, and $2$ are used to represent colors *Red*, *Green*, and *Blue* respectively. Judging from the numeric representation of these colors, *Red* differs more from *Blue* than it does from *Green*. Yet in actuality, it is not true that extent to which the color *Blue* is different from *Red* is greater than the extent to which *Green* is different from *Red*. Therefore, when handling a discrete feature that has $d$ possible values, we usually convert it to $d$ new features where each feature takes a binary value, $0$ or $1$, indicating whether the original value is absent or present. Alternatively, the discrete features can be mapped onto a continuous multi-dimensional vector through an embedding table. For our problem here, because CHAS itself is a binary discrete value, we do not need to do any preprocessing.
-
-#### Feature Normalization
-We also observe a huge difference among the value ranges of the 13 features (Figure 2). For instance, the values of feature *B* fall in $[0.32, 396.90]$, whereas those of feature *NOX* has a range of $[0.3850, 0.8170]$. An effective optimization would require data normalization. The goal of data normalization is to scale te values of each feature into roughly the same range, perhaps $[-0.5, 0.5]$. Here, we adopt a popular normalization technique where we substract the mean value from the feature value and divide the result by the width of the original range.
-
-There are at least three reasons for [Feature Normalization](https://en.wikipedia.org/wiki/Feature_scaling) (Feature Scaling):
-- A value range that is too large or too small might cause floating number overflow or underflow during computation.
-- Different value ranges might result in varying *importances* of different features to the model (at least in the beginning of the training process). This assumption about the data is often unreasonable, making the optimization difficult, which in turn results in increased training time.
-- Many machine learning techniques or models (e.g., *L1/L2 regularization* and *Vector Space Model*) assumes that all the features have roughly zero means and their value ranges are similar.
-
-<p align="center">
-    <img src = "image/ranges_en.png" width=550><br/>
-    Figure 2. The value ranges of the features
-</p>
-
-#### Prepare Training and Test Sets
-We split the dataset in two, one for adjusting the model parameters, namely, for model training, and the other for model testing. The model error on the former is called the **training error**, and the error on the latter is called the **test error**. Our goal in training a model is to find the statistical dependency between the outputs and the inputs, so that we can predict new outputs given new inputs. As a result, the test error reflects the performance of the model better than the training error does. We consider two things when deciding the ratio of the training set to the test set: 1) More training data will decrease the variance of the parameter estimation, yielding more reliable models; 2) More test data will decrease the variance of the test error, yielding more reliable test errors. One standard split ratio is $8:2$.
-
-
-When training complex models, we usually have one more split: the validation set. Complex models usually have [Hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_optimization) that need to be set before the training process, such as the number of layers in the network. Because hyperparameters are not part of the model parameters, they cannot be trained using the same loss function. Thus we will try several sets of hyperparameters to train several models and cross-validate them on the validation set to pick the best one; finally, the selected trained model is tested on the test set. Because our model is relatively simple, we will omit this validation process.
-
-
-## Training
-
-`fit_a_line/trainer.py` demonstrates the training using [PaddlePaddle](http://paddlepaddle.org).
-
-### Initialize PaddlePaddle
-
-```python
-paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### Model Configuration
-
-Logistic regression is essentially a fully-connected layer with linear activation:
-
-```python
-x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
-y_predict = paddle.layer.fc(input=x,
-                                size=1,
-                                act=paddle.activation.Linear())
-y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
-cost = paddle.layer.mse_cost(input=y_predict, label=y)
-```
-### Create Parameters
-
-```python
-parameters = paddle.parameters.create(cost)
-```
-
-### Create Trainer
-
-```python
-optimizer = paddle.optimizer.Momentum(momentum=0)
-
-trainer = paddle.trainer.SGD(cost=cost,
-                             parameters=parameters,
-                             update_equation=optimizer)
-```
-
-### Feeding Data
-
-PaddlePaddle provides the
-[reader mechanism](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
-for loadinng training data. A reader may return multiple columns, and we need a Python dictionary to specify the mapping from column index to data layers.
-
-```python
-feeding={'x': 0, 'y': 1}
-```
-
-Moreover, an event handler is provided to print the training progress:
-
-```python
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
-step=0
-
-train_costs=[],[]
-test_costs=[],[]
-
-def event_handler(event):
-    global step
-    global train_costs
-    global test_costs
-    if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
-        if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
-
-        if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(
-                reader=paddle.batch(
-                    uci_housing.test(), batch_size=2),
-                feeding=feeding)
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
-
-        if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
-        step += 1
-```
-
-### Start Training
-
-```python
-trainer.train(
-    reader=paddle.batch(
-        paddle.reader.shuffle(
-            uci_housing.train(), buf_size=500),
-        batch_size=2),
-    feeding=feeding,
-    event_handler=event_handler,
-    num_passes=30)
-```
-
-![png](./image/train-and-test.png)
-
-## Summary
-This chapter introduces *Linear Regression* and how to train and test this model with PaddlePaddle, using the UCI Housing Data Set. Because a large number of more complex models and techniques are derived from linear regression, it is important to understand its underlying theory and limitation.
-
-
-## References
-1. https://en.wikipedia.org/wiki/Linear_regression
-2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
-3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
-4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
-
-<br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.

01.fit_a_line/README.ipynb

-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "# 线性回归\n",
-        "让我们从经典的线性回归（Linear Regression \\[[1](#参考文献)\\]）模型开始这份教程。在这一章里，你将使用真实的数据集建立起一个房价预测模型，并且了解到机器学习中的若干重要概念。\n",
-        "\n",
-        "本教程源代码目录在[book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。\n",
-        "\n",
-        "## 背景介绍\n",
-        "给定一个大小为$n$的数据集  ${\\{y_{i}, x_{i1}, ..., x_{id}\\}}_{i=1}^{n}$，其中$x_{i1}, \\ldots, x_{id}$是第$i$个样本$d$个属性上的取值，$y_i$是该样本待预测的目标。线性回归模型假设目标$y_i$可以被属性间的线性组合描述，即\n",
-        "\n",
-        "$$y_i = \\omega_1x_{i1} + \\omega_2x_{i2} + \\ldots + \\omega_dx_{id} + b,  i=1,\\ldots,n$$\n",
-        "\n",
-        "例如，在我们将要建模的房价预测问题里，$x_{ij}$是描述房子$i$的各种属性（比如房间的个数、周围学校和医院的个数、交通状况等），而 $y_i$是房屋的价格。\n",
-        "\n",
-        "初看起来，这个假设实在过于简单了，变量间的真实关系很难是线性的。但由于线性回归模型有形式简单和易于建模分析的优点，它在实际问题中得到了大量的应用。很多经典的统计学习、机器学习书籍\\[[2,3,4](#参考文献)\\]也选择对线性模型独立成章重点讲解。\n",
-        "\n",
-        "## 效果展示\n",
-        "我们使用从[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)获得的波士顿房价数据集进行模型的训练和预测。下面的散点图展示了使用模型对部分房屋价格进行的预测。其中，每个点的横坐标表示同一类房屋真实价格的中位数，纵坐标表示线性回归模型根据特征预测的结果，当二者值完全相等的时候就会落在虚线上。所以模型预测得越准确，则点离虚线越近。\n",
-        "\u003cp align=\"center\"\u003e\n",
-        "    \u003cimg src = \"image/predictions.png\" width=400\u003e\u003cbr/\u003e\n",
-        "    图1. 预测值 V.S. 真实值\n",
-        "\u003c/p\u003e\n",
-        "\n",
-        "## 模型概览\n",
-        "\n",
-        "### 模型定义\n",
-        "\n",
-        "在波士顿房价数据集中，和房屋相关的值共有14个：前13个用来描述房屋相关的各种信息，即模型中的 $x_i$；最后一个值为我们要预测的该类房屋价格的中位数，即模型中的 $y_i$。因此，我们的模型就可以表示成：\n",
-        "\n",
-        "$$\\hat{Y} = \\omega_1X_{1} + \\omega_2X_{2} + \\ldots + \\omega_{13}X_{13} + b$$\n",
-        "\n",
-        "$\\hat{Y}$ 表示模型的预测结果，用来和真实值$Y$区分。模型要学习的参数即：$\\omega_1, \\ldots, \\omega_{13}, b$。\n",
-        "\n",
-        "建立模型后，我们需要给模型一个优化目标，使得学到的参数能够让预测值$\\hat{Y}$尽可能地接近真实值$Y$。这里我们引入损失函数（[Loss Function](https://en.wikipedia.org/wiki/Loss_function)，或Cost Function）这个概念。 输入任意一个数据样本的目标值$y_{i}$和模型给出的预测值$\\hat{y_{i}}$，损失函数输出一个非负的实值。这个实值通常用来反映模型误差的大小。\n",
-        "\n",
-        "对于线性回归模型来讲，最常见的损失函数就是均方误差（Mean Squared Error， [MSE](https://en.wikipedia.org/wiki/Mean_squared_error)）了，它的形式是：\n",
-        "\n",
-        "$$MSE=\\frac{1}{n}\\sum_{i=1}^{n}{(\\hat{Y_i}-Y_i)}^2$$\n",
-        "\n",
-        "即对于一个大小为$n$的测试集，$MSE$是$n$个数据预测结果误差平方的均值。\n",
-        "\n",
-        "### 训练过程\n",
-        "\n",
-        "定义好模型结构之后，我们要通过以下几个步骤进行模型训练\n",
-        " 1. 初始化参数，其中包括权重$\\omega_i$和偏置$b$，对其进行初始化（如0均值，1方差）。\n",
-        " 2. 网络正向传播计算网络输出和损失函数。\n",
-        " 3. 根据损失函数进行反向误差传播 （[backpropagation](https://en.wikipedia.org/wiki/Backpropagation)），将网络误差从输出层依次向前传递, 并更新网络中的参数。\n",
-        " 4. 重复2~3步骤，直至网络训练误差达到规定的程度或训练轮次达到设定值。\n",
-        "\n",
-        "## 数据集\n",
-        "\n",
-        "### 数据集接口的封装\n",
-        "首先加载需要的包\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "import paddle.v2 as paddle\n",
-        "import paddle.v2.dataset.uci_housing as uci_housing\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "我们通过uci_housing模块引入了数据集合[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)\n",
-        "\n",
-        "其中，在uci_housing模块中封装了：\n",
-        "\n",
-        "1. 数据下载的过程。下载数据保存在~/.cache/paddle/dataset/uci_housing/housing.data。\n",
-        "2. [数据预处理](#数据预处理)的过程。\n",
-        "\n",
-        "\n",
-        "### 数据集介绍\n",
-        "这份数据集共506行，每行包含了波士顿郊区的一类房屋的相关信息及该类房屋价格的中位数。其各维属性的意义如下：\n",
-        "\n",
-        "| 属性名 | 解释 | 类型 |\n",
-        "| ------| ------ | ------ |\n",
-        "| CRIM | 该镇的人均犯罪率 | 连续值 |\n",
-        "| ZN | 占地面积超过25,000平方呎的住宅用地比例 | 连续值 |\n",
-        "| INDUS | 非零售商业用地比例 | 连续值 |\n",
-        "| CHAS | 是否邻近 Charles River  | 离散值，1=邻近；0=不邻近 |\n",
-        "| NOX | 一氧化氮浓度 | 连续值 |\n",
-        "| RM | 每栋房屋的平均客房数 | 连续值 |\n",
-        "| AGE | 1940年之前建成的自用单位比例 | 连续值 |\n",
-        "| DIS | 到波士顿5个就业中心的加权距离 | 连续值 |\n",
-        "| RAD | 到径向公路的可达性指数 | 连续值 |\n",
-        "| TAX | 全值财产税率 | 连续值 |\n",
-        "| PTRATIO | 学生与教师的比例 | 连续值 |\n",
-        "| B | 1000(BK - 0.63)^2，其中BK为黑人占比 | 连续值 |\n",
-        "| LSTAT | 低收入人群占比 | 连续值 |\n",
-        "| MEDV | 同类房屋价格的中位数 | 连续值 |\n",
-        "\n",
-        "### 数据预处理\n",
-        "#### 连续值与离散值\n",
-        "观察一下数据，我们的第一个发现是：所有的13维属性中，有12维的连续值和1维的离散值（CHAS）。离散值虽然也常使用类似0、1、2这样的数字表示，但是其含义与连续值是不同的，因为这里的差值没有实际意义。例如，我们用0、1、2来分别表示红色、绿色和蓝色的话，我们并不能因此说“蓝色和红色”比“绿色和红色”的距离更远。所以通常对一个有$d$个可能取值的离散属性，我们会将它们转为$d$个取值为0或1的二值属性或者将每个可能取值映射为一个多维向量。不过就这里而言，因为CHAS本身就是一个二值属性，就省去了这个麻烦。\n",
-        "\n",
-        "#### 属性的归一化\n",
-        "另外一个稍加观察即可发现的事实是，各维属性的取值范围差别很大（如图2所示）。例如，属性B的取值范围是[0.32, 396.90]，而属性NOX的取值范围是[0.3850, 0.8170]。这里就要用到一个常见的操作-归一化（normalization）了。归一化的目标是把各位属性的取值范围放缩到差不多的区间，例如[-0.5,0.5]。这里我们使用一种很常见的操作方法：减掉均值，然后除以原取值范围。\n",
-        "\n",
-        "做归一化（或 [Feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)）至少有以下3个理由：\n",
-        "- 过大或过小的数值范围会导致计算时的浮点上溢或下溢。\n",
-        "- 不同的数值范围会导致不同属性对模型的重要性不同（至少在训练的初始阶段如此），而这个隐含的假设常常是不合理的。这会对优化的过程造成困难，使训练时间大大的加长。\n",
-        "- 很多的机器学习技巧/模型（例如L1，L2正则项，向量空间模型-Vector Space Model）都基于这样的假设：所有的属性取值都差不多是以0为均值且取值范围相近的。\n",
-        "\n",
-        "\u003cp align=\"center\"\u003e\n",
-        "    \u003cimg src = \"image/ranges.png\" width=550\u003e\u003cbr/\u003e\n",
-        "    图2. 各维属性的取值范围\n",
-        "\u003c/p\u003e\n",
-        "\n",
-        "#### 整理训练集与测试集\n",
-        "我们将数据集分割为两份：一份用于调整模型的参数，即进行模型的训练，模型在这份数据集上的误差被称为**训练误差**；另外一份被用来测试，模型在这份数据集上的误差被称为**测试误差**。我们训练模型的目的是为了通过从训练数据中找到规律来预测未知的新数据，所以测试误差是更能反映模型表现的指标。分割数据的比例要考虑到两个因素：更多的训练数据会降低参数估计的方差，从而得到更可信的模型；而更多的测试数据会降低测试误差的方差，从而得到更可信的测试误差。我们这个例子中设置的分割比例为$8:2$\n",
-        "\n",
-        "\n",
-        "在更复杂的模型训练过程中，我们往往还会多使用一种数据集：验证集。因为复杂的模型中常常还有一些超参数（[Hyperparameter](https://en.wikipedia.org/wiki/Hyperparameter_optimization)）需要调节，所以我们会尝试多种超参数的组合来分别训练多个模型，然后对比它们在验证集上的表现选择相对最好的一组超参数，最后才使用这组参数下训练的模型在测试集上评估测试误差。由于本章训练的模型比较简单，我们暂且忽略掉这个过程。\n",
-        "\n",
-        "## 训练\n",
-        "\n",
-        "`fit_a_line/trainer.py`演示了训练的整体过程。\n",
-        "\n",
-        "### 初始化PaddlePaddle\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "paddle.init(use_gpu=False, trainer_count=1)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### 模型配置\n",
-        "\n",
-        "线性回归的模型其实就是一个采用线性激活函数（linear activation，`LinearActivation`）的全连接层（fully-connected layer，`fc_layer`）：\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))\n",
-        "y_predict = paddle.layer.fc(input=x,\n",
-        "                                size=1,\n",
-        "                                act=paddle.activation.Linear())\n",
-        "y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))\n",
-        "cost = paddle.layer.mse_cost(input=y_predict, label=y)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "### 创建参数\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "parameters = paddle.parameters.create(cost)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### 创建Trainer\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "optimizer = paddle.optimizer.Momentum(momentum=0)\n",
-        "\n",
-        "trainer = paddle.trainer.SGD(cost=cost,\n",
-        "                             parameters=parameters,\n",
-        "                             update_equation=optimizer)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### 读取数据且打印训练的中间信息\n",
-        "\n",
-        "PaddlePaddle提供一个\n",
-        "[reader机制](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)\n",
-        "来读取数据。 Reader返回的数据可以包括多列，我们需要一个Python dict把列\n",
-        "序号映射到网络里的数据层。\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "feeding={'x': 0, 'y': 1}\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "此外，我们还可以提供一个 event handler，来打印训练的进度：\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "# event_handler to print training and testing info\n",
-        "import matplotlib.pyplot as plt\n",
-        "from IPython import display\n",
-        "import cPickle\n",
-        "\n",
-        "step=0\n",
-        "\n",
-        "train_costs=[],[]\n",
-        "test_costs=[],[]\n",
-        "\n",
-        "def event_handler(event):\n",
-        "    global step\n",
-        "    global train_costs\n",
-        "    global test_costs\n",
-        "    if isinstance(event, paddle.event.EndIteration):\n",
-        "        need_plot = False\n",
-        "        if step % 10 == 0:  # every 10 batches, record a train cost\n",
-        "            train_costs[0].append(step)\n",
-        "            train_costs[1].append(event.cost)\n",
-        "\n",
-        "        if step % 1000 == 0: # every 1000 batches, record a test cost\n",
-        "            result = trainer.test(\n",
-        "                reader=paddle.batch(\n",
-        "                    uci_housing.test(), batch_size=2),\n",
-        "                feeding=feeding)\n",
-        "            test_costs[0].append(step)\n",
-        "            test_costs[1].append(result.cost)\n",
-        "\n",
-        "        if step % 100 == 0: # every 100 batches, update cost plot\n",
-        "            plt.plot(*train_costs)\n",
-        "            plt.plot(*test_costs)\n",
-        "            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')\n",
-        "            display.clear_output(wait=True)\n",
-        "            display.display(plt.gcf())\n",
-        "            plt.gcf().clear()\n",
-        "        step += 1\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "### 开始训练\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "editable": true
-      },
-      "source": [
-        "trainer.train(\n",
-        "    reader=paddle.batch(\n",
-        "        paddle.reader.shuffle(\n",
-        "            uci_housing.train(), buf_size=500),\n",
-        "        batch_size=2),\n",
-        "    feeding=feeding,\n",
-        "    event_handler=event_handler,\n",
-        "    num_passes=30)\n"
-      ],
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "\n"
-          ]
-        }
-      ],
-      "execution_count": 1
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {},
-      "source": [
-        "\n",
-        "![png](./image/train-and-test.png)\n",
-        "\n",
-        "## 总结\n",
-        "在这章里，我们借助波士顿房价这一数据集，介绍了线性回归模型的基本概念，以及如何使用PaddlePaddle实现训练和测试的过程。很多的模型和技巧都是从简单的线性回归模型演化而来，因此弄清楚线性模型的原理和局限非常重要。\n",
-        "\n",
-        "\n",
-        "## 参考文献\n",
-        "1. https://en.wikipedia.org/wiki/Linear_regression\n",
-        "2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.\n",
-        "3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.\n",
-        "4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.\n",
-        "\n",
-        "\u003cbr/\u003e\n",
-        "\u003ca rel=\"license\" href=\"http://creativecommons.org/licenses/by-nc-sa/4.0/\"\u003e\u003cimg alt=\"知识共享许可协议\" style=\"border-width:0\" src=\"https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png\" /\u003e\u003c/a\u003e\u003cbr /\u003e\u003cspan xmlns:dct=\"http://purl.org/dc/terms/\" href=\"http://purl.org/dc/dcmitype/Text\" property=\"dct:title\" rel=\"dct:type\"\u003e本教程\u003c/span\u003e 由 \u003ca xmlns:cc=\"http://creativecommons.org/ns#\" href=\"http://book.paddlepaddle.org\" property=\"cc:attributionName\" rel=\"cc:attributionURL\"\u003ePaddlePaddle\u003c/a\u003e 创作，采用 \u003ca rel=\"license\" href=\"http://creativecommons.org/licenses/by-nc-sa/4.0/\"\u003e知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议\u003c/a\u003e进行许可。\n"
-      ]
-    }
-  ],
-  "metadata": {
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.0"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

01.fit_a_line/README.md

-# 线性回归
-让我们从经典的线性回归（Linear Regression \[[1](#参考文献)\]）模型开始这份教程。在这一章里，你将使用真实的数据集建立起一个房价预测模型，并且了解到机器学习中的若干重要概念。
+# Linear Regression
+Let us begin the tutorial with a classical problem called Linear Regression \[[1](#References)\]. In this chapter, we will train a model from a realistic dataset to predict home prices. Some important concepts in Machine Learning will be covered through this example.
 
-本教程源代码目录在[book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.md)。
+The source code for this tutorial lives on [book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line). For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
-给定一个大小为$n$的数据集  ${\{y_{i}, x_{i1}, ..., x_{id}\}}_{i=1}^{n}$，其中$x_{i1}, \ldots, x_{id}$是第$i$个样本$d$个属性上的取值，$y_i$是该样本待预测的目标。线性回归模型假设目标$y_i$可以被属性间的线性组合描述，即
+## Problem Setup
+Suppose we have a dataset of $n$ real estate properties. These real estate properties will be referred to as *homes* in this chapter for clarity.
 
-$$y_i = \omega_1x_{i1} + \omega_2x_{i2} + \ldots + \omega_dx_{id} + b,  i=1,\ldots,n$$
+Each home is associated with $d$ attributes. The attributes describe characteristics such the number of rooms in the home, the number of schools or hospitals in the neighborhood, and the traffic condition nearby.
 
-例如，在我们将要建模的房价预测问题里，$x_{ij}$是描述房子$i$的各种属性（比如房间的个数、周围学校和医院的个数、交通状况等），而 $y_i$是房屋的价格。
+In our problem setup, the attribute $x_{i,j}$ denotes the $j$th characteristic of the $i$th home. In addition, $y_i$ denotes the price of the $i$th home. Our task is to predict $y_i$ given a set of attributes $\{x_{i,1}, ..., x_{i,d}\}$. We assume that the price of a home is a linear combination of all of its attributes, namely,
 
-初看起来，这个假设实在过于简单了，变量间的真实关系很难是线性的。但由于线性回归模型有形式简单和易于建模分析的优点，它在实际问题中得到了大量的应用。很多经典的统计学习、机器学习书籍\[[2,3,4](#参考文献)\]也选择对线性模型独立成章重点讲解。
+$$y_i = \omega_1x_{i,1} + \omega_2x_{i,2} + \ldots + \omega_dx_{i,d} + b,  i=1,\ldots,n$$
 
-## 效果展示
-我们使用从[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)获得的波士顿房价数据集进行模型的训练和预测。下面的散点图展示了使用模型对部分房屋价格进行的预测。其中，每个点的横坐标表示同一类房屋真实价格的中位数，纵坐标表示线性回归模型根据特征预测的结果，当二者值完全相等的时候就会落在虚线上。所以模型预测得越准确，则点离虚线越近。
+where $\vec{\omega}$ and $b$ are the model parameters we want to estimate. Once they are learned, we will be able to predict the price of a home, given the attributes associated with it. We call this model **Linear Regression**. In other words, we want to regress a value against several values linearly. In practice, a linear model is often too simplistic to capture the real relationships between the variables. Yet, because Linear Regression is easy to train and analyze, it has been applied to a large number of real problems. As a result, it is an important topic in many classic Statistical Learning and Machine Learning textbooks \[[2,3,4](#References)\].
+
+## Results Demonstration
+We first show the result of our model. The dataset [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) is used to train a linear model to predict the home prices in Boston. The figure below shows the predictions the model makes for some home prices. The $X$-axis represents the median value of the prices of simlilar homes within a bin, while the $Y$-axis represents the home value our linear model predicts. The dotted line represents points where $X=Y$. When reading the diagram, the more precise the model predicts, the closer the point is to the dotted line.
 <p align="center">
-    <img src = "image/predictions.png" width=400><br/>
-    图1. 预测值 V.S. 真实值
+    <img src = "image/predictions_en.png" width=400><br/>
+    Figure 1. Predicted Value V.S. Actual Value
 </p>
 
-## 模型概览
+## Model Overview
 
-### 模型定义
+### Model Definition
 
-在波士顿房价数据集中，和房屋相关的值共有14个：前13个用来描述房屋相关的各种信息，即模型中的 $x_i$；最后一个值为我们要预测的该类房屋价格的中位数，即模型中的 $y_i$。因此，我们的模型就可以表示成：
+In the UCI Housing Data Set, there are 13 home attributes $\{x_{i,j}\}$ that are related to the median home price $y_i$, which we aim to predict. Thus, our model can be written as:
 
 $$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
 
-$\hat{Y}$ 表示模型的预测结果，用来和真实值$Y$区分。模型要学习的参数即：$\omega_1, \ldots, \omega_{13}, b$。
+where $\hat{Y}$ is the predicted value used to differentiate from actual value $Y$. The model learns parameters $\omega_1, \ldots, \omega_{13}, b$, where the entries of $\vec{\omega}$ are **weights** and $b$ is **bias**.
 
-建立模型后，我们需要给模型一个优化目标，使得学到的参数能够让预测值$\hat{Y}$尽可能地接近真实值$Y$。这里我们引入损失函数（[Loss Function](https://en.wikipedia.org/wiki/Loss_function)，或Cost Function）这个概念。 输入任意一个数据样本的目标值$y_{i}$和模型给出的预测值$\hat{y_{i}}$，损失函数输出一个非负的实值。这个实值通常用来反映模型误差的大小。
+Now we need an objective to optimize, so that the learned parameters can make $\hat{Y}$ as close to $Y$ as possible. Let's refer to the concept of [Loss Function (Cost Function)](https://en.wikipedia.org/wiki/Loss_function). A loss function must output a non-negative value, given any pair of the actual value $y_i$ and the predicted value $\hat{y_i}$. This value reflects the magnitutude of the model error.
 
-对于线性回归模型来讲，最常见的损失函数就是均方误差（Mean Squared Error， [MSE](https://en.wikipedia.org/wiki/Mean_squared_error)）了，它的形式是：
+For Linear Regression, the most common loss function is [Mean Square Error (MSE)](https://en.wikipedia.org/wiki/Mean_squared_error) which has the following form:
 
 $$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
 
-即对于一个大小为$n$的测试集，$MSE$是$n$个数据预测结果误差平方的均值。
+That is, for a dataset of size $n$, MSE is the average value of the the prediction sqaure errors.
+
+### Training
 
-### 训练过程
+After setting up our model, there are several major steps to go through to train it:
+1. Initialize the parameters including the weights $\vec{\omega}$ and the bias $b$. For example, we can set their mean values as $0$s, and their standard deviations as $1$s.
+2. Feedforward. Evaluate the network output and compute the corresponding loss.
+3. [Backpropagate](https://en.wikipedia.org/wiki/Backpropagation) the errors. The errors will be propagated from the output layer back to the input layer, during which the model parameters will be updated with the corresponding errors.
+4. Repeat steps 2~3, until the loss is below a predefined threshold or the maximum number of repeats is reached.
 
-定义好模型结构之后，我们要通过以下几个步骤进行模型训练
- 1. 初始化参数，其中包括权重$\omega_i$和偏置$b$，对其进行初始化（如0均值，1方差）。
- 2. 网络正向传播计算网络输出和损失函数。
- 3. 根据损失函数进行反向误差传播 （[backpropagation](https://en.wikipedia.org/wiki/Backpropagation)），将网络误差从输出层依次向前传递, 并更新网络中的参数。
- 4. 重复2~3步骤，直至网络训练误差达到规定的程度或训练轮次达到设定值。
+## Dataset
 
-## 数据集
+### Python Dataset Modules
 
-### 数据集接口的封装
-首先加载需要的包
+Our program starts with importing necessary packages:
 
 ```python
 import paddle.v2 as paddle
 import paddle.v2.dataset.uci_housing as uci_housing
 ```
 
-我们通过uci_housing模块引入了数据集合[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)
+We encapsulated the [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) in our Python module `uci_housing`.  This module can
 
-其中，在uci_housing模块中封装了：
+1. download the dataset to `~/.cache/paddle/dataset/uci_housing/housing.data`, if not yet, and
+2.  [preprocesses](#preprocessing) the dataset.
 
-1. 数据下载的过程。下载数据保存在~/.cache/paddle/dataset/uci_housing/housing.data。
-2. [数据预处理](#数据预处理)的过程。
+### An Introduction of the Dataset
 
+The UCI housing dataset has 506 instances. Each instance describes the attributes of a house in surburban Boston.  The attributes are explained below:
 
-### 数据集介绍
-这份数据集共506行，每行包含了波士顿郊区的一类房屋的相关信息及该类房屋价格的中位数。其各维属性的意义如下：
-
-| 属性名 | 解释 | 类型 |
+| Attribute Name | Characteristic | Data Type |
 | ------| ------ | ------ |
-| CRIM | 该镇的人均犯罪率 | 连续值 |
-| ZN | 占地面积超过25,000平方呎的住宅用地比例 | 连续值 |
-| INDUS | 非零售商业用地比例 | 连续值 |
-| CHAS | 是否邻近 Charles River  | 离散值，1=邻近；0=不邻近 |
-| NOX | 一氧化氮浓度 | 连续值 |
-| RM | 每栋房屋的平均客房数 | 连续值 |
-| AGE | 1940年之前建成的自用单位比例 | 连续值 |
-| DIS | 到波士顿5个就业中心的加权距离 | 连续值 |
-| RAD | 到径向公路的可达性指数 | 连续值 |
-| TAX | 全值财产税率 | 连续值 |
-| PTRATIO | 学生与教师的比例 | 连续值 |
-| B | 1000(BK - 0.63)^2，其中BK为黑人占比 | 连续值 |
-| LSTAT | 低收入人群占比 | 连续值 |
-| MEDV | 同类房屋价格的中位数 | 连续值 |
-
-### 数据预处理
-#### 连续值与离散值
-观察一下数据，我们的第一个发现是：所有的13维属性中，有12维的连续值和1维的离散值（CHAS）。离散值虽然也常使用类似0、1、2这样的数字表示，但是其含义与连续值是不同的，因为这里的差值没有实际意义。例如，我们用0、1、2来分别表示红色、绿色和蓝色的话，我们并不能因此说“蓝色和红色”比“绿色和红色”的距离更远。所以通常对一个有$d$个可能取值的离散属性，我们会将它们转为$d$个取值为0或1的二值属性或者将每个可能取值映射为一个多维向量。不过就这里而言，因为CHAS本身就是一个二值属性，就省去了这个麻烦。
-
-#### 属性的归一化
-另外一个稍加观察即可发现的事实是，各维属性的取值范围差别很大（如图2所示）。例如，属性B的取值范围是[0.32, 396.90]，而属性NOX的取值范围是[0.3850, 0.8170]。这里就要用到一个常见的操作-归一化（normalization）了。归一化的目标是把各位属性的取值范围放缩到差不多的区间，例如[-0.5,0.5]。这里我们使用一种很常见的操作方法：减掉均值，然后除以原取值范围。
-
-做归一化（或 [Feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)）至少有以下3个理由：
-- 过大或过小的数值范围会导致计算时的浮点上溢或下溢。
-- 不同的数值范围会导致不同属性对模型的重要性不同（至少在训练的初始阶段如此），而这个隐含的假设常常是不合理的。这会对优化的过程造成困难，使训练时间大大的加长。
-- 很多的机器学习技巧/模型（例如L1，L2正则项，向量空间模型-Vector Space Model）都基于这样的假设：所有的属性取值都差不多是以0为均值且取值范围相近的。
+| CRIM | per capita crime rate by town | Continuous|
+| ZN | proportion of residential land zoned for lots over 25,000 sq.ft. | Continuous |
+| INDUS | proportion of non-retail business acres per town | Continuous |
+| CHAS | Charles River dummy variable | Discrete, 1 if tract bounds river; 0 otherwise|
+| NOX | nitric oxides concentration (parts per 10 million) | Continuous |
+| RM | average number of rooms per dwelling | Continuous |
+| AGE | proportion of owner-occupied units built prior to 1940 | Continuous |
+| DIS | weighted distances to five Boston employment centres | Continuous |
+| RAD | index of accessibility to radial highways | Continuous |
+| TAX | full-value property-tax rate per $10,000 | Continuous |
+| PTRATIO | pupil-teacher ratio by town | Continuous |
+| B | 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town | Continuous |
+| LSTAT | % lower status of the population | Continuous |
+| MEDV | Median value of owner-occupied homes in $1000's | Continuous |
+
+The last entry is the median home price.
+
+### Preprocessing
+#### Continuous and Discrete Data
+We define a feature vector of length 13 for each home, where each entry corresponds to an attribute. Our first observation is that, among the 13 dimensions, there are 12 continuous dimensions and 1 discrete dimension.
+
+Note that although a discrete value is also written as numeric values such as 0, 1, or 2, its meaning differs from a continuous value drastically.  The linear difference between two discrete values has no meaning. For example, suppose $0$, $1$, and $2$ are used to represent colors *Red*, *Green*, and *Blue* respectively. Judging from the numeric representation of these colors, *Red* differs more from *Blue* than it does from *Green*. Yet in actuality, it is not true that extent to which the color *Blue* is different from *Red* is greater than the extent to which *Green* is different from *Red*. Therefore, when handling a discrete feature that has $d$ possible values, we usually convert it to $d$ new features where each feature takes a binary value, $0$ or $1$, indicating whether the original value is absent or present. Alternatively, the discrete features can be mapped onto a continuous multi-dimensional vector through an embedding table. For our problem here, because CHAS itself is a binary discrete value, we do not need to do any preprocessing.
+
+#### Feature Normalization
+We also observe a huge difference among the value ranges of the 13 features (Figure 2). For instance, the values of feature *B* fall in $[0.32, 396.90]$, whereas those of feature *NOX* has a range of $[0.3850, 0.8170]$. An effective optimization would require data normalization. The goal of data normalization is to scale te values of each feature into roughly the same range, perhaps $[-0.5, 0.5]$. Here, we adopt a popular normalization technique where we substract the mean value from the feature value and divide the result by the width of the original range.
+
+There are at least three reasons for [Feature Normalization](https://en.wikipedia.org/wiki/Feature_scaling) (Feature Scaling):
+- A value range that is too large or too small might cause floating number overflow or underflow during computation.
+- Different value ranges might result in varying *importances* of different features to the model (at least in the beginning of the training process). This assumption about the data is often unreasonable, making the optimization difficult, which in turn results in increased training time.
+- Many machine learning techniques or models (e.g., *L1/L2 regularization* and *Vector Space Model*) assumes that all the features have roughly zero means and their value ranges are similar.
 
 <p align="center">
-    <img src = "image/ranges.png" width=550><br/>
-    图2. 各维属性的取值范围
+    <img src = "image/ranges_en.png" width=550><br/>
+    Figure 2. The value ranges of the features
 </p>
 
-#### 整理训练集与测试集
-我们将数据集分割为两份：一份用于调整模型的参数，即进行模型的训练，模型在这份数据集上的误差被称为**训练误差**；另外一份被用来测试，模型在这份数据集上的误差被称为**测试误差**。我们训练模型的目的是为了通过从训练数据中找到规律来预测未知的新数据，所以测试误差是更能反映模型表现的指标。分割数据的比例要考虑到两个因素：更多的训练数据会降低参数估计的方差，从而得到更可信的模型；而更多的测试数据会降低测试误差的方差，从而得到更可信的测试误差。我们这个例子中设置的分割比例为$8:2$
+#### Prepare Training and Test Sets
+We split the dataset in two, one for adjusting the model parameters, namely, for model training, and the other for model testing. The model error on the former is called the **training error**, and the error on the latter is called the **test error**. Our goal in training a model is to find the statistical dependency between the outputs and the inputs, so that we can predict new outputs given new inputs. As a result, the test error reflects the performance of the model better than the training error does. We consider two things when deciding the ratio of the training set to the test set: 1) More training data will decrease the variance of the parameter estimation, yielding more reliable models; 2) More test data will decrease the variance of the test error, yielding more reliable test errors. One standard split ratio is $8:2$.
+
 
+When training complex models, we usually have one more split: the validation set. Complex models usually have [Hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_optimization) that need to be set before the training process, such as the number of layers in the network. Because hyperparameters are not part of the model parameters, they cannot be trained using the same loss function. Thus we will try several sets of hyperparameters to train several models and cross-validate them on the validation set to pick the best one; finally, the selected trained model is tested on the test set. Because our model is relatively simple, we will omit this validation process.
 
-在更复杂的模型训练过程中，我们往往还会多使用一种数据集：验证集。因为复杂的模型中常常还有一些超参数（[Hyperparameter](https://en.wikipedia.org/wiki/Hyperparameter_optimization)）需要调节，所以我们会尝试多种超参数的组合来分别训练多个模型，然后对比它们在验证集上的表现选择相对最好的一组超参数，最后才使用这组参数下训练的模型在测试集上评估测试误差。由于本章训练的模型比较简单，我们暂且忽略掉这个过程。
 
-## 训练
+## Training
 
-`fit_a_line/trainer.py`演示了训练的整体过程。
+`fit_a_line/trainer.py` demonstrates the training using [PaddlePaddle](http://paddlepaddle.org).
 
-### 初始化PaddlePaddle
+### Initialize PaddlePaddle
 
 ```python
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-### 模型配置
+### Model Configuration
 
-线性回归的模型其实就是一个采用线性激活函数（linear activation，`LinearActivation`）的全连接层（fully-connected layer，`fc_layer`）：
+Logistic regression is essentially a fully-connected layer with linear activation:
 
 ```python
 x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
@@ -128,13 +134,13 @@ y_predict = paddle.layer.fc(input=x,
 y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
 cost = paddle.layer.mse_cost(input=y_predict, label=y)
 ```
-### 创建参数
+### Create Parameters
 
 ```python
 parameters = paddle.parameters.create(cost)
 ```
 
-### 创建Trainer
+### Create Trainer
 
 ```python
 optimizer = paddle.optimizer.Momentum(momentum=0)
@@ -144,59 +150,69 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=optimizer)
 ```
 
-### 读取数据且打印训练的中间信息
+### Feeding Data
 
-PaddlePaddle提供一个
-[reader机制](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
-来读取数据。 Reader返回的数据可以包括多列，我们需要一个Python dict把列
-序号映射到网络里的数据层。
+PaddlePaddle provides the
+[reader mechanism](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
+for loadinng training data. A reader may return multiple columns, and we need a Python dictionary to specify the mapping from column index to data layers.
 
 ```python
 feeding={'x': 0, 'y': 1}
 ```
 
-此外，我们还可以提供一个 event handler，来打印训练的进度：
+Moreover, an event handler is provided to print the training progress:
 
 ```python
 # event_handler to print training and testing info
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f" % (
+                event.pass_id, event.batch_id, event.cost)
 
-step=0
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                uci_housing.test(), batch_size=2),
+            feeding=feeding)
+        print "Test %d, Cost %f" % (event.pass_id, result.cost)
+```
 
-train_costs=[],[]
-test_costs=[],[]
+```python
+# event_handler to print training and testing info
+from paddle.v2.plot import Ploter
 
-def event_handler(event):
+train_title = "Train cost"
+test_title = "Test cost"
+plot_cost = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            plot_cost.append(train_title, step, event.cost)
 
-        if step % 1000 == 0: # every 1000 batches, record a test cost
+        if step % 100 == 0: # every 100 batches, record a test cost
             result = trainer.test(
                 reader=paddle.batch(
                     uci_housing.test(), batch_size=2),
                 feeding=feeding)
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            plot_cost.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            plot_cost.plot()
+
         step += 1
+
+    if isinstance(event, paddle.event.EndPass):
+        if event.pass_id % 10 == 0:
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
 ```
 
-### 开始训练
+### Start Training
 
 ```python
 trainer.train(
@@ -205,21 +221,52 @@ trainer.train(
             uci_housing.train(), buf_size=500),
         batch_size=2),
     feeding=feeding,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     num_passes=30)
 ```
 
-![png](./image/train-and-test.png)
+![png](./image/train_and_test.png)
+
+### Apply model
+
+#### 1. generate testing data
+
+```python
+test_data_creator = paddle.dataset.uci_housing.test()
+test_data = []
+test_label = []
+
+for item in test_data_creator():
+    test_data.append((item[0],))
+    test_label.append(item[1])
+    if len(test_data) == 5:
+        break
+```
+
+#### 2. inference
+
+```python
+# load parameters from tar file.
+# users can remove the comments and change the model name
+# with open('params_pass_20.tar', 'r') as f:
+#     parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=y_predict, parameters=parameters, input=test_data)
+
+for i in xrange(len(probs)):
+    print "label=" + str(test_label[i][0]) + ", predict=" + str(probs[i][0])
+```
 
-## 总结
-在这章里，我们借助波士顿房价这一数据集，介绍了线性回归模型的基本概念，以及如何使用PaddlePaddle实现训练和测试的过程。很多的模型和技巧都是从简单的线性回归模型演化而来，因此弄清楚线性模型的原理和局限非常重要。
+## Summary
+This chapter introduces *Linear Regression* and how to train and test this model with PaddlePaddle, using the UCI Housing Data Set. Because a large number of more complex models and techniques are derived from linear regression, it is important to understand its underlying theory and limitation.
 
 
-## 参考文献
+## References
 1. https://en.wikipedia.org/wiki/Linear_regression
 2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
 3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
 4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

01.fit_a_line/index.cn.html

+
+<html>
+<head>
+  <script type="text/x-mathjax-config">
+  MathJax.Hub.Config({
+    extensions: ["tex2jax.js", "TeX/AMSsymbols.js", "TeX/AMSmath.js"],
+    jax: ["input/TeX", "output/HTML-CSS"],
+    tex2jax: {
+      inlineMath: [ ['$','$'] ],
+      displayMath: [ ['$$','$$'] ],
+      processEscapes: true
+    },
+    "HTML-CSS": { availableFonts: ["TeX"] }
+  });
+  </script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js" async></script>
+  <script type="text/javascript" src="../.tools/theme/marked.js">
+  </script>
+  <link href="http://cdn.bootcss.com/highlight.js/9.9.0/styles/darcula.min.css" rel="stylesheet">
+  <script src="http://cdn.bootcss.com/highlight.js/9.9.0/highlight.min.js"></script>
+  <link href="http://cdn.bootcss.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" rel="stylesheet">
+  <link href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" rel="stylesheet">
+  <link href="../.tools/theme/github-markdown.css" rel='stylesheet'>
+</head>
+<style type="text/css" >
+.markdown-body {
+    box-sizing: border-box;
+    min-width: 200px;
+    max-width: 980px;
+    margin: 0 auto;
+    padding: 45px;
+}
+</style>
+
+
+<body>
+
+<div id="context" class="container-fluid markdown-body">
+</div>
+
+<!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
+<div id="markdown" style='display:none'>
+# 线性回归
+让我们从经典的线性回归（Linear Regression \[[1](#参考文献)\]）模型开始这份教程。在这一章里，你将使用真实的数据集建立起一个房价预测模型，并且了解到机器学习中的若干重要概念。
+
+本教程源代码目录在[book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/137.html)。
+
+## 背景介绍
+给定一个大小为$n$的数据集  ${\{y_{i}, x_{i1}, ..., x_{id}\}}_{i=1}^{n}$，其中$x_{i1}, \ldots, x_{id}$是第$i$个样本$d$个属性上的取值，$y_i$是该样本待预测的目标。线性回归模型假设目标$y_i$可以被属性间的线性组合描述，即
+
+$$y_i = \omega_1x_{i1} + \omega_2x_{i2} + \ldots + \omega_dx_{id} + b,  i=1,\ldots,n$$
+
+例如，在我们将要建模的房价预测问题里，$x_{ij}$是描述房子$i$的各种属性（比如房间的个数、周围学校和医院的个数、交通状况等），而 $y_i$是房屋的价格。
+
+初看起来，这个假设实在过于简单了，变量间的真实关系很难是线性的。但由于线性回归模型有形式简单和易于建模分析的优点，它在实际问题中得到了大量的应用。很多经典的统计学习、机器学习书籍\[[2,3,4](#参考文献)\]也选择对线性模型独立成章重点讲解。
+
+## 效果展示
+我们使用从[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)获得的波士顿房价数据集进行模型的训练和预测。下面的散点图展示了使用模型对部分房屋价格进行的预测。其中，每个点的横坐标表示同一类房屋真实价格的中位数，纵坐标表示线性回归模型根据特征预测的结果，当二者值完全相等的时候就会落在虚线上。所以模型预测得越准确，则点离虚线越近。
+<p align="center">
+    <img src = "image/predictions.png" width=400><br/>
+    图1. 预测值 V.S. 真实值
+</p>
+
+## 模型概览
+
+### 模型定义
+
+在波士顿房价数据集中，和房屋相关的值共有14个：前13个用来描述房屋相关的各种信息，即模型中的 $x_i$；最后一个值为我们要预测的该类房屋价格的中位数，即模型中的 $y_i$。因此，我们的模型就可以表示成：
+
+$$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
+
+$\hat{Y}$ 表示模型的预测结果，用来和真实值$Y$区分。模型要学习的参数即：$\omega_1, \ldots, \omega_{13}, b$。
+
+建立模型后，我们需要给模型一个优化目标，使得学到的参数能够让预测值$\hat{Y}$尽可能地接近真实值$Y$。这里我们引入损失函数（[Loss Function](https://en.wikipedia.org/wiki/Loss_function)，或Cost Function）这个概念。 输入任意一个数据样本的目标值$y_{i}$和模型给出的预测值$\hat{y_{i}}$，损失函数输出一个非负的实值。这个实值通常用来反映模型误差的大小。
+
+对于线性回归模型来讲，最常见的损失函数就是均方误差（Mean Squared Error， [MSE](https://en.wikipedia.org/wiki/Mean_squared_error)）了，它的形式是：
+
+$$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
+
+即对于一个大小为$n$的测试集，$MSE$是$n$个数据预测结果误差平方的均值。
+
+### 训练过程
+
+定义好模型结构之后，我们要通过以下几个步骤进行模型训练
+ 1. 初始化参数，其中包括权重$\omega_i$和偏置$b$，对其进行初始化（如0均值，1方差）。
+ 2. 网络正向传播计算网络输出和损失函数。
+ 3. 根据损失函数进行反向误差传播 （[backpropagation](https://en.wikipedia.org/wiki/Backpropagation)），将网络误差从输出层依次向前传递, 并更新网络中的参数。
+ 4. 重复2~3步骤，直至网络训练误差达到规定的程度或训练轮次达到设定值。
+
+## 数据集
+
+### 数据集接口的封装
+首先加载需要的包
+
+```python
+import paddle.v2 as paddle
+import paddle.v2.dataset.uci_housing as uci_housing
+```
+
+我们通过uci_housing模块引入了数据集合[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)
+
+其中，在uci_housing模块中封装了：
+
+1. 数据下载的过程。下载数据保存在~/.cache/paddle/dataset/uci_housing/housing.data。
+2. [数据预处理](#数据预处理)的过程。
+
+
+### 数据集介绍
+这份数据集共506行，每行包含了波士顿郊区的一类房屋的相关信息及该类房屋价格的中位数。其各维属性的意义如下：
+
+| 属性名 | 解释 | 类型 |
+| ------| ------ | ------ |
+| CRIM | 该镇的人均犯罪率 | 连续值 |
+| ZN | 占地面积超过25,000平方呎的住宅用地比例 | 连续值 |
+| INDUS | 非零售商业用地比例 | 连续值 |
+| CHAS | 是否邻近 Charles River  | 离散值，1=邻近；0=不邻近 |
+| NOX | 一氧化氮浓度 | 连续值 |
+| RM | 每栋房屋的平均客房数 | 连续值 |
+| AGE | 1940年之前建成的自用单位比例 | 连续值 |
+| DIS | 到波士顿5个就业中心的加权距离 | 连续值 |
+| RAD | 到径向公路的可达性指数 | 连续值 |
+| TAX | 全值财产税率 | 连续值 |
+| PTRATIO | 学生与教师的比例 | 连续值 |
+| B | 1000(BK - 0.63)^2，其中BK为黑人占比 | 连续值 |
+| LSTAT | 低收入人群占比 | 连续值 |
+| MEDV | 同类房屋价格的中位数 | 连续值 |
+
+### 数据预处理
+#### 连续值与离散值
+观察一下数据，我们的第一个发现是：所有的13维属性中，有12维的连续值和1维的离散值（CHAS）。离散值虽然也常使用类似0、1、2这样的数字表示，但是其含义与连续值是不同的，因为这里的差值没有实际意义。例如，我们用0、1、2来分别表示红色、绿色和蓝色的话，我们并不能因此说“蓝色和红色”比“绿色和红色”的距离更远。所以通常对一个有$d$个可能取值的离散属性，我们会将它们转为$d$个取值为0或1的二值属性或者将每个可能取值映射为一个多维向量。不过就这里而言，因为CHAS本身就是一个二值属性，就省去了这个麻烦。
+
+#### 属性的归一化
+另外一个稍加观察即可发现的事实是，各维属性的取值范围差别很大（如图2所示）。例如，属性B的取值范围是[0.32, 396.90]，而属性NOX的取值范围是[0.3850, 0.8170]。这里就要用到一个常见的操作-归一化（normalization）了。归一化的目标是把各位属性的取值范围放缩到差不多的区间，例如[-0.5,0.5]。这里我们使用一种很常见的操作方法：减掉均值，然后除以原取值范围。
+
+做归一化（或 [Feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)）至少有以下3个理由：
+- 过大或过小的数值范围会导致计算时的浮点上溢或下溢。
+- 不同的数值范围会导致不同属性对模型的重要性不同（至少在训练的初始阶段如此），而这个隐含的假设常常是不合理的。这会对优化的过程造成困难，使训练时间大大的加长。
+- 很多的机器学习技巧/模型（例如L1，L2正则项，向量空间模型-Vector Space Model）都基于这样的假设：所有的属性取值都差不多是以0为均值且取值范围相近的。
+
+<p align="center">
+    <img src = "image/ranges.png" width=550><br/>
+    图2. 各维属性的取值范围
+</p>
+
+#### 整理训练集与测试集
+我们将数据集分割为两份：一份用于调整模型的参数，即进行模型的训练，模型在这份数据集上的误差被称为**训练误差**；另外一份被用来测试，模型在这份数据集上的误差被称为**测试误差**。我们训练模型的目的是为了通过从训练数据中找到规律来预测未知的新数据，所以测试误差是更能反映模型表现的指标。分割数据的比例要考虑到两个因素：更多的训练数据会降低参数估计的方差，从而得到更可信的模型；而更多的测试数据会降低测试误差的方差，从而得到更可信的测试误差。我们这个例子中设置的分割比例为$8:2$
+
+
+在更复杂的模型训练过程中，我们往往还会多使用一种数据集：验证集。因为复杂的模型中常常还有一些超参数（[Hyperparameter](https://en.wikipedia.org/wiki/Hyperparameter_optimization)）需要调节，所以我们会尝试多种超参数的组合来分别训练多个模型，然后对比它们在验证集上的表现选择相对最好的一组超参数，最后才使用这组参数下训练的模型在测试集上评估测试误差。由于本章训练的模型比较简单，我们暂且忽略掉这个过程。
+
+## 训练
+
+`fit_a_line/trainer.py`演示了训练的整体过程。
+
+### 初始化PaddlePaddle
+
+```python
+paddle.init(use_gpu=False, trainer_count=1)
+```
+
+### 模型配置
+
+线性回归的模型其实就是一个采用线性激活函数（linear activation，`LinearActivation`）的全连接层（fully-connected layer，`fc_layer`）：
+
+```python
+x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
+y_predict = paddle.layer.fc(input=x,
+                                size=1,
+                                act=paddle.activation.Linear())
+y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
+cost = paddle.layer.mse_cost(input=y_predict, label=y)
+```
+### 创建参数
+
+```python
+parameters = paddle.parameters.create(cost)
+```
+
+### 创建Trainer
+
+```python
+optimizer = paddle.optimizer.Momentum(momentum=0)
+
+trainer = paddle.trainer.SGD(cost=cost,
+                             parameters=parameters,
+                             update_equation=optimizer)
+```
+
+### 读取数据且打印训练的中间信息
+
+PaddlePaddle提供一个
+[reader机制](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
+来读取数据。 Reader返回的数据可以包括多列，我们需要一个Python dict把列
+序号映射到网络里的数据层。
+
+```python
+feeding={'x': 0, 'y': 1}
+```
+
+此外，我们还可以提供一个 event handler，来打印训练的进度：
+
+```python
+# event_handler to print training and testing info
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f" % (
+                event.pass_id, event.batch_id, event.cost)
+
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                uci_housing.test(), batch_size=2),
+            feeding=feeding)
+        print "Test %d, Cost %f" % (event.pass_id, result.cost)
+```
+
+```python
+# event_handler to print training and testing info
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 10 == 0:  # every 10 batches, record a train cost
+            cost_ploter.append(train_title, step, event.cost)
+
+        if step % 100 == 0: # every 100 batches, record a test cost
+            result = trainer.test(
+                reader=paddle.batch(
+                    uci_housing.test(), batch_size=2),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
+
+        if step % 100 == 0: # every 100 batches, update cost plot
+            cost_ploter.plot()
+
+        step += 1
+
+    if isinstance(event, paddle.event.EndPass):
+        if event.pass_id % 10 == 0:
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
+```
+
+### 开始训练
+
+```python
+trainer.train(
+    reader=paddle.batch(
+        paddle.reader.shuffle(
+            uci_housing.train(), buf_size=500),
+        batch_size=2),
+    feeding=feeding,
+    event_handler=event_handler_plot,
+    num_passes=30)
+```
+
+![png](./image/train_and_test.png)
+
+### 应用模型
+
+#### 1. 生成测试数据
+
+```python
+test_data_creator = paddle.dataset.uci_housing.test()
+test_data = []
+test_label = []
+
+for item in test_data_creator():
+    test_data.append((item[0],))
+    test_label.append(item[1])
+    if len(test_data) == 5:
+        break
+```
+
+#### 2. 推测 inference
+
+```python
+# load parameters from tar file.
+# users can remove the comments and change the model name
+# with open('params_pass_20.tar', 'r') as f:
+#     parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=y_predict, parameters=parameters, input=test_data)
+
+for i in xrange(len(probs)):
+    print "label=" + str(test_label[i][0]) + ", predict=" + str(probs[i][0])
+```
+
+## 总结
+在这章里，我们借助波士顿房价这一数据集，介绍了线性回归模型的基本概念，以及如何使用PaddlePaddle实现训练和测试的过程。很多的模型和技巧都是从简单的线性回归模型演化而来，因此弄清楚线性模型的原理和局限非常重要。
+
+
+## 参考文献
+1. https://en.wikipedia.org/wiki/Linear_regression
+2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
+3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
+4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
+
+<br/>
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
+
+</div>
+<!-- You can change the lines below now. -->
+
+<script type="text/javascript">
+marked.setOptions({
+  renderer: new marked.Renderer(),
+  gfm: true,
+  breaks: false,
+  smartypants: true,
+  highlight: function(code, lang) {
+    code = code.replace(/&amp;/g, "&")
+    code = code.replace(/&gt;/g, ">")
+    code = code.replace(/&lt;/g, "<")
+    code = code.replace(/&nbsp;/g, " ")
+    return hljs.highlightAuto(code, [lang]).value;
+  }
+});
+document.getElementById("context").innerHTML = marked(
+        document.getElementById("markdown").innerHTML)
+</script>
+</body>

01.fit_a_line/index.en.html

-
-<html>
-<head>
-  <script type="text/x-mathjax-config">
-  MathJax.Hub.Config({
-    extensions: ["tex2jax.js", "TeX/AMSsymbols.js", "TeX/AMSmath.js"],
-    jax: ["input/TeX", "output/HTML-CSS"],
-    tex2jax: {
-      inlineMath: [ ['$','$'] ],
-      displayMath: [ ['$$','$$'] ],
-      processEscapes: true
-    },
-    "HTML-CSS": { availableFonts: ["TeX"] }
-  });
-  </script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js" async></script>
-  <script type="text/javascript" src="../.tools/theme/marked.js">
-  </script>
-  <link href="http://cdn.bootcss.com/highlight.js/9.9.0/styles/darcula.min.css" rel="stylesheet">
-  <script src="http://cdn.bootcss.com/highlight.js/9.9.0/highlight.min.js"></script>
-  <link href="http://cdn.bootcss.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" rel="stylesheet">
-  <link href="https://cdn.jsdelivr.net/perfect-scrollbar/0.6.14/css/perfect-scrollbar.min.css" rel="stylesheet">
-  <link href="../.tools/theme/github-markdown.css" rel='stylesheet'>
-</head>
-<style type="text/css" >
-.markdown-body {
-    box-sizing: border-box;
-    min-width: 200px;
-    max-width: 980px;
-    margin: 0 auto;
-    padding: 45px;
-}
-</style>
-
-
-<body>
-
-<div id="context" class="container-fluid markdown-body">
-</div>
-
-<!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
-<div id="markdown" style='display:none'>
-# Linear Regression
-Let us begin the tutorial with a classical problem called Linear Regression \[[1](#References)\]. In this chapter, we will train a model from a realistic dataset to predict home prices. Some important concepts in Machine Learning will be covered through this example.
-
-The source code for this tutorial lives on [book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line). For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.en.md).
-
-## Problem Setup
-Suppose we have a dataset of $n$ real estate properties. These real estate properties will be referred to as *homes* in this chapter for clarity.
-
-Each home is associated with $d$ attributes. The attributes describe characteristics such the number of rooms in the home, the number of schools or hospitals in the neighborhood, and the traffic condition nearby.
-
-In our problem setup, the attribute $x_{i,j}$ denotes the $j$th characteristic of the $i$th home. In addition, $y_i$ denotes the price of the $i$th home. Our task is to predict $y_i$ given a set of attributes $\{x_{i,1}, ..., x_{i,d}\}$. We assume that the price of a home is a linear combination of all of its attributes, namely,
-
-$$y_i = \omega_1x_{i,1} + \omega_2x_{i,2} + \ldots + \omega_dx_{i,d} + b,  i=1,\ldots,n$$
-
-where $\vec{\omega}$ and $b$ are the model parameters we want to estimate. Once they are learned, we will be able to predict the price of a home, given the attributes associated with it. We call this model **Linear Regression**. In other words, we want to regress a value against several values linearly. In practice, a linear model is often too simplistic to capture the real relationships between the variables. Yet, because Linear Regression is easy to train and analyze, it has been applied to a large number of real problems. As a result, it is an important topic in many classic Statistical Learning and Machine Learning textbooks \[[2,3,4](#References)\].
-
-## Results Demonstration
-We first show the result of our model. The dataset [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) is used to train a linear model to predict the home prices in Boston. The figure below shows the predictions the model makes for some home prices. The $X$-axis represents the median value of the prices of simlilar homes within a bin, while the $Y$-axis represents the home value our linear model predicts. The dotted line represents points where $X=Y$. When reading the diagram, the more precise the model predicts, the closer the point is to the dotted line.
-<p align="center">
-    <img src = "image/predictions_en.png" width=400><br/>
-    Figure 1. Predicted Value V.S. Actual Value
-</p>
-
-## Model Overview
-
-### Model Definition
-
-In the UCI Housing Data Set, there are 13 home attributes $\{x_{i,j}\}$ that are related to the median home price $y_i$, which we aim to predict. Thus, our model can be written as:
-
-$$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
-
-where $\hat{Y}$ is the predicted value used to differentiate from actual value $Y$. The model learns parameters $\omega_1, \ldots, \omega_{13}, b$, where the entries of $\vec{\omega}$ are **weights** and $b$ is **bias**.
-
-Now we need an objective to optimize, so that the learned parameters can make $\hat{Y}$ as close to $Y$ as possible. Let's refer to the concept of [Loss Function (Cost Function)](https://en.wikipedia.org/wiki/Loss_function). A loss function must output a non-negative value, given any pair of the actual value $y_i$ and the predicted value $\hat{y_i}$. This value reflects the magnitutude of the model error.
-
-For Linear Regression, the most common loss function is [Mean Square Error (MSE)](https://en.wikipedia.org/wiki/Mean_squared_error) which has the following form:
-
-$$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
-
-That is, for a dataset of size $n$, MSE is the average value of the the prediction sqaure errors.
-
-### Training
-
-After setting up our model, there are several major steps to go through to train it:
-1. Initialize the parameters including the weights $\vec{\omega}$ and the bias $b$. For example, we can set their mean values as $0$s, and their standard deviations as $1$s.
-2. Feedforward. Evaluate the network output and compute the corresponding loss.
-3. [Backpropagate](https://en.wikipedia.org/wiki/Backpropagation) the errors. The errors will be propagated from the output layer back to the input layer, during which the model parameters will be updated with the corresponding errors.
-4. Repeat steps 2~3, until the loss is below a predefined threshold or the maximum number of repeats is reached.
-
-## Dataset
-
-### Python Dataset Modules
-
-Our program starts with importing necessary packages:
-
-```python
-import paddle.v2 as paddle
-import paddle.v2.dataset.uci_housing as uci_housing
-```
-
-We encapsulated the [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) in our Python module `uci_housing`.  This module can
-
-1. download the dataset to `~/.cache/paddle/dataset/uci_housing/housing.data`, if not yet, and
-2.  [preprocesses](#preprocessing) the dataset.
-
-### An Introduction of the Dataset
-
-The UCI housing dataset has 506 instances. Each instance describes the attributes of a house in surburban Boston.  The attributes are explained below:
-
-| Attribute Name | Characteristic | Data Type |
-| ------| ------ | ------ |
-| CRIM | per capita crime rate by town | Continuous|
-| ZN | proportion of residential land zoned for lots over 25,000 sq.ft. | Continuous |
-| INDUS | proportion of non-retail business acres per town | Continuous |
-| CHAS | Charles River dummy variable | Discrete, 1 if tract bounds river; 0 otherwise|
-| NOX | nitric oxides concentration (parts per 10 million) | Continuous |
-| RM | average number of rooms per dwelling | Continuous |
-| AGE | proportion of owner-occupied units built prior to 1940 | Continuous |
-| DIS | weighted distances to five Boston employment centres | Continuous |
-| RAD | index of accessibility to radial highways | Continuous |
-| TAX | full-value property-tax rate per $10,000 | Continuous |
-| PTRATIO | pupil-teacher ratio by town | Continuous |
-| B | 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town | Continuous |
-| LSTAT | % lower status of the population | Continuous |
-| MEDV | Median value of owner-occupied homes in $1000's | Continuous |
-
-The last entry is the median home price.
-
-### Preprocessing
-#### Continuous and Discrete Data
-We define a feature vector of length 13 for each home, where each entry corresponds to an attribute. Our first observation is that, among the 13 dimensions, there are 12 continuous dimensions and 1 discrete dimension.
-
-Note that although a discrete value is also written as numeric values such as 0, 1, or 2, its meaning differs from a continuous value drastically.  The linear difference between two discrete values has no meaning. For example, suppose $0$, $1$, and $2$ are used to represent colors *Red*, *Green*, and *Blue* respectively. Judging from the numeric representation of these colors, *Red* differs more from *Blue* than it does from *Green*. Yet in actuality, it is not true that extent to which the color *Blue* is different from *Red* is greater than the extent to which *Green* is different from *Red*. Therefore, when handling a discrete feature that has $d$ possible values, we usually convert it to $d$ new features where each feature takes a binary value, $0$ or $1$, indicating whether the original value is absent or present. Alternatively, the discrete features can be mapped onto a continuous multi-dimensional vector through an embedding table. For our problem here, because CHAS itself is a binary discrete value, we do not need to do any preprocessing.
-
-#### Feature Normalization
-We also observe a huge difference among the value ranges of the 13 features (Figure 2). For instance, the values of feature *B* fall in $[0.32, 396.90]$, whereas those of feature *NOX* has a range of $[0.3850, 0.8170]$. An effective optimization would require data normalization. The goal of data normalization is to scale te values of each feature into roughly the same range, perhaps $[-0.5, 0.5]$. Here, we adopt a popular normalization technique where we substract the mean value from the feature value and divide the result by the width of the original range.
-
-There are at least three reasons for [Feature Normalization](https://en.wikipedia.org/wiki/Feature_scaling) (Feature Scaling):
-- A value range that is too large or too small might cause floating number overflow or underflow during computation.
-- Different value ranges might result in varying *importances* of different features to the model (at least in the beginning of the training process). This assumption about the data is often unreasonable, making the optimization difficult, which in turn results in increased training time.
-- Many machine learning techniques or models (e.g., *L1/L2 regularization* and *Vector Space Model*) assumes that all the features have roughly zero means and their value ranges are similar.
-
-<p align="center">
-    <img src = "image/ranges_en.png" width=550><br/>
-    Figure 2. The value ranges of the features
-</p>
-
-#### Prepare Training and Test Sets
-We split the dataset in two, one for adjusting the model parameters, namely, for model training, and the other for model testing. The model error on the former is called the **training error**, and the error on the latter is called the **test error**. Our goal in training a model is to find the statistical dependency between the outputs and the inputs, so that we can predict new outputs given new inputs. As a result, the test error reflects the performance of the model better than the training error does. We consider two things when deciding the ratio of the training set to the test set: 1) More training data will decrease the variance of the parameter estimation, yielding more reliable models; 2) More test data will decrease the variance of the test error, yielding more reliable test errors. One standard split ratio is $8:2$.
-
-
-When training complex models, we usually have one more split: the validation set. Complex models usually have [Hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_optimization) that need to be set before the training process, such as the number of layers in the network. Because hyperparameters are not part of the model parameters, they cannot be trained using the same loss function. Thus we will try several sets of hyperparameters to train several models and cross-validate them on the validation set to pick the best one; finally, the selected trained model is tested on the test set. Because our model is relatively simple, we will omit this validation process.
-
-
-## Training
-
-`fit_a_line/trainer.py` demonstrates the training using [PaddlePaddle](http://paddlepaddle.org).
-
-### Initialize PaddlePaddle
-
-```python
-paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### Model Configuration
-
-Logistic regression is essentially a fully-connected layer with linear activation:
-
-```python
-x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
-y_predict = paddle.layer.fc(input=x,
-                                size=1,
-                                act=paddle.activation.Linear())
-y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
-cost = paddle.layer.mse_cost(input=y_predict, label=y)
-```
-### Create Parameters
-
-```python
-parameters = paddle.parameters.create(cost)
-```
-
-### Create Trainer
-
-```python
-optimizer = paddle.optimizer.Momentum(momentum=0)
-
-trainer = paddle.trainer.SGD(cost=cost,
-                             parameters=parameters,
-                             update_equation=optimizer)
-```
-
-### Feeding Data
-
-PaddlePaddle provides the
-[reader mechanism](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
-for loadinng training data. A reader may return multiple columns, and we need a Python dictionary to specify the mapping from column index to data layers.
-
-```python
-feeding={'x': 0, 'y': 1}
-```
-
-Moreover, an event handler is provided to print the training progress:
-
-```python
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
-step=0
-
-train_costs=[],[]
-test_costs=[],[]
-
-def event_handler(event):
-    global step
-    global train_costs
-    global test_costs
-    if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
-        if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
-
-        if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(
-                reader=paddle.batch(
-                    uci_housing.test(), batch_size=2),
-                feeding=feeding)
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
-
-        if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
-        step += 1
-```
-
-### Start Training
-
-```python
-trainer.train(
-    reader=paddle.batch(
-        paddle.reader.shuffle(
-            uci_housing.train(), buf_size=500),
-        batch_size=2),
-    feeding=feeding,
-    event_handler=event_handler,
-    num_passes=30)
-```
-
-![png](./image/train-and-test.png)
-
-## Summary
-This chapter introduces *Linear Regression* and how to train and test this model with PaddlePaddle, using the UCI Housing Data Set. Because a large number of more complex models and techniques are derived from linear regression, it is important to understand its underlying theory and limitation.
-
-
-## References
-1. https://en.wikipedia.org/wiki/Linear_regression
-2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
-3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
-4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
-
-<br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
-
-</div>
-<!-- You can change the lines below now. -->
-
-<script type="text/javascript">
-marked.setOptions({
-  renderer: new marked.Renderer(),
-  gfm: true,
-  breaks: false,
-  smartypants: true,
-  highlight: function(code, lang) {
-    code = code.replace(/&amp;/g, "&")
-    code = code.replace(/&gt;/g, ">")
-    code = code.replace(/&lt;/g, "<")
-    code = code.replace(/&nbsp;/g, " ")
-    return hljs.highlightAuto(code, [lang]).value;
-  }
-});
-document.getElementById("context").innerHTML = marked(
-        document.getElementById("markdown").innerHTML)
-</script>
-</body>

01.fit_a_line/index.html

@@ -40,127 +40,133 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-# 线性回归
-让我们从经典的线性回归（Linear Regression \[[1](#参考文献)\]）模型开始这份教程。在这一章里，你将使用真实的数据集建立起一个房价预测模型，并且了解到机器学习中的若干重要概念。
+# Linear Regression
+Let us begin the tutorial with a classical problem called Linear Regression \[[1](#References)\]. In this chapter, we will train a model from a realistic dataset to predict home prices. Some important concepts in Machine Learning will be covered through this example.
 
-本教程源代码目录在[book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.md)。
+The source code for this tutorial lives on [book/fit_a_line](https://github.com/PaddlePaddle/book/tree/develop/01.fit_a_line). For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
-给定一个大小为$n$的数据集  ${\{y_{i}, x_{i1}, ..., x_{id}\}}_{i=1}^{n}$，其中$x_{i1}, \ldots, x_{id}$是第$i$个样本$d$个属性上的取值，$y_i$是该样本待预测的目标。线性回归模型假设目标$y_i$可以被属性间的线性组合描述，即
+## Problem Setup
+Suppose we have a dataset of $n$ real estate properties. These real estate properties will be referred to as *homes* in this chapter for clarity.
 
-$$y_i = \omega_1x_{i1} + \omega_2x_{i2} + \ldots + \omega_dx_{id} + b,  i=1,\ldots,n$$
+Each home is associated with $d$ attributes. The attributes describe characteristics such the number of rooms in the home, the number of schools or hospitals in the neighborhood, and the traffic condition nearby.
 
-例如，在我们将要建模的房价预测问题里，$x_{ij}$是描述房子$i$的各种属性（比如房间的个数、周围学校和医院的个数、交通状况等），而 $y_i$是房屋的价格。
+In our problem setup, the attribute $x_{i,j}$ denotes the $j$th characteristic of the $i$th home. In addition, $y_i$ denotes the price of the $i$th home. Our task is to predict $y_i$ given a set of attributes $\{x_{i,1}, ..., x_{i,d}\}$. We assume that the price of a home is a linear combination of all of its attributes, namely,
 
-初看起来，这个假设实在过于简单了，变量间的真实关系很难是线性的。但由于线性回归模型有形式简单和易于建模分析的优点，它在实际问题中得到了大量的应用。很多经典的统计学习、机器学习书籍\[[2,3,4](#参考文献)\]也选择对线性模型独立成章重点讲解。
+$$y_i = \omega_1x_{i,1} + \omega_2x_{i,2} + \ldots + \omega_dx_{i,d} + b,  i=1,\ldots,n$$
 
-## 效果展示
-我们使用从[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)获得的波士顿房价数据集进行模型的训练和预测。下面的散点图展示了使用模型对部分房屋价格进行的预测。其中，每个点的横坐标表示同一类房屋真实价格的中位数，纵坐标表示线性回归模型根据特征预测的结果，当二者值完全相等的时候就会落在虚线上。所以模型预测得越准确，则点离虚线越近。
+where $\vec{\omega}$ and $b$ are the model parameters we want to estimate. Once they are learned, we will be able to predict the price of a home, given the attributes associated with it. We call this model **Linear Regression**. In other words, we want to regress a value against several values linearly. In practice, a linear model is often too simplistic to capture the real relationships between the variables. Yet, because Linear Regression is easy to train and analyze, it has been applied to a large number of real problems. As a result, it is an important topic in many classic Statistical Learning and Machine Learning textbooks \[[2,3,4](#References)\].
+
+## Results Demonstration
+We first show the result of our model. The dataset [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) is used to train a linear model to predict the home prices in Boston. The figure below shows the predictions the model makes for some home prices. The $X$-axis represents the median value of the prices of simlilar homes within a bin, while the $Y$-axis represents the home value our linear model predicts. The dotted line represents points where $X=Y$. When reading the diagram, the more precise the model predicts, the closer the point is to the dotted line.
 <p align="center">
-    <img src = "image/predictions.png" width=400><br/>
-    图1. 预测值 V.S. 真实值
+    <img src = "image/predictions_en.png" width=400><br/>
+    Figure 1. Predicted Value V.S. Actual Value
 </p>
 
-## 模型概览
+## Model Overview
 
-### 模型定义
+### Model Definition
 
-在波士顿房价数据集中，和房屋相关的值共有14个：前13个用来描述房屋相关的各种信息，即模型中的 $x_i$；最后一个值为我们要预测的该类房屋价格的中位数，即模型中的 $y_i$。因此，我们的模型就可以表示成：
+In the UCI Housing Data Set, there are 13 home attributes $\{x_{i,j}\}$ that are related to the median home price $y_i$, which we aim to predict. Thus, our model can be written as:
 
 $$\hat{Y} = \omega_1X_{1} + \omega_2X_{2} + \ldots + \omega_{13}X_{13} + b$$
 
-$\hat{Y}$ 表示模型的预测结果，用来和真实值$Y$区分。模型要学习的参数即：$\omega_1, \ldots, \omega_{13}, b$。
+where $\hat{Y}$ is the predicted value used to differentiate from actual value $Y$. The model learns parameters $\omega_1, \ldots, \omega_{13}, b$, where the entries of $\vec{\omega}$ are **weights** and $b$ is **bias**.
 
-建立模型后，我们需要给模型一个优化目标，使得学到的参数能够让预测值$\hat{Y}$尽可能地接近真实值$Y$。这里我们引入损失函数（[Loss Function](https://en.wikipedia.org/wiki/Loss_function)，或Cost Function）这个概念。 输入任意一个数据样本的目标值$y_{i}$和模型给出的预测值$\hat{y_{i}}$，损失函数输出一个非负的实值。这个实值通常用来反映模型误差的大小。
+Now we need an objective to optimize, so that the learned parameters can make $\hat{Y}$ as close to $Y$ as possible. Let's refer to the concept of [Loss Function (Cost Function)](https://en.wikipedia.org/wiki/Loss_function). A loss function must output a non-negative value, given any pair of the actual value $y_i$ and the predicted value $\hat{y_i}$. This value reflects the magnitutude of the model error.
 
-对于线性回归模型来讲，最常见的损失函数就是均方误差（Mean Squared Error， [MSE](https://en.wikipedia.org/wiki/Mean_squared_error)）了，它的形式是：
+For Linear Regression, the most common loss function is [Mean Square Error (MSE)](https://en.wikipedia.org/wiki/Mean_squared_error) which has the following form:
 
 $$MSE=\frac{1}{n}\sum_{i=1}^{n}{(\hat{Y_i}-Y_i)}^2$$
 
-即对于一个大小为$n$的测试集，$MSE$是$n$个数据预测结果误差平方的均值。
+That is, for a dataset of size $n$, MSE is the average value of the the prediction sqaure errors.
+
+### Training
 
-### 训练过程
+After setting up our model, there are several major steps to go through to train it:
+1. Initialize the parameters including the weights $\vec{\omega}$ and the bias $b$. For example, we can set their mean values as $0$s, and their standard deviations as $1$s.
+2. Feedforward. Evaluate the network output and compute the corresponding loss.
+3. [Backpropagate](https://en.wikipedia.org/wiki/Backpropagation) the errors. The errors will be propagated from the output layer back to the input layer, during which the model parameters will be updated with the corresponding errors.
+4. Repeat steps 2~3, until the loss is below a predefined threshold or the maximum number of repeats is reached.
 
-定义好模型结构之后，我们要通过以下几个步骤进行模型训练
- 1. 初始化参数，其中包括权重$\omega_i$和偏置$b$，对其进行初始化（如0均值，1方差）。
- 2. 网络正向传播计算网络输出和损失函数。
- 3. 根据损失函数进行反向误差传播 （[backpropagation](https://en.wikipedia.org/wiki/Backpropagation)），将网络误差从输出层依次向前传递, 并更新网络中的参数。
- 4. 重复2~3步骤，直至网络训练误差达到规定的程度或训练轮次达到设定值。
+## Dataset
 
-## 数据集
+### Python Dataset Modules
 
-### 数据集接口的封装
-首先加载需要的包
+Our program starts with importing necessary packages:
 
 ```python
 import paddle.v2 as paddle
 import paddle.v2.dataset.uci_housing as uci_housing
 ```
 
-我们通过uci_housing模块引入了数据集合[UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing)
+We encapsulated the [UCI Housing Data Set](https://archive.ics.uci.edu/ml/datasets/Housing) in our Python module `uci_housing`.  This module can
 
-其中，在uci_housing模块中封装了：
+1. download the dataset to `~/.cache/paddle/dataset/uci_housing/housing.data`, if not yet, and
+2.  [preprocesses](#preprocessing) the dataset.
 
-1. 数据下载的过程。下载数据保存在~/.cache/paddle/dataset/uci_housing/housing.data。
-2. [数据预处理](#数据预处理)的过程。
+### An Introduction of the Dataset
 
+The UCI housing dataset has 506 instances. Each instance describes the attributes of a house in surburban Boston.  The attributes are explained below:
 
-### 数据集介绍
-这份数据集共506行，每行包含了波士顿郊区的一类房屋的相关信息及该类房屋价格的中位数。其各维属性的意义如下：
-
-| 属性名 | 解释 | 类型 |
+| Attribute Name | Characteristic | Data Type |
 | ------| ------ | ------ |
-| CRIM | 该镇的人均犯罪率 | 连续值 |
-| ZN | 占地面积超过25,000平方呎的住宅用地比例 | 连续值 |
-| INDUS | 非零售商业用地比例 | 连续值 |
-| CHAS | 是否邻近 Charles River  | 离散值，1=邻近；0=不邻近 |
-| NOX | 一氧化氮浓度 | 连续值 |
-| RM | 每栋房屋的平均客房数 | 连续值 |
-| AGE | 1940年之前建成的自用单位比例 | 连续值 |
-| DIS | 到波士顿5个就业中心的加权距离 | 连续值 |
-| RAD | 到径向公路的可达性指数 | 连续值 |
-| TAX | 全值财产税率 | 连续值 |
-| PTRATIO | 学生与教师的比例 | 连续值 |
-| B | 1000(BK - 0.63)^2，其中BK为黑人占比 | 连续值 |
-| LSTAT | 低收入人群占比 | 连续值 |
-| MEDV | 同类房屋价格的中位数 | 连续值 |
-
-### 数据预处理
-#### 连续值与离散值
-观察一下数据，我们的第一个发现是：所有的13维属性中，有12维的连续值和1维的离散值（CHAS）。离散值虽然也常使用类似0、1、2这样的数字表示，但是其含义与连续值是不同的，因为这里的差值没有实际意义。例如，我们用0、1、2来分别表示红色、绿色和蓝色的话，我们并不能因此说“蓝色和红色”比“绿色和红色”的距离更远。所以通常对一个有$d$个可能取值的离散属性，我们会将它们转为$d$个取值为0或1的二值属性或者将每个可能取值映射为一个多维向量。不过就这里而言，因为CHAS本身就是一个二值属性，就省去了这个麻烦。
-
-#### 属性的归一化
-另外一个稍加观察即可发现的事实是，各维属性的取值范围差别很大（如图2所示）。例如，属性B的取值范围是[0.32, 396.90]，而属性NOX的取值范围是[0.3850, 0.8170]。这里就要用到一个常见的操作-归一化（normalization）了。归一化的目标是把各位属性的取值范围放缩到差不多的区间，例如[-0.5,0.5]。这里我们使用一种很常见的操作方法：减掉均值，然后除以原取值范围。
-
-做归一化（或 [Feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)）至少有以下3个理由：
-- 过大或过小的数值范围会导致计算时的浮点上溢或下溢。
-- 不同的数值范围会导致不同属性对模型的重要性不同（至少在训练的初始阶段如此），而这个隐含的假设常常是不合理的。这会对优化的过程造成困难，使训练时间大大的加长。
-- 很多的机器学习技巧/模型（例如L1，L2正则项，向量空间模型-Vector Space Model）都基于这样的假设：所有的属性取值都差不多是以0为均值且取值范围相近的。
+| CRIM | per capita crime rate by town | Continuous|
+| ZN | proportion of residential land zoned for lots over 25,000 sq.ft. | Continuous |
+| INDUS | proportion of non-retail business acres per town | Continuous |
+| CHAS | Charles River dummy variable | Discrete, 1 if tract bounds river; 0 otherwise|
+| NOX | nitric oxides concentration (parts per 10 million) | Continuous |
+| RM | average number of rooms per dwelling | Continuous |
+| AGE | proportion of owner-occupied units built prior to 1940 | Continuous |
+| DIS | weighted distances to five Boston employment centres | Continuous |
+| RAD | index of accessibility to radial highways | Continuous |
+| TAX | full-value property-tax rate per $10,000 | Continuous |
+| PTRATIO | pupil-teacher ratio by town | Continuous |
+| B | 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town | Continuous |
+| LSTAT | % lower status of the population | Continuous |
+| MEDV | Median value of owner-occupied homes in $1000's | Continuous |
+
+The last entry is the median home price.
+
+### Preprocessing
+#### Continuous and Discrete Data
+We define a feature vector of length 13 for each home, where each entry corresponds to an attribute. Our first observation is that, among the 13 dimensions, there are 12 continuous dimensions and 1 discrete dimension.
+
+Note that although a discrete value is also written as numeric values such as 0, 1, or 2, its meaning differs from a continuous value drastically.  The linear difference between two discrete values has no meaning. For example, suppose $0$, $1$, and $2$ are used to represent colors *Red*, *Green*, and *Blue* respectively. Judging from the numeric representation of these colors, *Red* differs more from *Blue* than it does from *Green*. Yet in actuality, it is not true that extent to which the color *Blue* is different from *Red* is greater than the extent to which *Green* is different from *Red*. Therefore, when handling a discrete feature that has $d$ possible values, we usually convert it to $d$ new features where each feature takes a binary value, $0$ or $1$, indicating whether the original value is absent or present. Alternatively, the discrete features can be mapped onto a continuous multi-dimensional vector through an embedding table. For our problem here, because CHAS itself is a binary discrete value, we do not need to do any preprocessing.
+
+#### Feature Normalization
+We also observe a huge difference among the value ranges of the 13 features (Figure 2). For instance, the values of feature *B* fall in $[0.32, 396.90]$, whereas those of feature *NOX* has a range of $[0.3850, 0.8170]$. An effective optimization would require data normalization. The goal of data normalization is to scale te values of each feature into roughly the same range, perhaps $[-0.5, 0.5]$. Here, we adopt a popular normalization technique where we substract the mean value from the feature value and divide the result by the width of the original range.
+
+There are at least three reasons for [Feature Normalization](https://en.wikipedia.org/wiki/Feature_scaling) (Feature Scaling):
+- A value range that is too large or too small might cause floating number overflow or underflow during computation.
+- Different value ranges might result in varying *importances* of different features to the model (at least in the beginning of the training process). This assumption about the data is often unreasonable, making the optimization difficult, which in turn results in increased training time.
+- Many machine learning techniques or models (e.g., *L1/L2 regularization* and *Vector Space Model*) assumes that all the features have roughly zero means and their value ranges are similar.
 
 <p align="center">
-    <img src = "image/ranges.png" width=550><br/>
-    图2. 各维属性的取值范围
+    <img src = "image/ranges_en.png" width=550><br/>
+    Figure 2. The value ranges of the features
 </p>
 
-#### 整理训练集与测试集
-我们将数据集分割为两份：一份用于调整模型的参数，即进行模型的训练，模型在这份数据集上的误差被称为**训练误差**；另外一份被用来测试，模型在这份数据集上的误差被称为**测试误差**。我们训练模型的目的是为了通过从训练数据中找到规律来预测未知的新数据，所以测试误差是更能反映模型表现的指标。分割数据的比例要考虑到两个因素：更多的训练数据会降低参数估计的方差，从而得到更可信的模型；而更多的测试数据会降低测试误差的方差，从而得到更可信的测试误差。我们这个例子中设置的分割比例为$8:2$
+#### Prepare Training and Test Sets
+We split the dataset in two, one for adjusting the model parameters, namely, for model training, and the other for model testing. The model error on the former is called the **training error**, and the error on the latter is called the **test error**. Our goal in training a model is to find the statistical dependency between the outputs and the inputs, so that we can predict new outputs given new inputs. As a result, the test error reflects the performance of the model better than the training error does. We consider two things when deciding the ratio of the training set to the test set: 1) More training data will decrease the variance of the parameter estimation, yielding more reliable models; 2) More test data will decrease the variance of the test error, yielding more reliable test errors. One standard split ratio is $8:2$.
+
 
+When training complex models, we usually have one more split: the validation set. Complex models usually have [Hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_optimization) that need to be set before the training process, such as the number of layers in the network. Because hyperparameters are not part of the model parameters, they cannot be trained using the same loss function. Thus we will try several sets of hyperparameters to train several models and cross-validate them on the validation set to pick the best one; finally, the selected trained model is tested on the test set. Because our model is relatively simple, we will omit this validation process.
 
-在更复杂的模型训练过程中，我们往往还会多使用一种数据集：验证集。因为复杂的模型中常常还有一些超参数（[Hyperparameter](https://en.wikipedia.org/wiki/Hyperparameter_optimization)）需要调节，所以我们会尝试多种超参数的组合来分别训练多个模型，然后对比它们在验证集上的表现选择相对最好的一组超参数，最后才使用这组参数下训练的模型在测试集上评估测试误差。由于本章训练的模型比较简单，我们暂且忽略掉这个过程。
 
-## 训练
+## Training
 
-`fit_a_line/trainer.py`演示了训练的整体过程。
+`fit_a_line/trainer.py` demonstrates the training using [PaddlePaddle](http://paddlepaddle.org).
 
-### 初始化PaddlePaddle
+### Initialize PaddlePaddle
 
 ```python
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-### 模型配置
+### Model Configuration
 
-线性回归的模型其实就是一个采用线性激活函数（linear activation，`LinearActivation`）的全连接层（fully-connected layer，`fc_layer`）：
+Logistic regression is essentially a fully-connected layer with linear activation:
 
 ```python
 x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
@@ -170,13 +176,13 @@ y_predict = paddle.layer.fc(input=x,
 y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
 cost = paddle.layer.mse_cost(input=y_predict, label=y)
 ```
-### 创建参数
+### Create Parameters
 
 ```python
 parameters = paddle.parameters.create(cost)
 ```
 
-### 创建Trainer
+### Create Trainer
 
 ```python
 optimizer = paddle.optimizer.Momentum(momentum=0)
@@ -186,59 +192,69 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=optimizer)
 ```
 
-### 读取数据且打印训练的中间信息
+### Feeding Data
 
-PaddlePaddle提供一个
-[reader机制](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
-来读取数据。 Reader返回的数据可以包括多列，我们需要一个Python dict把列
-序号映射到网络里的数据层。
+PaddlePaddle provides the
+[reader mechanism](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader)
+for loadinng training data. A reader may return multiple columns, and we need a Python dictionary to specify the mapping from column index to data layers.
 
 ```python
 feeding={'x': 0, 'y': 1}
 ```
 
-此外，我们还可以提供一个 event handler，来打印训练的进度：
+Moreover, an event handler is provided to print the training progress:
 
 ```python
 # event_handler to print training and testing info
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f" % (
+                event.pass_id, event.batch_id, event.cost)
 
-step=0
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                uci_housing.test(), batch_size=2),
+            feeding=feeding)
+        print "Test %d, Cost %f" % (event.pass_id, result.cost)
+```
 
-train_costs=[],[]
-test_costs=[],[]
+```python
+# event_handler to print training and testing info
+from paddle.v2.plot import Ploter
 
-def event_handler(event):
+train_title = "Train cost"
+test_title = "Test cost"
+plot_cost = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            plot_cost.append(train_title, step, event.cost)
 
-        if step % 1000 == 0: # every 1000 batches, record a test cost
+        if step % 100 == 0: # every 100 batches, record a test cost
             result = trainer.test(
                 reader=paddle.batch(
                     uci_housing.test(), batch_size=2),
                 feeding=feeding)
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            plot_cost.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            plot_cost.plot()
+
         step += 1
+
+    if isinstance(event, paddle.event.EndPass):
+        if event.pass_id % 10 == 0:
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
 ```
 
-### 开始训练
+### Start Training
 
 ```python
 trainer.train(
@@ -247,24 +263,55 @@ trainer.train(
             uci_housing.train(), buf_size=500),
         batch_size=2),
     feeding=feeding,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     num_passes=30)
 ```
 
-![png](./image/train-and-test.png)
+![png](./image/train_and_test.png)
+
+### Apply model
+
+#### 1. generate testing data
+
+```python
+test_data_creator = paddle.dataset.uci_housing.test()
+test_data = []
+test_label = []
+
+for item in test_data_creator():
+    test_data.append((item[0],))
+    test_label.append(item[1])
+    if len(test_data) == 5:
+        break
+```
+
+#### 2. inference
+
+```python
+# load parameters from tar file.
+# users can remove the comments and change the model name
+# with open('params_pass_20.tar', 'r') as f:
+#     parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=y_predict, parameters=parameters, input=test_data)
+
+for i in xrange(len(probs)):
+    print "label=" + str(test_label[i][0]) + ", predict=" + str(probs[i][0])
+```
 
-## 总结
-在这章里，我们借助波士顿房价这一数据集，介绍了线性回归模型的基本概念，以及如何使用PaddlePaddle实现训练和测试的过程。很多的模型和技巧都是从简单的线性回归模型演化而来，因此弄清楚线性模型的原理和局限非常重要。
+## Summary
+This chapter introduces *Linear Regression* and how to train and test this model with PaddlePaddle, using the UCI Housing Data Set. Because a large number of more complex models and techniques are derived from linear regression, it is important to understand its underlying theory and limitation.
 
 
-## 参考文献
+## References
 1. https://en.wikipedia.org/wiki/Linear_regression
 2. Friedman J, Hastie T, Tibshirani R. The elements of statistical learning[M]. Springer, Berlin: Springer series in statistics, 2001.
 3. Murphy K P. Machine learning: a probabilistic perspective[M]. MIT press, 2012.
 4. Bishop C M. Pattern recognition[J]. Machine Learning, 2006, 128.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

01.fit_a_line/train.py

@@ -10,7 +10,7 @@ def main():
     x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(13))
     y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear())
     y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1))
-    cost = paddle.layer.mse_cost(input=y_predict, label=y)
+    cost = paddle.layer.square_error_cost(input=y_predict, label=y)
 
     # create parameters
     parameters = paddle.parameters.create(cost)
@@ -21,6 +21,10 @@ def main():
     trainer = paddle.trainer.SGD(
         cost=cost, parameters=parameters, update_equation=optimizer)
 
+    # save model proto as file
+    with open("model.proto", "w") as f:
+        f.write(str(trainer.__topology_in_proto__))
+
     feeding = {'x': 0, 'y': 1}
 
     # event_handler to print training and testing info
@@ -31,6 +35,9 @@ def main():
                     event.pass_id, event.batch_id, event.cost)
 
         if isinstance(event, paddle.event.EndPass):
+            if event.pass_id % 10 == 0:
+                with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                    parameters.to_tar(f)
             result = trainer.test(
                 reader=paddle.batch(uci_housing.test(), batch_size=2),
                 feeding=feeding)
@@ -45,6 +52,28 @@ def main():
         event_handler=event_handler,
         num_passes=30)
 
+    # inference
+    test_data_creator = paddle.dataset.uci_housing.test()
+    test_data = []
+    test_label = []
+
+    for item in test_data_creator():
+        test_data.append((item[0], ))
+        test_label.append(item[1])
+        if len(test_data) == 5:
+            break
+
+    # load parameters from tar file.
+    # users can remove the comments and change the model name
+    # with open('params_pass_20.tar', 'r') as f:
+    #     parameters = paddle.parameters.Parameters.from_tar(f)
+
+    probs = paddle.infer(
+        output_layer=y_predict, parameters=parameters, input=test_data)
+
+    for i in xrange(len(probs)):
+        print "label=" + str(test_label[i][0]) + ", predict=" + str(probs[i][0])
+
 
 if __name__ == '__main__':
     main()

02.recognize_digits/README.cn.md

+# 识别数字
+
+本教程源代码目录在[book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/167.html)。
+
+## 背景介绍
+当我们学习编程的时候，编写的第一个程序一般是实现打印"Hello World"。而机器学习（或深度学习）的入门教程，一般都是 [MNIST](http://yann.lecun.com/exdb/mnist/) 数据库上的手写识别问题。原因是手写识别属于典型的图像分类问题，比较简单，同时MNIST数据集也很完备。MNIST数据集作为一个简单的计算机视觉数据集，包含一系列如图1所示的手写数字图片和对应的标签。图片是28x28的像素矩阵，标签则对应着0~9的10个数字。每张图片都经过了大小归一化和居中处理。
+
+<p align="center">
+<img src="image/mnist_example_image.png" width="400"><br/>
+图1. MNIST图片示例
+</p>
+
+MNIST数据集是从 [NIST](https://www.nist.gov/srd/nist-special-database-19) 的Special Database 3（SD-3）和Special Database 1（SD-1）构建而来。由于SD-3是由美国人口调查局的员工进行标注，SD-1是由美国高中生进行标注，因此SD-3比SD-1更干净也更容易识别。Yann LeCun等人从SD-1和SD-3中各取一半作为MNIST的训练集（60000条数据）和测试集（10000条数据），其中训练集来自250位不同的标注员，此外还保证了训练集和测试集的标注员是不完全相同的。
+
+Yann LeCun早先在手写字符识别上做了很多研究，并在研究过程中提出了卷积神经网络（Convolutional Neural Network），大幅度地提高了手写字符的识别能力，也因此成为了深度学习领域的奠基人之一。如今的深度学习领域，卷积神经网络占据了至关重要的地位，从最早Yann LeCun提出的简单LeNet，到如今ImageNet大赛上的优胜模型VGGNet、GoogLeNet、ResNet等（请参见[图像分类](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) 教程），人们在图像分类领域，利用卷积神经网络得到了一系列惊人的结果。
+
+有很多算法在MNIST上进行实验。1998年，LeCun分别用单层线性分类器、多层感知器（Multilayer Perceptron, MLP）和多层卷积神经网络LeNet进行实验，使得测试集上的误差不断下降（从12%下降到0.7%）\[[1](#参考文献)\]。此后，科学家们又基于K近邻（K-Nearest Neighbors）算法\[[2](#参考文献)\]、支持向量机（SVM）\[[3](#参考文献)\]、神经网络\[[4-7](#参考文献)\]和Boosting方法\[[8](#参考文献)\]等做了大量实验，并采用多种预处理方法（如去除歪曲、去噪、模糊等）来提高识别的准确率。
+
+本教程中，我们从简单的模型Softmax回归开始，带大家入门手写字符识别，并逐步进行模型优化。
+
+
+## 模型概览
+
+基于MNIST数据训练一个分类器，在介绍本教程使用的三个基本图像分类网络前，我们先给出一些定义：
+- $X$是输入：MNIST图片是$28\times28$ 的二维图像，为了进行计算，我们将其转化为$784$维向量，即$X=\left ( x_0, x_1, \dots, x_{783} \right )$。
+- $Y$是输出：分类器的输出是10类数字（0-9），即$Y=\left ( y_0, y_1, \dots, y_9 \right )$，每一维$y_i$代表图片分类为第$i$类数字的概率。
+- $L$是图片的真实标签：$L=\left ( l_0, l_1, \dots, l_9 \right )$也是10维，但只有一维为1，其他都为0。
+
+### Softmax回归(Softmax Regression)
+
+最简单的Softmax回归模型是先将输入层经过一个全连接层得到的特征，然后直接通过softmax 函数进行多分类\[[9](#参考文献)\]。
+
+输入层的数据$X$传到输出层，在激活操作之前，会乘以相应的权重 $W$ ，并加上偏置变量 $b$ ，具体如下：
+
+$$ y_i = \text{softmax}(\sum_j W_{i,j}x_j + b_i) $$
+
+其中 $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
+
+对于有 $N$ 个类别的多分类问题，指定 $N$ 个输出节点，$N$ 维结果向量经过softmax将归一化为 $N$ 个[0,1]范围内的实数值，分别表示该样本属于这 $N$ 个类别的概率。此处的 $y_i$ 即对应该图片为数字 $i$ 的预测概率。
+
+在分类问题中，我们一般采用交叉熵代价损失函数（cross entropy），公式如下：
+
+$$  \text{crossentropy}(label, y) = -\sum_i label_ilog(y_i) $$
+
+图2为softmax回归的网络图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+
+<p align="center">
+<img src="image/softmax_regression.png" width=400><br/>
+图2. softmax回归网络结构图<br/>
+</p>
+
+### 多层感知器(Multilayer Perceptron, MLP)
+
+Softmax回归模型采用了最简单的两层神经网络，即只有输入层和输出层，因此其拟合能力有限。为了达到更好的识别效果，我们考虑在输入层和输出层中间加上若干个隐藏层\[[10](#参考文献)\]。
+
+1.  经过第一个隐藏层，可以得到 $ H_1 = \phi(W_1X + b_1) $，其中$\phi$代表激活函数，常见的有sigmoid、tanh或ReLU等函数。
+2.  经过第二个隐藏层，可以得到 $ H_2 = \phi(W_2H_1 + b_2) $。
+3.  最后，再经过输出层，得到的$Y=\text{softmax}(W_3H_2 + b_3)$，即为最后的分类结果向量。
+
+
+图3为多层感知器的网络结构图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+
+<p align="center">
+<img src="image/mlp.png" width=500><br/>
+图3. 多层感知器网络结构图<br/>
+</p>
+
+### 卷积神经网络(Convolutional Neural Network, CNN)
+
+在多层感知器模型中，将图像展开成一维向量输入到网络中，忽略了图像的位置和结构信息，而卷积神经网络能够更好的利用图像的结构信息。[LeNet-5](http://yann.lecun.com/exdb/lenet/)是一个较简单的卷积神经网络。图4显示了其结构：输入的二维图像，先经过两次卷积层到池化层，再经过全连接层，最后使用softmax分类作为输出层。下面我们主要介绍卷积层和池化层。
+
+<p align="center">
+<img src="image/cnn.png"><br/>
+图4. LeNet-5卷积神经网络结构<br/>
+</p>
+
+#### 卷积层
+
+卷积层是卷积神经网络的核心基石。在图像识别里我们提到的卷积是二维卷积，即离散二维滤波器（也称作卷积核）与二维图像做卷积操作，简单的讲是二维滤波器滑动到二维图像上所有位置，并在每个位置上与该像素点及其领域像素点做内积。卷积操作被广泛应用与图像处理领域，不同卷积核可以提取不同的特征，例如边沿、线性、角等特征。在深层卷积神经网络中，通过卷积操作可以提取出图像低级到复杂的特征。
+
+<p align="center">
+<img src="image/conv_layer.png" width='750'><br/>
+图5. 卷积层图片<br/>
+</p>
+
+图5给出一个卷积计算过程的示例图，输入图像大小为$H=5,W=5,D=3$，即$5 \times 5$大小的3通道（RGB，也称作深度）彩色图像。这个示例图中包含两（用$K$表示）组卷积核，即图中滤波器$W_0$和$W_1$。在卷积计算中，通常对不同的输入通道采用不同的卷积核，如图示例中每组卷积核包含（$D=3）$个$3 \times 3$（用$F \times F$表示）大小的卷积核。另外，这个示例中卷积核在图像的水平方向（$W$方向）和垂直方向（$H$方向）的滑动步长为2（用$S$表示）；对输入图像周围各填充1（用$P$表示）个0，即图中输入层原始数据为蓝色部分，灰色部分是进行了大小为1的扩展，用0来进行扩展。经过卷积操作得到输出为$3 \times 3 \times 2$（用$H_{o} \times W_{o} \times K$表示）大小的特征图，即$3 \times 3$大小的2通道特征图，其中$H_o$计算公式为：$H_o = (H - F + 2 \times P)/S + 1$，$W_o$同理。 而输出特征图中的每个像素，是每组滤波器与输入图像每个特征图的内积再求和，再加上偏置$b_o$，偏置通常对于每个输出特征图是共享的。输出特征图$o[:,:,0]$中的最后一个$-2$计算如图5右下角公式所示。
+
+在卷积操作中卷积核是可学习的参数，经过上面示例介绍，每层卷积的参数大小为$D \times F \times F \times K$。在多层感知器模型中，神经元通常是全部连接，参数较多。而卷积层的参数较少，这也是由卷积层的主要特性即局部连接和共享权重所决定。
+
+- 局部连接：每个神经元仅与输入神经元的一块区域连接，这块局部区域称作感受野（receptive field）。在图像卷积操作中，即神经元在空间维度（spatial dimension，即上图示例H和W所在的平面）是局部连接，但在深度上是全部连接。对于二维图像本身而言，也是局部像素关联较强。这种局部连接保证了学习后的过滤器能够对于局部的输入特征有最强的响应。局部连接的思想，也是受启发于生物学里面的视觉系统结构，视觉皮层的神经元就是局部接受信息的。
+
+- 权重共享：计算同一个深度切片的神经元时采用的滤波器是共享的。例如图4中计算$o[:,:,0]$的每个每个神经元的滤波器均相同，都为$W_0$，这样可以很大程度上减少参数。共享权重在一定程度上讲是有意义的，例如图片的底层边缘特征与特征在图中的具体位置无关。但是在一些场景中是无意的，比如输入的图片是人脸，眼睛和头发位于不同的位置，希望在不同的位置学到不同的特征 (参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/))。请注意权重只是对于同一深度切片的神经元是共享的，在卷积层，通常采用多组卷积核提取不同特征，即对应不同深度切片的特征，不同深度切片的神经元权重是不共享。另外，偏重对同一深度切片的所有神经元都是共享的。
+
+通过介绍卷积计算过程及其特性，可以看出卷积是线性操作，并具有平移不变性（shift-invariant），平移不变性即在图像每个位置执行相同的操作。卷积层的局部连接和权重共享使得需要学习的参数大大减小，这样也有利于训练较大卷积神经网络。
+
+#### 池化层
+
+<p align="center">
+<img src="image/max_pooling.png" width="400px"><br/>
+图6. 池化层图片<br/>
+</p>
+
+池化是非线性下采样的一种形式，主要作用是通过减少网络的参数来减小计算量，并且能够在一定程度上控制过拟合。通常在卷积层的后面会加上一个池化层。池化包括最大池化、平均池化等。其中最大池化是用不重叠的矩形框将输入层分成不同的区域，对于每个矩形框的数取最大值作为输出层，如图6所示。
+
+更详细的关于卷积神经网络的具体知识可以参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/ )和[图像分类](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md)教程。
+
+### 常见激活函数介绍  
+- sigmoid激活函数： $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
+
+- tanh激活函数： $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
+
+  实际上，tanh函数只是规模变化的sigmoid函数，将sigmoid函数值放大2倍之后再向下平移1个单位：tanh(x) = 2sigmoid(2x) - 1 。
+
+- ReLU激活函数： $ f(x) = max(0, x) $
+
+更详细的介绍请参考[维基百科激活函数](https://en.wikipedia.org/wiki/Activation_function)。
+
+## 数据介绍
+
+PaddlePaddle在API中提供了自动加载[MNIST](http://yann.lecun.com/exdb/mnist/)数据的模块`paddle.dataset.mnist`。加载后的数据位于`/home/username/.cache/paddle/dataset/mnist`下：
+
+
+|    文件名称          |       说明              |
+|----------------------|-------------------------|
+|train-images-idx3-ubyte|  训练数据图片，60,000条数据 |
+|train-labels-idx1-ubyte|  训练数据标签，60,000条数据 |
+|t10k-images-idx3-ubyte |  测试数据图片，10,000条数据 |
+|t10k-labels-idx1-ubyte |  测试数据标签，10,000条数据 |
+
+## 配置说明
+
+首先，加载PaddlePaddle的V2 api包。
+
+```python
+import paddle.v2 as paddle
+```
+其次，定义三个不同的分类器：
+
+- Softmax回归：只通过一层简单的以softmax为激活函数的全连接层，就可以得到分类的结果。
+
+```python
+def softmax_regression(img):
+    predict = paddle.layer.fc(input=img,
+                              size=10,
+                              act=paddle.activation.Softmax())
+    return predict
+```
+- 多层感知器：下面代码实现了一个含有两个隐藏层（即全连接层）的多层感知器。其中两个隐藏层的激活函数均采用ReLU，输出层的激活函数用Softmax。
+
+```python
+def multilayer_perceptron(img):
+    # 第一个全连接层，激活函数为ReLU
+    hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
+    # 第二个全连接层，激活函数为ReLU
+    hidden2 = paddle.layer.fc(input=hidden1,
+                              size=64,
+                              act=paddle.activation.Relu())
+    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
+    predict = paddle.layer.fc(input=hidden2,
+                              size=10,
+                              act=paddle.activation.Softmax())
+    return predict
+```
+- 卷积神经网络LeNet-5: 输入的二维图像，首先经过两次卷积层到池化层，再经过全连接层，最后使用以softmax为激活函数的全连接层作为输出层。
+
+```python
+def convolutional_neural_network(img):
+    # 第一个卷积-池化层
+    conv_pool_1 = paddle.networks.simple_img_conv_pool(
+        input=img,
+        filter_size=5,
+        num_filters=20,
+        num_channel=1,
+        pool_size=2,
+        pool_stride=2,
+        act=paddle.activation.Relu())
+    # 第二个卷积-池化层
+    conv_pool_2 = paddle.networks.simple_img_conv_pool(
+        input=conv_pool_1,
+        filter_size=5,
+        num_filters=50,
+        num_channel=20,
+        pool_size=2,
+        pool_stride=2,
+        act=paddle.activation.Relu())
+    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
+    predict = paddle.layer.fc(input=conv_pool_2,
+                              size=10,
+                              act=paddle.activation.Softmax())
+    return predict
+```
+
+接着，通过`layer.data`调用来获取数据，然后调用分类器（这里我们提供了三个不同的分类器）得到分类结果。训练时，对该结果计算其损失函数，分类问题常常选择交叉熵损失函数。
+
+```python
+# 该模型运行在单个CPU上
+paddle.init(use_gpu=False, trainer_count=1)
+
+images = paddle.layer.data(
+    name='pixel', type=paddle.data_type.dense_vector(784))
+label = paddle.layer.data(
+    name='label', type=paddle.data_type.integer_value(10))
+
+# predict = softmax_regression(images) # Softmax回归
+# predict = multilayer_perceptron(images) #多层感知器
+predict = convolutional_neural_network(images) #LeNet5卷积神经网络
+
+cost = paddle.layer.classification_cost(input=predict, label=label)
+```
+
+然后，指定训练相关的参数。
+- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器 `Momentum` ，其中参数0.9代表动量优化每次保持前一次速度的0.9倍。
+- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
+- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+
+```python
+parameters = paddle.parameters.create(cost)
+
+optimizer = paddle.optimizer.Momentum(
+    learning_rate=0.1 / 128.0,
+    momentum=0.9,
+    regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128))
+
+trainer = paddle.trainer.SGD(cost=cost,
+                             parameters=parameters,
+                             update_equation=optimizer)
+```
+
+下一步，我们开始训练过程。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python yield generator。
+
+下面`shuffle`是一个reader decorator，它接受一个reader A，返回另一个reader B —— reader B 每次读入`buffer_size`条训练数据到一个buffer里，然后随机打乱其顺序，并且逐条输出。
+
+`batch`是一个特殊的decorator，它的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minibatch。
+
+`event_handler_plot`可以用来在训练过程中画图如下：
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+# event_handler to plot a figure
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 100 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=paddle.batch(
+            paddle.dataset.mnist.test(), batch_size=128))
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` 用来在训练过程中输出训练结果
+```python
+lists = []
+
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=paddle.batch(
+            paddle.dataset.mnist.test(), batch_size=128))
+        print "Test with Pass %d, Cost %f, %s\n" % (
+            event.pass_id, result.cost, result.metrics)
+        lists.append((event.pass_id, result.cost,
+                      result.metrics['classification_error_evaluator']))
+```
+
+```python
+trainer.train(
+    reader=paddle.batch(
+        paddle.reader.shuffle(
+            paddle.dataset.mnist.train(), buf_size=8192),
+        batch_size=128),
+    event_handler=event_handler_plot,
+    num_passes=5)
+```
+
+训练过程是完全自动的，event_handler里打印的日志类似如下所示：
+
+```
+# Pass 0, Batch 0, Cost 2.780790, {'classification_error_evaluator': 0.9453125}
+# Pass 0, Batch 100, Cost 0.635356, {'classification_error_evaluator': 0.2109375}
+# Pass 0, Batch 200, Cost 0.326094, {'classification_error_evaluator': 0.1328125}
+# Pass 0, Batch 300, Cost 0.361920, {'classification_error_evaluator': 0.1015625}
+# Pass 0, Batch 400, Cost 0.410101, {'classification_error_evaluator': 0.125}
+# Test with Pass 0, Cost 0.326659, {'classification_error_evaluator': 0.09470000118017197}
+```
+
+训练之后，检查模型的预测准确度。用 MNIST 训练的时候，一般 softmax回归模型的分类准确率为约为 92.34%，多层感知器为97.66%，卷积神经网络可以达到 99.20%。
+
+
+## 应用模型
+
+可以使用训练好的模型对手写体数字图片进行分类，下面程序展示了如何使用paddle.infer接口进行推断。
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file).convert('L')
+    im = im.resize((28, 28), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32).flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/infer_3.png'),))
+
+probs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/infer_3.png is: %d" % lab[0][0]
+```
+
+## 总结
+
+本教程的softmax回归、多层感知器和卷积神经网络是最基础的深度学习模型，后续章节中复杂的神经网络都是从它们衍生出来的，因此这几个模型对之后的学习大有裨益。同时，我们也观察到从最简单的softmax回归变换到稍复杂的卷积神经网络的时候，MNIST数据集上的识别准确率有了大幅度的提升，原因是卷积层具有局部连接和共享权重的特性。在之后学习新模型的时候，希望大家也要深入到新模型相比原模型带来效果提升的关键之处。此外，本教程还介绍了PaddlePaddle模型搭建的基本流程，从dataprovider的编写、网络层的构建，到最后的训练和预测。对这个流程熟悉以后，大家就可以用自己的数据，定义自己的网络模型，并完成自己的训练和预测任务了。
+
+## 参考文献
+
+1. LeCun, Yann, Léon Bottou, Yoshua Bengio, and Patrick Haffner. ["Gradient-based learning applied to document recognition."](http://ieeexplore.ieee.org/abstract/document/726791/) Proceedings of the IEEE 86, no. 11 (1998): 2278-2324.
+2. Wejéus, Samuel. ["A Neural Network Approach to Arbitrary SymbolRecognition on Modern Smartphones."](http://www.diva-portal.org/smash/record.jsf?pid=diva2%3A753279&dswid=-434) (2014).
+3. Decoste, Dennis, and Bernhard Schölkopf. ["Training invariant support vector machines."](http://link.springer.com/article/10.1023/A:1012454411458) Machine learning 46, no. 1-3 (2002): 161-190.
+4. Simard, Patrice Y., David Steinkraus, and John C. Platt. ["Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis."](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.160.8494&rep=rep1&type=pdf) In ICDAR, vol. 3, pp. 958-962. 2003.
+5. Salakhutdinov, Ruslan, and Geoffrey E. Hinton. ["Learning a Nonlinear Embedding by Preserving Class Neighbourhood Structure."](http://www.jmlr.org/proceedings/papers/v2/salakhutdinov07a/salakhutdinov07a.pdf) In AISTATS, vol. 11. 2007.
+6. Cireşan, Dan Claudiu, Ueli Meier, Luca Maria Gambardella, and Jürgen Schmidhuber. ["Deep, big, simple neural nets for handwritten digit recognition."](http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00052) Neural computation 22, no. 12 (2010): 3207-3220.
+7. Deng, Li, Michael L. Seltzer, Dong Yu, Alex Acero, Abdel-rahman Mohamed, and Geoffrey E. Hinton. ["Binary coding of speech spectrograms using a deep auto-encoder."](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.185.1908&rep=rep1&type=pdf) In Interspeech, pp. 1692-1695. 2010.
+8. Kégl, Balázs, and Róbert Busa-Fekete. ["Boosting products of base classifiers."](http://dl.acm.org/citation.cfm?id=1553439) In Proceedings of the 26th Annual International Conference on Machine Learning, pp. 497-504. ACM, 2009.
+9. Rosenblatt, Frank. ["The perceptron: A probabilistic model for information storage and organization in the brain."](http://psycnet.apa.org/journals/rev/65/6/386/) Psychological review 65, no. 6 (1958): 386.
+10. Bishop, Christopher M. ["Pattern recognition."](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) Machine Learning 128 (2006): 1-58.
+
+<br/>
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

02.recognize_digits/README.en.md

-# Recognize Digits
-
-The source code for this tutorial is live at [book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits). To get started with Paddle, please refer to [installation instructions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
-
-## Introduction
-Handwriting recognition is a classic image classification problem. In particular, recognizing hand-written digits on [MNIST](http://yann.lecun.com/exdb/mnist/) is the deep learning equivalence to a 'Hello World!' program.
-
-### MNIST
-
-MNIST is a complete dataset containing images of handwritten digits and their corresponding labels. Each of the input image is a $28\times28$ matrix, and the label is one of the digits from $0$ to $9$ (Fig. 1). All images are normalized, meaning that they are both scaled and centered.
-
-<p align="center">
-<img src="image/mnist_example_image.png" width="400"><br/>
-Fig. 1. Examples of MNIST images
-</p>
-
-The MNIST dataset is created from the [NIST](https://www.nist.gov/srd/nist-special-database-19) Special Database 3 (SD-3) and the Special Database 1 (SD-1). The SD-3 is labeled by the staff of the U.S. Census Bureau, while SD-1 is labeled by high school students the in U.S. Therefore the SD-3 is cleaner and easier to recognize than the SD-1 dataset. Yann LeCun et al. used half of the samples from each of SD-1 and SD-3 to create the MNIST training set (60,000 samples) and test set (10,000 samples), where training set was labeled by 250 different annotators, and it was guaranteed that there wasn't a complete overlap of annotators of training set and test set.
-
-Yann LeCun, one of the founders of Deep Learning, have previously made tremendous contributions to handwritten character recognition and proposed the **Convolutional Neural Network** (CNN), which drastically improved recognition capability for handwritten characters. CNNs are now a critical concept in Deep Learning. From the LeNet proposal by Yann LeCun, to those winning models in ImageNet competitions, such as VGGNet, GoogLeNet, and ResNet (See [Image Classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) tutorial), CNNs have achieved a series of impressive results in Image Classification tasks.
-
-Many algorithms are tested on MNIST. In 1998, LeCun experimented with single layer linear classifier, Multilayer Perceptron (MLP) and Multilayer CNN LeNet. These algorithms quickly reduced test error from 12% to 0.7% \[[1](#references)\]. Since then, researchers have worked on many algorithms such as **K-Nearest Neighbors** (k-NN) \[[2](#references)\], **Support Vector Machine** (SVM) \[[3](#references)\], **Neural Networks** \[[4-7](#references)\] and **Boosting** \[[8](#references)\]. Various preprocessing methods like distortion removal, noise removal, and blurring, have also been applied to increase recognition accuracy.
-
-In this tutorial, we tackle the task of handwritten character recognition. We start with a simple **softmax** regression model and guide our readers step-by-step to improve this model's performance on the task of recognition.
-
-
-## Model Overview
-
-Before introducing classification algorithms and training procedure, we define the following symbols:
-- $X$ is the input: Input is a $28\times 28$ MNIST image. It is flattened to a $784$ dimensional vector. $X=\left (x_0, x_1, \dots, x_{783} \right )$.
-- $Y$ is the output: Output of the classifier is 1 of the 10 classes (digits from 0 to 9). $Y=\left (y_0, y_1, \dots, y_9 \right )$. Each dimension $y_i$ represents the probability that the input image belongs to class $i$.
-- $L$ is the ground truth label: $L=\left ( l_0, l_1, \dots, l_9 \right )$. It is also 10 dimensional, but only one entry is $1$ and all others are $0$s.
-
-### Softmax Regression
-
-In a simple softmax regression model, the input is first fed to fully connected layers. Then, a softmax function is applied to output probabilities of multiple output classes\[[9](#references)\].
-
-The input $X$ is multiplied by weights $W$ and then added to the bias $b$ to generate activations.
-
-$$ y_i = \text{softmax}(\sum_j W_{i,j}x_j + b_i) $$
-
-where $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
-
-For an $N$-class classification problem with $N$ output nodes, Softmax normalizes the resulting $N$ dimensional vector so that each of its entries falls in the range $[0,1]\in\math{R}$, representing the probability that the sample belongs to a certain class. Here $y_i$ denotes the predicted probability that an image is of digit $i$.
-
-In such a classification problem, we usually use the cross entropy loss function:
-
-$$  \text{crossentropy}(label, y) = -\sum_i label_ilog(y_i) $$
-
-Fig. 2 illustrates a softmax regression network, with the weights in blue, and the bias in red. `+1` indicates that the bias is $1$.
-
-<p align="center">
-<img src="image/softmax_regression_en.png" width=400><br/>
-Fig. 2. Softmax regression network architecture<br/>
-</p>
-
-### Multilayer Perceptron
-
-The softmax regression model described above uses the simplest two-layer neural network. That is, it only contains an input layer and an output layer, with limited regression capability. To achieve better recognition results, consider adding several hidden layers\[[10](#references)\] between the input layer and the output layer.
-
-1.  After the first hidden layer, we get $ H_1 = \phi(W_1X + b_1) $, where $\phi$ denotes the activation function. Some [common ones](###list-of-common-activation-functions) are sigmoid, tanh and ReLU.
-2.  After the second hidden layer, we get $ H_2 = \phi(W_2H_1 + b_2) $.
-3.  Finally, the output layer outputs $Y=\text{softmax}(W_3H_2 + b_3)$, the vector denoting our classification result.
-
-Fig. 3. shows a Multilayer Perceptron network, with the weights in blue, and the bias in red. +1 indicates that the bias is $1$.
-
-<p align="center">
-<img src="image/mlp_en.png" width=500><br/>
-Fig. 3. Multilayer Perceptron network architecture<br/>
-
-</p>
-
-### Convolutional Neural Network
-
-#### Convolutional Layer
-
-<p align="center">
-<img src="image/conv_layer.png" width='750'><br/>
-Fig. 4. Convolutional layer<br/>
-</p>
-
-The **convolutional layer** is the core of a Convolutional Neural Network. The parameters in this layer are composed of a set of filters, also called kernels. We could visualize the convolution step in the following fashion: Each kernel slides horizontally and vertically till it covers the whole image. At every window, we compute the dot product of the kernel and the input. Then, we add the bias and apply an activation function. The result is a two-dimensional activation map. For example, some kernel may recognize corners, and some may recognize circles. These convolution kernels may respond strongly to the corresponding features.
-
-Fig. 4 illustrates the dynamic programming of a convolutional layer, where depths are flattened for simplicity. The input is $W_1=5$, $H_1=5$, $D_1=3$. In fact, this is a common representation for colored images. $W_1$ and $H_1$ correspond to the width and height in a colored image. $D_1$ corresponds to the 3 color channels for RGB. The parameters of the convolutional layer are $K=2$, $F=3$, $S=2$, $P=1$. $K$ denotes the number of kernels; specifically, $Filter$ $W_0$ and $Filter$ $W_1$ are the kernels. $F$ is kernel size while $W0$ and $W1$ are both $F\timesF = 3\times3$ matrices in all depths. $S$ is the stride, which is the width of the sliding window; here, kernels move leftwards or downwards by 2 units each time. $P$ is the width of the padding, which denotes an extension of the input; here, the gray area shows zero padding with size 1.
-
-#### Pooling Layer
-
-<p align="center">
-<img src="image/max_pooling_en.png" width="400px"><br/>
-Fig. 5 Pooling layer using max-pooling<br/>
-</p>
-
-A **pooling layer** performs downsampling. The main functionality of this layer is to reduce computation by reducing the network parameters. It also prevents over-fitting to some extent. Usually, a pooling layer is added after a convolutional layer. Pooling layer can use various techniques, such as max pooling and average pooling. As shown in Fig.5, max pooling uses rectangles to segment the input layer into several parts and computes the maximum value in each part as the output.
-
-#### LeNet-5 Network
-
-<p align="center">
-<img src="image/cnn_en.png"><br/>
-Fig. 6. LeNet-5 Convolutional Neural Network architecture<br/>
-</p>
-
-[**LeNet-5**](http://yann.lecun.com/exdb/lenet/) is one of the simplest Convolutional Neural Networks. Fig. 6. shows its architecture: A 2-dimensional input image is fed into two sets of convolutional layers and pooling layers. This output is then fed to a fully connected layer and a softmax classifier. Compared to multilayer, fully connected perceptrons, the LeNet-5 can recognize images better. This is due to the following three properties of the convolution:
-
-- The 3D nature of the neurons: a convolutional layer is organized by width, height and depth. Neurons in each layer are connected to only a small region in the previous layer. This region is called the receptive field.
-- Local connectivity: A CNN utilizes the local space correlation by connecting local neurons. This design guarantees that the learned filter has a strong response to local input features. Stacking many such layers generates a non-linear filter that is more global. This enables the network to first obtain good representation for small parts of input and then combine them to represent a larger region.
-- Weight sharing: In a CNN, computation is iterated on shared parameters (weights and bias) to form a feature map. This means that all the neurons in the same depth of the output respond to the same feature. This allows the network to detect a feature regardless of its position in the input. In other words, it is shift invariant.
-
-For more details on Convolutional Neural Networks, please refer to the tutorial on [Image Classification](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md) and the [relevant lecture](http://cs231n.github.io/convolutional-networks/) from a Stanford open course.
-
-### List of Common Activation Functions  
-- Sigmoid activation function: $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
-
-- Tanh activation function: $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
-
-  In fact, tanh function is just a rescaled version of the sigmoid function. It is obtained by magnifying the value of the sigmoid function and moving it downwards by 1.
-
-- ReLU activation function: $ f(x) = max(0, x) $
-
-For more information, please refer to [Activation functions on Wikipedia](https://en.wikipedia.org/wiki/Activation_function).
-
-## Data Preparation
-
-PaddlePaddle provides a Python module, `paddle.dataset.mnist`, which downloads and caches the [MNIST dataset](http://yann.lecun.com/exdb/mnist/).  The cache is under `/home/username/.cache/paddle/dataset/mnist`:
-
-
-|    File name          |       Description | Size            |
-|----------------------|--------------|-----------|
-|train-images-idx3-ubyte|  Training images | 60,000 |
-|train-labels-idx1-ubyte|  Training labels | 60,000 |
-|t10k-images-idx3-ubyte |  Evaluation images | 10,000 |
-|t10k-labels-idx1-ubyte |  Evaluation labels | 10,000 |
-
-
-## Model Configuration
-
-A PaddlePaddle program starts from importing the API package:
-
-```python
-import paddle.v2 as paddle
-```
-
-We want to use this program to demonstrate three different classifiers, each defined as a Python function:
-
-- Softmax regression: the network has a fully-connection layer with softmax activation:
-
-```python
-def softmax_regression(img):
-    predict = paddle.layer.fc(input=img,
-                              size=10,
-                              act=paddle.activation.Softmax())
-    return predict
-```
-
-- Multi-Layer Perceptron: this network has two hidden fully-connected layers, one with ReLU and the other with softmax activation:
-
-```python
-def multilayer_perceptron(img):
-    hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
-    hidden2 = paddle.layer.fc(input=hidden1,
-                              size=64,
-                              act=paddle.activation.Relu())
-    predict = paddle.layer.fc(input=hidden2,
-                              size=10,
-                              act=paddle.activation.Softmax())
-    return predict
-```
-
-- Convolution network LeNet-5: the input image is fed through two convolution-pooling layers, a fully-connected layer, and the softmax output layer:
-
-```python
-def convolutional_neural_network(img):
-
-    conv_pool_1 = paddle.networks.simple_img_conv_pool(
-        input=img,
-        filter_size=5,
-        num_filters=20,
-        num_channel=1,
-        pool_size=2,
-        pool_stride=2,
-        act=paddle.activation.Tanh())
-
-    conv_pool_2 = paddle.networks.simple_img_conv_pool(
-        input=conv_pool_1,
-        filter_size=5,
-        num_filters=50,
-        num_channel=20,
-        pool_size=2,
-        pool_stride=2,
-        act=paddle.activation.Tanh())
-
-    fc1 = paddle.layer.fc(input=conv_pool_2,
-                          size=128,
-                          act=paddle.activation.Tanh())
-
-    predict = paddle.layer.fc(input=fc1,
-                              size=10,
-                              act=paddle.activation.Softmax())
-    return predict
-```
-
-PaddlePaddle provides a special layer `layer.data` for reading data. Let us create a data layer for reading images and connect it to a classification network created using one of above three functions.  We also need a cost layer for training the model.
-
-```python
-paddle.init(use_gpu=False, trainer_count=1)
-
-images = paddle.layer.data(
-    name='pixel', type=paddle.data_type.dense_vector(784))
-label = paddle.layer.data(
-    name='label', type=paddle.data_type.integer_value(10))
-
-predict = softmax_regression(images)
-#predict = multilayer_perceptron(images) # uncomment for MLP
-#predict = convolutional_neural_network(images) # uncomment for LeNet5
-
-cost = paddle.layer.classification_cost(input=predict, label=label)
-```
-
-Now, it is time to specify training parameters. In the following `Momentum` optimizer, `momentum=0.9` means that 90% of the current momentum comes from that of the previous iteration. The learning rate relates to the speed at which the network training converges. Regularization is meant to prevent over-fitting; here we use the L2 regularization.
-
-```python
-parameters = paddle.parameters.create(cost)
-
-optimizer = paddle.optimizer.Momentum(
-    learning_rate=0.1 / 128.0,
-    momentum=0.9,
-    regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128))
-
-trainer = paddle.trainer.SGD(cost=cost,
-                             parameters=parameters,
-                             update_equation=optimizer)
-```
-
-Then we specify the training data `paddle.dataset.movielens.train()` and testing data `paddle.dataset.movielens.test()`. These two methods are *reader creators*. Once called, a reader creator returns a *reader*.  A reader is a Python method, which, once called, returns a Python generator, which yields instances of data.
-
-`shuffle` is a reader decorator. It takes in a reader A as input and returns a new reader B. Under the hood, B calls A to read data in the following fashion: it copies in `buffer_size` instances at a time into a buffer, shuffles the data, and yields the shuffled instances one at a time. A large buffer size would yield very shuffled data.
-
-`batch` is a special decorator, which takes in reader and outputs a *batch reader*, which doesn't yield an instance, but a minibatch at a time.
-
-```python
-lists = []
-
-def event_handler(event):
-    if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f, %s" % (
-                event.pass_id, event.batch_id, event.cost, event.metrics)
-    if isinstance(event, paddle.event.EndPass):
-        result = trainer.test(reader=paddle.batch(
-            paddle.dataset.mnist.test(), batch_size=128))
-        print "Test with Pass %d, Cost %f, %s\n" % (
-            event.pass_id, result.cost, result.metrics)
-        lists.append((event.pass_id, result.cost,
-                      result.metrics['classification_error_evaluator']))
-
-trainer.train(
-    reader=paddle.batch(
-        paddle.reader.shuffle(
-            paddle.dataset.mnist.train(), buf_size=8192),
-        batch_size=128),
-    event_handler=event_handler,
-    num_passes=100)
-```
-
-During training, `trainer.train` invokes `event_handler` for certain events. This gives us a chance to print the training progress.
-
-```
-# Pass 0, Batch 0, Cost 2.780790, {'classification_error_evaluator': 0.9453125}
-# Pass 0, Batch 100, Cost 0.635356, {'classification_error_evaluator': 0.2109375}
-# Pass 0, Batch 200, Cost 0.326094, {'classification_error_evaluator': 0.1328125}
-# Pass 0, Batch 300, Cost 0.361920, {'classification_error_evaluator': 0.1015625}
-# Pass 0, Batch 400, Cost 0.410101, {'classification_error_evaluator': 0.125}
-# Test with Pass 0, Cost 0.326659, {'classification_error_evaluator': 0.09470000118017197}
-```
-
-After the training, we can check the model's prediction accuracy.
-
-```
-# find the best pass
-best = sorted(lists, key=lambda list: float(list[1]))[0]
-print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
-print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
-```
-
-Usually, with MNIST data, the softmax regression model achieves an accuracy around 92.34%, the MLP 97.66%, and the convolution network around 99.20%. Convolution layers have been widely considered a great invention for image processing.
-
-
-## Conclusion
-This tutorial describes a few common deep learning models using **Softmax regression**, **Multilayer Perceptron Network**, and **Convolutional Neural Network**. Understanding these models is crucial for future learning; the subsequent tutorials derive more sophisticated networks by building on top of them.
-
-When our model evolves from a simple softmax regression to a slightly complex Convolutional Neural Network, the recognition accuracy on the MNIST data set achieves a large improvement in accuracy. This is due to the Convolutional layers' local connections and parameter sharing. While learning new models in the future, we encourage the readers to understand the key ideas that lead a new model to improve the results of an old one.
-
-Moreover, this tutorial introduces the basic flow of PaddlePaddle model design, which starts with a *dataprovider*, a model layer construction, and finally training and prediction. Motivated readers can leverage the flow used in this MNIST handwritten digit classification example and experiment with different data and network architectures to train models for classification tasks of their choice.
-
-## References
-
-1. LeCun, Yann, Léon Bottou, Yoshua Bengio, and Patrick Haffner. ["Gradient-based learning applied to document recognition."](http://ieeexplore.ieee.org/abstract/document/726791/) Proceedings of the IEEE 86, no. 11 (1998): 2278-2324.
-2. Wejéus, Samuel. ["A Neural Network Approach to Arbitrary SymbolRecognition on Modern Smartphones."](http://www.diva-portal.org/smash/record.jsf?pid=diva2%3A753279&dswid=-434) (2014).
-3. Decoste, Dennis, and Bernhard Schölkopf. ["Training invariant support vector machines."](http://link.springer.com/article/10.1023/A:1012454411458) Machine learning 46, no. 1-3 (2002): 161-190.
-4. Simard, Patrice Y., David Steinkraus, and John C. Platt. ["Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis."](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.160.8494&rep=rep1&type=pdf) In ICDAR, vol. 3, pp. 958-962. 2003.
-5. Salakhutdinov, Ruslan, and Geoffrey E. Hinton. ["Learning a Nonlinear Embedding by Preserving Class Neighbourhood Structure."](http://www.jmlr.org/proceedings/papers/v2/salakhutdinov07a/salakhutdinov07a.pdf) In AISTATS, vol. 11. 2007.
-6. Cireşan, Dan Claudiu, Ueli Meier, Luca Maria Gambardella, and Jürgen Schmidhuber. ["Deep, big, simple neural nets for handwritten digit recognition."](http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00052) Neural computation 22, no. 12 (2010): 3207-3220.
-7. Deng, Li, Michael L. Seltzer, Dong Yu, Alex Acero, Abdel-rahman Mohamed, and Geoffrey E. Hinton. ["Binary coding of speech spectrograms using a deep auto-encoder."](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.185.1908&rep=rep1&type=pdf) In Interspeech, pp. 1692-1695. 2010.
-8. Kégl, Balázs, and Róbert Busa-Fekete. ["Boosting products of base classifiers."](http://dl.acm.org/citation.cfm?id=1553439) In Proceedings of the 26th Annual International Conference on Machine Learning, pp. 497-504. ACM, 2009.
-9. Rosenblatt, Frank. ["The perceptron: A probabilistic model for information storage and organization in the brain."](http://psycnet.apa.org/journals/rev/65/6/386/) Psychological review 65, no. 6 (1958): 386.
-10. Bishop, Christopher M. ["Pattern recognition."](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) Machine Learning 128 (2006): 1-58.
-
-<br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.

02.recognize_digits/README.md

-# 识别数字
+# Recognize Digits
 
-本教程源代码目录在[book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+The source code for this tutorial is live at [book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits). For instructions on getting started with Paddle, please refer to [installation instructions](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
-当我们学习编程的时候，编写的第一个程序一般是实现打印"Hello World"。而机器学习（或深度学习）的入门教程，一般都是 [MNIST](http://yann.lecun.com/exdb/mnist/) 数据库上的手写识别问题。原因是手写识别属于典型的图像分类问题，比较简单，同时MNIST数据集也很完备。MNIST数据集作为一个简单的计算机视觉数据集，包含一系列如图1所示的手写数字图片和对应的标签。图片是28x28的像素矩阵，标签则对应着0~9的10个数字。每张图片都经过了大小归一化和居中处理。
+## Introduction
+When one learns to program, the first task is usually to write a program that prints "Hello World!". In Machine Learning or Deep Learning, the equivalent task is to train a model to recognize hand-written digits on the dataset [MNIST](http://yann.lecun.com/exdb/mnist/). Handwriting recognition is a classic image classification problem. The problem is relatively easy and MNIST is a complete dataset. As a simple Computer Vision dataset, MNIST contains images of handwritten digits and their corresponding labels (Fig. 1). The input image is a $28\times28$ matrix, and the label is one of the digits from $0$ to $9$. All images are normalized, meaning that they are both rescaled and centered.
 
 <p align="center">
 <img src="image/mnist_example_image.png" width="400"><br/>
-图1. MNIST图片示例
+Fig. 1. Examples of MNIST images
 </p>
 
-MNIST数据集是从 [NIST](https://www.nist.gov/srd/nist-special-database-19) 的Special Database 3（SD-3）和Special Database 1（SD-1）构建而来。由于SD-3是由美国人口调查局的员工进行标注，SD-1是由美国高中生进行标注，因此SD-3比SD-1更干净也更容易识别。Yann LeCun等人从SD-1和SD-3中各取一半作为MNIST的训练集（60000条数据）和测试集（10000条数据），其中训练集来自250位不同的标注员，此外还保证了训练集和测试集的标注员是不完全相同的。
+The MNIST dataset is created from the [NIST](https://www.nist.gov/srd/nist-special-database-19) Special Database 3 (SD-3) and the Special Database 1 (SD-1). The SD-3 is labeled by the staff of the U.S. Census Bureau, while SD-1 is labeled by high school students the in U.S. Therefore the SD-3 is cleaner and easier to recognize than the SD-1 dataset. Yann LeCun et al. used half of the samples from each of SD-1 and SD-3 to create the MNIST training set (60,000 samples) and test set (10,000 samples), where training set was labeled by 250 different annotators, and it was guaranteed that there wasn't a complete overlap of annotators of training set and test set.
 
-Yann LeCun早先在手写字符识别上做了很多研究，并在研究过程中提出了卷积神经网络（Convolutional Neural Network），大幅度地提高了手写字符的识别能力，也因此成为了深度学习领域的奠基人之一。如今的深度学习领域，卷积神经网络占据了至关重要的地位，从最早Yann LeCun提出的简单LeNet，到如今ImageNet大赛上的优胜模型VGGNet、GoogLeNet、ResNet等（请参见[图像分类](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) 教程），人们在图像分类领域，利用卷积神经网络得到了一系列惊人的结果。
+Yann LeCun, one of the founders of Deep Learning, have previously made tremendous contributions to handwritten character recognition and proposed the **Convolutional Neural Network** (CNN), which drastically improved recognition capability for handwritten characters. CNNs are now a critical concept in Deep Learning. From the LeNet proposal by Yann LeCun, to those winning models in ImageNet competitions, such as VGGNet, GoogLeNet, and ResNet (See [Image Classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) tutorial), CNNs have achieved a series of impressive results in Image Classification tasks.
 
-有很多算法在MNIST上进行实验。1998年，LeCun分别用单层线性分类器、多层感知器（Multilayer Perceptron, MLP）和多层卷积神经网络LeNet进行实验，使得测试集上的误差不断下降（从12%下降到0.7%）\[[1](#参考文献)\]。此后，科学家们又基于K近邻（K-Nearest Neighbors）算法\[[2](#参考文献)\]、支持向量机（SVM）\[[3](#参考文献)\]、神经网络\[[4-7](#参考文献)\]和Boosting方法\[[8](#参考文献)\]等做了大量实验，并采用多种预处理方法（如去除歪曲、去噪、模糊等）来提高识别的准确率。
+Many algorithms are tested on MNIST. In 1998, LeCun experimented with single layer linear classifier, Multilayer Perceptron (MLP) and Multilayer CNN LeNet. These algorithms quickly reduced test error from 12% to 0.7% \[[1](#references)\]. Since then, researchers have worked on many algorithms such as **K-Nearest Neighbors** (k-NN) \[[2](#references)\], **Support Vector Machine** (SVM) \[[3](#references)\], **Neural Networks** \[[4-7](#references)\] and **Boosting** \[[8](#references)\]. Various preprocessing methods like distortion removal, noise removal, and blurring, have also been applied to increase recognition accuracy.
 
-本教程中，我们从简单的模型Softmax回归开始，带大家入门手写字符识别，并逐步进行模型优化。
+In this tutorial, we tackle the task of handwritten character recognition. We start with a simple **softmax** regression model and guide our readers step-by-step to improve this model's performance on the task of recognition.
 
 
-## 模型概览
+## Model Overview
 
-基于MNIST数据训练一个分类器，在介绍本教程使用的三个基本图像分类网络前，我们先给出一些定义：
-- $X$是输入：MNIST图片是$28\times28$ 的二维图像，为了进行计算，我们将其转化为$784$维向量，即$X=\left ( x_0, x_1, \dots, x_{783} \right )$。
-- $Y$是输出：分类器的输出是10类数字（0-9），即$Y=\left ( y_0, y_1, \dots, y_9 \right )$，每一维$y_i$代表图片分类为第$i$类数字的概率。
-- $L$是图片的真实标签：$L=\left ( l_0, l_1, \dots, l_9 \right )$也是10维，但只有一维为1，其他都为0。
+Before introducing classification algorithms and training procedure, we define the following symbols:
+- $X$ is the input: Input is a $28\times 28$ MNIST image. It is flattened to a $784$ dimensional vector. $X=\left (x_0, x_1, \dots, x_{783} \right )$.
+- $Y$ is the output: Output of the classifier is 1 of the 10 classes (digits from 0 to 9). $Y=\left (y_0, y_1, \dots, y_9 \right )$. Each dimension $y_i$ represents the probability that the input image belongs to class $i$.
+- $L$ is the ground truth label: $L=\left ( l_0, l_1, \dots, l_9 \right )$. It is also 10 dimensional, but only one entry is $1$ and all others are $0$s.
 
-### Softmax回归(Softmax Regression)
+### Softmax Regression
 
-最简单的Softmax回归模型是先将输入层经过一个全连接层得到的特征，然后直接通过softmax 函数进行多分类\[[9](#参考文献)\]。
+In a simple softmax regression model, the input is first fed to fully connected layers. Then, a softmax function is applied to output probabilities of multiple output classes\[[9](#references)\].
 
-输入层的数据$X$传到输出层，在激活操作之前，会乘以相应的权重 $W$ ，并加上偏置变量 $b$ ，具体如下：
+The input $X$ is multiplied by weights $W$ and then added to the bias $b$ to generate activations.
 
 $$ y_i = \text{softmax}(\sum_j W_{i,j}x_j + b_i) $$
 
-其中 $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
+where $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
 
-对于有 $N$ 个类别的多分类问题，指定 $N$ 个输出节点，$N$ 维结果向量经过softmax将归一化为 $N$ 个[0,1]范围内的实数值，分别表示该样本属于这 $N$ 个类别的概率。此处的 $y_i$ 即对应该图片为数字 $i$ 的预测概率。
+For an $N$-class classification problem with $N$ output nodes, Softmax normalizes the resulting $N$ dimensional vector so that each of its entries falls in the range $[0,1]\in\math{R}$, representing the probability that the sample belongs to a certain class. Here $y_i$ denotes the predicted probability that an image is of digit $i$.
 
-在分类问题中，我们一般采用交叉熵代价损失函数（cross entropy），公式如下：
+In such a classification problem, we usually use the cross entropy loss function:
 
 $$  \text{crossentropy}(label, y) = -\sum_i label_ilog(y_i) $$
 
-图2为softmax回归的网络图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+Fig. 2 illustrates a softmax regression network, with the weights in blue, and the bias in red. `+1` indicates that the bias is $1$.
 
 <p align="center">
-<img src="image/softmax_regression.png" width=400><br/>
-图2. softmax回归网络结构图<br/>
+<img src="image/softmax_regression_en.png" width=400><br/>
+Fig. 2. Softmax regression network architecture<br/>
 </p>
 
-### 多层感知器(Multilayer Perceptron, MLP)
+### Multilayer Perceptron
 
-Softmax回归模型采用了最简单的两层神经网络，即只有输入层和输出层，因此其拟合能力有限。为了达到更好的识别效果，我们考虑在输入层和输出层中间加上若干个隐藏层\[[10](#参考文献)\]。
+The softmax regression model described above uses the simplest two-layer neural network. That is, it only contains an input layer and an output layer, with limited regression capability. To achieve better recognition results, consider adding several hidden layers\[[10](#references)\] between the input layer and the output layer.
 
-1.  经过第一个隐藏层，可以得到 $ H_1 = \phi(W_1X + b_1) $，其中$\phi$代表激活函数，常见的有sigmoid、tanh或ReLU等函数。
-2.  经过第二个隐藏层，可以得到 $ H_2 = \phi(W_2H_1 + b_2) $。
-3.  最后，再经过输出层，得到的$Y=\text{softmax}(W_3H_2 + b_3)$，即为最后的分类结果向量。
+1.  After the first hidden layer, we get $ H_1 = \phi(W_1X + b_1) $, where $\phi$ denotes the activation function. Some [common ones](###list-of-common-activation-functions) are sigmoid, tanh and ReLU.
+2.  After the second hidden layer, we get $ H_2 = \phi(W_2H_1 + b_2) $.
+3.  Finally, the output layer outputs $Y=\text{softmax}(W_3H_2 + b_3)$, the vector denoting our classification result.
 
-
-图3为多层感知器的网络结构图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+Fig. 3. shows a Multilayer Perceptron network, with the weights in blue, and the bias in red. +1 indicates that the bias is $1$.
 
 <p align="center">
-<img src="image/mlp.png" width=500><br/>
-图3. 多层感知器网络结构图<br/>
-</p>
-
-### 卷积神经网络(Convolutional Neural Network, CNN)
+<img src="image/mlp_en.png" width=500><br/>
+Fig. 3. Multilayer Perceptron network architecture<br/>
 
-在多层感知器模型中，将图像展开成一维向量输入到网络中，忽略了图像的位置和结构信息，而卷积神经网络能够更好的利用图像的结构信息。[LeNet-5](http://yann.lecun.com/exdb/lenet/)是一个较简单的卷积神经网络。图4显示了其结构：输入的二维图像，先经过两次卷积层到池化层，再经过全连接层，最后使用softmax分类作为输出层。下面我们主要介绍卷积层和池化层。
-
-<p align="center">
-<img src="image/cnn.png"><br/>
-图4. LeNet-5卷积神经网络结构<br/>
 </p>
 
-#### 卷积层
+### Convolutional Neural Network
 
-卷积层是卷积神经网络的核心基石。在图像识别里我们提到的卷积是二维卷积，即离散二维滤波器（也称作卷积核）与二维图像做卷积操作，简单的讲是二维滤波器滑动到二维图像上所有位置，并在每个位置上与该像素点及其领域像素点做内积。卷积操作被广泛应用与图像处理领域，不同卷积核可以提取不同的特征，例如边沿、线性、角等特征。在深层卷积神经网络中，通过卷积操作可以提取出图像低级到复杂的特征。
+#### Convolutional Layer
 
 <p align="center">
 <img src="image/conv_layer.png" width='750'><br/>
-图5. 卷积层图片<br/>
+Fig. 4. Convolutional layer<br/>
 </p>
 
-图5给出一个卷积计算过程的示例图，输入图像大小为$H=5,W=5,D=3$，即$5 \times 5$大小的3通道（RGB，也称作深度）彩色图像。这个示例图中包含两（用$K$表示）组卷积核，即图中滤波器$W_0$和$W_1$。在卷积计算中，通常对不同的输入通道采用不同的卷积核，如图示例中每组卷积核包含（$D=3）$个$3 \times 3$（用$F \times F$表示）大小的卷积核。另外，这个示例中卷积核在图像的水平方向（$W$方向）和垂直方向（$H$方向）的滑动步长为2（用$S$表示）；对输入图像周围各填充1（用$P$表示）个0，即图中输入层原始数据为蓝色部分，灰色部分是进行了大小为1的扩展，用0来进行扩展。经过卷积操作得到输出为$3 \times 3 \times 2$（用$H_{o} \times W_{o} \times K$表示）大小的特征图，即$3 \times 3$大小的2通道特征图，其中$H_o$计算公式为：$H_o = (H - F + 2 \times P)/S + 1$，$W_o$同理。 而输出特征图中的每个像素，是每组滤波器与输入图像每个特征图的内积再求和，再加上偏置$b_o$，偏置通常对于每个输出特征图是共享的。输出特征图$o[:,:,0]$中的最后一个$-2$计算如图5右下角公式所示。
+The **convolutional layer** is the core of a Convolutional Neural Network. The parameters in this layer are composed of a set of filters, also called kernels. We could visualize the convolution step in the following fashion: Each kernel slides horizontally and vertically till it covers the whole image. At every window, we compute the dot product of the kernel and the input. Then, we add the bias and apply an activation function. The result is a two-dimensional activation map. For example, some kernel may recognize corners, and some may recognize circles. These convolution kernels may respond strongly to the corresponding features.
 
-在卷积操作中卷积核是可学习的参数，经过上面示例介绍，每层卷积的参数大小为$D \times F \times F \times K$。在多层感知器模型中，神经元通常是全部连接，参数较多。而卷积层的参数较少，这也是由卷积层的主要特性即局部连接和共享权重所决定。
+Fig. 4 illustrates the dynamic programming of a convolutional layer, where depths are flattened for simplicity. The input is $W_1=5$, $H_1=5$, $D_1=3$. In fact, this is a common representation for colored images. $W_1$ and $H_1$ correspond to the width and height in a colored image. $D_1$ corresponds to the 3 color channels for RGB. The parameters of the convolutional layer are $K=2$, $F=3$, $S=2$, $P=1$. $K$ denotes the number of kernels; specifically, $Filter$ $W_0$ and $Filter$ $W_1$ are the kernels. $F$ is kernel size while $W0$ and $W1$ are both $F\timesF = 3\times3$ matrices in all depths. $S$ is the stride, which is the width of the sliding window; here, kernels move leftwards or downwards by 2 units each time. $P$ is the width of the padding, which denotes an extension of the input; here, the gray area shows zero padding with size 1.
 
-- 局部连接：每个神经元仅与输入神经元的一块区域连接，这块局部区域称作感受野（receptive field）。在图像卷积操作中，即神经元在空间维度（spatial dimension，即上图示例H和W所在的平面）是局部连接，但在深度上是全部连接。对于二维图像本身而言，也是局部像素关联较强。这种局部连接保证了学习后的过滤器能够对于局部的输入特征有最强的响应。局部连接的思想，也是受启发于生物学里面的视觉系统结构，视觉皮层的神经元就是局部接受信息的。
+#### Pooling Layer
 
-- 权重共享：计算同一个深度切片的神经元时采用的滤波器是共享的。例如图4中计算$o[:,:,0]$的每个每个神经元的滤波器均相同，都为$W_0$，这样可以很大程度上减少参数。共享权重在一定程度上讲是有意义的，例如图片的底层边缘特征与特征在图中的具体位置无关。但是在一些场景中是无意的，比如输入的图片是人脸，眼睛和头发位于不同的位置，希望在不同的位置学到不同的特征 (参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/))。请注意权重只是对于同一深度切片的神经元是共享的，在卷积层，通常采用多组卷积核提取不同特征，即对应不同深度切片的特征，不同深度切片的神经元权重是不共享。另外，偏重对同一深度切片的所有神经元都是共享的。
+<p align="center">
+<img src="image/max_pooling_en.png" width="400px"><br/>
+Fig. 5 Pooling layer using max-pooling<br/>
+</p>
 
-通过介绍卷积计算过程及其特性，可以看出卷积是线性操作，并具有平移不变性（shift-invariant），平移不变性即在图像每个位置执行相同的操作。卷积层的局部连接和权重共享使得需要学习的参数大大减小，这样也有利于训练较大卷积神经网络。
+A **pooling layer** performs downsampling. The main functionality of this layer is to reduce computation by reducing the network parameters. It also prevents over-fitting to some extent. Usually, a pooling layer is added after a convolutional layer. Pooling layer can use various techniques, such as max pooling and average pooling. As shown in Fig.5, max pooling uses rectangles to segment the input layer into several parts and computes the maximum value in each part as the output.
 
-#### 池化层
+#### LeNet-5 Network
 
 <p align="center">
-<img src="image/max_pooling.png" width="400px"><br/>
-图6. 池化层图片<br/>
+<img src="image/cnn_en.png"><br/>
+Fig. 6. LeNet-5 Convolutional Neural Network architecture<br/>
 </p>
 
-池化是非线性下采样的一种形式，主要作用是通过减少网络的参数来减小计算量，并且能够在一定程度上控制过拟合。通常在卷积层的后面会加上一个池化层。池化包括最大池化、平均池化等。其中最大池化是用不重叠的矩形框将输入层分成不同的区域，对于每个矩形框的数取最大值作为输出层，如图6所示。
+[**LeNet-5**](http://yann.lecun.com/exdb/lenet/) is one of the simplest Convolutional Neural Networks. Fig. 6. shows its architecture: A 2-dimensional input image is fed into two sets of convolutional layers and pooling layers. This output is then fed to a fully connected layer and a softmax classifier. Compared to multilayer, fully connected perceptrons, the LeNet-5 can recognize images better. This is due to the following three properties of the convolution:
+
+- The 3D nature of the neurons: a convolutional layer is organized by width, height and depth. Neurons in each layer are connected to only a small region in the previous layer. This region is called the receptive field.
+- Local connectivity: A CNN utilizes the local space correlation by connecting local neurons. This design guarantees that the learned filter has a strong response to local input features. Stacking many such layers generates a non-linear filter that is more global. This enables the network to first obtain good representation for small parts of input and then combine them to represent a larger region.
+- Weight sharing: In a CNN, computation is iterated on shared parameters (weights and bias) to form a feature map. This means that all the neurons in the same depth of the output respond to the same feature. This allows the network to detect a feature regardless of its position in the input. In other words, it is shift invariant.
+
+For more details on Convolutional Neural Networks, please refer to the tutorial on [Image Classification](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md) and the [relevant lecture](http://cs231n.github.io/convolutional-networks/) from a Stanford open course.
 
-更详细的关于卷积神经网络的具体知识可以参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/ )和[图像分类](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md)教程。
+### List of Common Activation Functions
+- Sigmoid activation function: $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
 
-### 常见激活函数介绍  
-- sigmoid激活函数： $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
+- Tanh activation function: $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
 
-- tanh激活函数： $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
+  In fact, tanh function is just a rescaled version of the sigmoid function. It is obtained by magnifying the value of the sigmoid function and moving it downwards by 1.
 
-  实际上，tanh函数只是规模变化的sigmoid函数，将sigmoid函数值放大2倍之后再向下平移1个单位：tanh(x) = 2sigmoid(2x) - 1 。
+- ReLU activation function: $ f(x) = max(0, x) $
 
-- ReLU激活函数： $ f(x) = max(0, x) $
+For more information, please refer to [Activation functions on Wikipedia](https://en.wikipedia.org/wiki/Activation_function).
 
-更详细的介绍请参考[维基百科激活函数](https://en.wikipedia.org/wiki/Activation_function)。
+## Data Preparation
 
-## 数据介绍
+PaddlePaddle provides a Python module, `paddle.dataset.mnist`, which downloads and caches the [MNIST dataset](http://yann.lecun.com/exdb/mnist/).  The cache is under `/home/username/.cache/paddle/dataset/mnist`:
 
-PaddlePaddle在API中提供了自动加载[MNIST](http://yann.lecun.com/exdb/mnist/)数据的模块`paddle.dataset.mnist`。加载后的数据位于`/home/username/.cache/paddle/dataset/mnist`下：
 
+|    File name          |       Description | Size            |
+|----------------------|--------------|-----------|
+|train-images-idx3-ubyte|  Training images | 60,000 |
+|train-labels-idx1-ubyte|  Training labels | 60,000 |
+|t10k-images-idx3-ubyte |  Evaluation images | 10,000 |
+|t10k-labels-idx1-ubyte |  Evaluation labels | 10,000 |
 
-|    文件名称          |       说明              |
-|----------------------|-------------------------|
-|train-images-idx3-ubyte|  训练数据图片，60,000条数据 |
-|train-labels-idx1-ubyte|  训练数据标签，60,000条数据 |
-|t10k-images-idx3-ubyte |  测试数据图片，10,000条数据 |
-|t10k-labels-idx1-ubyte |  测试数据标签，10,000条数据 |
 
-## 配置说明
+## Model Configuration
 
-首先，加载PaddlePaddle的V2 api包。
+A PaddlePaddle program starts from importing the API package:
 
 ```python
 import paddle.v2 as paddle
 ```
-其次，定义三个不同的分类器：
 
-- Softmax回归：只通过一层简单的以softmax为激活函数的全连接层，就可以得到分类的结果。
+We want to use this program to demonstrate three different classifiers, each defined as a Python function:
+
+- Softmax regression: the network has a fully-connection layer with softmax activation:
 
 ```python
 def softmax_regression(img):
@@ -145,27 +145,26 @@ def softmax_regression(img):
                               act=paddle.activation.Softmax())
     return predict
 ```
-- 多层感知器：下面代码实现了一个含有两个隐藏层（即全连接层）的多层感知器。其中两个隐藏层的激活函数均采用ReLU，输出层的激活函数用Softmax。
+
+- Multi-Layer Perceptron: this network has two hidden fully-connected layers, one with ReLU and the other with softmax activation:
 
 ```python
 def multilayer_perceptron(img):
-    # 第一个全连接层，激活函数为ReLU
     hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
-    # 第二个全连接层，激活函数为ReLU
     hidden2 = paddle.layer.fc(input=hidden1,
                               size=64,
                               act=paddle.activation.Relu())
-    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
     predict = paddle.layer.fc(input=hidden2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
-- 卷积神经网络LeNet-5: 输入的二维图像，首先经过两次卷积层到池化层，再经过全连接层，最后使用以softmax为激活函数的全连接层作为输出层。
+
+- Convolution network LeNet-5: the input image is fed through two convolution-pooling layers, a fully-connected layer, and the softmax output layer:
 
 ```python
 def convolutional_neural_network(img):
-    # 第一个卷积-池化层
+
     conv_pool_1 = paddle.networks.simple_img_conv_pool(
         input=img,
         filter_size=5,
@@ -173,8 +172,8 @@ def convolutional_neural_network(img):
         num_channel=1,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-    # 第二个卷积-池化层
+        act=paddle.activation.Relu())
+
     conv_pool_2 = paddle.networks.simple_img_conv_pool(
         input=conv_pool_1,
         filter_size=5,
@@ -182,22 +181,17 @@ def convolutional_neural_network(img):
         num_channel=20,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-    # 全连接层
-    fc1 = paddle.layer.fc(input=conv_pool_2,
-                          size=128,
-                          act=paddle.activation.Tanh())
-    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
-    predict = paddle.layer.fc(input=fc1,
+        act=paddle.activation.Relu())
+
+    predict = paddle.layer.fc(input=conv_pool_2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
 
-接着，通过`layer.data`调用来获取数据，然后调用分类器（这里我们提供了三个不同的分类器）得到分类结果。训练时，对该结果计算其损失函数，分类问题常常选择交叉熵损失函数。
+PaddlePaddle provides a special layer `layer.data` for reading data. Let us create a data layer for reading images and connect it to a classification network created using one of above three functions.  We also need a cost layer for training the model.
 
 ```python
-# 该模型运行在单个CPU上
 paddle.init(use_gpu=False, trainer_count=1)
 
 images = paddle.layer.data(
@@ -205,17 +199,14 @@ images = paddle.layer.data(
 label = paddle.layer.data(
     name='label', type=paddle.data_type.integer_value(10))
 
-predict = softmax_regression(images) # Softmax回归
-#predict = multilayer_perceptron(images) #多层感知器
-#predict = convolutional_neural_network(images) #LeNet5卷积神经网络
+# predict = softmax_regression(images)
+# predict = multilayer_perceptron(images) # uncomment for MLP
+predict = convolutional_neural_network(images) # uncomment for LeNet5
 
 cost = paddle.layer.classification_cost(input=predict, label=label)
 ```
 
-然后，指定训练相关的参数。
-- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器 `Momentum` ，其中参数0.9代表动量优化每次保持前一次速度的0.9倍。
-- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
-- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+Now, it is time to specify training parameters. In the following `Momentum` optimizer, `momentum=0.9` means that 90% of the current momentum comes from that of the previous iteration. The learning rate relates to the speed at which the network training converges. Regularization is meant to prevent over-fitting; here we use the L2 regularization.
 
 ```python
 parameters = paddle.parameters.create(cost)
@@ -230,11 +221,44 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=optimizer)
 ```
 
-下一步，我们开始训练过程。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python yield generator。
+Then we specify the training data `paddle.dataset.movielens.train()` and testing data `paddle.dataset.movielens.test()`. These two methods are *reader creators*. Once called, a reader creator returns a *reader*.  A reader is a Python method, which, once called, returns a Python generator, which yields instances of data.
+
+`shuffle` is a reader decorator. It takes in a reader A as input and returns a new reader B. Under the hood, B calls A to read data in the following fashion: it copies in `buffer_size` instances at a time into a buffer, shuffles the data, and yields the shuffled instances one at a time. A large buffer size would yield very shuffled data.
 
-下面`shuffle`是一个reader decorator，它接受一个reader A，返回另一个reader B —— reader B 每次读入`buffer_size`条训练数据到一个buffer里，然后随机打乱其顺序，并且逐条输出。
+`batch` is a special decorator, which takes in reader and outputs a *batch reader*, which doesn't yield an instance, but a minibatch at a time.
 
-`batch`是一个特殊的decorator，它的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minibatch。
+`event_handler_plot` is used to plot a figure like below：
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+# event_handler to plot a figure
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 100 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=paddle.batch(
+            paddle.dataset.mnist.test(), batch_size=128))
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` is used to plot some text data when training.
 
 ```python
 lists = []
@@ -245,23 +269,29 @@ def event_handler(event):
             print "Pass %d, Batch %d, Cost %f, %s" % (
                 event.pass_id, event.batch_id, event.cost, event.metrics)
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(reader=paddle.batch(
             paddle.dataset.mnist.test(), batch_size=128))
         print "Test with Pass %d, Cost %f, %s\n" % (
             event.pass_id, result.cost, result.metrics)
         lists.append((event.pass_id, result.cost,
                       result.metrics['classification_error_evaluator']))
+```
 
+```python
 trainer.train(
     reader=paddle.batch(
         paddle.reader.shuffle(
             paddle.dataset.mnist.train(), buf_size=8192),
         batch_size=128),
-    event_handler=event_handler,
-    num_passes=100)
+    event_handler=event_handler_plot,
+    num_passes=5)
 ```
 
-训练过程是完全自动的，event_handler里打印的日志类似如下所示：
+During training, `trainer.train` invokes `event_handler` for certain events. This gives us a chance to print the training progress.
 
 ```
 # Pass 0, Batch 0, Cost 2.780790, {'classification_error_evaluator': 0.9453125}
@@ -272,13 +302,53 @@ trainer.train(
 # Test with Pass 0, Cost 0.326659, {'classification_error_evaluator': 0.09470000118017197}
 ```
 
-训练之后，检查模型的预测准确度。用 MNIST 训练的时候，一般 softmax回归模型的分类准确率为约为 92.34%，多层感知器为97.66%，卷积神经网络可以达到 99.20%。
+After the training, we can check the model's prediction accuracy.
+
+```
+# find the best pass
+best = sorted(lists, key=lambda list: float(list[1]))[0]
+print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
+print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
+```
+
+Usually, with MNIST data, the softmax regression model achieves an accuracy around 92.34%, the MLP 97.66%, and the convolution network around 99.20%. Convolution layers have been widely considered a great invention for image processing.
+
+## Application
+
+After training is done, user can use the trained model to classify images. The following code shows how to inference MNIST images through `paddle.infer` interface.
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file).convert('L')
+    im = im.resize((28, 28), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32).flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/infer_3.png'),))
+
+probs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/infer_3.png is: %d" % lab[0][0]
+```
+
+
+## Conclusion
+
+This tutorial describes a few common deep learning models using **Softmax regression**, **Multilayer Perceptron Network**, and **Convolutional Neural Network**. Understanding these models is crucial for future learning; the subsequent tutorials derive more sophisticated networks by building on top of them.
+
+When our model evolves from a simple softmax regression to a slightly complex Convolutional Neural Network, the recognition accuracy on the MNIST data set achieves a large improvement in accuracy. This is due to the Convolutional layers' local connections and parameter sharing. While learning new models in the future, we encourage the readers to understand the key ideas that lead a new model to improve the results of an old one.
 
-## 总结
+Moreover, this tutorial introduces the basic flow of PaddlePaddle model design, which starts with a *dataprovider*, a model layer construction, and finally training and prediction. Motivated readers can leverage the flow used in this MNIST handwritten digit classification example and experiment with different data and network architectures to train models for classification tasks of their choice.
 
-本教程的softmax回归、多层感知器和卷积神经网络是最基础的深度学习模型，后续章节中复杂的神经网络都是从它们衍生出来的，因此这几个模型对之后的学习大有裨益。同时，我们也观察到从最简单的softmax回归变换到稍复杂的卷积神经网络的时候，MNIST数据集上的识别准确率有了大幅度的提升，原因是卷积层具有局部连接和共享权重的特性。在之后学习新模型的时候，希望大家也要深入到新模型相比原模型带来效果提升的关键之处。此外，本教程还介绍了PaddlePaddle模型搭建的基本流程，从dataprovider的编写、网络层的构建，到最后的训练和预测。对这个流程熟悉以后，大家就可以用自己的数据，定义自己的网络模型，并完成自己的训练和预测任务了。
 
-## 参考文献
+## References
 
 1. LeCun, Yann, Léon Bottou, Yoshua Bengio, and Patrick Haffner. ["Gradient-based learning applied to document recognition."](http://ieeexplore.ieee.org/abstract/document/726791/) Proceedings of the IEEE 86, no. 11 (1998): 2278-2324.
 2. Wejéus, Samuel. ["A Neural Network Approach to Arbitrary SymbolRecognition on Modern Smartphones."](http://www.diva-portal.org/smash/record.jsf?pid=diva2%3A753279&dswid=-434) (2014).
@@ -292,4 +362,4 @@ trainer.train(
 10. Bishop, Christopher M. ["Pattern recognition."](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) Machine Learning 128 (2006): 1-58.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

02.recognize_digits/index.en.html → 02.recognize_digits/index.cn.html

@@ -40,149 +40,145 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-# Recognize Digits
+# 识别数字
 
-The source code for this tutorial is live at [book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits). To get started with Paddle, please refer to [installation instructions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+本教程源代码目录在[book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/167.html)。
 
-## Introduction
-Handwriting recognition is a classic image classification problem. In particular, recognizing hand-written digits on [MNIST](http://yann.lecun.com/exdb/mnist/) is the deep learning equivalence to a 'Hello World!' program.
-
-### MNIST
-
-MNIST is a complete dataset containing images of handwritten digits and their corresponding labels. Each of the input image is a $28\times28$ matrix, and the label is one of the digits from $0$ to $9$ (Fig. 1). All images are normalized, meaning that they are both scaled and centered.
+## 背景介绍
+当我们学习编程的时候，编写的第一个程序一般是实现打印"Hello World"。而机器学习（或深度学习）的入门教程，一般都是 [MNIST](http://yann.lecun.com/exdb/mnist/) 数据库上的手写识别问题。原因是手写识别属于典型的图像分类问题，比较简单，同时MNIST数据集也很完备。MNIST数据集作为一个简单的计算机视觉数据集，包含一系列如图1所示的手写数字图片和对应的标签。图片是28x28的像素矩阵，标签则对应着0~9的10个数字。每张图片都经过了大小归一化和居中处理。
 
 <p align="center">
 <img src="image/mnist_example_image.png" width="400"><br/>
-Fig. 1. Examples of MNIST images
+图1. MNIST图片示例
 </p>
 
-The MNIST dataset is created from the [NIST](https://www.nist.gov/srd/nist-special-database-19) Special Database 3 (SD-3) and the Special Database 1 (SD-1). The SD-3 is labeled by the staff of the U.S. Census Bureau, while SD-1 is labeled by high school students the in U.S. Therefore the SD-3 is cleaner and easier to recognize than the SD-1 dataset. Yann LeCun et al. used half of the samples from each of SD-1 and SD-3 to create the MNIST training set (60,000 samples) and test set (10,000 samples), where training set was labeled by 250 different annotators, and it was guaranteed that there wasn't a complete overlap of annotators of training set and test set.
+MNIST数据集是从 [NIST](https://www.nist.gov/srd/nist-special-database-19) 的Special Database 3（SD-3）和Special Database 1（SD-1）构建而来。由于SD-3是由美国人口调查局的员工进行标注，SD-1是由美国高中生进行标注，因此SD-3比SD-1更干净也更容易识别。Yann LeCun等人从SD-1和SD-3中各取一半作为MNIST的训练集（60000条数据）和测试集（10000条数据），其中训练集来自250位不同的标注员，此外还保证了训练集和测试集的标注员是不完全相同的。
 
-Yann LeCun, one of the founders of Deep Learning, have previously made tremendous contributions to handwritten character recognition and proposed the **Convolutional Neural Network** (CNN), which drastically improved recognition capability for handwritten characters. CNNs are now a critical concept in Deep Learning. From the LeNet proposal by Yann LeCun, to those winning models in ImageNet competitions, such as VGGNet, GoogLeNet, and ResNet (See [Image Classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) tutorial), CNNs have achieved a series of impressive results in Image Classification tasks.
+Yann LeCun早先在手写字符识别上做了很多研究，并在研究过程中提出了卷积神经网络（Convolutional Neural Network），大幅度地提高了手写字符的识别能力，也因此成为了深度学习领域的奠基人之一。如今的深度学习领域，卷积神经网络占据了至关重要的地位，从最早Yann LeCun提出的简单LeNet，到如今ImageNet大赛上的优胜模型VGGNet、GoogLeNet、ResNet等（请参见[图像分类](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) 教程），人们在图像分类领域，利用卷积神经网络得到了一系列惊人的结果。
 
-Many algorithms are tested on MNIST. In 1998, LeCun experimented with single layer linear classifier, Multilayer Perceptron (MLP) and Multilayer CNN LeNet. These algorithms quickly reduced test error from 12% to 0.7% \[[1](#references)\]. Since then, researchers have worked on many algorithms such as **K-Nearest Neighbors** (k-NN) \[[2](#references)\], **Support Vector Machine** (SVM) \[[3](#references)\], **Neural Networks** \[[4-7](#references)\] and **Boosting** \[[8](#references)\]. Various preprocessing methods like distortion removal, noise removal, and blurring, have also been applied to increase recognition accuracy.
+有很多算法在MNIST上进行实验。1998年，LeCun分别用单层线性分类器、多层感知器（Multilayer Perceptron, MLP）和多层卷积神经网络LeNet进行实验，使得测试集上的误差不断下降（从12%下降到0.7%）\[[1](#参考文献)\]。此后，科学家们又基于K近邻（K-Nearest Neighbors）算法\[[2](#参考文献)\]、支持向量机（SVM）\[[3](#参考文献)\]、神经网络\[[4-7](#参考文献)\]和Boosting方法\[[8](#参考文献)\]等做了大量实验，并采用多种预处理方法（如去除歪曲、去噪、模糊等）来提高识别的准确率。
 
-In this tutorial, we tackle the task of handwritten character recognition. We start with a simple **softmax** regression model and guide our readers step-by-step to improve this model's performance on the task of recognition.
+本教程中，我们从简单的模型Softmax回归开始，带大家入门手写字符识别，并逐步进行模型优化。
 
 
-## Model Overview
+## 模型概览
 
-Before introducing classification algorithms and training procedure, we define the following symbols:
-- $X$ is the input: Input is a $28\times 28$ MNIST image. It is flattened to a $784$ dimensional vector. $X=\left (x_0, x_1, \dots, x_{783} \right )$.
-- $Y$ is the output: Output of the classifier is 1 of the 10 classes (digits from 0 to 9). $Y=\left (y_0, y_1, \dots, y_9 \right )$. Each dimension $y_i$ represents the probability that the input image belongs to class $i$.
-- $L$ is the ground truth label: $L=\left ( l_0, l_1, \dots, l_9 \right )$. It is also 10 dimensional, but only one entry is $1$ and all others are $0$s.
+基于MNIST数据训练一个分类器，在介绍本教程使用的三个基本图像分类网络前，我们先给出一些定义：
+- $X$是输入：MNIST图片是$28\times28$ 的二维图像，为了进行计算，我们将其转化为$784$维向量，即$X=\left ( x_0, x_1, \dots, x_{783} \right )$。
+- $Y$是输出：分类器的输出是10类数字（0-9），即$Y=\left ( y_0, y_1, \dots, y_9 \right )$，每一维$y_i$代表图片分类为第$i$类数字的概率。
+- $L$是图片的真实标签：$L=\left ( l_0, l_1, \dots, l_9 \right )$也是10维，但只有一维为1，其他都为0。
 
-### Softmax Regression
+### Softmax回归(Softmax Regression)
 
-In a simple softmax regression model, the input is first fed to fully connected layers. Then, a softmax function is applied to output probabilities of multiple output classes\[[9](#references)\].
+最简单的Softmax回归模型是先将输入层经过一个全连接层得到的特征，然后直接通过softmax 函数进行多分类\[[9](#参考文献)\]。
 
-The input $X$ is multiplied by weights $W$ and then added to the bias $b$ to generate activations.
+输入层的数据$X$传到输出层，在激活操作之前，会乘以相应的权重 $W$ ，并加上偏置变量 $b$ ，具体如下：
 
 $$ y_i = \text{softmax}(\sum_j W_{i,j}x_j + b_i) $$
 
-where $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
+其中 $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
 
-For an $N$-class classification problem with $N$ output nodes, Softmax normalizes the resulting $N$ dimensional vector so that each of its entries falls in the range $[0,1]\in\math{R}$, representing the probability that the sample belongs to a certain class. Here $y_i$ denotes the predicted probability that an image is of digit $i$.
+对于有 $N$ 个类别的多分类问题，指定 $N$ 个输出节点，$N$ 维结果向量经过softmax将归一化为 $N$ 个[0,1]范围内的实数值，分别表示该样本属于这 $N$ 个类别的概率。此处的 $y_i$ 即对应该图片为数字 $i$ 的预测概率。
 
-In such a classification problem, we usually use the cross entropy loss function:
+在分类问题中，我们一般采用交叉熵代价损失函数（cross entropy），公式如下：
 
 $$  \text{crossentropy}(label, y) = -\sum_i label_ilog(y_i) $$
 
-Fig. 2 illustrates a softmax regression network, with the weights in blue, and the bias in red. `+1` indicates that the bias is $1$.
+图2为softmax回归的网络图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
 
 <p align="center">
-<img src="image/softmax_regression_en.png" width=400><br/>
-Fig. 2. Softmax regression network architecture<br/>
+<img src="image/softmax_regression.png" width=400><br/>
+图2. softmax回归网络结构图<br/>
 </p>
 
-### Multilayer Perceptron
+### 多层感知器(Multilayer Perceptron, MLP)
+
+Softmax回归模型采用了最简单的两层神经网络，即只有输入层和输出层，因此其拟合能力有限。为了达到更好的识别效果，我们考虑在输入层和输出层中间加上若干个隐藏层\[[10](#参考文献)\]。
 
-The softmax regression model described above uses the simplest two-layer neural network. That is, it only contains an input layer and an output layer, with limited regression capability. To achieve better recognition results, consider adding several hidden layers\[[10](#references)\] between the input layer and the output layer.
+1.  经过第一个隐藏层，可以得到 $ H_1 = \phi(W_1X + b_1) $，其中$\phi$代表激活函数，常见的有sigmoid、tanh或ReLU等函数。
+2.  经过第二个隐藏层，可以得到 $ H_2 = \phi(W_2H_1 + b_2) $。
+3.  最后，再经过输出层，得到的$Y=\text{softmax}(W_3H_2 + b_3)$，即为最后的分类结果向量。
 
-1.  After the first hidden layer, we get $ H_1 = \phi(W_1X + b_1) $, where $\phi$ denotes the activation function. Some [common ones](###list-of-common-activation-functions) are sigmoid, tanh and ReLU.
-2.  After the second hidden layer, we get $ H_2 = \phi(W_2H_1 + b_2) $.
-3.  Finally, the output layer outputs $Y=\text{softmax}(W_3H_2 + b_3)$, the vector denoting our classification result.
 
-Fig. 3. shows a Multilayer Perceptron network, with the weights in blue, and the bias in red. +1 indicates that the bias is $1$.
+图3为多层感知器的网络结构图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
 
 <p align="center">
-<img src="image/mlp_en.png" width=500><br/>
-Fig. 3. Multilayer Perceptron network architecture<br/>
+<img src="image/mlp.png" width=500><br/>
+图3. 多层感知器网络结构图<br/>
+</p>
+
+### 卷积神经网络(Convolutional Neural Network, CNN)
 
+在多层感知器模型中，将图像展开成一维向量输入到网络中，忽略了图像的位置和结构信息，而卷积神经网络能够更好的利用图像的结构信息。[LeNet-5](http://yann.lecun.com/exdb/lenet/)是一个较简单的卷积神经网络。图4显示了其结构：输入的二维图像，先经过两次卷积层到池化层，再经过全连接层，最后使用softmax分类作为输出层。下面我们主要介绍卷积层和池化层。
+
+<p align="center">
+<img src="image/cnn.png"><br/>
+图4. LeNet-5卷积神经网络结构<br/>
 </p>
 
-### Convolutional Neural Network
+#### 卷积层
 
-#### Convolutional Layer
+卷积层是卷积神经网络的核心基石。在图像识别里我们提到的卷积是二维卷积，即离散二维滤波器（也称作卷积核）与二维图像做卷积操作，简单的讲是二维滤波器滑动到二维图像上所有位置，并在每个位置上与该像素点及其领域像素点做内积。卷积操作被广泛应用与图像处理领域，不同卷积核可以提取不同的特征，例如边沿、线性、角等特征。在深层卷积神经网络中，通过卷积操作可以提取出图像低级到复杂的特征。
 
 <p align="center">
 <img src="image/conv_layer.png" width='750'><br/>
-Fig. 4. Convolutional layer<br/>
+图5. 卷积层图片<br/>
 </p>
 
-The **convolutional layer** is the core of a Convolutional Neural Network. The parameters in this layer are composed of a set of filters, also called kernels. We could visualize the convolution step in the following fashion: Each kernel slides horizontally and vertically till it covers the whole image. At every window, we compute the dot product of the kernel and the input. Then, we add the bias and apply an activation function. The result is a two-dimensional activation map. For example, some kernel may recognize corners, and some may recognize circles. These convolution kernels may respond strongly to the corresponding features.
+图5给出一个卷积计算过程的示例图，输入图像大小为$H=5,W=5,D=3$，即$5 \times 5$大小的3通道（RGB，也称作深度）彩色图像。这个示例图中包含两（用$K$表示）组卷积核，即图中滤波器$W_0$和$W_1$。在卷积计算中，通常对不同的输入通道采用不同的卷积核，如图示例中每组卷积核包含（$D=3）$个$3 \times 3$（用$F \times F$表示）大小的卷积核。另外，这个示例中卷积核在图像的水平方向（$W$方向）和垂直方向（$H$方向）的滑动步长为2（用$S$表示）；对输入图像周围各填充1（用$P$表示）个0，即图中输入层原始数据为蓝色部分，灰色部分是进行了大小为1的扩展，用0来进行扩展。经过卷积操作得到输出为$3 \times 3 \times 2$（用$H_{o} \times W_{o} \times K$表示）大小的特征图，即$3 \times 3$大小的2通道特征图，其中$H_o$计算公式为：$H_o = (H - F + 2 \times P)/S + 1$，$W_o$同理。 而输出特征图中的每个像素，是每组滤波器与输入图像每个特征图的内积再求和，再加上偏置$b_o$，偏置通常对于每个输出特征图是共享的。输出特征图$o[:,:,0]$中的最后一个$-2$计算如图5右下角公式所示。
 
-Fig. 4 illustrates the dynamic programming of a convolutional layer, where depths are flattened for simplicity. The input is $W_1=5$, $H_1=5$, $D_1=3$. In fact, this is a common representation for colored images. $W_1$ and $H_1$ correspond to the width and height in a colored image. $D_1$ corresponds to the 3 color channels for RGB. The parameters of the convolutional layer are $K=2$, $F=3$, $S=2$, $P=1$. $K$ denotes the number of kernels; specifically, $Filter$ $W_0$ and $Filter$ $W_1$ are the kernels. $F$ is kernel size while $W0$ and $W1$ are both $F\timesF = 3\times3$ matrices in all depths. $S$ is the stride, which is the width of the sliding window; here, kernels move leftwards or downwards by 2 units each time. $P$ is the width of the padding, which denotes an extension of the input; here, the gray area shows zero padding with size 1.
+在卷积操作中卷积核是可学习的参数，经过上面示例介绍，每层卷积的参数大小为$D \times F \times F \times K$。在多层感知器模型中，神经元通常是全部连接，参数较多。而卷积层的参数较少，这也是由卷积层的主要特性即局部连接和共享权重所决定。
 
-#### Pooling Layer
+- 局部连接：每个神经元仅与输入神经元的一块区域连接，这块局部区域称作感受野（receptive field）。在图像卷积操作中，即神经元在空间维度（spatial dimension，即上图示例H和W所在的平面）是局部连接，但在深度上是全部连接。对于二维图像本身而言，也是局部像素关联较强。这种局部连接保证了学习后的过滤器能够对于局部的输入特征有最强的响应。局部连接的思想，也是受启发于生物学里面的视觉系统结构，视觉皮层的神经元就是局部接受信息的。
 
-<p align="center">
-<img src="image/max_pooling_en.png" width="400px"><br/>
-Fig. 5 Pooling layer using max-pooling<br/>
-</p>
+- 权重共享：计算同一个深度切片的神经元时采用的滤波器是共享的。例如图4中计算$o[:,:,0]$的每个每个神经元的滤波器均相同，都为$W_0$，这样可以很大程度上减少参数。共享权重在一定程度上讲是有意义的，例如图片的底层边缘特征与特征在图中的具体位置无关。但是在一些场景中是无意的，比如输入的图片是人脸，眼睛和头发位于不同的位置，希望在不同的位置学到不同的特征 (参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/))。请注意权重只是对于同一深度切片的神经元是共享的，在卷积层，通常采用多组卷积核提取不同特征，即对应不同深度切片的特征，不同深度切片的神经元权重是不共享。另外，偏重对同一深度切片的所有神经元都是共享的。
 
-A **pooling layer** performs downsampling. The main functionality of this layer is to reduce computation by reducing the network parameters. It also prevents over-fitting to some extent. Usually, a pooling layer is added after a convolutional layer. Pooling layer can use various techniques, such as max pooling and average pooling. As shown in Fig.5, max pooling uses rectangles to segment the input layer into several parts and computes the maximum value in each part as the output.
+通过介绍卷积计算过程及其特性，可以看出卷积是线性操作，并具有平移不变性（shift-invariant），平移不变性即在图像每个位置执行相同的操作。卷积层的局部连接和权重共享使得需要学习的参数大大减小，这样也有利于训练较大卷积神经网络。
 
-#### LeNet-5 Network
+#### 池化层
 
 <p align="center">
-<img src="image/cnn_en.png"><br/>
-Fig. 6. LeNet-5 Convolutional Neural Network architecture<br/>
+<img src="image/max_pooling.png" width="400px"><br/>
+图6. 池化层图片<br/>
 </p>
 
-[**LeNet-5**](http://yann.lecun.com/exdb/lenet/) is one of the simplest Convolutional Neural Networks. Fig. 6. shows its architecture: A 2-dimensional input image is fed into two sets of convolutional layers and pooling layers. This output is then fed to a fully connected layer and a softmax classifier. Compared to multilayer, fully connected perceptrons, the LeNet-5 can recognize images better. This is due to the following three properties of the convolution:
-
-- The 3D nature of the neurons: a convolutional layer is organized by width, height and depth. Neurons in each layer are connected to only a small region in the previous layer. This region is called the receptive field.
-- Local connectivity: A CNN utilizes the local space correlation by connecting local neurons. This design guarantees that the learned filter has a strong response to local input features. Stacking many such layers generates a non-linear filter that is more global. This enables the network to first obtain good representation for small parts of input and then combine them to represent a larger region.
-- Weight sharing: In a CNN, computation is iterated on shared parameters (weights and bias) to form a feature map. This means that all the neurons in the same depth of the output respond to the same feature. This allows the network to detect a feature regardless of its position in the input. In other words, it is shift invariant.
+池化是非线性下采样的一种形式，主要作用是通过减少网络的参数来减小计算量，并且能够在一定程度上控制过拟合。通常在卷积层的后面会加上一个池化层。池化包括最大池化、平均池化等。其中最大池化是用不重叠的矩形框将输入层分成不同的区域，对于每个矩形框的数取最大值作为输出层，如图6所示。
 
-For more details on Convolutional Neural Networks, please refer to the tutorial on [Image Classification](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md) and the [relevant lecture](http://cs231n.github.io/convolutional-networks/) from a Stanford open course.
+更详细的关于卷积神经网络的具体知识可以参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/ )和[图像分类](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md)教程。
 
-### List of Common Activation Functions  
-- Sigmoid activation function: $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
+### 常见激活函数介绍  
+- sigmoid激活函数： $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
 
-- Tanh activation function: $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
+- tanh激活函数： $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
 
-  In fact, tanh function is just a rescaled version of the sigmoid function. It is obtained by magnifying the value of the sigmoid function and moving it downwards by 1.
+  实际上，tanh函数只是规模变化的sigmoid函数，将sigmoid函数值放大2倍之后再向下平移1个单位：tanh(x) = 2sigmoid(2x) - 1 。
 
-- ReLU activation function: $ f(x) = max(0, x) $
+- ReLU激活函数： $ f(x) = max(0, x) $
 
-For more information, please refer to [Activation functions on Wikipedia](https://en.wikipedia.org/wiki/Activation_function).
+更详细的介绍请参考[维基百科激活函数](https://en.wikipedia.org/wiki/Activation_function)。
 
-## Data Preparation
+## 数据介绍
 
-PaddlePaddle provides a Python module, `paddle.dataset.mnist`, which downloads and caches the [MNIST dataset](http://yann.lecun.com/exdb/mnist/).  The cache is under `/home/username/.cache/paddle/dataset/mnist`:
+PaddlePaddle在API中提供了自动加载[MNIST](http://yann.lecun.com/exdb/mnist/)数据的模块`paddle.dataset.mnist`。加载后的数据位于`/home/username/.cache/paddle/dataset/mnist`下：
 
 
-|    File name          |       Description | Size            |
-|----------------------|--------------|-----------|
-|train-images-idx3-ubyte|  Training images | 60,000 |
-|train-labels-idx1-ubyte|  Training labels | 60,000 |
-|t10k-images-idx3-ubyte |  Evaluation images | 10,000 |
-|t10k-labels-idx1-ubyte |  Evaluation labels | 10,000 |
+|    文件名称          |       说明              |
+|----------------------|-------------------------|
+|train-images-idx3-ubyte|  训练数据图片，60,000条数据 |
+|train-labels-idx1-ubyte|  训练数据标签，60,000条数据 |
+|t10k-images-idx3-ubyte |  测试数据图片，10,000条数据 |
+|t10k-labels-idx1-ubyte |  测试数据标签，10,000条数据 |
 
+## 配置说明
 
-## Model Configuration
-
-A PaddlePaddle program starts from importing the API package:
+首先，加载PaddlePaddle的V2 api包。
 
 ```python
 import paddle.v2 as paddle
 ```
+其次，定义三个不同的分类器：
 
-We want to use this program to demonstrate three different classifiers, each defined as a Python function:
-
-- Softmax regression: the network has a fully-connection layer with softmax activation:
+- Softmax回归：只通过一层简单的以softmax为激活函数的全连接层，就可以得到分类的结果。
 
 ```python
 def softmax_regression(img):
@@ -191,26 +187,27 @@ def softmax_regression(img):
                               act=paddle.activation.Softmax())
     return predict
 ```
-
-- Multi-Layer Perceptron: this network has two hidden fully-connected layers, one with ReLU and the other with softmax activation:
+- 多层感知器：下面代码实现了一个含有两个隐藏层（即全连接层）的多层感知器。其中两个隐藏层的激活函数均采用ReLU，输出层的激活函数用Softmax。
 
 ```python
 def multilayer_perceptron(img):
+    # 第一个全连接层，激活函数为ReLU
     hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
+    # 第二个全连接层，激活函数为ReLU
     hidden2 = paddle.layer.fc(input=hidden1,
                               size=64,
                               act=paddle.activation.Relu())
+    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
     predict = paddle.layer.fc(input=hidden2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
-
-- Convolution network LeNet-5: the input image is fed through two convolution-pooling layers, a fully-connected layer, and the softmax output layer:
+- 卷积神经网络LeNet-5: 输入的二维图像，首先经过两次卷积层到池化层，再经过全连接层，最后使用以softmax为激活函数的全连接层作为输出层。
 
 ```python
 def convolutional_neural_network(img):
-
+    # 第一个卷积-池化层
     conv_pool_1 = paddle.networks.simple_img_conv_pool(
         input=img,
         filter_size=5,
@@ -218,8 +215,8 @@ def convolutional_neural_network(img):
         num_channel=1,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-
+        act=paddle.activation.Relu())
+    # 第二个卷积-池化层
     conv_pool_2 = paddle.networks.simple_img_conv_pool(
         input=conv_pool_1,
         filter_size=5,
@@ -227,21 +224,18 @@ def convolutional_neural_network(img):
         num_channel=20,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-
-    fc1 = paddle.layer.fc(input=conv_pool_2,
-                          size=128,
-                          act=paddle.activation.Tanh())
-
-    predict = paddle.layer.fc(input=fc1,
+        act=paddle.activation.Relu())
+    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
+    predict = paddle.layer.fc(input=conv_pool_2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
 
-PaddlePaddle provides a special layer `layer.data` for reading data. Let us create a data layer for reading images and connect it to a classification network created using one of above three functions.  We also need a cost layer for training the model.
+接着，通过`layer.data`调用来获取数据，然后调用分类器（这里我们提供了三个不同的分类器）得到分类结果。训练时，对该结果计算其损失函数，分类问题常常选择交叉熵损失函数。
 
 ```python
+# 该模型运行在单个CPU上
 paddle.init(use_gpu=False, trainer_count=1)
 
 images = paddle.layer.data(
@@ -249,14 +243,17 @@ images = paddle.layer.data(
 label = paddle.layer.data(
     name='label', type=paddle.data_type.integer_value(10))
 
-predict = softmax_regression(images)
-#predict = multilayer_perceptron(images) # uncomment for MLP
-#predict = convolutional_neural_network(images) # uncomment for LeNet5
+# predict = softmax_regression(images) # Softmax回归
+# predict = multilayer_perceptron(images) #多层感知器
+predict = convolutional_neural_network(images) #LeNet5卷积神经网络
 
 cost = paddle.layer.classification_cost(input=predict, label=label)
 ```
 
-Now, it is time to specify training parameters. In the following `Momentum` optimizer, `momentum=0.9` means that 90% of the current momentum comes from that of the previous iteration. The learning rate relates to the speed at which the network training converges. Regularization is meant to prevent over-fitting; here we use the L2 regularization.
+然后，指定训练相关的参数。
+- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器 `Momentum` ，其中参数0.9代表动量优化每次保持前一次速度的0.9倍。
+- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
+- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
 
 ```python
 parameters = paddle.parameters.create(cost)
@@ -271,12 +268,44 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=optimizer)
 ```
 
-Then we specify the training data `paddle.dataset.movielens.train()` and testing data `paddle.dataset.movielens.test()`. These two methods are *reader creators*. Once called, a reader creator returns a *reader*.  A reader is a Python method, which, once called, returns a Python generator, which yields instances of data.
+下一步，我们开始训练过程。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python yield generator。
 
-`shuffle` is a reader decorator. It takes in a reader A as input and returns a new reader B. Under the hood, B calls A to read data in the following fashion: it copies in `buffer_size` instances at a time into a buffer, shuffles the data, and yields the shuffled instances one at a time. A large buffer size would yield very shuffled data.
+下面`shuffle`是一个reader decorator，它接受一个reader A，返回另一个reader B —— reader B 每次读入`buffer_size`条训练数据到一个buffer里，然后随机打乱其顺序，并且逐条输出。
 
-`batch` is a special decorator, which takes in reader and outputs a *batch reader*, which doesn't yield an instance, but a minibatch at a time.
+`batch`是一个特殊的decorator，它的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minibatch。
 
+`event_handler_plot`可以用来在训练过程中画图如下：
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+# event_handler to plot a figure
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 100 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=paddle.batch(
+            paddle.dataset.mnist.test(), batch_size=128))
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` 用来在训练过程中输出训练结果
 ```python
 lists = []
 
@@ -286,23 +315,29 @@ def event_handler(event):
             print "Pass %d, Batch %d, Cost %f, %s" % (
                 event.pass_id, event.batch_id, event.cost, event.metrics)
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(reader=paddle.batch(
             paddle.dataset.mnist.test(), batch_size=128))
         print "Test with Pass %d, Cost %f, %s\n" % (
             event.pass_id, result.cost, result.metrics)
         lists.append((event.pass_id, result.cost,
                       result.metrics['classification_error_evaluator']))
+```
 
+```python
 trainer.train(
     reader=paddle.batch(
         paddle.reader.shuffle(
             paddle.dataset.mnist.train(), buf_size=8192),
         batch_size=128),
-    event_handler=event_handler,
-    num_passes=100)
+    event_handler=event_handler_plot,
+    num_passes=5)
 ```
 
-During training, `trainer.train` invokes `event_handler` for certain events. This gives us a chance to print the training progress.
+训练过程是完全自动的，event_handler里打印的日志类似如下所示：
 
 ```
 # Pass 0, Batch 0, Cost 2.780790, {'classification_error_evaluator': 0.9453125}
@@ -313,26 +348,39 @@ During training, `trainer.train` invokes `event_handler` for certain events. Thi
 # Test with Pass 0, Cost 0.326659, {'classification_error_evaluator': 0.09470000118017197}
 ```
 
-After the training, we can check the model's prediction accuracy.
+训练之后，检查模型的预测准确度。用 MNIST 训练的时候，一般 softmax回归模型的分类准确率为约为 92.34%，多层感知器为97.66%，卷积神经网络可以达到 99.20%。
 
-```
-# find the best pass
-best = sorted(lists, key=lambda list: float(list[1]))[0]
-print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
-print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
-```
 
-Usually, with MNIST data, the softmax regression model achieves an accuracy around 92.34%, the MLP 97.66%, and the convolution network around 99.20%. Convolution layers have been widely considered a great invention for image processing.
+## 应用模型
 
+可以使用训练好的模型对手写体数字图片进行分类，下面程序展示了如何使用paddle.infer接口进行推断。
 
-## Conclusion
-This tutorial describes a few common deep learning models using **Softmax regression**, **Multilayer Perceptron Network**, and **Convolutional Neural Network**. Understanding these models is crucial for future learning; the subsequent tutorials derive more sophisticated networks by building on top of them.
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file).convert('L')
+    im = im.resize((28, 28), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32).flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/infer_3.png'),))
+
+probs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/infer_3.png is: %d" % lab[0][0]
+```
 
-When our model evolves from a simple softmax regression to a slightly complex Convolutional Neural Network, the recognition accuracy on the MNIST data set achieves a large improvement in accuracy. This is due to the Convolutional layers' local connections and parameter sharing. While learning new models in the future, we encourage the readers to understand the key ideas that lead a new model to improve the results of an old one.
+## 总结
 
-Moreover, this tutorial introduces the basic flow of PaddlePaddle model design, which starts with a *dataprovider*, a model layer construction, and finally training and prediction. Motivated readers can leverage the flow used in this MNIST handwritten digit classification example and experiment with different data and network architectures to train models for classification tasks of their choice.
+本教程的softmax回归、多层感知器和卷积神经网络是最基础的深度学习模型，后续章节中复杂的神经网络都是从它们衍生出来的，因此这几个模型对之后的学习大有裨益。同时，我们也观察到从最简单的softmax回归变换到稍复杂的卷积神经网络的时候，MNIST数据集上的识别准确率有了大幅度的提升，原因是卷积层具有局部连接和共享权重的特性。在之后学习新模型的时候，希望大家也要深入到新模型相比原模型带来效果提升的关键之处。此外，本教程还介绍了PaddlePaddle模型搭建的基本流程，从dataprovider的编写、网络层的构建，到最后的训练和预测。对这个流程熟悉以后，大家就可以用自己的数据，定义自己的网络模型，并完成自己的训练和预测任务了。
 
-## References
+## 参考文献
 
 1. LeCun, Yann, Léon Bottou, Yoshua Bengio, and Patrick Haffner. ["Gradient-based learning applied to document recognition."](http://ieeexplore.ieee.org/abstract/document/726791/) Proceedings of the IEEE 86, no. 11 (1998): 2278-2324.
 2. Wejéus, Samuel. ["A Neural Network Approach to Arbitrary SymbolRecognition on Modern Smartphones."](http://www.diva-portal.org/smash/record.jsf?pid=diva2%3A753279&dswid=-434) (2014).
@@ -346,7 +394,7 @@ Moreover, this tutorial introduces the basic flow of PaddlePaddle model design,
 10. Bishop, Christopher M. ["Pattern recognition."](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) Machine Learning 128 (2006): 1-58.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

02.recognize_digits/index.html

@@ -40,145 +40,145 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-# 识别数字
+# Recognize Digits
 
-本教程源代码目录在[book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+The source code for this tutorial is live at [book/recognize_digits](https://github.com/PaddlePaddle/book/tree/develop/02.recognize_digits). For instructions on getting started with Paddle, please refer to [installation instructions](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
-当我们学习编程的时候，编写的第一个程序一般是实现打印"Hello World"。而机器学习（或深度学习）的入门教程，一般都是 [MNIST](http://yann.lecun.com/exdb/mnist/) 数据库上的手写识别问题。原因是手写识别属于典型的图像分类问题，比较简单，同时MNIST数据集也很完备。MNIST数据集作为一个简单的计算机视觉数据集，包含一系列如图1所示的手写数字图片和对应的标签。图片是28x28的像素矩阵，标签则对应着0~9的10个数字。每张图片都经过了大小归一化和居中处理。
+## Introduction
+When one learns to program, the first task is usually to write a program that prints "Hello World!". In Machine Learning or Deep Learning, the equivalent task is to train a model to recognize hand-written digits on the dataset [MNIST](http://yann.lecun.com/exdb/mnist/). Handwriting recognition is a classic image classification problem. The problem is relatively easy and MNIST is a complete dataset. As a simple Computer Vision dataset, MNIST contains images of handwritten digits and their corresponding labels (Fig. 1). The input image is a $28\times28$ matrix, and the label is one of the digits from $0$ to $9$. All images are normalized, meaning that they are both rescaled and centered.
 
 <p align="center">
 <img src="image/mnist_example_image.png" width="400"><br/>
-图1. MNIST图片示例
+Fig. 1. Examples of MNIST images
 </p>
 
-MNIST数据集是从 [NIST](https://www.nist.gov/srd/nist-special-database-19) 的Special Database 3（SD-3）和Special Database 1（SD-1）构建而来。由于SD-3是由美国人口调查局的员工进行标注，SD-1是由美国高中生进行标注，因此SD-3比SD-1更干净也更容易识别。Yann LeCun等人从SD-1和SD-3中各取一半作为MNIST的训练集（60000条数据）和测试集（10000条数据），其中训练集来自250位不同的标注员，此外还保证了训练集和测试集的标注员是不完全相同的。
+The MNIST dataset is created from the [NIST](https://www.nist.gov/srd/nist-special-database-19) Special Database 3 (SD-3) and the Special Database 1 (SD-1). The SD-3 is labeled by the staff of the U.S. Census Bureau, while SD-1 is labeled by high school students the in U.S. Therefore the SD-3 is cleaner and easier to recognize than the SD-1 dataset. Yann LeCun et al. used half of the samples from each of SD-1 and SD-3 to create the MNIST training set (60,000 samples) and test set (10,000 samples), where training set was labeled by 250 different annotators, and it was guaranteed that there wasn't a complete overlap of annotators of training set and test set.
 
-Yann LeCun早先在手写字符识别上做了很多研究，并在研究过程中提出了卷积神经网络（Convolutional Neural Network），大幅度地提高了手写字符的识别能力，也因此成为了深度学习领域的奠基人之一。如今的深度学习领域，卷积神经网络占据了至关重要的地位，从最早Yann LeCun提出的简单LeNet，到如今ImageNet大赛上的优胜模型VGGNet、GoogLeNet、ResNet等（请参见[图像分类](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) 教程），人们在图像分类领域，利用卷积神经网络得到了一系列惊人的结果。
+Yann LeCun, one of the founders of Deep Learning, have previously made tremendous contributions to handwritten character recognition and proposed the **Convolutional Neural Network** (CNN), which drastically improved recognition capability for handwritten characters. CNNs are now a critical concept in Deep Learning. From the LeNet proposal by Yann LeCun, to those winning models in ImageNet competitions, such as VGGNet, GoogLeNet, and ResNet (See [Image Classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification) tutorial), CNNs have achieved a series of impressive results in Image Classification tasks.
 
-有很多算法在MNIST上进行实验。1998年，LeCun分别用单层线性分类器、多层感知器（Multilayer Perceptron, MLP）和多层卷积神经网络LeNet进行实验，使得测试集上的误差不断下降（从12%下降到0.7%）\[[1](#参考文献)\]。此后，科学家们又基于K近邻（K-Nearest Neighbors）算法\[[2](#参考文献)\]、支持向量机（SVM）\[[3](#参考文献)\]、神经网络\[[4-7](#参考文献)\]和Boosting方法\[[8](#参考文献)\]等做了大量实验，并采用多种预处理方法（如去除歪曲、去噪、模糊等）来提高识别的准确率。
+Many algorithms are tested on MNIST. In 1998, LeCun experimented with single layer linear classifier, Multilayer Perceptron (MLP) and Multilayer CNN LeNet. These algorithms quickly reduced test error from 12% to 0.7% \[[1](#references)\]. Since then, researchers have worked on many algorithms such as **K-Nearest Neighbors** (k-NN) \[[2](#references)\], **Support Vector Machine** (SVM) \[[3](#references)\], **Neural Networks** \[[4-7](#references)\] and **Boosting** \[[8](#references)\]. Various preprocessing methods like distortion removal, noise removal, and blurring, have also been applied to increase recognition accuracy.
 
-本教程中，我们从简单的模型Softmax回归开始，带大家入门手写字符识别，并逐步进行模型优化。
+In this tutorial, we tackle the task of handwritten character recognition. We start with a simple **softmax** regression model and guide our readers step-by-step to improve this model's performance on the task of recognition.
 
 
-## 模型概览
+## Model Overview
 
-基于MNIST数据训练一个分类器，在介绍本教程使用的三个基本图像分类网络前，我们先给出一些定义：
-- $X$是输入：MNIST图片是$28\times28$ 的二维图像，为了进行计算，我们将其转化为$784$维向量，即$X=\left ( x_0, x_1, \dots, x_{783} \right )$。
-- $Y$是输出：分类器的输出是10类数字（0-9），即$Y=\left ( y_0, y_1, \dots, y_9 \right )$，每一维$y_i$代表图片分类为第$i$类数字的概率。
-- $L$是图片的真实标签：$L=\left ( l_0, l_1, \dots, l_9 \right )$也是10维，但只有一维为1，其他都为0。
+Before introducing classification algorithms and training procedure, we define the following symbols:
+- $X$ is the input: Input is a $28\times 28$ MNIST image. It is flattened to a $784$ dimensional vector. $X=\left (x_0, x_1, \dots, x_{783} \right )$.
+- $Y$ is the output: Output of the classifier is 1 of the 10 classes (digits from 0 to 9). $Y=\left (y_0, y_1, \dots, y_9 \right )$. Each dimension $y_i$ represents the probability that the input image belongs to class $i$.
+- $L$ is the ground truth label: $L=\left ( l_0, l_1, \dots, l_9 \right )$. It is also 10 dimensional, but only one entry is $1$ and all others are $0$s.
 
-### Softmax回归(Softmax Regression)
+### Softmax Regression
 
-最简单的Softmax回归模型是先将输入层经过一个全连接层得到的特征，然后直接通过softmax 函数进行多分类\[[9](#参考文献)\]。
+In a simple softmax regression model, the input is first fed to fully connected layers. Then, a softmax function is applied to output probabilities of multiple output classes\[[9](#references)\].
 
-输入层的数据$X$传到输出层，在激活操作之前，会乘以相应的权重 $W$ ，并加上偏置变量 $b$ ，具体如下：
+The input $X$ is multiplied by weights $W$ and then added to the bias $b$ to generate activations.
 
 $$ y_i = \text{softmax}(\sum_j W_{i,j}x_j + b_i) $$
 
-其中 $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
+where $ \text{softmax}(x_i) = \frac{e^{x_i}}{\sum_j e^{x_j}} $
 
-对于有 $N$ 个类别的多分类问题，指定 $N$ 个输出节点，$N$ 维结果向量经过softmax将归一化为 $N$ 个[0,1]范围内的实数值，分别表示该样本属于这 $N$ 个类别的概率。此处的 $y_i$ 即对应该图片为数字 $i$ 的预测概率。
+For an $N$-class classification problem with $N$ output nodes, Softmax normalizes the resulting $N$ dimensional vector so that each of its entries falls in the range $[0,1]\in\math{R}$, representing the probability that the sample belongs to a certain class. Here $y_i$ denotes the predicted probability that an image is of digit $i$.
 
-在分类问题中，我们一般采用交叉熵代价损失函数（cross entropy），公式如下：
+In such a classification problem, we usually use the cross entropy loss function:
 
 $$  \text{crossentropy}(label, y) = -\sum_i label_ilog(y_i) $$
 
-图2为softmax回归的网络图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+Fig. 2 illustrates a softmax regression network, with the weights in blue, and the bias in red. `+1` indicates that the bias is $1$.
 
 <p align="center">
-<img src="image/softmax_regression.png" width=400><br/>
-图2. softmax回归网络结构图<br/>
+<img src="image/softmax_regression_en.png" width=400><br/>
+Fig. 2. Softmax regression network architecture<br/>
 </p>
 
-### 多层感知器(Multilayer Perceptron, MLP)
+### Multilayer Perceptron
 
-Softmax回归模型采用了最简单的两层神经网络，即只有输入层和输出层，因此其拟合能力有限。为了达到更好的识别效果，我们考虑在输入层和输出层中间加上若干个隐藏层\[[10](#参考文献)\]。
+The softmax regression model described above uses the simplest two-layer neural network. That is, it only contains an input layer and an output layer, with limited regression capability. To achieve better recognition results, consider adding several hidden layers\[[10](#references)\] between the input layer and the output layer.
 
-1.  经过第一个隐藏层，可以得到 $ H_1 = \phi(W_1X + b_1) $，其中$\phi$代表激活函数，常见的有sigmoid、tanh或ReLU等函数。
-2.  经过第二个隐藏层，可以得到 $ H_2 = \phi(W_2H_1 + b_2) $。
-3.  最后，再经过输出层，得到的$Y=\text{softmax}(W_3H_2 + b_3)$，即为最后的分类结果向量。
+1.  After the first hidden layer, we get $ H_1 = \phi(W_1X + b_1) $, where $\phi$ denotes the activation function. Some [common ones](###list-of-common-activation-functions) are sigmoid, tanh and ReLU.
+2.  After the second hidden layer, we get $ H_2 = \phi(W_2H_1 + b_2) $.
+3.  Finally, the output layer outputs $Y=\text{softmax}(W_3H_2 + b_3)$, the vector denoting our classification result.
 
-
-图3为多层感知器的网络结构图，图中权重用蓝线表示、偏置用红线表示、+1代表偏置参数的系数为1。
+Fig. 3. shows a Multilayer Perceptron network, with the weights in blue, and the bias in red. +1 indicates that the bias is $1$.
 
 <p align="center">
-<img src="image/mlp.png" width=500><br/>
-图3. 多层感知器网络结构图<br/>
-</p>
-
-### 卷积神经网络(Convolutional Neural Network, CNN)
+<img src="image/mlp_en.png" width=500><br/>
+Fig. 3. Multilayer Perceptron network architecture<br/>
 
-在多层感知器模型中，将图像展开成一维向量输入到网络中，忽略了图像的位置和结构信息，而卷积神经网络能够更好的利用图像的结构信息。[LeNet-5](http://yann.lecun.com/exdb/lenet/)是一个较简单的卷积神经网络。图4显示了其结构：输入的二维图像，先经过两次卷积层到池化层，再经过全连接层，最后使用softmax分类作为输出层。下面我们主要介绍卷积层和池化层。
-
-<p align="center">
-<img src="image/cnn.png"><br/>
-图4. LeNet-5卷积神经网络结构<br/>
 </p>
 
-#### 卷积层
+### Convolutional Neural Network
 
-卷积层是卷积神经网络的核心基石。在图像识别里我们提到的卷积是二维卷积，即离散二维滤波器（也称作卷积核）与二维图像做卷积操作，简单的讲是二维滤波器滑动到二维图像上所有位置，并在每个位置上与该像素点及其领域像素点做内积。卷积操作被广泛应用与图像处理领域，不同卷积核可以提取不同的特征，例如边沿、线性、角等特征。在深层卷积神经网络中，通过卷积操作可以提取出图像低级到复杂的特征。
+#### Convolutional Layer
 
 <p align="center">
 <img src="image/conv_layer.png" width='750'><br/>
-图5. 卷积层图片<br/>
+Fig. 4. Convolutional layer<br/>
 </p>
 
-图5给出一个卷积计算过程的示例图，输入图像大小为$H=5,W=5,D=3$，即$5 \times 5$大小的3通道（RGB，也称作深度）彩色图像。这个示例图中包含两（用$K$表示）组卷积核，即图中滤波器$W_0$和$W_1$。在卷积计算中，通常对不同的输入通道采用不同的卷积核，如图示例中每组卷积核包含（$D=3）$个$3 \times 3$（用$F \times F$表示）大小的卷积核。另外，这个示例中卷积核在图像的水平方向（$W$方向）和垂直方向（$H$方向）的滑动步长为2（用$S$表示）；对输入图像周围各填充1（用$P$表示）个0，即图中输入层原始数据为蓝色部分，灰色部分是进行了大小为1的扩展，用0来进行扩展。经过卷积操作得到输出为$3 \times 3 \times 2$（用$H_{o} \times W_{o} \times K$表示）大小的特征图，即$3 \times 3$大小的2通道特征图，其中$H_o$计算公式为：$H_o = (H - F + 2 \times P)/S + 1$，$W_o$同理。 而输出特征图中的每个像素，是每组滤波器与输入图像每个特征图的内积再求和，再加上偏置$b_o$，偏置通常对于每个输出特征图是共享的。输出特征图$o[:,:,0]$中的最后一个$-2$计算如图5右下角公式所示。
+The **convolutional layer** is the core of a Convolutional Neural Network. The parameters in this layer are composed of a set of filters, also called kernels. We could visualize the convolution step in the following fashion: Each kernel slides horizontally and vertically till it covers the whole image. At every window, we compute the dot product of the kernel and the input. Then, we add the bias and apply an activation function. The result is a two-dimensional activation map. For example, some kernel may recognize corners, and some may recognize circles. These convolution kernels may respond strongly to the corresponding features.
 
-在卷积操作中卷积核是可学习的参数，经过上面示例介绍，每层卷积的参数大小为$D \times F \times F \times K$。在多层感知器模型中，神经元通常是全部连接，参数较多。而卷积层的参数较少，这也是由卷积层的主要特性即局部连接和共享权重所决定。
+Fig. 4 illustrates the dynamic programming of a convolutional layer, where depths are flattened for simplicity. The input is $W_1=5$, $H_1=5$, $D_1=3$. In fact, this is a common representation for colored images. $W_1$ and $H_1$ correspond to the width and height in a colored image. $D_1$ corresponds to the 3 color channels for RGB. The parameters of the convolutional layer are $K=2$, $F=3$, $S=2$, $P=1$. $K$ denotes the number of kernels; specifically, $Filter$ $W_0$ and $Filter$ $W_1$ are the kernels. $F$ is kernel size while $W0$ and $W1$ are both $F\timesF = 3\times3$ matrices in all depths. $S$ is the stride, which is the width of the sliding window; here, kernels move leftwards or downwards by 2 units each time. $P$ is the width of the padding, which denotes an extension of the input; here, the gray area shows zero padding with size 1.
 
-- 局部连接：每个神经元仅与输入神经元的一块区域连接，这块局部区域称作感受野（receptive field）。在图像卷积操作中，即神经元在空间维度（spatial dimension，即上图示例H和W所在的平面）是局部连接，但在深度上是全部连接。对于二维图像本身而言，也是局部像素关联较强。这种局部连接保证了学习后的过滤器能够对于局部的输入特征有最强的响应。局部连接的思想，也是受启发于生物学里面的视觉系统结构，视觉皮层的神经元就是局部接受信息的。
+#### Pooling Layer
 
-- 权重共享：计算同一个深度切片的神经元时采用的滤波器是共享的。例如图4中计算$o[:,:,0]$的每个每个神经元的滤波器均相同，都为$W_0$，这样可以很大程度上减少参数。共享权重在一定程度上讲是有意义的，例如图片的底层边缘特征与特征在图中的具体位置无关。但是在一些场景中是无意的，比如输入的图片是人脸，眼睛和头发位于不同的位置，希望在不同的位置学到不同的特征 (参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/))。请注意权重只是对于同一深度切片的神经元是共享的，在卷积层，通常采用多组卷积核提取不同特征，即对应不同深度切片的特征，不同深度切片的神经元权重是不共享。另外，偏重对同一深度切片的所有神经元都是共享的。
+<p align="center">
+<img src="image/max_pooling_en.png" width="400px"><br/>
+Fig. 5 Pooling layer using max-pooling<br/>
+</p>
 
-通过介绍卷积计算过程及其特性，可以看出卷积是线性操作，并具有平移不变性（shift-invariant），平移不变性即在图像每个位置执行相同的操作。卷积层的局部连接和权重共享使得需要学习的参数大大减小，这样也有利于训练较大卷积神经网络。
+A **pooling layer** performs downsampling. The main functionality of this layer is to reduce computation by reducing the network parameters. It also prevents over-fitting to some extent. Usually, a pooling layer is added after a convolutional layer. Pooling layer can use various techniques, such as max pooling and average pooling. As shown in Fig.5, max pooling uses rectangles to segment the input layer into several parts and computes the maximum value in each part as the output.
 
-#### 池化层
+#### LeNet-5 Network
 
 <p align="center">
-<img src="image/max_pooling.png" width="400px"><br/>
-图6. 池化层图片<br/>
+<img src="image/cnn_en.png"><br/>
+Fig. 6. LeNet-5 Convolutional Neural Network architecture<br/>
 </p>
 
-池化是非线性下采样的一种形式，主要作用是通过减少网络的参数来减小计算量，并且能够在一定程度上控制过拟合。通常在卷积层的后面会加上一个池化层。池化包括最大池化、平均池化等。其中最大池化是用不重叠的矩形框将输入层分成不同的区域，对于每个矩形框的数取最大值作为输出层，如图6所示。
+[**LeNet-5**](http://yann.lecun.com/exdb/lenet/) is one of the simplest Convolutional Neural Networks. Fig. 6. shows its architecture: A 2-dimensional input image is fed into two sets of convolutional layers and pooling layers. This output is then fed to a fully connected layer and a softmax classifier. Compared to multilayer, fully connected perceptrons, the LeNet-5 can recognize images better. This is due to the following three properties of the convolution:
+
+- The 3D nature of the neurons: a convolutional layer is organized by width, height and depth. Neurons in each layer are connected to only a small region in the previous layer. This region is called the receptive field.
+- Local connectivity: A CNN utilizes the local space correlation by connecting local neurons. This design guarantees that the learned filter has a strong response to local input features. Stacking many such layers generates a non-linear filter that is more global. This enables the network to first obtain good representation for small parts of input and then combine them to represent a larger region.
+- Weight sharing: In a CNN, computation is iterated on shared parameters (weights and bias) to form a feature map. This means that all the neurons in the same depth of the output respond to the same feature. This allows the network to detect a feature regardless of its position in the input. In other words, it is shift invariant.
+
+For more details on Convolutional Neural Networks, please refer to the tutorial on [Image Classification](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md) and the [relevant lecture](http://cs231n.github.io/convolutional-networks/) from a Stanford open course.
 
-更详细的关于卷积神经网络的具体知识可以参考[斯坦福大学公开课]( http://cs231n.github.io/convolutional-networks/ )和[图像分类](https://github.com/PaddlePaddle/book/blob/develop/image_classification/README.md)教程。
+### List of Common Activation Functions
+- Sigmoid activation function: $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
 
-### 常见激活函数介绍  
-- sigmoid激活函数： $ f(x) = sigmoid(x) = \frac{1}{1+e^{-x}} $
+- Tanh activation function: $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
 
-- tanh激活函数： $ f(x) = tanh(x) = \frac{e^x-e^{-x}}{e^x+e^{-x}} $
+  In fact, tanh function is just a rescaled version of the sigmoid function. It is obtained by magnifying the value of the sigmoid function and moving it downwards by 1.
 
-  实际上，tanh函数只是规模变化的sigmoid函数，将sigmoid函数值放大2倍之后再向下平移1个单位：tanh(x) = 2sigmoid(2x) - 1 。
+- ReLU activation function: $ f(x) = max(0, x) $
 
-- ReLU激活函数： $ f(x) = max(0, x) $
+For more information, please refer to [Activation functions on Wikipedia](https://en.wikipedia.org/wiki/Activation_function).
 
-更详细的介绍请参考[维基百科激活函数](https://en.wikipedia.org/wiki/Activation_function)。
+## Data Preparation
 
-## 数据介绍
+PaddlePaddle provides a Python module, `paddle.dataset.mnist`, which downloads and caches the [MNIST dataset](http://yann.lecun.com/exdb/mnist/).  The cache is under `/home/username/.cache/paddle/dataset/mnist`:
 
-PaddlePaddle在API中提供了自动加载[MNIST](http://yann.lecun.com/exdb/mnist/)数据的模块`paddle.dataset.mnist`。加载后的数据位于`/home/username/.cache/paddle/dataset/mnist`下：
 
+|    File name          |       Description | Size            |
+|----------------------|--------------|-----------|
+|train-images-idx3-ubyte|  Training images | 60,000 |
+|train-labels-idx1-ubyte|  Training labels | 60,000 |
+|t10k-images-idx3-ubyte |  Evaluation images | 10,000 |
+|t10k-labels-idx1-ubyte |  Evaluation labels | 10,000 |
 
-|    文件名称          |       说明              |
-|----------------------|-------------------------|
-|train-images-idx3-ubyte|  训练数据图片，60,000条数据 |
-|train-labels-idx1-ubyte|  训练数据标签，60,000条数据 |
-|t10k-images-idx3-ubyte |  测试数据图片，10,000条数据 |
-|t10k-labels-idx1-ubyte |  测试数据标签，10,000条数据 |
 
-## 配置说明
+## Model Configuration
 
-首先，加载PaddlePaddle的V2 api包。
+A PaddlePaddle program starts from importing the API package:
 
 ```python
 import paddle.v2 as paddle
 ```
-其次，定义三个不同的分类器：
 
-- Softmax回归：只通过一层简单的以softmax为激活函数的全连接层，就可以得到分类的结果。
+We want to use this program to demonstrate three different classifiers, each defined as a Python function:
+
+- Softmax regression: the network has a fully-connection layer with softmax activation:
 
 ```python
 def softmax_regression(img):
@@ -187,27 +187,26 @@ def softmax_regression(img):
                               act=paddle.activation.Softmax())
     return predict
 ```
-- 多层感知器：下面代码实现了一个含有两个隐藏层（即全连接层）的多层感知器。其中两个隐藏层的激活函数均采用ReLU，输出层的激活函数用Softmax。
+
+- Multi-Layer Perceptron: this network has two hidden fully-connected layers, one with ReLU and the other with softmax activation:
 
 ```python
 def multilayer_perceptron(img):
-    # 第一个全连接层，激活函数为ReLU
     hidden1 = paddle.layer.fc(input=img, size=128, act=paddle.activation.Relu())
-    # 第二个全连接层，激活函数为ReLU
     hidden2 = paddle.layer.fc(input=hidden1,
                               size=64,
                               act=paddle.activation.Relu())
-    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
     predict = paddle.layer.fc(input=hidden2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
-- 卷积神经网络LeNet-5: 输入的二维图像，首先经过两次卷积层到池化层，再经过全连接层，最后使用以softmax为激活函数的全连接层作为输出层。
+
+- Convolution network LeNet-5: the input image is fed through two convolution-pooling layers, a fully-connected layer, and the softmax output layer:
 
 ```python
 def convolutional_neural_network(img):
-    # 第一个卷积-池化层
+
     conv_pool_1 = paddle.networks.simple_img_conv_pool(
         input=img,
         filter_size=5,
@@ -215,8 +214,8 @@ def convolutional_neural_network(img):
         num_channel=1,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-    # 第二个卷积-池化层
+        act=paddle.activation.Relu())
+
     conv_pool_2 = paddle.networks.simple_img_conv_pool(
         input=conv_pool_1,
         filter_size=5,
@@ -224,22 +223,17 @@ def convolutional_neural_network(img):
         num_channel=20,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-    # 全连接层
-    fc1 = paddle.layer.fc(input=conv_pool_2,
-                          size=128,
-                          act=paddle.activation.Tanh())
-    # 以softmax为激活函数的全连接输出层，输出层的大小必须为数字的个数10
-    predict = paddle.layer.fc(input=fc1,
+        act=paddle.activation.Relu())
+
+    predict = paddle.layer.fc(input=conv_pool_2,
                               size=10,
                               act=paddle.activation.Softmax())
     return predict
 ```
 
-接着，通过`layer.data`调用来获取数据，然后调用分类器（这里我们提供了三个不同的分类器）得到分类结果。训练时，对该结果计算其损失函数，分类问题常常选择交叉熵损失函数。
+PaddlePaddle provides a special layer `layer.data` for reading data. Let us create a data layer for reading images and connect it to a classification network created using one of above three functions.  We also need a cost layer for training the model.
 
 ```python
-# 该模型运行在单个CPU上
 paddle.init(use_gpu=False, trainer_count=1)
 
 images = paddle.layer.data(
@@ -247,17 +241,14 @@ images = paddle.layer.data(
 label = paddle.layer.data(
     name='label', type=paddle.data_type.integer_value(10))
 
-predict = softmax_regression(images) # Softmax回归
-#predict = multilayer_perceptron(images) #多层感知器
-#predict = convolutional_neural_network(images) #LeNet5卷积神经网络
+# predict = softmax_regression(images)
+# predict = multilayer_perceptron(images) # uncomment for MLP
+predict = convolutional_neural_network(images) # uncomment for LeNet5
 
 cost = paddle.layer.classification_cost(input=predict, label=label)
 ```
 
-然后，指定训练相关的参数。
-- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器 `Momentum` ，其中参数0.9代表动量优化每次保持前一次速度的0.9倍。
-- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
-- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+Now, it is time to specify training parameters. In the following `Momentum` optimizer, `momentum=0.9` means that 90% of the current momentum comes from that of the previous iteration. The learning rate relates to the speed at which the network training converges. Regularization is meant to prevent over-fitting; here we use the L2 regularization.
 
 ```python
 parameters = paddle.parameters.create(cost)
@@ -272,11 +263,44 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=optimizer)
 ```
 
-下一步，我们开始训练过程。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python yield generator。
+Then we specify the training data `paddle.dataset.movielens.train()` and testing data `paddle.dataset.movielens.test()`. These two methods are *reader creators*. Once called, a reader creator returns a *reader*.  A reader is a Python method, which, once called, returns a Python generator, which yields instances of data.
+
+`shuffle` is a reader decorator. It takes in a reader A as input and returns a new reader B. Under the hood, B calls A to read data in the following fashion: it copies in `buffer_size` instances at a time into a buffer, shuffles the data, and yields the shuffled instances one at a time. A large buffer size would yield very shuffled data.
 
-下面`shuffle`是一个reader decorator，它接受一个reader A，返回另一个reader B —— reader B 每次读入`buffer_size`条训练数据到一个buffer里，然后随机打乱其顺序，并且逐条输出。
+`batch` is a special decorator, which takes in reader and outputs a *batch reader*, which doesn't yield an instance, but a minibatch at a time.
 
-`batch`是一个特殊的decorator，它的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minibatch。
+`event_handler_plot` is used to plot a figure like below：
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+# event_handler to plot a figure
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 100 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=paddle.batch(
+            paddle.dataset.mnist.test(), batch_size=128))
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` is used to plot some text data when training.
 
 ```python
 lists = []
@@ -287,23 +311,29 @@ def event_handler(event):
             print "Pass %d, Batch %d, Cost %f, %s" % (
                 event.pass_id, event.batch_id, event.cost, event.metrics)
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(reader=paddle.batch(
             paddle.dataset.mnist.test(), batch_size=128))
         print "Test with Pass %d, Cost %f, %s\n" % (
             event.pass_id, result.cost, result.metrics)
         lists.append((event.pass_id, result.cost,
                       result.metrics['classification_error_evaluator']))
+```
 
+```python
 trainer.train(
     reader=paddle.batch(
         paddle.reader.shuffle(
             paddle.dataset.mnist.train(), buf_size=8192),
         batch_size=128),
-    event_handler=event_handler,
-    num_passes=100)
+    event_handler=event_handler_plot,
+    num_passes=5)
 ```
 
-训练过程是完全自动的，event_handler里打印的日志类似如下所示：
+During training, `trainer.train` invokes `event_handler` for certain events. This gives us a chance to print the training progress.
 
 ```
 # Pass 0, Batch 0, Cost 2.780790, {'classification_error_evaluator': 0.9453125}
@@ -314,13 +344,53 @@ trainer.train(
 # Test with Pass 0, Cost 0.326659, {'classification_error_evaluator': 0.09470000118017197}
 ```
 
-训练之后，检查模型的预测准确度。用 MNIST 训练的时候，一般 softmax回归模型的分类准确率为约为 92.34%，多层感知器为97.66%，卷积神经网络可以达到 99.20%。
+After the training, we can check the model's prediction accuracy.
+
+```
+# find the best pass
+best = sorted(lists, key=lambda list: float(list[1]))[0]
+print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
+print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
+```
+
+Usually, with MNIST data, the softmax regression model achieves an accuracy around 92.34%, the MLP 97.66%, and the convolution network around 99.20%. Convolution layers have been widely considered a great invention for image processing.
+
+## Application
+
+After training is done, user can use the trained model to classify images. The following code shows how to inference MNIST images through `paddle.infer` interface.
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file).convert('L')
+    im = im.resize((28, 28), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32).flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/infer_3.png'),))
+
+probs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/infer_3.png is: %d" % lab[0][0]
+```
+
+
+## Conclusion
+
+This tutorial describes a few common deep learning models using **Softmax regression**, **Multilayer Perceptron Network**, and **Convolutional Neural Network**. Understanding these models is crucial for future learning; the subsequent tutorials derive more sophisticated networks by building on top of them.
+
+When our model evolves from a simple softmax regression to a slightly complex Convolutional Neural Network, the recognition accuracy on the MNIST data set achieves a large improvement in accuracy. This is due to the Convolutional layers' local connections and parameter sharing. While learning new models in the future, we encourage the readers to understand the key ideas that lead a new model to improve the results of an old one.
 
-## 总结
+Moreover, this tutorial introduces the basic flow of PaddlePaddle model design, which starts with a *dataprovider*, a model layer construction, and finally training and prediction. Motivated readers can leverage the flow used in this MNIST handwritten digit classification example and experiment with different data and network architectures to train models for classification tasks of their choice.
 
-本教程的softmax回归、多层感知器和卷积神经网络是最基础的深度学习模型，后续章节中复杂的神经网络都是从它们衍生出来的，因此这几个模型对之后的学习大有裨益。同时，我们也观察到从最简单的softmax回归变换到稍复杂的卷积神经网络的时候，MNIST数据集上的识别准确率有了大幅度的提升，原因是卷积层具有局部连接和共享权重的特性。在之后学习新模型的时候，希望大家也要深入到新模型相比原模型带来效果提升的关键之处。此外，本教程还介绍了PaddlePaddle模型搭建的基本流程，从dataprovider的编写、网络层的构建，到最后的训练和预测。对这个流程熟悉以后，大家就可以用自己的数据，定义自己的网络模型，并完成自己的训练和预测任务了。
 
-## 参考文献
+## References
 
 1. LeCun, Yann, Léon Bottou, Yoshua Bengio, and Patrick Haffner. ["Gradient-based learning applied to document recognition."](http://ieeexplore.ieee.org/abstract/document/726791/) Proceedings of the IEEE 86, no. 11 (1998): 2278-2324.
 2. Wejéus, Samuel. ["A Neural Network Approach to Arbitrary SymbolRecognition on Modern Smartphones."](http://www.diva-portal.org/smash/record.jsf?pid=diva2%3A753279&dswid=-434) (2014).
@@ -334,7 +404,7 @@ trainer.train(
 10. Bishop, Christopher M. ["Pattern recognition."](http://users.isr.ist.utl.pt/~wurmd/Livros/school/Bishop%20-%20Pattern%20Recognition%20And%20Machine%20Learning%20-%20Springer%20%202006.pdf) Machine Learning 128 (2006): 1-58.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

02.recognize_digits/train.py

+import os
+from PIL import Image
+import numpy as np
 import paddle.v2 as paddle
 
 
@@ -29,7 +32,7 @@ def convolutional_neural_network(img):
         num_channel=1,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
+        act=paddle.activation.Relu())
     # second conv layer
     conv_pool_2 = paddle.networks.simple_img_conv_pool(
         input=conv_pool_1,
@@ -38,67 +41,87 @@ def convolutional_neural_network(img):
         num_channel=20,
         pool_size=2,
         pool_stride=2,
-        act=paddle.activation.Tanh())
-    # The first fully-connected layer
-    fc1 = paddle.layer.fc(
-        input=conv_pool_2, size=128, act=paddle.activation.Tanh())
-    # The softmax layer, note that the hidden size should be 10,
-    # which is the number of unique digits
+        act=paddle.activation.Relu())
+    # fully-connected layer
     predict = paddle.layer.fc(
-        input=fc1, size=10, act=paddle.activation.Softmax())
+        input=conv_pool_2, size=10, act=paddle.activation.Softmax())
     return predict
 
 
-paddle.init(use_gpu=False, trainer_count=1)
+def main():
+    paddle.init(use_gpu=False, trainer_count=1)
 
-# define network topology
-images = paddle.layer.data(
+    # define network topology
+    images = paddle.layer.data(
         name='pixel', type=paddle.data_type.dense_vector(784))
-label = paddle.layer.data(name='label', type=paddle.data_type.integer_value(10))
+    label = paddle.layer.data(
+        name='label', type=paddle.data_type.integer_value(10))
 
-# Here we can build the prediction network in different ways. Please
-# choose one by uncomment corresponding line.
-predict = softmax_regression(images)
-#predict = multilayer_perceptron(images)
-#predict = convolutional_neural_network(images)
+    # Here we can build the prediction network in different ways. Please
+    # choose one by uncomment corresponding line.
+    # predict = softmax_regression(images)
+    # predict = multilayer_perceptron(images)
+    predict = convolutional_neural_network(images)
 
-cost = paddle.layer.classification_cost(input=predict, label=label)
+    cost = paddle.layer.classification_cost(input=predict, label=label)
 
-parameters = paddle.parameters.create(cost)
+    parameters = paddle.parameters.create(cost)
 
-optimizer = paddle.optimizer.Momentum(
+    optimizer = paddle.optimizer.Momentum(
         learning_rate=0.1 / 128.0,
         momentum=0.9,
         regularization=paddle.optimizer.L2Regularization(rate=0.0005 * 128))
 
-trainer = paddle.trainer.SGD(
+    trainer = paddle.trainer.SGD(
         cost=cost, parameters=parameters, update_equation=optimizer)
 
-lists = []
-
+    lists = []
 
-def event_handler(event):
+    def event_handler(event):
         if isinstance(event, paddle.event.EndIteration):
             if event.batch_id % 100 == 0:
                 print "Pass %d, Batch %d, Cost %f, %s" % (
                     event.pass_id, event.batch_id, event.cost, event.metrics)
         if isinstance(event, paddle.event.EndPass):
+            # save parameters
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
+
             result = trainer.test(reader=paddle.batch(
                 paddle.dataset.mnist.test(), batch_size=128))
-        print "Test with Pass %d, Cost %f, %s\n" % (event.pass_id, result.cost,
-                                                    result.metrics)
+            print "Test with Pass %d, Cost %f, %s\n" % (
+                event.pass_id, result.cost, result.metrics)
             lists.append((event.pass_id, result.cost,
                           result.metrics['classification_error_evaluator']))
 
-
-trainer.train(
+    trainer.train(
         reader=paddle.batch(
             paddle.reader.shuffle(paddle.dataset.mnist.train(), buf_size=8192),
             batch_size=128),
         event_handler=event_handler,
-    num_passes=100)
+        num_passes=5)
+
+    # find the best pass
+    best = sorted(lists, key=lambda list: float(list[1]))[0]
+    print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
+    print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
+
+    def load_image(file):
+        im = Image.open(file).convert('L')
+        im = im.resize((28, 28), Image.ANTIALIAS)
+        im = np.array(im).astype(np.float32).flatten()
+        im = im / 255.0
+        return im
+
+    test_data = []
+    cur_dir = os.path.dirname(os.path.realpath(__file__))
+    test_data.append((load_image(cur_dir + '/image/infer_3.png'), ))
+
+    probs = paddle.infer(
+        output_layer=predict, parameters=parameters, input=test_data)
+    lab = np.argsort(-probs)  # probs and lab are the results of one batch data
+    print "Label of image/infer_3.png is: %d" % lab[0][0]
+
 
-# find the best pass
-best = sorted(lists, key=lambda list: float(list[1]))[0]
-print 'Best pass is %s, testing Avgcost is %s' % (best[0], best[1])
-print 'The classification accuracy is %.2f%%' % (100 - float(best[2]) * 100)
+if __name__ == '__main__':
+    main()

03.image_classification/README.en.md → 03.image_classification/README.cn.md

-Image Classification
-=======================
+# 图像分类
 
-The source code for this chapter is at [book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification). First-time users, please refer to PaddlePaddle [Installation Tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst) for installation instructions.
+本教程源代码目录在[book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/168.html)。
 
-## Background
+## 背景介绍
 
-Compared to words, images provide much more vivid and easier to understand information with an artistic sense. They are an important source for people to express and exchange ideas. In this chapter, we focus on one of the essential problems in image recognition -- image classification.
+图像相比文字能够提供更加生动、容易理解及更具艺术感的信息，是人们转递与交换信息的重要来源。在本教程中，我们专注于图像识别领域的一个重要问题，即图像分类。
 
-Image classification is the task of distinguishing images in different categories based on their semantic meaning. It is a core problem in computer vision and is also the foundation of other higher level computer vision tasks such as object detection, image segmentation, object tracking, action recognition, etc. Image classification has applications in many areas such as face recognition, intelligent video analysis in security systems, traffic scene recognition in transportation systems, content-based image retrieval and automatic photo indexing in web services, image classification in medicine, etc.
+图像分类是根据图像的语义信息将不同类别图像区分开来，是计算机视觉中重要的基本问题，也是图像检测、图像分割、物体跟踪、行为分析等其他高层视觉任务的基础。图像分类在很多领域有广泛应用，包括安防领域的人脸识别和智能视频分析等，交通领域的交通场景识别，互联网领域基于内容的图像检索和相册自动归类，医学领域的图像识别等。
 
-To classify an image we first encode the entire image using handcrafted or learned features and then determine the category using a classifier. Thus, feature extraction plays an important role in image classification. Prior to deep learning the BoW(Bag of Words) model was the most widely used method for classifying an image as well as an object. The BoW technique was introduced in Natural Language Processing where a training sentence is represented as a bag of words. In the context of image classification, the BoW model requires constructing a dictionary. The simplest BoW framework can be designed with three steps: **feature extraction**, **feature encoding** and **classifier design**.
 
-Using Deep learning, image classification can be framed as a supervised or unsupervised learning problem that uses hierarchical features automatically without any need for manually crafted features from the image. In recent years, Convolutional Neural Networks (CNNs) have made significant progress in image classification. CNNs use raw image pixels as input, extract low-level and high-level abstract features through convolution operations, and directly output the classification results from the model. This style of end-to-end learning has lead to not only increased performance but also wider adoption various applications.
+一般来说，图像分类通过手工特征或特征学习方法对整个图像进行全部描述，然后使用分类器判别物体类别，因此如何提取图像的特征至关重要。在深度学习算法之前使用较多的是基于词袋(Bag of Words)模型的物体分类方法。词袋方法从自然语言处理中引入，即一句话可以用一个装了词的袋子表示其特征，袋子中的词为句子中的单词、短语或字。对于图像而言，词袋方法需要构建字典。最简单的词袋模型框架可以设计为**底层特征抽取**、**特征编码**、**分类器设计**三个过程。
 
-In this chapter, we introduce deep-learning-based image classification methods and explain how to train a CNN model using PaddlePaddle.
+而基于深度学习的图像分类方法，可以通过有监督或无监督的方式**学习**层次化的特征描述，从而取代了手工设计或选择图像特征的工作。深度学习模型中的卷积神经网络(Convolution Neural Network, CNN)近年来在图像领域取得了惊人的成绩，CNN直接利用图像像素信息作为输入，最大程度上保留了输入图像的所有信息，通过卷积操作进行特征的提取和高层抽象，模型输出直接是图像识别的结果。这种基于"输入-输出"直接端到端的学习方法取得了非常好的效果，得到了广泛的应用。
 
-## Demonstration
+本教程主要介绍图像分类的深度学习模型，以及如何使用PaddlePaddle训练CNN模型。
 
-An image can be classified by a general as well as fine-grained image classifier.
+## 效果展示
 
-
-Figure 1 shows the results of a general image classifier -- the trained model can correctly recognize the main objects in the images.
+图像分类包括通用图像分类、细粒度图像分类等。图1展示了通用图像分类效果，即模型可以正确识别图像上的主要物体。
 
 <p align="center">
 <img src="image/dog_cat.png "  width="350" ><br/>
-Figure 1. General image classification
+图1. 通用图像分类展示
 </p>
 
 
-Figure 2 shows the results of a fine-grained image classifier. This task of flower recognition requires correctly recognizing of the flower's categories.
+图2展示了细粒度图像分类-花卉识别的效果，要求模型可以正确识别花的类别。
+
 
 <p align="center">
 <img src="image/flowers.png" width="400" ><br/>
-Figure 2. Fine-grained image classification
+图2. 细粒度图像分类展示
 </p>
 
 
-A good model should recognize objects of different categories correctly. The results of such a model should not vary due to viewpoint variation, illumination conditions, object distortion or occlusion.
-Figure 3 shows some images with various disturbances. A good model should classify these images correctly like humans.
+一个好的模型既要对不同类别识别正确，同时也应该能够对不同视角、光照、背景、变形或部分遮挡的图像正确识别(这里我们统一称作图像扰动)。图3展示了一些图像的扰动，较好的模型会像聪明的人类一样能够正确识别。
 
 <p align="center">
-<img src="image/variations_en.png" width="550" ><br/>
-Figure 3. Disturbed images [22]
+<img src="image/variations.png" width="550" ><br/>
+图3. 扰动图片展示[22]
 </p>
 
-## Model Overview
-
-A large amount of research in image classification is built upon public datasets such as [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/), [ImageNet](http://image-net.org/) etc. Many image classification algorithms are usually evaluated and compared on these datasets. PASCAL VOC is a computer vision competition started in 2005, and ImageNet is a dataset for Large Scale Visual Recognition Challenge (ILSVRC) started in 2010. In this chapter, we introduce some image classification models from the submissions to these competitions.
-
-Before 2012, traditional image classification was accomplished with the three steps described in the background section. A complete model construction usually involves the following stages: low-level feature extraction, feature encoding, spatial constraint or feature clustering, classifier design, model ensemble.
-
-  1). **Low-level feature extraction**: This step extracts large amounts of local features according to fixed strides and scales. Popular local features include Scale-Invariant Feature Transform (SIFT)[1], Histogram of Oriented Gradient(HOG)[2], Local Binary Pattern(LBP)[3], etc. A common practice is to employ multiple feature descriptors in order to avoid missing a lot of information.
-
-  2). **Feature encoding**: Low-level features contain a large amount of redundancy and noise. In order to improve the robustness of features, it is necessary to employ a feature transformation to encode low-level features. This is called feature encoding. Common feature encoding methods include vector quantization [4], sparse coding [5], locality-constrained linear coding [6], Fisher vector encoding [7], etc.
+## 模型概览
 
-  3). **Spatial constraint**: Spatial constraint or feature clustering is usually adopted after feature encoding for extracting the maximum or average of each dimension in the spatial domain. Pyramid feature matching--a popular feature clustering method--divides an image uniformly into patches and performs feature clustering in each patch.
+图像识别领域大量的研究成果都是建立在[PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)、[ImageNet](http://image-net.org/)等公开的数据集上，很多图像识别算法通常在这些数据集上进行测试和比较。PASCAL VOC是2005年发起的一个视觉挑战赛，ImageNet是2010年发起的大规模视觉识别竞赛(ILSVRC)的数据集，在本章中我们基于这些竞赛的一些论文介绍图像分类模型。
 
-  4). **Classification**: In the above steps an image can be described by a vector of fixed dimension. Then a classifier can be used to classify the image into categories. Common classifiers include Support Vector Machine(SVM), random forest etc. Kernel SVM is the most popular classifier and has achieved very good performance in traditional image classification tasks.
+在2012年之前的传统图像分类方法可以用背景描述中提到的三步完成，但通常完整建立图像识别模型一般包括底层特征学习、特征编码、空间约束、分类器设计、模型融合等几个阶段。
+  1). **底层特征提取**: 通常从图像中按照固定步长、尺度提取大量局部特征描述。常用的局部特征包括SIFT(Scale-Invariant Feature Transform, 尺度不变特征转换) \[[1](#参考文献)\]、HOG(Histogram of Oriented Gradient, 方向梯度直方图) \[[2](#参考文献)\]、LBP(Local Bianray Pattern, 局部二值模式) \[[3](#参考文献)\] 等，一般也采用多种特征描述子，防止丢失过多的有用信息。
+  2). **特征编码**: 底层特征中包含了大量冗余与噪声，为了提高特征表达的鲁棒性，需要使用一种特征变换算法对底层特征进行编码，称作特征编码。常用的特征编码包括向量量化编码 \[[4](#参考文献)\]、稀疏编码 \[[5](#参考文献)\]、局部线性约束编码 \[[6](#参考文献)\]、Fisher向量编码 \[[7](#参考文献)\] 等。
+  3). **空间特征约束**: 特征编码之后一般会经过空间特征约束，也称作**特征汇聚**。特征汇聚是指在一个空间范围内，对每一维特征取最大值或者平均值，可以获得一定特征不变形的特征表达。金字塔特征匹配是一种常用的特征聚会方法，这种方法提出将图像均匀分块，在分块内做特征汇聚。
+  4). **通过分类器分类**: 经过前面步骤之后一张图像可以用一个固定维度的向量进行描述，接下来就是经过分类器对图像进行分类。通常使用的分类器包括SVM(Support Vector Machine, 支持向量机)、随机森林等。而使用核方法的SVM是最为广泛的分类器，在传统图像分类任务上性能很好。
 
-This method has been used widely as image classification algorithm in PASCAL VOC [18]. NEC Labs(http://www.nec-labs.com/) won the championship by employing SIFT and LBP features, two non-linear encoders and SVM in ILSVRC 2010 [8].
+这种方法在PASCAL VOC竞赛中的图像分类算法中被广泛使用 \[[18](#参考文献)\]。[NEC实验室](http://www.nec-labs.com/)在ILSVRC2010中采用SIFT和LBP特征，两个非线性编码器以及SVM分类器获得图像分类的冠军 \[[8](#参考文献)\]。
 
-The CNN model--AlexNet proposed by Alex Krizhevsky et al.[9], made a breakthrough in ILSVRC 2012. It dramatically outperformed traditional methods and won the ILSVRC championship in 2012. This was also the first time that a deep learning method was used for large-scale image classification. Since AlexNet, a series of CNN models have been proposed that have advanced the state of the art steadily on Imagenet as shown in Figure 4. With deeper and more sophisticated architectures, Top-5 error rate is getting lower and lower (to around 3.5%). The error rate of human raters on the same Imagenet dataset is 5.1%, which means that the image classification capability of a deep learning model has surpassed human raters.
+Alex Krizhevsky在2012年ILSVRC提出的CNN模型 \[[9](#参考文献)\] 取得了历史性的突破，效果大幅度超越传统方法，获得了ILSVRC2012冠军，该模型被称作AlexNet。这也是首次将深度学习用于大规模图像分类中。从AlexNet之后，涌现了一系列CNN模型，不断地在ImageNet上刷新成绩，如图4展示。随着模型变得越来越深以及精妙的结构设计，Top-5的错误率也越来越低，降到了3.5%附近。而在同样的ImageNet数据集上，人眼的辨识错误率大概在5.1%，也就是目前的深度学习模型的识别能力已经超过了人眼。
 
 <p align="center">
 <img src="image/ilsvrc.png" width="500" ><br/>
-Figure 4. Top-5 error rates on ILSVRC image classification
+图4. ILSVRC图像分类Top-5错误率
 </p>
 
 ### CNN
 
-Traditional CNNs consist of convolutional and fully-connected layers and use the softmax multi-category classifier with the cross-entropy loss function. Figure 5 shows a typical CNN. We first introduce the common components of a CNN.
+传统CNN包含卷积层、全连接层等组件，并采用softmax多类别分类器和多类交叉熵损失函数，一个典型的卷积神经网络如图5所示，我们先介绍用来构造CNN的常见组件。
 
 <p align="center">
-<img src="image/lenet_en.png"><br/>
-Figure 5. A CNN example [20]
+<img src="image/lenet.png"><br/>
+图5. CNN网络示例[20]
 </p>
 
-- convolutional layer: this layer uses the convolution operation to extract (low-level and high-level) features and to discover local correlation and spatial invariance.
+- 卷积层(convolution layer): 执行卷积操作提取底层到高层的特征，发掘出图片局部关联性质和空间不变性质。
+- 池化层(pooling layer): 执行降采样操作。通过取卷积输出特征图中局部区块的最大值(max-pooling)或者均值(avg-pooling)。降采样也是图像处理中常见的一种操作，可以过滤掉一些不重要的高频信息。
+- 全连接层(fully-connected layer，或者fc layer): 输入层到隐藏层的神经元是全部连接的。
+- 非线性变化: 卷积层、全连接层后面一般都会接非线性变化层，例如Sigmoid、Tanh、ReLu等来增强网络的表达能力，在CNN里最常使用的为ReLu激活函数。
+- Dropout \[[10](#参考文献)\] : 在模型训练阶段随机让一些隐层节点权重不工作，提高网络的泛化能力，一定程度上防止过拟合。
 
-- pooling layer: this layer down samples feature maps by extracting local max (max-pooling) or average (avg-pooling) value of each patch in the feature map. Down-sampling is a common operation in image processing and is used to filter out high-frequency information.
+另外，在训练过程中由于每层参数不断更新，会导致下一次输入分布发生变化，这样导致训练过程需要精心设计超参数。如2015年Sergey Ioffe和Christian Szegedy提出了Batch Normalization (BN)算法 \[[14](#参考文献)\] 中，每个batch对网络中的每一层特征都做归一化，使得每层分布相对稳定。BN算法不仅起到一定的正则作用，而且弱化了一些超参数的设计。经过实验证明，BN算法加速了模型收敛过程，在后来较深的模型中被广泛使用。
 
-- fully-connected layer: this layer fully connects neurons between two adjacent layers.
-
-- non-linear activation: Convolutional and fully-connected layers are usually followed by some non-linear activation layers. Non-linearities enhance the expression capability of the network. Some examples of non-linear activation functions are Sigmoid, Tanh and ReLU. ReLU is the most commonly used activation function in CNN.
-
-- Dropout [10]: At each training stage, individual nodes are dropped out of the network with a certain probability. This improves the network's ability to generalize and avoids overfitting.
-
-Parameter updates at each layer during training causes input layer distributions to change and in turn requires hyper-parameters to be careful tuned. In 2015, Sergey Ioffe and Christian Szegedy proposed a Batch Normalization (BN) algorithm [14], which normalizes the features of each batch in a layer, and enables relatively stable distribution in each layer. Not only does BN algorithm act as a regularizer, but also reduces the need for careful hyper-parameter design. Experiments demonstrate that BN algorithm accelerates the training convergence and has been widely used in later deeper models.
-
-In the following sections, we will introduce the following network architectures - VGG, GoogleNet and ResNets.
+接下来我们主要介绍VGG，GoogleNet和ResNet网络结构。
 
 ### VGG
 
-The Oxford Visual Geometry Group (VGG) proposed the VGG network in ILSVRC 2014 [11]. This model is deeper and wider than previous neural architectures. It consists of five main groups of convolution operations. Adjacent convolution groups are connected via max-pooling layers. Each group contains a series of 3x3 convolutional layers (i.e. kernels). The number of convolution kernels stays the same within the group and increases from 64 in the first group to 512 in the last one. The total number of learnable layers could be 11, 13, 16, or 19 depending on the number of convolutional layers in each group. Figure 6 illustrates a 16-layer VGG. The neural architecture of VGG is relatively simple and has been adopted by many papers such as the first one that surpassed human-level performance on ImageNet [19].
+牛津大学VGG(Visual Geometry Group)组在2014年ILSVRC提出的模型被称作VGG模型 \[[11](#参考文献)\] 。该模型相比以往模型进一步加宽和加深了网络结构，它的核心是五组卷积操作，每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积，卷积核的数目由较浅组的64增多到最深组的512，同一组内的卷积核数目是一样的。卷积之后接两层全连接层，之后是分类层。由于每组内卷积层的不同，有11、13、16、19层这几种模型，下图展示一个16层的网络结构。VGG模型结构相对简洁，提出之后也有很多文章基于此模型进行研究，如在ImageNet上首次公开超过人眼识别的模型\[[19](#参考文献)\]就是借鉴VGG模型的结构。
 
 <p align="center">
 <img src="image/vgg16.png" width="750" ><br/>
-Figure 6. VGG16 model for ImageNet
+图6. 基于ImageNet的VGG16模型
 </p>
 
 ### GoogleNet
 
-GoogleNet [12] won the ILSVRC championship in 2014. GoogleNet borrowed some ideas from the Network in Network(NIN) model [13] and is built on the Inception blocks. Let us first familiarize ourselves with these first.
-
-The two main characteristics of the NIN model are:
+GoogleNet \[[12](#参考文献)\] 在2014年ILSVRC的获得了冠军，在介绍该模型之前我们先来了解NIN(Network in Network)模型 \[[13](#参考文献)\] 和Inception模块，因为GoogleNet模型由多组Inception模块组成，模型设计借鉴了NIN的一些思想。
 
-1) A single-layer convolutional network is replaced with a Multi-Layer Perceptron Convolution (MLPconv). MLPconv is a tiny multi-layer convolutional network. It enhances non-linearity by adding several 1x1 convolutional layers after linear ones.
+NIN模型主要有两个特点：1) 引入了多层感知卷积网络(Multi-Layer Perceptron Convolution, MLPconv)代替一层线性卷积网络。MLPconv是一个微小的多层卷积网络，即在线性卷积后面增加若干层1x1的卷积，这样可以提取出高度非线性特征。2) 传统的CNN最后几层一般都是全连接层，参数较多。而NIN模型设计最后一层卷积层包含类别维度大小的特征图，然后采用全局均值池化(Avg-Pooling)替代全连接层，得到类别维度大小的向量，再进行分类。这种替代全连接层的方式有利于减少参数。
 
-2) In traditional CNNs, the last fewer layers are usually fully-connected with a large number of parameters. In contrast, NIN replaces all fully-connected layers with convolutional layers with feature maps of the same size as the category dimension and a global average pooling. This replacement of fully-connected layers significantly reduces the number of parameters.
-
-Figure 7 depicts two Inception blocks. Figure 7(a) is the simplest design. The output is a concatenation of features from three convolutional layers and one pooling layer. The disadvantage of this design is that the pooling layer does not change the number of filters and leads to an increase in the number of outputs. After several of such blocks, the number of outputs and parameters become larger and larger and lead to higher computation complexity. To overcome this drawback, the Inception block in Figure 7(b) employs three 1x1 convolutional layers. These reduce dimensions or the number of channels but improve the non-linearity of the network.
+Inception模块如下图7所示，图(a)是最简单的设计，输出是3个卷积层和一个池化层的特征拼接。这种设计的缺点是池化层不会改变特征通道数，拼接后会导致特征的通道数较大，经过几层这样的模块堆积后，通道数会越来越大，导致参数和计算量也随之增大。为了改善这个缺点，图(b)引入3个1x1卷积层进行降维，所谓的降维就是减少通道数，同时如NIN模型中提到的1x1卷积也可以修正线性特征。
 
 <p align="center">
-<img src="image/inception_en.png" width="800" ><br/>
-Figure 7. Inception block
+<img src="image/inception.png" width="800" ><br/>
+图7. Inception模块
 </p>
 
-GoogleNet consists of multiple stacked Inception blocks followed by an avg-pooling layer as in NIN instead of traditional fully connected layers. The difference between GoogleNet and NIN is that GoogleNet adds a fully connected layer after avg-pooling layer to output a vector of category size. Besides these two characteristics, the features from middle layers of a GoogleNet are also very discriminative. Therefore, GoogeleNet inserts two auxiliary classifiers in the model for enhancing gradient and regularization when doing backpropagation. The loss function of the whole network is the weighted sum of these three classifiers.
+GoogleNet由多组Inception模块堆积而成。另外，在网络最后也没有采用传统的多层全连接层，而是像NIN网络一样采用了均值池化层；但与NIN不同的是，池化层后面接了一层到类别数映射的全连接层。除了这两个特点之外，由于网络中间层特征也很有判别性，GoogleNet在中间层添加了两个辅助分类器，在后向传播中增强梯度并且增强正则化，而整个网络的损失函数是这个三个分类器的损失加权求和。
 
-Figure 8 illustrates the neural architecture of a GoogleNet which consists of 22 layers: it starts with three regular convolutional layers followed by three groups of sub-networks -- the first group contains two Inception blocks, the second one five, and the third one two. It ends up with an average pooling and a fully-connected layer.
+GoogleNet整体网络结构如图8所示，总共22层网络：开始由3层普通的卷积组成；接下来由三组子网络组成，第一组子网络包含2个Inception模块，第二组包含5个Inception模块，第三组包含2个Inception模块；然后接均值池化层、全连接层。
 
 <p align="center">
 <img src="image/googlenet.jpeg" ><br/>
-Figure 8. GoogleNet[12]
+图8. GoogleNet[12]
 </p>
 
-The above model is the first version of GoogleNet or GoogelNet-v1. GoogleNet-v2 [14] introduced BN layer; GoogleNet-v3 [16] further split some convolutional layers, which increases non-linearity and network depth; GoogelNet-v4 [17] leads to the design idea of ResNet which will be introduced in the next section. The evolution from v1 to v4 improved the accuracy rate consistently. We will not go into details of the neural architectures of v2 to v4.
+
+上面介绍的是GoogleNet第一版模型(称作GoogleNet-v1)。GoogleNet-v2 \[[14](#参考文献)\] 引入BN层；GoogleNet-v3 \[[16](#参考文献)\] 对一些卷积层做了分解，进一步提高网络非线性能力和加深网络；GoogleNet-v4 \[[17](#参考文献)\] 引入下面要讲的ResNet设计思路。从v1到v4每一版的改进都会带来准确度的提升，介于篇幅，这里不再详细介绍v2到v4的结构。
+
 
 ### ResNet
 
-Residual Network(ResNet)[15] won the 2015 championship on three ImageNet competitions -- image classification, object localization, and object detection. The main challenge in training deeper networks is that accuracy degrades with network depth. The authors of ResNet proposed a residual learning approach to ease the difficulty of training deeper networks. Based on the design ideas of BN, small convolutional kernels, full convolutional network, ResNets reformulate the layers as residual blocks, with each block containing two branches, one directly connecting input to the output, the other performing two to three convolutions and calculating the residual function with reference to the layer inputs. The outputs of these two branches are then added up.
+ResNet(Residual Network) \[[15](#参考文献)\] 是2015年ImageNet图像分类、图像物体定位和图像物体检测比赛的冠军。针对训练卷积神经网络时加深网络导致准确度下降的问题，ResNet提出了采用残差学习。在已有设计思路(BN, 小卷积核，全卷积网络)的基础上，引入了残差模块。每个残差模块包含两条路径，其中一条路径是输入特征的直连通路，另一条路径对该特征做两到三次卷积操作得到该特征的残差，最后再将两条路径上的特征相加。
 
-Figure 9 illustrates the ResNet architecture. To the left is the basic building block, it consists of two 3x3 convolutional layers of the same channels. To the right is a Bottleneck block. The bottleneck is a 1x1 convolutional layer used to reduce dimension from 256 to 64. The other 1x1 convolutional layer is used to increase dimension from 64 to 256. Thus, the number of input and output channels of the middle 3x3 convolutional layer is 64, which is relatively small.
+残差模块如图9所示，左边是基本模块连接方式，由两个输出通道数相同的3x3卷积组成。右边是瓶颈模块(Bottleneck)连接方式，之所以称为瓶颈，是因为上面的1x1卷积用来降维(图示例即256->64)，下面的1x1卷积用来升维(图示例即64->256)，这样中间3x3卷积的输入和输出通道数都较小(图示例即64->64)。
 
 <p align="center">
 <img src="image/resnet_block.jpg" width="400"><br/>
-Figure 9. Residual block
+图9. 残差模块
 </p>
 
-Figure 10 illustrates ResNets with 50, 101, 152 layers, respectively. All three networks use bottleneck blocks of different numbers of repetitions. ResNet converges very fast and can be trained with hundreds or thousands of layers.
+图10展示了50、101、152层网络连接示意图，使用的是瓶颈模块。这三个模型的区别在于每组中残差模块的重复次数不同(见图右上角)。ResNet训练收敛较快，成功的训练了上百乃至近千层的卷积神经网络。
 
 <p align="center">
 <img src="image/resnet.png"><br/>
-Figure 10. ResNet model for ImageNet
+图10. 基于ImageNet的ResNet模型
 </p>
 
 
-## Dataset
+## 数据准备
 
-Commonly used public datasets for image classification are [CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html), [ImageNet](http://image-net.org/), [COCO](http://mscoco.org/), etc. Those used for fine-grained image classification are [CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html), [Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/), [Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/), etc. Among these, the ImageNet dataset is the largest. Most research results are reported on ImageNet as mentioned in the Model Overview section. Since 2010, the ImageNet dataset has gone through some changes. The commonly used ImageNet-2012 dataset contains 1000 categories. There are 1,281,167 training images, ranging from 732 to 1200 images per category, and 50,000 validation images with 50 images per category in average.
+通用图像分类公开的标准数据集常用的有[CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html)、[ImageNet](http://image-net.org/)、[COCO](http://mscoco.org/)等，常用的细粒度图像分类数据集包括[CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html)、[Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/)、[Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/)等。其中ImageNet数据集规模相对较大，如[模型概览](#模型概览)一章所讲，大量研究成果基于ImageNet。ImageNet数据从2010年来稍有变化，常用的是ImageNet-2012数据集，该数据集包含1000个类别：训练集包含1,281,167张图片，每个类别数据732至1300张不等，验证集包含50,000张图片，平均每个类别50张图片。
 
-Since ImageNet is too large to be downloaded and trained efficiently, we use [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) in this tutorial. The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. Figure 11 shows all the classes in CIFAR-10 as well as 10 images randomly sampled from each category.
+由于ImageNet数据集较大，下载和训练较慢，为了方便大家学习，我们使用[CIFAR10](<https://www.cs.toronto.edu/~kriz/cifar.html>)数据集。CIFAR10数据集包含60,000张32x32的彩色图片，10个类别，每个类包含6,000张。其中50,000张图片作为训练集，10000张作为测试集。图11从每个类别中随机抽取了10张图片，展示了所有的类别。
 
 <p align="center">
 <img src="image/cifar.png" width="350"><br/>
-Figure 11. CIFAR10 dataset[21]
+图11. CIFAR10数据集[21]
 </p>
 
- `paddle.datasets` package encapsulates multiple public datasets, including `cifar`, `imdb`, `mnist`, `moivelens` and `wmt14`, etc. There's no need to manually download and preprocess CIFAR-10.
+Paddle API提供了自动加载cifar数据集模块 `paddle.dataset.cifar`。
 
-After issuing a command `python train.py`, training will start immediately. The following sections describe the details:
+通过输入`python train.py`，就可以开始训练模型了，以下小节将详细介绍`train.py`的相关内容。
 
-## Model Structure
+### 模型结构
 
-### Initialize PaddlePaddle
+#### Paddle 初始化
 
-We must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
+通过 `paddle.init`，初始化Paddle是否使用GPU，trainer的数目等等。
 
 ```python
 import sys
@@ -177,29 +164,30 @@ from resnet import resnet_cifar10
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-As mentioned in section [Model Overview](#model-overview), here we provide the implementations of the VGG and ResNet models.
+本教程中我们提供了VGG和ResNet两个模型的配置。
 
-### VGG
+#### VGG
 
-First, we use a VGG network. Since the image size and amount of CIFAR10 are relatively small comparing to ImageNet, we use a small version of VGG network for CIFAR10. Convolution groups incorporate BN and dropout operations.
+首先介绍VGG模型结构，由于CIFAR10图片大小和数量相比ImageNet数据小很多，因此这里的模型针对CIFAR10数据做了一定的适配。卷积部分引入了BN和Dropout操作。
 
-1. Define input data and its dimension
+1. 定义数据输入及其维度
 
-    The input to the network is defined as `paddle.layer.data`, or image pixels in the context of image classification. The images in CIFAR10 are 32x32 color images of three channels. Therefore, the size of the input data is 3072 (3x32x32), and the number of categories is 10.
+    网络输入定义为 `data_layer` (数据层)，在图像分类中即为图像像素信息。CIFRAR10是RGB 3通道32x32大小的彩色图，因此输入数据大小为3072(3x32x32)，类别大小为10，即10分类。
 
     ```python
     datadim = 3 * 32 * 32
     classdim = 10
+
     image = paddle.layer.data(
         name="image", type=paddle.data_type.dense_vector(datadim))
     ```
 
-2. Define VGG main module
+2. 定义VGG网络核心模块
 
     ```python
     net = vgg_bn_drop(image)
     ```
-    The input to VGG main module is from the data layer. `vgg_bn_drop` defines a 16-layer VGG network, with each convolutional layer followed by BN and dropout layers. Here is the definition in detail:
+    VGG核心模块的输入是数据层，`vgg_bn_drop` 定义了16层VGG结构，每层卷积后面引入BN层和Dropout层，详细的定义如下：
 
     ```python
     def vgg_bn_drop(input):
@@ -232,15 +220,15 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
         return fc2
     ```
 
-    2.1. First, define a convolution block or conv_block. The default convolution kernel is 3x3, and the default pooling size is 2x2 with stride 2. Dropout specifies the probability in dropout operation. Function `img_conv_group` is defined in `paddle.networks` consisting of a series of `Conv->BN->ReLu->Dropout` and a `Pooling`.
+    2.1. 首先定义了一组卷积网络，即conv_block。卷积核大小为3x3，池化窗口大小为2x2，窗口滑动大小为2，groups决定每组VGG模块是几次连续的卷积操作，dropouts指定Dropout操作的概率。所使用的`img_conv_group`是在`paddle.networks`中预定义的模块，由若干组 Conv->BN->ReLu->Dropout 和 一组 Pooling 组成。
 
-    2.2. Five groups of convolutions. The first two groups perform two convolutions, while the last three groups perform three convolutions. The dropout rate of the last convolution in each group is set to 0, which means there is no dropout for this layer.
+    2.2. 五组卷积操作，即 5个conv_block。 第一、二组采用两次连续的卷积操作。第三、四、五组采用三次连续的卷积操作。每组最后一个卷积后面Dropout概率为0，即不使用Dropout操作。
 
-    2.3. The last two layers are fully-connected layers of dimension 512.
+    2.3. 最后接两层512维的全连接。
 
-3. Define Classifier
+3. 定义分类器
 
-    The above VGG network extracts high-level features and maps them to a vector of the same size as the categories. Softmax function or classifier is then used for calculating the probability of the image belonging to each category.
+    通过上面VGG网络提取高层特征，然后经过全连接层映射到类别维度大小的向量，再通过Softmax归一化得到每个类别的概率，也可称作分类器。
 
     ```python
     out = paddle.layer.fc(input=net,
@@ -248,9 +236,9 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
                           act=paddle.activation.Softmax())
     ```
 
-4. Define Loss Function and Outputs
+4. 定义损失函数和网络输出
 
-    In the context of supervised learning, labels of training images are defined in `paddle.layer.data` as well. During training, the cross-entropy loss function is used and the loss is the output of the network. During testing, the outputs are the probabilities calculated in the classifier.
+    在有监督训练中需要输入图像对应的类别信息，同样通过`paddle.layer.data`来定义。训练中采用多类交叉熵作为损失函数，并作为网络的输出，预测阶段定义网络的输出为分类器得到的概率信息。
 
     ```python
     lbl = paddle.layer.data(
@@ -260,20 +248,19 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
 
 ### ResNet
 
-The first, third and fourth steps of a ResNet are the same as a VGG. The second one is the main module.
+ResNet模型的第1、3、4步和VGG模型相同，这里不再介绍。主要介绍第2步即CIFAR10数据集上ResNet核心模块。
 
 ```python
 net = resnet_cifar10(image, depth=56)
 ```
 
-Here are some basic functions used in `resnet_cifar10`:
-
-  - `conv_bn_layer` : convolutional layer followed by BN.
-  - `shortcut` : the shortcut branch in a residual block. There are two kinds of shortcuts: 1x1 convolution used when the number of channels between input and output is different; direct connection used otherwise.
+先介绍`resnet_cifar10`中的一些基本函数，再介绍网络连接过程。
 
-  - `basicblock` : a basic residual module as shown in the left of Figure 9, it consists of two sequential 3x3 convolutions and one "shortcut" branch.
-  - `bottleneck` : a bottleneck module as shown in the right of Figure 9, it consists of two 1x1 convolutions with one 3x3 convolution in between branch and a "shortcut" branch.
-  - `layer_warp` : a group of residual modules consisting of several stacking blocks. In each group, the sliding window size of the first residual block could be different from the rest of blocks, in order to reduce the size of feature maps along horizontal and vertical directions.
+  - `conv_bn_layer` : 带BN的卷积层。
+  - `shortcut` : 残差模块的"直连"路径，"直连"实际分两种形式：残差模块输入和输出特征通道数不等时，采用1x1卷积的升维操作；残差模块输入和输出通道相等时，采用直连操作。
+  - `basicblock` : 一个基础残差模块，即图9左边所示，由两组3x3卷积组成的路径和一条"直连"路径组成。
+  - `bottleneck` : 一个瓶颈残差模块，即图9右边所示，由上下1x1卷积和中间3x3卷积组成的路径和一条"直连"路径组成。
+  - `layer_warp` : 一组残差模块，由若干个残差模块堆积而成。每组中第一个残差模块滑动窗口大小与其他可以不同，以用来减少特征图在垂直和水平方向的大小。
 
 ```python
 def conv_bn_layer(input,
@@ -315,13 +302,13 @@ def layer_warp(block_func, ipt, features, count, stride):
     return tmp
 ```
 
-The following are the components of `resnet_cifar10`:
+`resnet_cifar10` 的连接结构主要有以下几个过程。
 
-1. The lowest level is `conv_bn_layer`.
-2. The middle level consists of three `layer_warp`, each of which uses the left residual block in Figure 9.
-3. The last level is average pooling layer.
+1. 底层输入连接一层 `conv_bn_layer`，即带BN的卷积层。
+2. 然后连接3组残差模块即下面配置3组 `layer_warp` ，每组采用图 10 左边残差模块组成。
+3. 最后对网络做均值池化并返回该层。
 
-Note: besides the first convolutional layer and the last fully-connected layer, the total number of layers in three `layer_warp` should be dividable by 6, that is the depth of `resnet_cifar10` should satisfy $(depth - 2) % 6 == 0$.
+注意：除过第一层卷积层和最后一层全连接层之外，要求三组 `layer_warp` 总的含参层数能够被6整除，即 `resnet_cifar10` 的 depth 要满足 $(depth - 2) % 6 == 0$ 。
 
 ```python
 def resnet_cifar10(ipt, depth=32):
@@ -339,21 +326,26 @@ def resnet_cifar10(ipt, depth=32):
     return pool
 ```
 
-## Model Training
+## 训练模型
 
-### Define Parameters
+### 定义参数
 
-First, we create the model parameters according to the previous model configuration `cost`.
+首先依据模型配置的`cost`定义模型参数。
 
 ```python
 # Create parameters
 parameters = paddle.parameters.create(cost)
 ```
 
-### Create Trainer
+可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
 
-Before creating a training module, it is necessary to set the algorithm.
-Here we specify `Momentum` optimization algorithm via `paddle.optimizer`.
+```python
+print parameters.keys()
+```
+
+### 构造训练(Trainer)
+
+根据网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的Momentum方法，同时设定了学习率、正则等。
 
 ```python
 # Create optimizer
@@ -363,8 +355,7 @@ momentum_optimizer = paddle.optimizer.Momentum(
     learning_rate=0.1 / 128.0,
     learning_rate_decay_a=0.1,
     learning_rate_decay_b=50000 * 100,
-    learning_rate_schedule='discexp',
-    batch_size=128)
+    learning_rate_schedule='discexp')
 
 # Create trainer
 trainer = paddle.trainer.SGD(cost=cost,
@@ -372,13 +363,14 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=momentum_optimizer)
 ```
 
-The learning rate adjustment policy can be defined with variables `learning_rate_decay_a`($a$), `learning_rate_decay_b`($b$) and `learning_rate_schedule`. In this example, discrete exponential method is used for adjusting learning rate. The formula is as follows,
+通过 `learning_rate_decay_a` (简写$a$） 、`learning_rate_decay_b` (简写$b$) 和 `learning_rate_schedule` 指定学习率调整策略，这里采用离散指数的方式调节学习率，计算公式如下， $n$ 代表已经处理过的累计总样本数，$lr_{0}$ 即为 `settings` 里设置的 `learning_rate`。
+
 $$  lr = lr_{0} * a^ {\lfloor \frac{n}{ b}\rfloor} $$
-where $n$ is the number of processed samples, $lr_{0}$ is the learning_rate.
 
-### Training
 
-`cifar.train10()` will yield records during each pass, after shuffling, a batch input is generated for training.
+### 训练
+
+cifar.train10()每次产生一条样本，在完成shuffle和batch之后，作为训练的输入。
 
 ```python
 reader=paddle.batch(
@@ -387,19 +379,47 @@ reader=paddle.batch(
         batch_size=128)
 ```
 
-`feeding` is devoted to specifying the correspondence between each yield record and `paddle.layer.data`. For instance,
- the first column of data generated by `cifar.train10()` corresponds to image layer's feature.
+通过`feeding`来指定每一个数据和`paddle.layer.data`的对应关系。例如: `cifar.train10()`产生数据的第0列对应image层的特征。
 
 ```python
 feeding={'image': 0,
          'label': 1}
 ```
 
-Callback function `event_handler` will be called during training when a pre-defined event happens.
+可以使用`event_handler`回调函数来观察训练过程，或进行测试等, 该回调函数是`trainer.train`函数里设定。
+
+`event_handler_plot`可以用来利用回调数据来打点画图:
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 1 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+
+        result = trainer.test(
+            reader=paddle.batch(
+                paddle.dataset.cifar.test10(), batch_size=128),
+            feeding=feeding)
+        cost_ploter.append(test_title, step, result.cost)
+```
 
+`event_handler` 用来在训练过程中输出文本日志
 
 ```python
-# event handler to track training and testing process
+# End batch and end pass event handler
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -409,6 +429,10 @@ def event_handler(event):
             sys.stdout.write('.')
             sys.stdout.flush()
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(
             reader=paddle.batch(
                 paddle.dataset.cifar.test10(), batch_size=128),
@@ -416,17 +440,17 @@ def event_handler(event):
         print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
-Finally, we can invoke `trainer.train` to start training:
+通过`trainer.train`函数训练:
 
 ```python
 trainer.train(
     reader=reader,
     num_passes=200,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding)
 ```
 
-Here is an example log after training for one pass. The average error rates are 0.6875 on the training set and 0.8852 on the validation set.
+一轮训练log示例如下所示，经过1个pass， 训练集上平均error为0.6875 ，测试集上平均error为0.8852 。
 
 ```text
 Pass 0, Batch 0, Cost 2.473182, {'classification_error_evaluator': 0.9140625}
@@ -440,21 +464,55 @@ Pass 0, Batch 300, Cost 1.668833, {'classification_error_evaluator': 0.6875}
 Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 ```
 
-Figure 12 shows the curve of training error rate, which indicates it converges at Pass 200 with error rate 8.54%.
+图12是训练的分类错误率曲线图，运行到第200个pass后基本收敛，最终得到测试集上分类错误率为8.54%。
+
 <p align="center">
-<img src="image/plot_en.png" width="400" ><br/>
-Figure 12. The error rate of VGG model on CIFAR10
+<img src="image/plot.png" width="400" ><br/>
+图12. CIFAR10数据集上VGG模型的分类错误率
 </p>
 
+## 应用模型
+
+可以使用训练好的模型对图片进行分类，下面程序展示了如何使用`paddle.infer`接口进行推断，可以打开注释，更改加载的模型。
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file)
+    im = im.resize((32, 32), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32)
+    # PIL打开图片存储顺序为H(高度)，W(宽度)，C(通道)。
+    # PaddlePaddle要求数据顺序为CHW，所以需要转换顺序。
+    im = im.transpose((2, 0, 1)) # CHW
+    # CIFAR训练图片通道顺序为B(蓝),G(绿),R(红),
+    # 而PIL打开图片默认通道顺序为RGB,因为需要交换通道。
+    im = im[(2, 1, 0),:,:] # BGR
+    im = im.flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/dog.png'),))
+
+# with open('params_pass_50.tar', 'r') as f:
+#    parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=out, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/dog.png is: %d" % lab[0][0]
+```
 
-After training is done, the model from each pass is saved in `output/pass-%05d`. For example, the model of Pass 300 is saved in `output/pass-00299`.
 
-## Conclusion
+## 总结
 
-Traditional image classification methods have complicated frameworks that involve multiple stages of processing. In contrast, CNN models can be trained end-to-end with a significant increase in classification accuracy. In this chapter, we introduced three models -- VGG, GoogleNet, ResNet and provided PaddlePaddle config files for training VGG and ResNet on CIFAR10. We also explained how to perform prediction and feature extraction using the PaddlePaddle API. For other datasets such as ImageNet, the procedure for config and training are the same and you are welcome to give it a try.
+传统图像分类方法由多个阶段构成，框架较为复杂，而端到端的CNN模型结构可一步到位，而且大幅度提升了分类准确率。本文我们首先介绍VGG、GoogleNet、ResNet三个经典的模型；然后基于CIFAR10数据集，介绍如何使用PaddlePaddle配置和训练CNN模型，尤其是VGG和ResNet模型；最后介绍如何使用PaddlePaddle的API接口对图片进行预测和特征提取。对于其他数据集比如ImageNet，配置和训练流程是同样的，大家可以自行进行实验。
 
 
-## Reference
+## 参考文献
 
 [1] D. G. Lowe, [Distinctive image features from scale-invariant keypoints](http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf). IJCV, 60(2):91-110, 2004.
 
@@ -501,4 +559,4 @@ Traditional image classification methods have complicated frameworks that involv
 [22] http://cs231n.github.io/classification/
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

03.image_classification/README.md

-# 图像分类
+Image Classification
+=======================
 
-本教程源代码目录在[book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+The source code for this chapter is at [book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification). First-time users, please refer to PaddlePaddle [Installation Tutorial](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book) for installation instructions.
 
-## 背景介绍
+## Background
 
-图像相比文字能够提供更加生动、容易理解及更具艺术感的信息，是人们转递与交换信息的重要来源。在本教程中，我们专注于图像识别领域的一个重要问题，即图像分类。
+Compared to words, images provide much more vivid and easier to understand information with an artistic sense. They are an important source for people to express and exchange ideas. In this chapter, we focus on one of the essential problems in image recognition -- image classification.
 
-图像分类是根据图像的语义信息将不同类别图像区分开来，是计算机视觉中重要的基本问题，也是图像检测、图像分割、物体跟踪、行为分析等其他高层视觉任务的基础。图像分类在很多领域有广泛应用，包括安防领域的人脸识别和智能视频分析等，交通领域的交通场景识别，互联网领域基于内容的图像检索和相册自动归类，医学领域的图像识别等。
+Image classification is the task of distinguishing images in different categories based on their semantic meaning. It is a core problem in computer vision and is also the foundation of other higher level computer vision tasks such as object detection, image segmentation, object tracking, action recognition, etc. Image classification has applications in many areas such as face recognition, intelligent video analysis in security systems, traffic scene recognition in transportation systems, content-based image retrieval and automatic photo indexing in web services, image classification in medicine, etc.
 
+To classify an image we first encode the entire image using handcrafted or learned features and then determine the category using a classifier. Thus, feature extraction plays an important role in image classification. Prior to deep learning the BoW(Bag of Words) model was the most widely used method for classifying an image as well as an object. The BoW technique was introduced in Natural Language Processing where a training sentence is represented as a bag of words. In the context of image classification, the BoW model requires constructing a dictionary. The simplest BoW framework can be designed with three steps: **feature extraction**, **feature encoding** and **classifier design**.
 
-一般来说，图像分类通过手工特征或特征学习方法对整个图像进行全部描述，然后使用分类器判别物体类别，因此如何提取图像的特征至关重要。在深度学习算法之前使用较多的是基于词袋(Bag of Words)模型的物体分类方法。词袋方法从自然语言处理中引入，即一句话可以用一个装了词的袋子表示其特征，袋子中的词为句子中的单词、短语或字。对于图像而言，词袋方法需要构建字典。最简单的词袋模型框架可以设计为**底层特征抽取**、**特征编码**、**分类器设计**三个过程。
+Using Deep learning, image classification can be framed as a supervised or unsupervised learning problem that uses hierarchical features automatically without any need for manually crafted features from the image. In recent years, Convolutional Neural Networks (CNNs) have made significant progress in image classification. CNNs use raw image pixels as input, extract low-level and high-level abstract features through convolution operations, and directly output the classification results from the model. This style of end-to-end learning has lead to not only increased performance but also wider adoption various applications.
 
-而基于深度学习的图像分类方法，可以通过有监督或无监督的方式**学习**层次化的特征描述，从而取代了手工设计或选择图像特征的工作。深度学习模型中的卷积神经网络(Convolution Neural Network, CNN)近年来在图像领域取得了惊人的成绩，CNN直接利用图像像素信息作为输入，最大程度上保留了输入图像的所有信息，通过卷积操作进行特征的提取和高层抽象，模型输出直接是图像识别的结果。这种基于"输入-输出"直接端到端的学习方法取得了非常好的效果，得到了广泛的应用。
+In this chapter, we introduce deep-learning-based image classification methods and explain how to train a CNN model using PaddlePaddle.
 
-本教程主要介绍图像分类的深度学习模型，以及如何使用PaddlePaddle训练CNN模型。
+## Demonstration
 
-## 效果展示
+An image can be classified by a general as well as fine-grained image classifier.
 
-图像分类包括通用图像分类、细粒度图像分类等。图1展示了通用图像分类效果，即模型可以正确识别图像上的主要物体。
+
+Figure 1 shows the results of a general image classifier -- the trained model can correctly recognize the main objects in the images.
 
 <p align="center">
 <img src="image/dog_cat.png "  width="350" ><br/>
-图1. 通用图像分类展示
+Figure 1. General image classification
 </p>
 
 
-图2展示了细粒度图像分类-花卉识别的效果，要求模型可以正确识别花的类别。
-
+Figure 2 shows the results of a fine-grained image classifier. This task of flower recognition requires correctly recognizing of the flower's categories.
 
 <p align="center">
 <img src="image/flowers.png" width="400" ><br/>
-图2. 细粒度图像分类展示
+Figure 2. Fine-grained image classification
 </p>
 
 
-一个好的模型既要对不同类别识别正确，同时也应该能够对不同视角、光照、背景、变形或部分遮挡的图像正确识别(这里我们统一称作图像扰动)。图3展示了一些图像的扰动，较好的模型会像聪明的人类一样能够正确识别。
+A good model should recognize objects of different categories correctly. The results of such a model should not vary due to viewpoint variation, illumination conditions, object distortion or occlusion.
+Figure 3 shows some images with various disturbances. A good model should classify these images correctly like humans.
 
 <p align="center">
-<img src="image/variations.png" width="550" ><br/>
-图3. 扰动图片展示[22]
+<img src="image/variations_en.png" width="550" ><br/>
+Figure 3. Disturbed images [22]
 </p>
 
-## 模型概览
+## Model Overview
+
+A large amount of research in image classification is built upon public datasets such as [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/), [ImageNet](http://image-net.org/) etc. Many image classification algorithms are usually evaluated and compared on these datasets. PASCAL VOC is a computer vision competition started in 2005, and ImageNet is a dataset for Large Scale Visual Recognition Challenge (ILSVRC) started in 2010. In this chapter, we introduce some image classification models from the submissions to these competitions.
+
+Before 2012, traditional image classification was accomplished with the three steps described in the background section. A complete model construction usually involves the following stages: low-level feature extraction, feature encoding, spatial constraint or feature clustering, classifier design, model ensemble.
+
+  1). **Low-level feature extraction**: This step extracts large amounts of local features according to fixed strides and scales. Popular local features include Scale-Invariant Feature Transform (SIFT)[1], Histogram of Oriented Gradient(HOG)[2], Local Binary Pattern(LBP)[3], etc. A common practice is to employ multiple feature descriptors in order to avoid missing a lot of information.
+
+  2). **Feature encoding**: Low-level features contain a large amount of redundancy and noise. In order to improve the robustness of features, it is necessary to employ a feature transformation to encode low-level features. This is called feature encoding. Common feature encoding methods include vector quantization [4], sparse coding [5], locality-constrained linear coding [6], Fisher vector encoding [7], etc.
 
-图像识别领域大量的研究成果都是建立在[PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)、[ImageNet](http://image-net.org/)等公开的数据集上，很多图像识别算法通常在这些数据集上进行测试和比较。PASCAL VOC是2005年发起的一个视觉挑战赛，ImageNet是2010年发起的大规模视觉识别竞赛(ILSVRC)的数据集，在本章中我们基于这些竞赛的一些论文介绍图像分类模型。
+  3). **Spatial constraint**: Spatial constraint or feature clustering is usually adopted after feature encoding for extracting the maximum or average of each dimension in the spatial domain. Pyramid feature matching--a popular feature clustering method--divides an image uniformly into patches and performs feature clustering in each patch.
 
-在2012年之前的传统图像分类方法可以用背景描述中提到的三步完成，但通常完整建立图像识别模型一般包括底层特征学习、特征编码、空间约束、分类器设计、模型融合等几个阶段。
-  1). **底层特征提取**: 通常从图像中按照固定步长、尺度提取大量局部特征描述。常用的局部特征包括SIFT(Scale-Invariant Feature Transform, 尺度不变特征转换) \[[1](#参考文献)\]、HOG(Histogram of Oriented Gradient, 方向梯度直方图) \[[2](#参考文献)\]、LBP(Local Bianray Pattern, 局部二值模式) \[[3](#参考文献)\] 等，一般也采用多种特征描述子，防止丢失过多的有用信息。
-  2). **特征编码**: 底层特征中包含了大量冗余与噪声，为了提高特征表达的鲁棒性，需要使用一种特征变换算法对底层特征进行编码，称作特征编码。常用的特征编码包括向量量化编码 \[[4](#参考文献)\]、稀疏编码 \[[5](#参考文献)\]、局部线性约束编码 \[[6](#参考文献)\]、Fisher向量编码 \[[7](#参考文献)\] 等。
-  3). **空间特征约束**: 特征编码之后一般会经过空间特征约束，也称作**特征汇聚**。特征汇聚是指在一个空间范围内，对每一维特征取最大值或者平均值，可以获得一定特征不变形的特征表达。金字塔特征匹配是一种常用的特征聚会方法，这种方法提出将图像均匀分块，在分块内做特征汇聚。
-  4). **通过分类器分类**: 经过前面步骤之后一张图像可以用一个固定维度的向量进行描述，接下来就是经过分类器对图像进行分类。通常使用的分类器包括SVM(Support Vector Machine, 支持向量机)、随机森林等。而使用核方法的SVM是最为广泛的分类器，在传统图像分类任务上性能很好。
+  4). **Classification**: In the above steps an image can be described by a vector of fixed dimension. Then a classifier can be used to classify the image into categories. Common classifiers include Support Vector Machine(SVM), random forest etc. Kernel SVM is the most popular classifier and has achieved very good performance in traditional image classification tasks.
 
-这种方法在PASCAL VOC竞赛中的图像分类算法中被广泛使用 \[[18](#参考文献)\]。[NEC实验室](http://www.nec-labs.com/)在ILSVRC2010中采用SIFT和LBP特征，两个非线性编码器以及SVM分类器获得图像分类的冠军 \[[8](#参考文献)\]。
+This method has been used widely as image classification algorithm in PASCAL VOC [18]. NEC Labs(http://www.nec-labs.com/) won the championship by employing SIFT and LBP features, two non-linear encoders and SVM in ILSVRC 2010 [8].
 
-Alex Krizhevsky在2012年ILSVRC提出的CNN模型 \[[9](#参考文献)\] 取得了历史性的突破，效果大幅度超越传统方法，获得了ILSVRC2012冠军，该模型被称作AlexNet。这也是首次将深度学习用于大规模图像分类中。从AlexNet之后，涌现了一系列CNN模型，不断地在ImageNet上刷新成绩，如图4展示。随着模型变得越来越深以及精妙的结构设计，Top-5的错误率也越来越低，降到了3.5%附近。而在同样的ImageNet数据集上，人眼的辨识错误率大概在5.1%，也就是目前的深度学习模型的识别能力已经超过了人眼。
+The CNN model--AlexNet proposed by Alex Krizhevsky et al.[9], made a breakthrough in ILSVRC 2012. It dramatically outperformed traditional methods and won the ILSVRC championship in 2012. This was also the first time that a deep learning method was used for large-scale image classification. Since AlexNet, a series of CNN models have been proposed that have advanced the state of the art steadily on Imagenet as shown in Figure 4. With deeper and more sophisticated architectures, Top-5 error rate is getting lower and lower (to around 3.5%). The error rate of human raters on the same Imagenet dataset is 5.1%, which means that the image classification capability of a deep learning model has surpassed human raters.
 
 <p align="center">
 <img src="image/ilsvrc.png" width="500" ><br/>
-图4. ILSVRC图像分类Top-5错误率
+Figure 4. Top-5 error rates on ILSVRC image classification
 </p>
 
 ### CNN
 
-传统CNN包含卷积层、全连接层等组件，并采用softmax多类别分类器和多类交叉熵损失函数，一个典型的卷积神经网络如图5所示，我们先介绍用来构造CNN的常见组件。
+Traditional CNNs consist of convolutional and fully-connected layers and use the softmax multi-category classifier with the cross-entropy loss function. Figure 5 shows a typical CNN. We first introduce the common components of a CNN.
 
 <p align="center">
-<img src="image/lenet.png"><br/>
-图5. CNN网络示例[20]
+<img src="image/lenet_en.png"><br/>
+Figure 5. A CNN example [20]
 </p>
 
-- 卷积层(convolution layer): 执行卷积操作提取底层到高层的特征，发掘出图片局部关联性质和空间不变性质。
-- 池化层(pooling layer): 执行降采样操作。通过取卷积输出特征图中局部区块的最大值(max-pooling)或者均值(avg-pooling)。降采样也是图像处理中常见的一种操作，可以过滤掉一些不重要的高频信息。
-- 全连接层(fully-connected layer，或者fc layer): 输入层到隐藏层的神经元是全部连接的。
-- 非线性变化: 卷积层、全连接层后面一般都会接非线性变化层，例如Sigmoid、Tanh、ReLu等来增强网络的表达能力，在CNN里最常使用的为ReLu激活函数。
-- Dropout \[[10](#参考文献)\] : 在模型训练阶段随机让一些隐层节点权重不工作，提高网络的泛化能力，一定程度上防止过拟合。
+- convolutional layer: this layer uses the convolution operation to extract (low-level and high-level) features and to discover local correlation and spatial invariance.
 
-另外，在训练过程中由于每层参数不断更新，会导致下一次输入分布发生变化，这样导致训练过程需要精心设计超参数。如2015年Sergey Ioffe和Christian Szegedy提出了Batch Normalization (BN)算法 \[[14](#参考文献)\] 中，每个batch对网络中的每一层特征都做归一化，使得每层分布相对稳定。BN算法不仅起到一定的正则作用，而且弱化了一些超参数的设计。经过实验证明，BN算法加速了模型收敛过程，在后来较深的模型中被广泛使用。
+- pooling layer: this layer down samples feature maps by extracting local max (max-pooling) or average (avg-pooling) value of each patch in the feature map. Down-sampling is a common operation in image processing and is used to filter out high-frequency information.
 
-接下来我们主要介绍VGG，GoogleNet和ResNet网络结构。
+- fully-connected layer: this layer fully connects neurons between two adjacent layers.
+
+- non-linear activation: Convolutional and fully-connected layers are usually followed by some non-linear activation layers. Non-linearities enhance the expression capability of the network. Some examples of non-linear activation functions are Sigmoid, Tanh and ReLU. ReLU is the most commonly used activation function in CNN.
+
+- Dropout [10]: At each training stage, individual nodes are dropped out of the network with a certain probability. This improves the network's ability to generalize and avoids overfitting.
+
+Parameter updates at each layer during training causes input layer distributions to change and in turn requires hyper-parameters to be careful tuned. In 2015, Sergey Ioffe and Christian Szegedy proposed a Batch Normalization (BN) algorithm [14], which normalizes the features of each batch in a layer, and enables relatively stable distribution in each layer. Not only does BN algorithm act as a regularizer, but also reduces the need for careful hyper-parameter design. Experiments demonstrate that BN algorithm accelerates the training convergence and has been widely used in later deeper models.
+
+In the following sections, we will introduce the following network architectures - VGG, GoogleNet and ResNets.
 
 ### VGG
 
-牛津大学VGG(Visual Geometry Group)组在2014年ILSVRC提出的模型被称作VGG模型 \[[11](#参考文献)\] 。该模型相比以往模型进一步加宽和加深了网络结构，它的核心是五组卷积操作，每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积，卷积核的数目由较浅组的64增多到最深组的512，同一组内的卷积核数目是一样的。卷积之后接两层全连接层，之后是分类层。由于每组内卷积层的不同，有11、13、16、19层这几种模型，下图展示一个16层的网络结构。VGG模型结构相对简洁，提出之后也有很多文章基于此模型进行研究，如在ImageNet上首次公开超过人眼识别的模型\[[19](#参考文献)\]就是借鉴VGG模型的结构。
+The Oxford Visual Geometry Group (VGG) proposed the VGG network in ILSVRC 2014 [11]. This model is deeper and wider than previous neural architectures. It consists of five main groups of convolution operations. Adjacent convolution groups are connected via max-pooling layers. Each group contains a series of 3x3 convolutional layers (i.e. kernels). The number of convolution kernels stays the same within the group and increases from 64 in the first group to 512 in the last one. The total number of learnable layers could be 11, 13, 16, or 19 depending on the number of convolutional layers in each group. Figure 6 illustrates a 16-layer VGG. The neural architecture of VGG is relatively simple and has been adopted by many papers such as the first one that surpassed human-level performance on ImageNet [19].
 
 <p align="center">
 <img src="image/vgg16.png" width="750" ><br/>
-图6. 基于ImageNet的VGG16模型
+Figure 6. VGG16 model for ImageNet
 </p>
 
 ### GoogleNet
 
-GoogleNet \[[12](#参考文献)\] 在2014年ILSVRC的获得了冠军，在介绍该模型之前我们先来了解NIN(Network in Network)模型 \[[13](#参考文献)\] 和Inception模块，因为GoogleNet模型由多组Inception模块组成，模型设计借鉴了NIN的一些思想。
+GoogleNet [12] won the ILSVRC championship in 2014. GoogleNet borrowed some ideas from the Network in Network(NIN) model [13] and is built on the Inception blocks. Let us first familiarize ourselves with these first.
+
+The two main characteristics of the NIN model are:
+
+1) A single-layer convolutional network is replaced with a Multi-Layer Perceptron Convolution (MLPconv). MLPconv is a tiny multi-layer convolutional network. It enhances non-linearity by adding several 1x1 convolutional layers after linear ones.
 
-NIN模型主要有两个特点：1) 引入了多层感知卷积网络(Multi-Layer Perceptron Convolution, MLPconv)代替一层线性卷积网络。MLPconv是一个微小的多层卷积网络，即在线性卷积后面增加若干层1x1的卷积，这样可以提取出高度非线性特征。2) 传统的CNN最后几层一般都是全连接层，参数较多。而NIN模型设计最后一层卷积层包含类别维度大小的特征图，然后采用全局均值池化(Avg-Pooling)替代全连接层，得到类别维度大小的向量，再进行分类。这种替代全连接层的方式有利于减少参数。
+2) In traditional CNNs, the last fewer layers are usually fully-connected with a large number of parameters. In contrast, NIN replaces all fully-connected layers with convolutional layers with feature maps of the same size as the category dimension and a global average pooling. This replacement of fully-connected layers significantly reduces the number of parameters.
 
-Inception模块如下图7所示，图(a)是最简单的设计，输出是3个卷积层和一个池化层的特征拼接。这种设计的缺点是池化层不会改变特征通道数，拼接后会导致特征的通道数较大，经过几层这样的模块堆积后，通道数会越来越大，导致参数和计算量也随之增大。为了改善这个缺点，图(b)引入3个1x1卷积层进行降维，所谓的降维就是减少通道数，同时如NIN模型中提到的1x1卷积也可以修正线性特征。
+Figure 7 depicts two Inception blocks. Figure 7(a) is the simplest design. The output is a concatenation of features from three convolutional layers and one pooling layer. The disadvantage of this design is that the pooling layer does not change the number of filters and leads to an increase in the number of outputs. After several of such blocks, the number of outputs and parameters become larger and larger and lead to higher computation complexity. To overcome this drawback, the Inception block in Figure 7(b) employs three 1x1 convolutional layers. These reduce dimensions or the number of channels but improve the non-linearity of the network.
 
 <p align="center">
-<img src="image/inception.png" width="800" ><br/>
-图7. Inception模块
+<img src="image/inception_en.png" width="800" ><br/>
+Figure 7. Inception block
 </p>
 
-GoogleNet由多组Inception模块堆积而成。另外，在网络最后也没有采用传统的多层全连接层，而是像NIN网络一样采用了均值池化层；但与NIN不同的是，池化层后面接了一层到类别数映射的全连接层。除了这两个特点之外，由于网络中间层特征也很有判别性，GoogleNet在中间层添加了两个辅助分类器，在后向传播中增强梯度并且增强正则化，而整个网络的损失函数是这个三个分类器的损失加权求和。
+GoogleNet consists of multiple stacked Inception blocks followed by an avg-pooling layer as in NIN instead of traditional fully connected layers. The difference between GoogleNet and NIN is that GoogleNet adds a fully connected layer after avg-pooling layer to output a vector of category size. Besides these two characteristics, the features from middle layers of a GoogleNet are also very discriminative. Therefore, GoogeleNet inserts two auxiliary classifiers in the model for enhancing gradient and regularization when doing backpropagation. The loss function of the whole network is the weighted sum of these three classifiers.
 
-GoogleNet整体网络结构如图8所示，总共22层网络：开始由3层普通的卷积组成；接下来由三组子网络组成，第一组子网络包含2个Inception模块，第二组包含5个Inception模块，第三组包含2个Inception模块；然后接均值池化层、全连接层。
+Figure 8 illustrates the neural architecture of a GoogleNet which consists of 22 layers: it starts with three regular convolutional layers followed by three groups of sub-networks -- the first group contains two Inception blocks, the second one five, and the third one two. It ends up with an average pooling and a fully-connected layer.
 
 <p align="center">
 <img src="image/googlenet.jpeg" ><br/>
-图8. GoogleNet[12]
+Figure 8. GoogleNet[12]
 </p>
 
-
-上面介绍的是GoogleNet第一版模型(称作GoogleNet-v1)。GoogleNet-v2 \[[14](#参考文献)\] 引入BN层；GoogleNet-v3 \[[16](#参考文献)\] 对一些卷积层做了分解，进一步提高网络非线性能力和加深网络；GoogleNet-v4 \[[17](#参考文献)\] 引入下面要讲的ResNet设计思路。从v1到v4每一版的改进都会带来准确度的提升，介于篇幅，这里不再详细介绍v2到v4的结构。
-
+The above model is the first version of GoogleNet or GoogelNet-v1. GoogleNet-v2 [14] introduced BN layer; GoogleNet-v3 [16] further split some convolutional layers, which increases non-linearity and network depth; GoogelNet-v4 [17] leads to the design idea of ResNet which will be introduced in the next section. The evolution from v1 to v4 improved the accuracy rate consistently. We will not go into details of the neural architectures of v2 to v4.
 
 ### ResNet
 
-ResNet(Residual Network) \[[15](#参考文献)\] 是2015年ImageNet图像分类、图像物体定位和图像物体检测比赛的冠军。针对训练卷积神经网络时加深网络导致准确度下降的问题，ResNet提出了采用残差学习。在已有设计思路(BN, 小卷积核，全卷积网络)的基础上，引入了残差模块。每个残差模块包含两条路径，其中一条路径是输入特征的直连通路，另一条路径对该特征做两到三次卷积操作得到该特征的残差，最后再将两条路径上的特征相加。
+Residual Network(ResNet)[15] won the 2015 championship on three ImageNet competitions -- image classification, object localization, and object detection. The main challenge in training deeper networks is that accuracy degrades with network depth. The authors of ResNet proposed a residual learning approach to ease the difficulty of training deeper networks. Based on the design ideas of BN, small convolutional kernels, full convolutional network, ResNets reformulate the layers as residual blocks, with each block containing two branches, one directly connecting input to the output, the other performing two to three convolutions and calculating the residual function with reference to the layer inputs. The outputs of these two branches are then added up.
 
-残差模块如图9所示，左边是基本模块连接方式，由两个输出通道数相同的3x3卷积组成。右边是瓶颈模块(Bottleneck)连接方式，之所以称为瓶颈，是因为上面的1x1卷积用来降维(图示例即256->64)，下面的1x1卷积用来升维(图示例即64->256)，这样中间3x3卷积的输入和输出通道数都较小(图示例即64->64)。
+Figure 9 illustrates the ResNet architecture. To the left is the basic building block, it consists of two 3x3 convolutional layers of the same channels. To the right is a Bottleneck block. The bottleneck is a 1x1 convolutional layer used to reduce dimension from 256 to 64. The other 1x1 convolutional layer is used to increase dimension from 64 to 256. Thus, the number of input and output channels of the middle 3x3 convolutional layer is 64, which is relatively small.
 
 <p align="center">
 <img src="image/resnet_block.jpg" width="400"><br/>
-图9. 残差模块
+Figure 9. Residual block
 </p>
 
-图10展示了50、101、152层网络连接示意图，使用的是瓶颈模块。这三个模型的区别在于每组中残差模块的重复次数不同(见图右上角)。ResNet训练收敛较快，成功的训练了上百乃至近千层的卷积神经网络。
+Figure 10 illustrates ResNets with 50, 101, 152 layers, respectively. All three networks use bottleneck blocks of different numbers of repetitions. ResNet converges very fast and can be trained with hundreds or thousands of layers.
 
 <p align="center">
 <img src="image/resnet.png"><br/>
-图10. 基于ImageNet的ResNet模型
+Figure 10. ResNet model for ImageNet
 </p>
 
 
-## 数据准备
+## Dataset
 
-通用图像分类公开的标准数据集常用的有[CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html)、[ImageNet](http://image-net.org/)、[COCO](http://mscoco.org/)等，常用的细粒度图像分类数据集包括[CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html)、[Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/)、[Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/)等。其中ImageNet数据集规模相对较大，如[模型概览](#模型概览)一章所讲，大量研究成果基于ImageNet。ImageNet数据从2010年来稍有变化，常用的是ImageNet-2012数据集，该数据集包含1000个类别：训练集包含1,281,167张图片，每个类别数据732至1300张不等，验证集包含50,000张图片，平均每个类别50张图片。
+Commonly used public datasets for image classification are [CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html), [ImageNet](http://image-net.org/), [COCO](http://mscoco.org/), etc. Those used for fine-grained image classification are [CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html), [Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/), [Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/), etc. Among these, the ImageNet dataset is the largest. Most research results are reported on ImageNet as mentioned in the Model Overview section. Since 2010, the ImageNet dataset has gone through some changes. The commonly used ImageNet-2012 dataset contains 1000 categories. There are 1,281,167 training images, ranging from 732 to 1200 images per category, and 50,000 validation images with 50 images per category in average.
 
-由于ImageNet数据集较大，下载和训练较慢，为了方便大家学习，我们使用[CIFAR10](<https://www.cs.toronto.edu/~kriz/cifar.html>)数据集。CIFAR10数据集包含60,000张32x32的彩色图片，10个类别，每个类包含6,000张。其中50,000张图片作为训练集，10000张作为测试集。图11从每个类别中随机抽取了10张图片，展示了所有的类别。
+Since ImageNet is too large to be downloaded and trained efficiently, we use [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) in this tutorial. The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. Figure 11 shows all the classes in CIFAR-10 as well as 10 images randomly sampled from each category.
 
 <p align="center">
 <img src="image/cifar.png" width="350"><br/>
-图11. CIFAR10数据集[21]
+Figure 11. CIFAR10 dataset[21]
 </p>
 
-Paddle API提供了自动加载cifar数据集模块 `paddle.dataset.cifar`。
+ `paddle.datasets` package encapsulates multiple public datasets, including `cifar`, `imdb`, `mnist`, `moivelens` and `wmt14`, etc. There's no need to manually download and preprocess CIFAR-10.
 
-通过输入`python train.py`，就可以开始训练模型了，以下小节将详细介绍`train.py`的相关内容。
+After issuing a command `python train.py`, training will start immediately. The following sections describe the details:
 
-### 模型结构
+## Model Structure
 
-#### Paddle 初始化
+### Initialize PaddlePaddle
 
-通过 `paddle.init`，初始化Paddle是否使用GPU，trainer的数目等等。
+We must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
 
 ```python
 import sys
@@ -164,30 +177,29 @@ from resnet import resnet_cifar10
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-本教程中我们提供了VGG和ResNet两个模型的配置。
+As mentioned in section [Model Overview](#model-overview), here we provide the implementations of the VGG and ResNet models.
 
-#### VGG
+### VGG
 
-首先介绍VGG模型结构，由于CIFAR10图片大小和数量相比ImageNet数据小很多，因此这里的模型针对CIFAR10数据做了一定的适配。卷积部分引入了BN和Dropout操作。
+First, we use a VGG network. Since the image size and amount of CIFAR10 are relatively small comparing to ImageNet, we use a small version of VGG network for CIFAR10. Convolution groups incorporate BN and dropout operations.
 
-1. 定义数据输入及其维度
+1. Define input data and its dimension
 
-    网络输入定义为 `data_layer` (数据层)，在图像分类中即为图像像素信息。CIFRAR10是RGB 3通道32x32大小的彩色图，因此输入数据大小为3072(3x32x32)，类别大小为10，即10分类。
+    The input to the network is defined as `paddle.layer.data`, or image pixels in the context of image classification. The images in CIFAR10 are 32x32 color images of three channels. Therefore, the size of the input data is 3072 (3x32x32), and the number of categories is 10.
 
     ```python
     datadim = 3 * 32 * 32
     classdim = 10
-
     image = paddle.layer.data(
         name="image", type=paddle.data_type.dense_vector(datadim))
     ```
 
-2. 定义VGG网络核心模块
+2. Define VGG main module
 
     ```python
     net = vgg_bn_drop(image)
     ```
-    VGG核心模块的输入是数据层，`vgg_bn_drop` 定义了16层VGG结构，每层卷积后面引入BN层和Dropout层，详细的定义如下：
+    The input to VGG main module is from the data layer. `vgg_bn_drop` defines a 16-layer VGG network, with each convolutional layer followed by BN and dropout layers. Here is the definition in detail:
 
     ```python
     def vgg_bn_drop(input):
@@ -220,15 +232,15 @@ paddle.init(use_gpu=False, trainer_count=1)
         return fc2
     ```
 
-    2.1. 首先定义了一组卷积网络，即conv_block。卷积核大小为3x3，池化窗口大小为2x2，窗口滑动大小为2，groups决定每组VGG模块是几次连续的卷积操作，dropouts指定Dropout操作的概率。所使用的`img_conv_group`是在`paddle.networks`中预定义的模块，由若干组 Conv->BN->ReLu->Dropout 和 一组 Pooling 组成。
+    2.1. First, define a convolution block or conv_block. The default convolution kernel is 3x3, and the default pooling size is 2x2 with stride 2. Dropout specifies the probability in dropout operation. Function `img_conv_group` is defined in `paddle.networks` consisting of a series of `Conv->BN->ReLu->Dropout` and a `Pooling`.
 
-    2.2. 五组卷积操作，即 5个conv_block。 第一、二组采用两次连续的卷积操作。第三、四、五组采用三次连续的卷积操作。每组最后一个卷积后面Dropout概率为0，即不使用Dropout操作。
+    2.2. Five groups of convolutions. The first two groups perform two convolutions, while the last three groups perform three convolutions. The dropout rate of the last convolution in each group is set to 0, which means there is no dropout for this layer.
 
-    2.3. 最后接两层512维的全连接。
+    2.3. The last two layers are fully-connected layers of dimension 512.
 
-3. 定义分类器
+3. Define Classifier
 
-    通过上面VGG网络提取高层特征，然后经过全连接层映射到类别维度大小的向量，再通过Softmax归一化得到每个类别的概率，也可称作分类器。
+    The above VGG network extracts high-level features and maps them to a vector of the same size as the categories. Softmax function or classifier is then used for calculating the probability of the image belonging to each category.
 
     ```python
     out = paddle.layer.fc(input=net,
@@ -236,9 +248,9 @@ paddle.init(use_gpu=False, trainer_count=1)
                           act=paddle.activation.Softmax())
     ```
 
-4. 定义损失函数和网络输出
+4. Define Loss Function and Outputs
 
-    在有监督训练中需要输入图像对应的类别信息，同样通过`paddle.layer.data`来定义。训练中采用多类交叉熵作为损失函数，并作为网络的输出，预测阶段定义网络的输出为分类器得到的概率信息。
+    In the context of supervised learning, labels of training images are defined in `paddle.layer.data` as well. During training, the cross-entropy loss function is used and the loss is the output of the network. During testing, the outputs are the probabilities calculated in the classifier.
 
     ```python
     lbl = paddle.layer.data(
@@ -248,19 +260,20 @@ paddle.init(use_gpu=False, trainer_count=1)
 
 ### ResNet
 
-ResNet模型的第1、3、4步和VGG模型相同，这里不再介绍。主要介绍第2步即CIFAR10数据集上ResNet核心模块。
+The first, third and fourth steps of a ResNet are the same as a VGG. The second one is the main module.
 
 ```python
 net = resnet_cifar10(image, depth=56)
 ```
 
-先介绍`resnet_cifar10`中的一些基本函数，再介绍网络连接过程。
+Here are some basic functions used in `resnet_cifar10`:
+
+  - `conv_bn_layer` : convolutional layer followed by BN.
+  - `shortcut` : the shortcut branch in a residual block. There are two kinds of shortcuts: 1x1 convolution used when the number of channels between input and output is different; direct connection used otherwise.
 
-  - `conv_bn_layer` : 带BN的卷积层。
-  - `shortcut` : 残差模块的"直连"路径，"直连"实际分两种形式：残差模块输入和输出特征通道数不等时，采用1x1卷积的升维操作；残差模块输入和输出通道相等时，采用直连操作。
-  - `basicblock` : 一个基础残差模块，即图9左边所示，由两组3x3卷积组成的路径和一条"直连"路径组成。
-  - `bottleneck` : 一个瓶颈残差模块，即图9右边所示，由上下1x1卷积和中间3x3卷积组成的路径和一条"直连"路径组成。
-  - `layer_warp` : 一组残差模块，由若干个残差模块堆积而成。每组中第一个残差模块滑动窗口大小与其他可以不同，以用来减少特征图在垂直和水平方向的大小。
+  - `basicblock` : a basic residual module as shown in the left of Figure 9, it consists of two sequential 3x3 convolutions and one "shortcut" branch.
+  - `bottleneck` : a bottleneck module as shown in the right of Figure 9, it consists of two 1x1 convolutions with one 3x3 convolution in between branch and a "shortcut" branch.
+  - `layer_warp` : a group of residual modules consisting of several stacking blocks. In each group, the sliding window size of the first residual block could be different from the rest of blocks, in order to reduce the size of feature maps along horizontal and vertical directions.
 
 ```python
 def conv_bn_layer(input,
@@ -302,13 +315,13 @@ def layer_warp(block_func, ipt, features, count, stride):
     return tmp
 ```
 
-`resnet_cifar10` 的连接结构主要有以下几个过程。
+The following are the components of `resnet_cifar10`:
 
-1. 底层输入连接一层 `conv_bn_layer`，即带BN的卷积层。
-2. 然后连接3组残差模块即下面配置3组 `layer_warp` ，每组采用图 10 左边残差模块组成。
-3. 最后对网络做均值池化并返回该层。
+1. The lowest level is `conv_bn_layer`.
+2. The middle level consists of three `layer_warp`, each of which uses the left residual block in Figure 9.
+3. The last level is average pooling layer.
 
-注意：除过第一层卷积层和最后一层全连接层之外，要求三组 `layer_warp` 总的含参层数能够被6整除，即 `resnet_cifar10` 的 depth 要满足 $(depth - 2) % 6 == 0$ 。
+Note: besides the first convolutional layer and the last fully-connected layer, the total number of layers in three `layer_warp` should be dividable by 6, that is the depth of `resnet_cifar10` should satisfy $(depth - 2) % 6 == 0$.
 
 ```python
 def resnet_cifar10(ipt, depth=32):
@@ -326,26 +339,21 @@ def resnet_cifar10(ipt, depth=32):
     return pool
 ```
 
-## 训练模型
+## Model Training
 
-### 定义参数
+### Define Parameters
 
-首先依据模型配置的`cost`定义模型参数。
+First, we create the model parameters according to the previous model configuration `cost`.
 
 ```python
 # Create parameters
 parameters = paddle.parameters.create(cost)
 ```
 
-可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
-
-```python
-print parameters.keys()
-```
-
-### 构造训练(Trainer)
+### Create Trainer
 
-根据网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的Momentum方法，同时设定了学习率、正则等。
+Before creating a training module, it is necessary to set the algorithm.
+Here we specify `Momentum` optimization algorithm via `paddle.optimizer`.
 
 ```python
 # Create optimizer
@@ -355,8 +363,7 @@ momentum_optimizer = paddle.optimizer.Momentum(
     learning_rate=0.1 / 128.0,
     learning_rate_decay_a=0.1,
     learning_rate_decay_b=50000 * 100,
-    learning_rate_schedule='discexp',
-    batch_size=128)
+    learning_rate_schedule='discexp')
 
 # Create trainer
 trainer = paddle.trainer.SGD(cost=cost,
@@ -364,14 +371,13 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=momentum_optimizer)
 ```
 
-通过 `learning_rate_decay_a` (简写$a$） 、`learning_rate_decay_b` (简写$b$) 和 `learning_rate_schedule` 指定学习率调整策略，这里采用离散指数的方式调节学习率，计算公式如下， $n$ 代表已经处理过的累计总样本数，$lr_{0}$ 即为 `settings` 里设置的 `learning_rate`。
-
+The learning rate adjustment policy can be defined with variables `learning_rate_decay_a`($a$), `learning_rate_decay_b`($b$) and `learning_rate_schedule`. In this example, discrete exponential method is used for adjusting learning rate. The formula is as follows,
 $$  lr = lr_{0} * a^ {\lfloor \frac{n}{ b}\rfloor} $$
+where $n$ is the number of processed samples, $lr_{0}$ is the learning_rate.
 
+### Training
 
-### 训练
-
-cifar.train10()每次产生一条样本，在完成shuffle和batch之后，作为训练的输入。
+`cifar.train10()` will yield records during each pass, after shuffling, a batch input is generated for training.
 
 ```python
 reader=paddle.batch(
@@ -380,17 +386,47 @@ reader=paddle.batch(
         batch_size=128)
 ```
 
-通过`feeding`来指定每一个数据和`paddle.layer.data`的对应关系。例如: `cifar.train10()`产生数据的第0列对应image层的特征。
+`feeding` is devoted to specifying the correspondence between each yield record and `paddle.layer.data`. For instance,
+ the first column of data generated by `cifar.train10()` corresponds to image layer's feature.
 
 ```python
 feeding={'image': 0,
          'label': 1}
 ```
 
-可以使用`event_handler`回调函数来观察训练过程，或进行测试等, 该回调函数是`trainer.train`函数里设定。
+Callback function `event_handler` will be called during training when a pre-defined event happens.
+
+`event_handler_plot`is used to plot a figure like below：
+
+![png](./image/train_and_test.png)
 
 ```python
-# End batch and end pass event handler
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 1 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                paddle.dataset.cifar.test10(), batch_size=128),
+            feeding=feeding)
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` is used to plot some text data when training.
+
+```python
+# event handler to track training and testing process
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -400,6 +436,10 @@ def event_handler(event):
             sys.stdout.write('.')
             sys.stdout.flush()
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(
             reader=paddle.batch(
                 paddle.dataset.cifar.test10(), batch_size=128),
@@ -407,17 +447,17 @@ def event_handler(event):
         print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
-通过`trainer.train`函数训练:
+Finally, we can invoke `trainer.train` to start training:
 
 ```python
 trainer.train(
     reader=reader,
     num_passes=200,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding)
 ```
 
-一轮训练log示例如下所示，经过1个pass， 训练集上平均error为0.6875 ，测试集上平均error为0.8852 。
+Here is an example log after training for one pass. The average error rates are 0.6875 on the training set and 0.8852 on the validation set.
 
 ```text
 Pass 0, Batch 0, Cost 2.473182, {'classification_error_evaluator': 0.9140625}
@@ -431,20 +471,58 @@ Pass 0, Batch 300, Cost 1.668833, {'classification_error_evaluator': 0.6875}
 Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 ```
 
-图12是训练的分类错误率曲线图，运行到第200个pass后基本收敛，最终得到测试集上分类错误率为8.54%。
-
+Figure 12 shows the curve of training error rate, which indicates it converges at Pass 200 with error rate 8.54%.
 <p align="center">
-<img src="image/plot.png" width="400" ><br/>
-图12. CIFAR10数据集上VGG模型的分类错误率
+<img src="image/plot_en.png" width="400" ><br/>
+Figure 12. The error rate of VGG model on CIFAR10
 </p>
 
 
-## 总结
 
-传统图像分类方法由多个阶段构成，框架较为复杂，而端到端的CNN模型结构可一步到位，而且大幅度提升了分类准确率。本文我们首先介绍VGG、GoogleNet、ResNet三个经典的模型；然后基于CIFAR10数据集，介绍如何使用PaddlePaddle配置和训练CNN模型，尤其是VGG和ResNet模型；最后介绍如何使用PaddlePaddle的API接口对图片进行预测和特征提取。对于其他数据集比如ImageNet，配置和训练流程是同样的，大家可以自行进行实验。
+## Application
+
+After training is done, users can use the trained model to classify images. The following code shows how to infer through `paddle.infer` interface. You can remove the comments to change the model name.
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file)
+    im = im.resize((32, 32), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32)
+    # The storage order of the loaded image is W(widht),
+    # H(height), C(channel). PaddlePaddle requires
+    # the CHW order, so transpose them.
+    im = im.transpose((2, 0, 1)) # CHW
+    # In the training phase, the channel order of CIFAR
+    # image is B(Blue), G(green), R(Red). But PIL open
+    # image in RGB mode. It must swap the channel order.
+    im = im[(2, 1, 0),:,:] # BGR
+    im = im.flatten()
+    im = im / 255.0
+    return im
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/dog.png'),))
+
+# users can remove the comments and change the model name
+# with open('params_pass_50.tar', 'r') as f:
+#    parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=out, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/dog.png is: %d" % lab[0][0]
+```
+
+
+## Conclusion
+
+Traditional image classification methods have complicated frameworks that involve multiple stages of processing. In contrast, CNN models can be trained end-to-end with a significant increase in classification accuracy. In this chapter, we introduced three models -- VGG, GoogleNet, ResNet and provided PaddlePaddle config files for training VGG and ResNet on CIFAR10. We also explained how to perform prediction and feature extraction using the PaddlePaddle API. For other datasets such as ImageNet, the procedure for config and training are the same and you are welcome to give it a try.
 
 
-## 参考文献
+## Reference
 
 [1] D. G. Lowe, [Distinctive image features from scale-invariant keypoints](http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf). IJCV, 60(2):91-110, 2004.
 
@@ -491,4 +569,4 @@ Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 [22] http://cs231n.github.io/classification/
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

03.image_classification/index.en.html → 03.image_classification/index.cn.html

@@ -40,174 +40,161 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-Image Classification
-=======================
+# 图像分类
 
-The source code for this chapter is at [book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification). First-time users, please refer to PaddlePaddle [Installation Tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst) for installation instructions.
+本教程源代码目录在[book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/168.html)。
 
-## Background
+## 背景介绍
 
-Compared to words, images provide much more vivid and easier to understand information with an artistic sense. They are an important source for people to express and exchange ideas. In this chapter, we focus on one of the essential problems in image recognition -- image classification.
+图像相比文字能够提供更加生动、容易理解及更具艺术感的信息，是人们转递与交换信息的重要来源。在本教程中，我们专注于图像识别领域的一个重要问题，即图像分类。
 
-Image classification is the task of distinguishing images in different categories based on their semantic meaning. It is a core problem in computer vision and is also the foundation of other higher level computer vision tasks such as object detection, image segmentation, object tracking, action recognition, etc. Image classification has applications in many areas such as face recognition, intelligent video analysis in security systems, traffic scene recognition in transportation systems, content-based image retrieval and automatic photo indexing in web services, image classification in medicine, etc.
+图像分类是根据图像的语义信息将不同类别图像区分开来，是计算机视觉中重要的基本问题，也是图像检测、图像分割、物体跟踪、行为分析等其他高层视觉任务的基础。图像分类在很多领域有广泛应用，包括安防领域的人脸识别和智能视频分析等，交通领域的交通场景识别，互联网领域基于内容的图像检索和相册自动归类，医学领域的图像识别等。
 
-To classify an image we first encode the entire image using handcrafted or learned features and then determine the category using a classifier. Thus, feature extraction plays an important role in image classification. Prior to deep learning the BoW(Bag of Words) model was the most widely used method for classifying an image as well as an object. The BoW technique was introduced in Natural Language Processing where a training sentence is represented as a bag of words. In the context of image classification, the BoW model requires constructing a dictionary. The simplest BoW framework can be designed with three steps: **feature extraction**, **feature encoding** and **classifier design**.
 
-Using Deep learning, image classification can be framed as a supervised or unsupervised learning problem that uses hierarchical features automatically without any need for manually crafted features from the image. In recent years, Convolutional Neural Networks (CNNs) have made significant progress in image classification. CNNs use raw image pixels as input, extract low-level and high-level abstract features through convolution operations, and directly output the classification results from the model. This style of end-to-end learning has lead to not only increased performance but also wider adoption various applications.
+一般来说，图像分类通过手工特征或特征学习方法对整个图像进行全部描述，然后使用分类器判别物体类别，因此如何提取图像的特征至关重要。在深度学习算法之前使用较多的是基于词袋(Bag of Words)模型的物体分类方法。词袋方法从自然语言处理中引入，即一句话可以用一个装了词的袋子表示其特征，袋子中的词为句子中的单词、短语或字。对于图像而言，词袋方法需要构建字典。最简单的词袋模型框架可以设计为**底层特征抽取**、**特征编码**、**分类器设计**三个过程。
 
-In this chapter, we introduce deep-learning-based image classification methods and explain how to train a CNN model using PaddlePaddle.
+而基于深度学习的图像分类方法，可以通过有监督或无监督的方式**学习**层次化的特征描述，从而取代了手工设计或选择图像特征的工作。深度学习模型中的卷积神经网络(Convolution Neural Network, CNN)近年来在图像领域取得了惊人的成绩，CNN直接利用图像像素信息作为输入，最大程度上保留了输入图像的所有信息，通过卷积操作进行特征的提取和高层抽象，模型输出直接是图像识别的结果。这种基于"输入-输出"直接端到端的学习方法取得了非常好的效果，得到了广泛的应用。
 
-## Demonstration
+本教程主要介绍图像分类的深度学习模型，以及如何使用PaddlePaddle训练CNN模型。
 
-An image can be classified by a general as well as fine-grained image classifier.
+## 效果展示
 
-
-Figure 1 shows the results of a general image classifier -- the trained model can correctly recognize the main objects in the images.
+图像分类包括通用图像分类、细粒度图像分类等。图1展示了通用图像分类效果，即模型可以正确识别图像上的主要物体。
 
 <p align="center">
 <img src="image/dog_cat.png "  width="350" ><br/>
-Figure 1. General image classification
+图1. 通用图像分类展示
 </p>
 
 
-Figure 2 shows the results of a fine-grained image classifier. This task of flower recognition requires correctly recognizing of the flower's categories.
+图2展示了细粒度图像分类-花卉识别的效果，要求模型可以正确识别花的类别。
+
 
 <p align="center">
 <img src="image/flowers.png" width="400" ><br/>
-Figure 2. Fine-grained image classification
+图2. 细粒度图像分类展示
 </p>
 
 
-A good model should recognize objects of different categories correctly. The results of such a model should not vary due to viewpoint variation, illumination conditions, object distortion or occlusion.
-Figure 3 shows some images with various disturbances. A good model should classify these images correctly like humans.
+一个好的模型既要对不同类别识别正确，同时也应该能够对不同视角、光照、背景、变形或部分遮挡的图像正确识别(这里我们统一称作图像扰动)。图3展示了一些图像的扰动，较好的模型会像聪明的人类一样能够正确识别。
 
 <p align="center">
-<img src="image/variations_en.png" width="550" ><br/>
-Figure 3. Disturbed images [22]
+<img src="image/variations.png" width="550" ><br/>
+图3. 扰动图片展示[22]
 </p>
 
-## Model Overview
-
-A large amount of research in image classification is built upon public datasets such as [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/), [ImageNet](http://image-net.org/) etc. Many image classification algorithms are usually evaluated and compared on these datasets. PASCAL VOC is a computer vision competition started in 2005, and ImageNet is a dataset for Large Scale Visual Recognition Challenge (ILSVRC) started in 2010. In this chapter, we introduce some image classification models from the submissions to these competitions.
-
-Before 2012, traditional image classification was accomplished with the three steps described in the background section. A complete model construction usually involves the following stages: low-level feature extraction, feature encoding, spatial constraint or feature clustering, classifier design, model ensemble.
-
-  1). **Low-level feature extraction**: This step extracts large amounts of local features according to fixed strides and scales. Popular local features include Scale-Invariant Feature Transform (SIFT)[1], Histogram of Oriented Gradient(HOG)[2], Local Binary Pattern(LBP)[3], etc. A common practice is to employ multiple feature descriptors in order to avoid missing a lot of information.
-
-  2). **Feature encoding**: Low-level features contain a large amount of redundancy and noise. In order to improve the robustness of features, it is necessary to employ a feature transformation to encode low-level features. This is called feature encoding. Common feature encoding methods include vector quantization [4], sparse coding [5], locality-constrained linear coding [6], Fisher vector encoding [7], etc.
+## 模型概览
 
-  3). **Spatial constraint**: Spatial constraint or feature clustering is usually adopted after feature encoding for extracting the maximum or average of each dimension in the spatial domain. Pyramid feature matching--a popular feature clustering method--divides an image uniformly into patches and performs feature clustering in each patch.
+图像识别领域大量的研究成果都是建立在[PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)、[ImageNet](http://image-net.org/)等公开的数据集上，很多图像识别算法通常在这些数据集上进行测试和比较。PASCAL VOC是2005年发起的一个视觉挑战赛，ImageNet是2010年发起的大规模视觉识别竞赛(ILSVRC)的数据集，在本章中我们基于这些竞赛的一些论文介绍图像分类模型。
 
-  4). **Classification**: In the above steps an image can be described by a vector of fixed dimension. Then a classifier can be used to classify the image into categories. Common classifiers include Support Vector Machine(SVM), random forest etc. Kernel SVM is the most popular classifier and has achieved very good performance in traditional image classification tasks.
+在2012年之前的传统图像分类方法可以用背景描述中提到的三步完成，但通常完整建立图像识别模型一般包括底层特征学习、特征编码、空间约束、分类器设计、模型融合等几个阶段。
+  1). **底层特征提取**: 通常从图像中按照固定步长、尺度提取大量局部特征描述。常用的局部特征包括SIFT(Scale-Invariant Feature Transform, 尺度不变特征转换) \[[1](#参考文献)\]、HOG(Histogram of Oriented Gradient, 方向梯度直方图) \[[2](#参考文献)\]、LBP(Local Bianray Pattern, 局部二值模式) \[[3](#参考文献)\] 等，一般也采用多种特征描述子，防止丢失过多的有用信息。
+  2). **特征编码**: 底层特征中包含了大量冗余与噪声，为了提高特征表达的鲁棒性，需要使用一种特征变换算法对底层特征进行编码，称作特征编码。常用的特征编码包括向量量化编码 \[[4](#参考文献)\]、稀疏编码 \[[5](#参考文献)\]、局部线性约束编码 \[[6](#参考文献)\]、Fisher向量编码 \[[7](#参考文献)\] 等。
+  3). **空间特征约束**: 特征编码之后一般会经过空间特征约束，也称作**特征汇聚**。特征汇聚是指在一个空间范围内，对每一维特征取最大值或者平均值，可以获得一定特征不变形的特征表达。金字塔特征匹配是一种常用的特征聚会方法，这种方法提出将图像均匀分块，在分块内做特征汇聚。
+  4). **通过分类器分类**: 经过前面步骤之后一张图像可以用一个固定维度的向量进行描述，接下来就是经过分类器对图像进行分类。通常使用的分类器包括SVM(Support Vector Machine, 支持向量机)、随机森林等。而使用核方法的SVM是最为广泛的分类器，在传统图像分类任务上性能很好。
 
-This method has been used widely as image classification algorithm in PASCAL VOC [18]. NEC Labs(http://www.nec-labs.com/) won the championship by employing SIFT and LBP features, two non-linear encoders and SVM in ILSVRC 2010 [8].
+这种方法在PASCAL VOC竞赛中的图像分类算法中被广泛使用 \[[18](#参考文献)\]。[NEC实验室](http://www.nec-labs.com/)在ILSVRC2010中采用SIFT和LBP特征，两个非线性编码器以及SVM分类器获得图像分类的冠军 \[[8](#参考文献)\]。
 
-The CNN model--AlexNet proposed by Alex Krizhevsky et al.[9], made a breakthrough in ILSVRC 2012. It dramatically outperformed traditional methods and won the ILSVRC championship in 2012. This was also the first time that a deep learning method was used for large-scale image classification. Since AlexNet, a series of CNN models have been proposed that have advanced the state of the art steadily on Imagenet as shown in Figure 4. With deeper and more sophisticated architectures, Top-5 error rate is getting lower and lower (to around 3.5%). The error rate of human raters on the same Imagenet dataset is 5.1%, which means that the image classification capability of a deep learning model has surpassed human raters.
+Alex Krizhevsky在2012年ILSVRC提出的CNN模型 \[[9](#参考文献)\] 取得了历史性的突破，效果大幅度超越传统方法，获得了ILSVRC2012冠军，该模型被称作AlexNet。这也是首次将深度学习用于大规模图像分类中。从AlexNet之后，涌现了一系列CNN模型，不断地在ImageNet上刷新成绩，如图4展示。随着模型变得越来越深以及精妙的结构设计，Top-5的错误率也越来越低，降到了3.5%附近。而在同样的ImageNet数据集上，人眼的辨识错误率大概在5.1%，也就是目前的深度学习模型的识别能力已经超过了人眼。
 
 <p align="center">
 <img src="image/ilsvrc.png" width="500" ><br/>
-Figure 4. Top-5 error rates on ILSVRC image classification
+图4. ILSVRC图像分类Top-5错误率
 </p>
 
 ### CNN
 
-Traditional CNNs consist of convolutional and fully-connected layers and use the softmax multi-category classifier with the cross-entropy loss function. Figure 5 shows a typical CNN. We first introduce the common components of a CNN.
+传统CNN包含卷积层、全连接层等组件，并采用softmax多类别分类器和多类交叉熵损失函数，一个典型的卷积神经网络如图5所示，我们先介绍用来构造CNN的常见组件。
 
 <p align="center">
-<img src="image/lenet_en.png"><br/>
-Figure 5. A CNN example [20]
+<img src="image/lenet.png"><br/>
+图5. CNN网络示例[20]
 </p>
 
-- convolutional layer: this layer uses the convolution operation to extract (low-level and high-level) features and to discover local correlation and spatial invariance.
+- 卷积层(convolution layer): 执行卷积操作提取底层到高层的特征，发掘出图片局部关联性质和空间不变性质。
+- 池化层(pooling layer): 执行降采样操作。通过取卷积输出特征图中局部区块的最大值(max-pooling)或者均值(avg-pooling)。降采样也是图像处理中常见的一种操作，可以过滤掉一些不重要的高频信息。
+- 全连接层(fully-connected layer，或者fc layer): 输入层到隐藏层的神经元是全部连接的。
+- 非线性变化: 卷积层、全连接层后面一般都会接非线性变化层，例如Sigmoid、Tanh、ReLu等来增强网络的表达能力，在CNN里最常使用的为ReLu激活函数。
+- Dropout \[[10](#参考文献)\] : 在模型训练阶段随机让一些隐层节点权重不工作，提高网络的泛化能力，一定程度上防止过拟合。
 
-- pooling layer: this layer down samples feature maps by extracting local max (max-pooling) or average (avg-pooling) value of each patch in the feature map. Down-sampling is a common operation in image processing and is used to filter out high-frequency information.
+另外，在训练过程中由于每层参数不断更新，会导致下一次输入分布发生变化，这样导致训练过程需要精心设计超参数。如2015年Sergey Ioffe和Christian Szegedy提出了Batch Normalization (BN)算法 \[[14](#参考文献)\] 中，每个batch对网络中的每一层特征都做归一化，使得每层分布相对稳定。BN算法不仅起到一定的正则作用，而且弱化了一些超参数的设计。经过实验证明，BN算法加速了模型收敛过程，在后来较深的模型中被广泛使用。
 
-- fully-connected layer: this layer fully connects neurons between two adjacent layers.
-
-- non-linear activation: Convolutional and fully-connected layers are usually followed by some non-linear activation layers. Non-linearities enhance the expression capability of the network. Some examples of non-linear activation functions are Sigmoid, Tanh and ReLU. ReLU is the most commonly used activation function in CNN.
-
-- Dropout [10]: At each training stage, individual nodes are dropped out of the network with a certain probability. This improves the network's ability to generalize and avoids overfitting.
-
-Parameter updates at each layer during training causes input layer distributions to change and in turn requires hyper-parameters to be careful tuned. In 2015, Sergey Ioffe and Christian Szegedy proposed a Batch Normalization (BN) algorithm [14], which normalizes the features of each batch in a layer, and enables relatively stable distribution in each layer. Not only does BN algorithm act as a regularizer, but also reduces the need for careful hyper-parameter design. Experiments demonstrate that BN algorithm accelerates the training convergence and has been widely used in later deeper models.
-
-In the following sections, we will introduce the following network architectures - VGG, GoogleNet and ResNets.
+接下来我们主要介绍VGG，GoogleNet和ResNet网络结构。
 
 ### VGG
 
-The Oxford Visual Geometry Group (VGG) proposed the VGG network in ILSVRC 2014 [11]. This model is deeper and wider than previous neural architectures. It consists of five main groups of convolution operations. Adjacent convolution groups are connected via max-pooling layers. Each group contains a series of 3x3 convolutional layers (i.e. kernels). The number of convolution kernels stays the same within the group and increases from 64 in the first group to 512 in the last one. The total number of learnable layers could be 11, 13, 16, or 19 depending on the number of convolutional layers in each group. Figure 6 illustrates a 16-layer VGG. The neural architecture of VGG is relatively simple and has been adopted by many papers such as the first one that surpassed human-level performance on ImageNet [19].
+牛津大学VGG(Visual Geometry Group)组在2014年ILSVRC提出的模型被称作VGG模型 \[[11](#参考文献)\] 。该模型相比以往模型进一步加宽和加深了网络结构，它的核心是五组卷积操作，每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积，卷积核的数目由较浅组的64增多到最深组的512，同一组内的卷积核数目是一样的。卷积之后接两层全连接层，之后是分类层。由于每组内卷积层的不同，有11、13、16、19层这几种模型，下图展示一个16层的网络结构。VGG模型结构相对简洁，提出之后也有很多文章基于此模型进行研究，如在ImageNet上首次公开超过人眼识别的模型\[[19](#参考文献)\]就是借鉴VGG模型的结构。
 
 <p align="center">
 <img src="image/vgg16.png" width="750" ><br/>
-Figure 6. VGG16 model for ImageNet
+图6. 基于ImageNet的VGG16模型
 </p>
 
 ### GoogleNet
 
-GoogleNet [12] won the ILSVRC championship in 2014. GoogleNet borrowed some ideas from the Network in Network(NIN) model [13] and is built on the Inception blocks. Let us first familiarize ourselves with these first.
-
-The two main characteristics of the NIN model are:
+GoogleNet \[[12](#参考文献)\] 在2014年ILSVRC的获得了冠军，在介绍该模型之前我们先来了解NIN(Network in Network)模型 \[[13](#参考文献)\] 和Inception模块，因为GoogleNet模型由多组Inception模块组成，模型设计借鉴了NIN的一些思想。
 
-1) A single-layer convolutional network is replaced with a Multi-Layer Perceptron Convolution (MLPconv). MLPconv is a tiny multi-layer convolutional network. It enhances non-linearity by adding several 1x1 convolutional layers after linear ones.
+NIN模型主要有两个特点：1) 引入了多层感知卷积网络(Multi-Layer Perceptron Convolution, MLPconv)代替一层线性卷积网络。MLPconv是一个微小的多层卷积网络，即在线性卷积后面增加若干层1x1的卷积，这样可以提取出高度非线性特征。2) 传统的CNN最后几层一般都是全连接层，参数较多。而NIN模型设计最后一层卷积层包含类别维度大小的特征图，然后采用全局均值池化(Avg-Pooling)替代全连接层，得到类别维度大小的向量，再进行分类。这种替代全连接层的方式有利于减少参数。
 
-2) In traditional CNNs, the last fewer layers are usually fully-connected with a large number of parameters. In contrast, NIN replaces all fully-connected layers with convolutional layers with feature maps of the same size as the category dimension and a global average pooling. This replacement of fully-connected layers significantly reduces the number of parameters.
-
-Figure 7 depicts two Inception blocks. Figure 7(a) is the simplest design. The output is a concatenation of features from three convolutional layers and one pooling layer. The disadvantage of this design is that the pooling layer does not change the number of filters and leads to an increase in the number of outputs. After several of such blocks, the number of outputs and parameters become larger and larger and lead to higher computation complexity. To overcome this drawback, the Inception block in Figure 7(b) employs three 1x1 convolutional layers. These reduce dimensions or the number of channels but improve the non-linearity of the network.
+Inception模块如下图7所示，图(a)是最简单的设计，输出是3个卷积层和一个池化层的特征拼接。这种设计的缺点是池化层不会改变特征通道数，拼接后会导致特征的通道数较大，经过几层这样的模块堆积后，通道数会越来越大，导致参数和计算量也随之增大。为了改善这个缺点，图(b)引入3个1x1卷积层进行降维，所谓的降维就是减少通道数，同时如NIN模型中提到的1x1卷积也可以修正线性特征。
 
 <p align="center">
-<img src="image/inception_en.png" width="800" ><br/>
-Figure 7. Inception block
+<img src="image/inception.png" width="800" ><br/>
+图7. Inception模块
 </p>
 
-GoogleNet consists of multiple stacked Inception blocks followed by an avg-pooling layer as in NIN instead of traditional fully connected layers. The difference between GoogleNet and NIN is that GoogleNet adds a fully connected layer after avg-pooling layer to output a vector of category size. Besides these two characteristics, the features from middle layers of a GoogleNet are also very discriminative. Therefore, GoogeleNet inserts two auxiliary classifiers in the model for enhancing gradient and regularization when doing backpropagation. The loss function of the whole network is the weighted sum of these three classifiers.
+GoogleNet由多组Inception模块堆积而成。另外，在网络最后也没有采用传统的多层全连接层，而是像NIN网络一样采用了均值池化层；但与NIN不同的是，池化层后面接了一层到类别数映射的全连接层。除了这两个特点之外，由于网络中间层特征也很有判别性，GoogleNet在中间层添加了两个辅助分类器，在后向传播中增强梯度并且增强正则化，而整个网络的损失函数是这个三个分类器的损失加权求和。
 
-Figure 8 illustrates the neural architecture of a GoogleNet which consists of 22 layers: it starts with three regular convolutional layers followed by three groups of sub-networks -- the first group contains two Inception blocks, the second one five, and the third one two. It ends up with an average pooling and a fully-connected layer.
+GoogleNet整体网络结构如图8所示，总共22层网络：开始由3层普通的卷积组成；接下来由三组子网络组成，第一组子网络包含2个Inception模块，第二组包含5个Inception模块，第三组包含2个Inception模块；然后接均值池化层、全连接层。
 
 <p align="center">
 <img src="image/googlenet.jpeg" ><br/>
-Figure 8. GoogleNet[12]
+图8. GoogleNet[12]
 </p>
 
-The above model is the first version of GoogleNet or GoogelNet-v1. GoogleNet-v2 [14] introduced BN layer; GoogleNet-v3 [16] further split some convolutional layers, which increases non-linearity and network depth; GoogelNet-v4 [17] leads to the design idea of ResNet which will be introduced in the next section. The evolution from v1 to v4 improved the accuracy rate consistently. We will not go into details of the neural architectures of v2 to v4.
+
+上面介绍的是GoogleNet第一版模型(称作GoogleNet-v1)。GoogleNet-v2 \[[14](#参考文献)\] 引入BN层；GoogleNet-v3 \[[16](#参考文献)\] 对一些卷积层做了分解，进一步提高网络非线性能力和加深网络；GoogleNet-v4 \[[17](#参考文献)\] 引入下面要讲的ResNet设计思路。从v1到v4每一版的改进都会带来准确度的提升，介于篇幅，这里不再详细介绍v2到v4的结构。
+
 
 ### ResNet
 
-Residual Network(ResNet)[15] won the 2015 championship on three ImageNet competitions -- image classification, object localization, and object detection. The main challenge in training deeper networks is that accuracy degrades with network depth. The authors of ResNet proposed a residual learning approach to ease the difficulty of training deeper networks. Based on the design ideas of BN, small convolutional kernels, full convolutional network, ResNets reformulate the layers as residual blocks, with each block containing two branches, one directly connecting input to the output, the other performing two to three convolutions and calculating the residual function with reference to the layer inputs. The outputs of these two branches are then added up.
+ResNet(Residual Network) \[[15](#参考文献)\] 是2015年ImageNet图像分类、图像物体定位和图像物体检测比赛的冠军。针对训练卷积神经网络时加深网络导致准确度下降的问题，ResNet提出了采用残差学习。在已有设计思路(BN, 小卷积核，全卷积网络)的基础上，引入了残差模块。每个残差模块包含两条路径，其中一条路径是输入特征的直连通路，另一条路径对该特征做两到三次卷积操作得到该特征的残差，最后再将两条路径上的特征相加。
 
-Figure 9 illustrates the ResNet architecture. To the left is the basic building block, it consists of two 3x3 convolutional layers of the same channels. To the right is a Bottleneck block. The bottleneck is a 1x1 convolutional layer used to reduce dimension from 256 to 64. The other 1x1 convolutional layer is used to increase dimension from 64 to 256. Thus, the number of input and output channels of the middle 3x3 convolutional layer is 64, which is relatively small.
+残差模块如图9所示，左边是基本模块连接方式，由两个输出通道数相同的3x3卷积组成。右边是瓶颈模块(Bottleneck)连接方式，之所以称为瓶颈，是因为上面的1x1卷积用来降维(图示例即256->64)，下面的1x1卷积用来升维(图示例即64->256)，这样中间3x3卷积的输入和输出通道数都较小(图示例即64->64)。
 
 <p align="center">
 <img src="image/resnet_block.jpg" width="400"><br/>
-Figure 9. Residual block
+图9. 残差模块
 </p>
 
-Figure 10 illustrates ResNets with 50, 101, 152 layers, respectively. All three networks use bottleneck blocks of different numbers of repetitions. ResNet converges very fast and can be trained with hundreds or thousands of layers.
+图10展示了50、101、152层网络连接示意图，使用的是瓶颈模块。这三个模型的区别在于每组中残差模块的重复次数不同(见图右上角)。ResNet训练收敛较快，成功的训练了上百乃至近千层的卷积神经网络。
 
 <p align="center">
 <img src="image/resnet.png"><br/>
-Figure 10. ResNet model for ImageNet
+图10. 基于ImageNet的ResNet模型
 </p>
 
 
-## Dataset
+## 数据准备
 
-Commonly used public datasets for image classification are [CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html), [ImageNet](http://image-net.org/), [COCO](http://mscoco.org/), etc. Those used for fine-grained image classification are [CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html), [Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/), [Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/), etc. Among these, the ImageNet dataset is the largest. Most research results are reported on ImageNet as mentioned in the Model Overview section. Since 2010, the ImageNet dataset has gone through some changes. The commonly used ImageNet-2012 dataset contains 1000 categories. There are 1,281,167 training images, ranging from 732 to 1200 images per category, and 50,000 validation images with 50 images per category in average.
+通用图像分类公开的标准数据集常用的有[CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html)、[ImageNet](http://image-net.org/)、[COCO](http://mscoco.org/)等，常用的细粒度图像分类数据集包括[CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html)、[Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/)、[Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/)等。其中ImageNet数据集规模相对较大，如[模型概览](#模型概览)一章所讲，大量研究成果基于ImageNet。ImageNet数据从2010年来稍有变化，常用的是ImageNet-2012数据集，该数据集包含1000个类别：训练集包含1,281,167张图片，每个类别数据732至1300张不等，验证集包含50,000张图片，平均每个类别50张图片。
 
-Since ImageNet is too large to be downloaded and trained efficiently, we use [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) in this tutorial. The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. Figure 11 shows all the classes in CIFAR-10 as well as 10 images randomly sampled from each category.
+由于ImageNet数据集较大，下载和训练较慢，为了方便大家学习，我们使用[CIFAR10](<https://www.cs.toronto.edu/~kriz/cifar.html>)数据集。CIFAR10数据集包含60,000张32x32的彩色图片，10个类别，每个类包含6,000张。其中50,000张图片作为训练集，10000张作为测试集。图11从每个类别中随机抽取了10张图片，展示了所有的类别。
 
 <p align="center">
 <img src="image/cifar.png" width="350"><br/>
-Figure 11. CIFAR10 dataset[21]
+图11. CIFAR10数据集[21]
 </p>
 
- `paddle.datasets` package encapsulates multiple public datasets, including `cifar`, `imdb`, `mnist`, `moivelens` and `wmt14`, etc. There's no need to manually download and preprocess CIFAR-10.
+Paddle API提供了自动加载cifar数据集模块 `paddle.dataset.cifar`。
 
-After issuing a command `python train.py`, training will start immediately. The following sections describe the details:
+通过输入`python train.py`，就可以开始训练模型了，以下小节将详细介绍`train.py`的相关内容。
 
-## Model Structure
+### 模型结构
 
-### Initialize PaddlePaddle
+#### Paddle 初始化
 
-We must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
+通过 `paddle.init`，初始化Paddle是否使用GPU，trainer的数目等等。
 
 ```python
 import sys
@@ -219,29 +206,30 @@ from resnet import resnet_cifar10
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-As mentioned in section [Model Overview](#model-overview), here we provide the implementations of the VGG and ResNet models.
+本教程中我们提供了VGG和ResNet两个模型的配置。
 
-### VGG
+#### VGG
 
-First, we use a VGG network. Since the image size and amount of CIFAR10 are relatively small comparing to ImageNet, we use a small version of VGG network for CIFAR10. Convolution groups incorporate BN and dropout operations.
+首先介绍VGG模型结构，由于CIFAR10图片大小和数量相比ImageNet数据小很多，因此这里的模型针对CIFAR10数据做了一定的适配。卷积部分引入了BN和Dropout操作。
 
-1. Define input data and its dimension
+1. 定义数据输入及其维度
 
-    The input to the network is defined as `paddle.layer.data`, or image pixels in the context of image classification. The images in CIFAR10 are 32x32 color images of three channels. Therefore, the size of the input data is 3072 (3x32x32), and the number of categories is 10.
+    网络输入定义为 `data_layer` (数据层)，在图像分类中即为图像像素信息。CIFRAR10是RGB 3通道32x32大小的彩色图，因此输入数据大小为3072(3x32x32)，类别大小为10，即10分类。
 
     ```python
     datadim = 3 * 32 * 32
     classdim = 10
+
     image = paddle.layer.data(
         name="image", type=paddle.data_type.dense_vector(datadim))
     ```
 
-2. Define VGG main module
+2. 定义VGG网络核心模块
 
     ```python
     net = vgg_bn_drop(image)
     ```
-    The input to VGG main module is from the data layer. `vgg_bn_drop` defines a 16-layer VGG network, with each convolutional layer followed by BN and dropout layers. Here is the definition in detail:
+    VGG核心模块的输入是数据层，`vgg_bn_drop` 定义了16层VGG结构，每层卷积后面引入BN层和Dropout层，详细的定义如下：
 
     ```python
     def vgg_bn_drop(input):
@@ -274,15 +262,15 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
         return fc2
     ```
 
-    2.1. First, define a convolution block or conv_block. The default convolution kernel is 3x3, and the default pooling size is 2x2 with stride 2. Dropout specifies the probability in dropout operation. Function `img_conv_group` is defined in `paddle.networks` consisting of a series of `Conv->BN->ReLu->Dropout` and a `Pooling`.
+    2.1. 首先定义了一组卷积网络，即conv_block。卷积核大小为3x3，池化窗口大小为2x2，窗口滑动大小为2，groups决定每组VGG模块是几次连续的卷积操作，dropouts指定Dropout操作的概率。所使用的`img_conv_group`是在`paddle.networks`中预定义的模块，由若干组 Conv->BN->ReLu->Dropout 和 一组 Pooling 组成。
 
-    2.2. Five groups of convolutions. The first two groups perform two convolutions, while the last three groups perform three convolutions. The dropout rate of the last convolution in each group is set to 0, which means there is no dropout for this layer.
+    2.2. 五组卷积操作，即 5个conv_block。 第一、二组采用两次连续的卷积操作。第三、四、五组采用三次连续的卷积操作。每组最后一个卷积后面Dropout概率为0，即不使用Dropout操作。
 
-    2.3. The last two layers are fully-connected layers of dimension 512.
+    2.3. 最后接两层512维的全连接。
 
-3. Define Classifier
+3. 定义分类器
 
-    The above VGG network extracts high-level features and maps them to a vector of the same size as the categories. Softmax function or classifier is then used for calculating the probability of the image belonging to each category.
+    通过上面VGG网络提取高层特征，然后经过全连接层映射到类别维度大小的向量，再通过Softmax归一化得到每个类别的概率，也可称作分类器。
 
     ```python
     out = paddle.layer.fc(input=net,
@@ -290,9 +278,9 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
                           act=paddle.activation.Softmax())
     ```
 
-4. Define Loss Function and Outputs
+4. 定义损失函数和网络输出
 
-    In the context of supervised learning, labels of training images are defined in `paddle.layer.data` as well. During training, the cross-entropy loss function is used and the loss is the output of the network. During testing, the outputs are the probabilities calculated in the classifier.
+    在有监督训练中需要输入图像对应的类别信息，同样通过`paddle.layer.data`来定义。训练中采用多类交叉熵作为损失函数，并作为网络的输出，预测阶段定义网络的输出为分类器得到的概率信息。
 
     ```python
     lbl = paddle.layer.data(
@@ -302,20 +290,19 @@ First, we use a VGG network. Since the image size and amount of CIFAR10 are rela
 
 ### ResNet
 
-The first, third and fourth steps of a ResNet are the same as a VGG. The second one is the main module.
+ResNet模型的第1、3、4步和VGG模型相同，这里不再介绍。主要介绍第2步即CIFAR10数据集上ResNet核心模块。
 
 ```python
 net = resnet_cifar10(image, depth=56)
 ```
 
-Here are some basic functions used in `resnet_cifar10`:
-
-  - `conv_bn_layer` : convolutional layer followed by BN.
-  - `shortcut` : the shortcut branch in a residual block. There are two kinds of shortcuts: 1x1 convolution used when the number of channels between input and output is different; direct connection used otherwise.
+先介绍`resnet_cifar10`中的一些基本函数，再介绍网络连接过程。
 
-  - `basicblock` : a basic residual module as shown in the left of Figure 9, it consists of two sequential 3x3 convolutions and one "shortcut" branch.
-  - `bottleneck` : a bottleneck module as shown in the right of Figure 9, it consists of two 1x1 convolutions with one 3x3 convolution in between branch and a "shortcut" branch.
-  - `layer_warp` : a group of residual modules consisting of several stacking blocks. In each group, the sliding window size of the first residual block could be different from the rest of blocks, in order to reduce the size of feature maps along horizontal and vertical directions.
+  - `conv_bn_layer` : 带BN的卷积层。
+  - `shortcut` : 残差模块的"直连"路径，"直连"实际分两种形式：残差模块输入和输出特征通道数不等时，采用1x1卷积的升维操作；残差模块输入和输出通道相等时，采用直连操作。
+  - `basicblock` : 一个基础残差模块，即图9左边所示，由两组3x3卷积组成的路径和一条"直连"路径组成。
+  - `bottleneck` : 一个瓶颈残差模块，即图9右边所示，由上下1x1卷积和中间3x3卷积组成的路径和一条"直连"路径组成。
+  - `layer_warp` : 一组残差模块，由若干个残差模块堆积而成。每组中第一个残差模块滑动窗口大小与其他可以不同，以用来减少特征图在垂直和水平方向的大小。
 
 ```python
 def conv_bn_layer(input,
@@ -357,13 +344,13 @@ def layer_warp(block_func, ipt, features, count, stride):
     return tmp
 ```
 
-The following are the components of `resnet_cifar10`:
+`resnet_cifar10` 的连接结构主要有以下几个过程。
 
-1. The lowest level is `conv_bn_layer`.
-2. The middle level consists of three `layer_warp`, each of which uses the left residual block in Figure 9.
-3. The last level is average pooling layer.
+1. 底层输入连接一层 `conv_bn_layer`，即带BN的卷积层。
+2. 然后连接3组残差模块即下面配置3组 `layer_warp` ，每组采用图 10 左边残差模块组成。
+3. 最后对网络做均值池化并返回该层。
 
-Note: besides the first convolutional layer and the last fully-connected layer, the total number of layers in three `layer_warp` should be dividable by 6, that is the depth of `resnet_cifar10` should satisfy $(depth - 2) % 6 == 0$.
+注意：除过第一层卷积层和最后一层全连接层之外，要求三组 `layer_warp` 总的含参层数能够被6整除，即 `resnet_cifar10` 的 depth 要满足 $(depth - 2) % 6 == 0$ 。
 
 ```python
 def resnet_cifar10(ipt, depth=32):
@@ -381,21 +368,26 @@ def resnet_cifar10(ipt, depth=32):
     return pool
 ```
 
-## Model Training
+## 训练模型
 
-### Define Parameters
+### 定义参数
 
-First, we create the model parameters according to the previous model configuration `cost`.
+首先依据模型配置的`cost`定义模型参数。
 
 ```python
 # Create parameters
 parameters = paddle.parameters.create(cost)
 ```
 
-### Create Trainer
+可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
 
-Before creating a training module, it is necessary to set the algorithm.
-Here we specify `Momentum` optimization algorithm via `paddle.optimizer`.
+```python
+print parameters.keys()
+```
+
+### 构造训练(Trainer)
+
+根据网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的Momentum方法，同时设定了学习率、正则等。
 
 ```python
 # Create optimizer
@@ -405,8 +397,7 @@ momentum_optimizer = paddle.optimizer.Momentum(
     learning_rate=0.1 / 128.0,
     learning_rate_decay_a=0.1,
     learning_rate_decay_b=50000 * 100,
-    learning_rate_schedule='discexp',
-    batch_size=128)
+    learning_rate_schedule='discexp')
 
 # Create trainer
 trainer = paddle.trainer.SGD(cost=cost,
@@ -414,13 +405,14 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=momentum_optimizer)
 ```
 
-The learning rate adjustment policy can be defined with variables `learning_rate_decay_a`($a$), `learning_rate_decay_b`($b$) and `learning_rate_schedule`. In this example, discrete exponential method is used for adjusting learning rate. The formula is as follows,
+通过 `learning_rate_decay_a` (简写$a$） 、`learning_rate_decay_b` (简写$b$) 和 `learning_rate_schedule` 指定学习率调整策略，这里采用离散指数的方式调节学习率，计算公式如下， $n$ 代表已经处理过的累计总样本数，$lr_{0}$ 即为 `settings` 里设置的 `learning_rate`。
+
 $$  lr = lr_{0} * a^ {\lfloor \frac{n}{ b}\rfloor} $$
-where $n$ is the number of processed samples, $lr_{0}$ is the learning_rate.
 
-### Training
 
-`cifar.train10()` will yield records during each pass, after shuffling, a batch input is generated for training.
+### 训练
+
+cifar.train10()每次产生一条样本，在完成shuffle和batch之后，作为训练的输入。
 
 ```python
 reader=paddle.batch(
@@ -429,19 +421,47 @@ reader=paddle.batch(
         batch_size=128)
 ```
 
-`feeding` is devoted to specifying the correspondence between each yield record and `paddle.layer.data`. For instance,
- the first column of data generated by `cifar.train10()` corresponds to image layer's feature.
+通过`feeding`来指定每一个数据和`paddle.layer.data`的对应关系。例如: `cifar.train10()`产生数据的第0列对应image层的特征。
 
 ```python
 feeding={'image': 0,
          'label': 1}
 ```
 
-Callback function `event_handler` will be called during training when a pre-defined event happens.
+可以使用`event_handler`回调函数来观察训练过程，或进行测试等, 该回调函数是`trainer.train`函数里设定。
+
+`event_handler_plot`可以用来利用回调数据来打点画图:
+
+![png](./image/train_and_test.png)
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 1 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+
+        result = trainer.test(
+            reader=paddle.batch(
+                paddle.dataset.cifar.test10(), batch_size=128),
+            feeding=feeding)
+        cost_ploter.append(test_title, step, result.cost)
+```
 
+`event_handler` 用来在训练过程中输出文本日志
 
 ```python
-# event handler to track training and testing process
+# End batch and end pass event handler
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -451,6 +471,10 @@ def event_handler(event):
             sys.stdout.write('.')
             sys.stdout.flush()
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(
             reader=paddle.batch(
                 paddle.dataset.cifar.test10(), batch_size=128),
@@ -458,17 +482,17 @@ def event_handler(event):
         print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
-Finally, we can invoke `trainer.train` to start training:
+通过`trainer.train`函数训练:
 
 ```python
 trainer.train(
     reader=reader,
     num_passes=200,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding)
 ```
 
-Here is an example log after training for one pass. The average error rates are 0.6875 on the training set and 0.8852 on the validation set.
+一轮训练log示例如下所示，经过1个pass， 训练集上平均error为0.6875 ，测试集上平均error为0.8852 。
 
 ```text
 Pass 0, Batch 0, Cost 2.473182, {'classification_error_evaluator': 0.9140625}
@@ -482,21 +506,55 @@ Pass 0, Batch 300, Cost 1.668833, {'classification_error_evaluator': 0.6875}
 Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 ```
 
-Figure 12 shows the curve of training error rate, which indicates it converges at Pass 200 with error rate 8.54%.
+图12是训练的分类错误率曲线图，运行到第200个pass后基本收敛，最终得到测试集上分类错误率为8.54%。
+
 <p align="center">
-<img src="image/plot_en.png" width="400" ><br/>
-Figure 12. The error rate of VGG model on CIFAR10
+<img src="image/plot.png" width="400" ><br/>
+图12. CIFAR10数据集上VGG模型的分类错误率
 </p>
 
+## 应用模型
+
+可以使用训练好的模型对图片进行分类，下面程序展示了如何使用`paddle.infer`接口进行推断，可以打开注释，更改加载的模型。
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file)
+    im = im.resize((32, 32), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32)
+    # PIL打开图片存储顺序为H(高度)，W(宽度)，C(通道)。
+    # PaddlePaddle要求数据顺序为CHW，所以需要转换顺序。
+    im = im.transpose((2, 0, 1)) # CHW
+    # CIFAR训练图片通道顺序为B(蓝),G(绿),R(红),
+    # 而PIL打开图片默认通道顺序为RGB,因为需要交换通道。
+    im = im[(2, 1, 0),:,:] # BGR
+    im = im.flatten()
+    im = im / 255.0
+    return im
+
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/dog.png'),))
+
+# with open('params_pass_50.tar', 'r') as f:
+#    parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=out, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/dog.png is: %d" % lab[0][0]
+```
 
-After training is done, the model from each pass is saved in `output/pass-%05d`. For example, the model of Pass 300 is saved in `output/pass-00299`.
 
-## Conclusion
+## 总结
 
-Traditional image classification methods have complicated frameworks that involve multiple stages of processing. In contrast, CNN models can be trained end-to-end with a significant increase in classification accuracy. In this chapter, we introduced three models -- VGG, GoogleNet, ResNet and provided PaddlePaddle config files for training VGG and ResNet on CIFAR10. We also explained how to perform prediction and feature extraction using the PaddlePaddle API. For other datasets such as ImageNet, the procedure for config and training are the same and you are welcome to give it a try.
+传统图像分类方法由多个阶段构成，框架较为复杂，而端到端的CNN模型结构可一步到位，而且大幅度提升了分类准确率。本文我们首先介绍VGG、GoogleNet、ResNet三个经典的模型；然后基于CIFAR10数据集，介绍如何使用PaddlePaddle配置和训练CNN模型，尤其是VGG和ResNet模型；最后介绍如何使用PaddlePaddle的API接口对图片进行预测和特征提取。对于其他数据集比如ImageNet，配置和训练流程是同样的，大家可以自行进行实验。
 
 
-## Reference
+## 参考文献
 
 [1] D. G. Lowe, [Distinctive image features from scale-invariant keypoints](http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf). IJCV, 60(2):91-110, 2004.
 
@@ -543,7 +601,7 @@ Traditional image classification methods have complicated frameworks that involv
 [22] http://cs231n.github.io/classification/
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

03.image_classification/index.html

@@ -40,161 +40,174 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-# 图像分类
+Image Classification
+=======================
 
-本教程源代码目录在[book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+The source code for this chapter is at [book/image_classification](https://github.com/PaddlePaddle/book/tree/develop/03.image_classification). First-time users, please refer to PaddlePaddle [Installation Tutorial](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book) for installation instructions.
 
-## 背景介绍
+## Background
 
-图像相比文字能够提供更加生动、容易理解及更具艺术感的信息，是人们转递与交换信息的重要来源。在本教程中，我们专注于图像识别领域的一个重要问题，即图像分类。
+Compared to words, images provide much more vivid and easier to understand information with an artistic sense. They are an important source for people to express and exchange ideas. In this chapter, we focus on one of the essential problems in image recognition -- image classification.
 
-图像分类是根据图像的语义信息将不同类别图像区分开来，是计算机视觉中重要的基本问题，也是图像检测、图像分割、物体跟踪、行为分析等其他高层视觉任务的基础。图像分类在很多领域有广泛应用，包括安防领域的人脸识别和智能视频分析等，交通领域的交通场景识别，互联网领域基于内容的图像检索和相册自动归类，医学领域的图像识别等。
+Image classification is the task of distinguishing images in different categories based on their semantic meaning. It is a core problem in computer vision and is also the foundation of other higher level computer vision tasks such as object detection, image segmentation, object tracking, action recognition, etc. Image classification has applications in many areas such as face recognition, intelligent video analysis in security systems, traffic scene recognition in transportation systems, content-based image retrieval and automatic photo indexing in web services, image classification in medicine, etc.
 
+To classify an image we first encode the entire image using handcrafted or learned features and then determine the category using a classifier. Thus, feature extraction plays an important role in image classification. Prior to deep learning the BoW(Bag of Words) model was the most widely used method for classifying an image as well as an object. The BoW technique was introduced in Natural Language Processing where a training sentence is represented as a bag of words. In the context of image classification, the BoW model requires constructing a dictionary. The simplest BoW framework can be designed with three steps: **feature extraction**, **feature encoding** and **classifier design**.
 
-一般来说，图像分类通过手工特征或特征学习方法对整个图像进行全部描述，然后使用分类器判别物体类别，因此如何提取图像的特征至关重要。在深度学习算法之前使用较多的是基于词袋(Bag of Words)模型的物体分类方法。词袋方法从自然语言处理中引入，即一句话可以用一个装了词的袋子表示其特征，袋子中的词为句子中的单词、短语或字。对于图像而言，词袋方法需要构建字典。最简单的词袋模型框架可以设计为**底层特征抽取**、**特征编码**、**分类器设计**三个过程。
+Using Deep learning, image classification can be framed as a supervised or unsupervised learning problem that uses hierarchical features automatically without any need for manually crafted features from the image. In recent years, Convolutional Neural Networks (CNNs) have made significant progress in image classification. CNNs use raw image pixels as input, extract low-level and high-level abstract features through convolution operations, and directly output the classification results from the model. This style of end-to-end learning has lead to not only increased performance but also wider adoption various applications.
 
-而基于深度学习的图像分类方法，可以通过有监督或无监督的方式**学习**层次化的特征描述，从而取代了手工设计或选择图像特征的工作。深度学习模型中的卷积神经网络(Convolution Neural Network, CNN)近年来在图像领域取得了惊人的成绩，CNN直接利用图像像素信息作为输入，最大程度上保留了输入图像的所有信息，通过卷积操作进行特征的提取和高层抽象，模型输出直接是图像识别的结果。这种基于"输入-输出"直接端到端的学习方法取得了非常好的效果，得到了广泛的应用。
+In this chapter, we introduce deep-learning-based image classification methods and explain how to train a CNN model using PaddlePaddle.
 
-本教程主要介绍图像分类的深度学习模型，以及如何使用PaddlePaddle训练CNN模型。
+## Demonstration
 
-## 效果展示
+An image can be classified by a general as well as fine-grained image classifier.
 
-图像分类包括通用图像分类、细粒度图像分类等。图1展示了通用图像分类效果，即模型可以正确识别图像上的主要物体。
+
+Figure 1 shows the results of a general image classifier -- the trained model can correctly recognize the main objects in the images.
 
 <p align="center">
 <img src="image/dog_cat.png "  width="350" ><br/>
-图1. 通用图像分类展示
+Figure 1. General image classification
 </p>
 
 
-图2展示了细粒度图像分类-花卉识别的效果，要求模型可以正确识别花的类别。
-
+Figure 2 shows the results of a fine-grained image classifier. This task of flower recognition requires correctly recognizing of the flower's categories.
 
 <p align="center">
 <img src="image/flowers.png" width="400" ><br/>
-图2. 细粒度图像分类展示
+Figure 2. Fine-grained image classification
 </p>
 
 
-一个好的模型既要对不同类别识别正确，同时也应该能够对不同视角、光照、背景、变形或部分遮挡的图像正确识别(这里我们统一称作图像扰动)。图3展示了一些图像的扰动，较好的模型会像聪明的人类一样能够正确识别。
+A good model should recognize objects of different categories correctly. The results of such a model should not vary due to viewpoint variation, illumination conditions, object distortion or occlusion.
+Figure 3 shows some images with various disturbances. A good model should classify these images correctly like humans.
 
 <p align="center">
-<img src="image/variations.png" width="550" ><br/>
-图3. 扰动图片展示[22]
+<img src="image/variations_en.png" width="550" ><br/>
+Figure 3. Disturbed images [22]
 </p>
 
-## 模型概览
+## Model Overview
+
+A large amount of research in image classification is built upon public datasets such as [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/), [ImageNet](http://image-net.org/) etc. Many image classification algorithms are usually evaluated and compared on these datasets. PASCAL VOC is a computer vision competition started in 2005, and ImageNet is a dataset for Large Scale Visual Recognition Challenge (ILSVRC) started in 2010. In this chapter, we introduce some image classification models from the submissions to these competitions.
+
+Before 2012, traditional image classification was accomplished with the three steps described in the background section. A complete model construction usually involves the following stages: low-level feature extraction, feature encoding, spatial constraint or feature clustering, classifier design, model ensemble.
+
+  1). **Low-level feature extraction**: This step extracts large amounts of local features according to fixed strides and scales. Popular local features include Scale-Invariant Feature Transform (SIFT)[1], Histogram of Oriented Gradient(HOG)[2], Local Binary Pattern(LBP)[3], etc. A common practice is to employ multiple feature descriptors in order to avoid missing a lot of information.
+
+  2). **Feature encoding**: Low-level features contain a large amount of redundancy and noise. In order to improve the robustness of features, it is necessary to employ a feature transformation to encode low-level features. This is called feature encoding. Common feature encoding methods include vector quantization [4], sparse coding [5], locality-constrained linear coding [6], Fisher vector encoding [7], etc.
 
-图像识别领域大量的研究成果都是建立在[PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)、[ImageNet](http://image-net.org/)等公开的数据集上，很多图像识别算法通常在这些数据集上进行测试和比较。PASCAL VOC是2005年发起的一个视觉挑战赛，ImageNet是2010年发起的大规模视觉识别竞赛(ILSVRC)的数据集，在本章中我们基于这些竞赛的一些论文介绍图像分类模型。
+  3). **Spatial constraint**: Spatial constraint or feature clustering is usually adopted after feature encoding for extracting the maximum or average of each dimension in the spatial domain. Pyramid feature matching--a popular feature clustering method--divides an image uniformly into patches and performs feature clustering in each patch.
 
-在2012年之前的传统图像分类方法可以用背景描述中提到的三步完成，但通常完整建立图像识别模型一般包括底层特征学习、特征编码、空间约束、分类器设计、模型融合等几个阶段。
-  1). **底层特征提取**: 通常从图像中按照固定步长、尺度提取大量局部特征描述。常用的局部特征包括SIFT(Scale-Invariant Feature Transform, 尺度不变特征转换) \[[1](#参考文献)\]、HOG(Histogram of Oriented Gradient, 方向梯度直方图) \[[2](#参考文献)\]、LBP(Local Bianray Pattern, 局部二值模式) \[[3](#参考文献)\] 等，一般也采用多种特征描述子，防止丢失过多的有用信息。
-  2). **特征编码**: 底层特征中包含了大量冗余与噪声，为了提高特征表达的鲁棒性，需要使用一种特征变换算法对底层特征进行编码，称作特征编码。常用的特征编码包括向量量化编码 \[[4](#参考文献)\]、稀疏编码 \[[5](#参考文献)\]、局部线性约束编码 \[[6](#参考文献)\]、Fisher向量编码 \[[7](#参考文献)\] 等。
-  3). **空间特征约束**: 特征编码之后一般会经过空间特征约束，也称作**特征汇聚**。特征汇聚是指在一个空间范围内，对每一维特征取最大值或者平均值，可以获得一定特征不变形的特征表达。金字塔特征匹配是一种常用的特征聚会方法，这种方法提出将图像均匀分块，在分块内做特征汇聚。
-  4). **通过分类器分类**: 经过前面步骤之后一张图像可以用一个固定维度的向量进行描述，接下来就是经过分类器对图像进行分类。通常使用的分类器包括SVM(Support Vector Machine, 支持向量机)、随机森林等。而使用核方法的SVM是最为广泛的分类器，在传统图像分类任务上性能很好。
+  4). **Classification**: In the above steps an image can be described by a vector of fixed dimension. Then a classifier can be used to classify the image into categories. Common classifiers include Support Vector Machine(SVM), random forest etc. Kernel SVM is the most popular classifier and has achieved very good performance in traditional image classification tasks.
 
-这种方法在PASCAL VOC竞赛中的图像分类算法中被广泛使用 \[[18](#参考文献)\]。[NEC实验室](http://www.nec-labs.com/)在ILSVRC2010中采用SIFT和LBP特征，两个非线性编码器以及SVM分类器获得图像分类的冠军 \[[8](#参考文献)\]。
+This method has been used widely as image classification algorithm in PASCAL VOC [18]. NEC Labs(http://www.nec-labs.com/) won the championship by employing SIFT and LBP features, two non-linear encoders and SVM in ILSVRC 2010 [8].
 
-Alex Krizhevsky在2012年ILSVRC提出的CNN模型 \[[9](#参考文献)\] 取得了历史性的突破，效果大幅度超越传统方法，获得了ILSVRC2012冠军，该模型被称作AlexNet。这也是首次将深度学习用于大规模图像分类中。从AlexNet之后，涌现了一系列CNN模型，不断地在ImageNet上刷新成绩，如图4展示。随着模型变得越来越深以及精妙的结构设计，Top-5的错误率也越来越低，降到了3.5%附近。而在同样的ImageNet数据集上，人眼的辨识错误率大概在5.1%，也就是目前的深度学习模型的识别能力已经超过了人眼。
+The CNN model--AlexNet proposed by Alex Krizhevsky et al.[9], made a breakthrough in ILSVRC 2012. It dramatically outperformed traditional methods and won the ILSVRC championship in 2012. This was also the first time that a deep learning method was used for large-scale image classification. Since AlexNet, a series of CNN models have been proposed that have advanced the state of the art steadily on Imagenet as shown in Figure 4. With deeper and more sophisticated architectures, Top-5 error rate is getting lower and lower (to around 3.5%). The error rate of human raters on the same Imagenet dataset is 5.1%, which means that the image classification capability of a deep learning model has surpassed human raters.
 
 <p align="center">
 <img src="image/ilsvrc.png" width="500" ><br/>
-图4. ILSVRC图像分类Top-5错误率
+Figure 4. Top-5 error rates on ILSVRC image classification
 </p>
 
 ### CNN
 
-传统CNN包含卷积层、全连接层等组件，并采用softmax多类别分类器和多类交叉熵损失函数，一个典型的卷积神经网络如图5所示，我们先介绍用来构造CNN的常见组件。
+Traditional CNNs consist of convolutional and fully-connected layers and use the softmax multi-category classifier with the cross-entropy loss function. Figure 5 shows a typical CNN. We first introduce the common components of a CNN.
 
 <p align="center">
-<img src="image/lenet.png"><br/>
-图5. CNN网络示例[20]
+<img src="image/lenet_en.png"><br/>
+Figure 5. A CNN example [20]
 </p>
 
-- 卷积层(convolution layer): 执行卷积操作提取底层到高层的特征，发掘出图片局部关联性质和空间不变性质。
-- 池化层(pooling layer): 执行降采样操作。通过取卷积输出特征图中局部区块的最大值(max-pooling)或者均值(avg-pooling)。降采样也是图像处理中常见的一种操作，可以过滤掉一些不重要的高频信息。
-- 全连接层(fully-connected layer，或者fc layer): 输入层到隐藏层的神经元是全部连接的。
-- 非线性变化: 卷积层、全连接层后面一般都会接非线性变化层，例如Sigmoid、Tanh、ReLu等来增强网络的表达能力，在CNN里最常使用的为ReLu激活函数。
-- Dropout \[[10](#参考文献)\] : 在模型训练阶段随机让一些隐层节点权重不工作，提高网络的泛化能力，一定程度上防止过拟合。
+- convolutional layer: this layer uses the convolution operation to extract (low-level and high-level) features and to discover local correlation and spatial invariance.
 
-另外，在训练过程中由于每层参数不断更新，会导致下一次输入分布发生变化，这样导致训练过程需要精心设计超参数。如2015年Sergey Ioffe和Christian Szegedy提出了Batch Normalization (BN)算法 \[[14](#参考文献)\] 中，每个batch对网络中的每一层特征都做归一化，使得每层分布相对稳定。BN算法不仅起到一定的正则作用，而且弱化了一些超参数的设计。经过实验证明，BN算法加速了模型收敛过程，在后来较深的模型中被广泛使用。
+- pooling layer: this layer down samples feature maps by extracting local max (max-pooling) or average (avg-pooling) value of each patch in the feature map. Down-sampling is a common operation in image processing and is used to filter out high-frequency information.
 
-接下来我们主要介绍VGG，GoogleNet和ResNet网络结构。
+- fully-connected layer: this layer fully connects neurons between two adjacent layers.
+
+- non-linear activation: Convolutional and fully-connected layers are usually followed by some non-linear activation layers. Non-linearities enhance the expression capability of the network. Some examples of non-linear activation functions are Sigmoid, Tanh and ReLU. ReLU is the most commonly used activation function in CNN.
+
+- Dropout [10]: At each training stage, individual nodes are dropped out of the network with a certain probability. This improves the network's ability to generalize and avoids overfitting.
+
+Parameter updates at each layer during training causes input layer distributions to change and in turn requires hyper-parameters to be careful tuned. In 2015, Sergey Ioffe and Christian Szegedy proposed a Batch Normalization (BN) algorithm [14], which normalizes the features of each batch in a layer, and enables relatively stable distribution in each layer. Not only does BN algorithm act as a regularizer, but also reduces the need for careful hyper-parameter design. Experiments demonstrate that BN algorithm accelerates the training convergence and has been widely used in later deeper models.
+
+In the following sections, we will introduce the following network architectures - VGG, GoogleNet and ResNets.
 
 ### VGG
 
-牛津大学VGG(Visual Geometry Group)组在2014年ILSVRC提出的模型被称作VGG模型 \[[11](#参考文献)\] 。该模型相比以往模型进一步加宽和加深了网络结构，它的核心是五组卷积操作，每两组之间做Max-Pooling空间降维。同一组内采用多次连续的3X3卷积，卷积核的数目由较浅组的64增多到最深组的512，同一组内的卷积核数目是一样的。卷积之后接两层全连接层，之后是分类层。由于每组内卷积层的不同，有11、13、16、19层这几种模型，下图展示一个16层的网络结构。VGG模型结构相对简洁，提出之后也有很多文章基于此模型进行研究，如在ImageNet上首次公开超过人眼识别的模型\[[19](#参考文献)\]就是借鉴VGG模型的结构。
+The Oxford Visual Geometry Group (VGG) proposed the VGG network in ILSVRC 2014 [11]. This model is deeper and wider than previous neural architectures. It consists of five main groups of convolution operations. Adjacent convolution groups are connected via max-pooling layers. Each group contains a series of 3x3 convolutional layers (i.e. kernels). The number of convolution kernels stays the same within the group and increases from 64 in the first group to 512 in the last one. The total number of learnable layers could be 11, 13, 16, or 19 depending on the number of convolutional layers in each group. Figure 6 illustrates a 16-layer VGG. The neural architecture of VGG is relatively simple and has been adopted by many papers such as the first one that surpassed human-level performance on ImageNet [19].
 
 <p align="center">
 <img src="image/vgg16.png" width="750" ><br/>
-图6. 基于ImageNet的VGG16模型
+Figure 6. VGG16 model for ImageNet
 </p>
 
 ### GoogleNet
 
-GoogleNet \[[12](#参考文献)\] 在2014年ILSVRC的获得了冠军，在介绍该模型之前我们先来了解NIN(Network in Network)模型 \[[13](#参考文献)\] 和Inception模块，因为GoogleNet模型由多组Inception模块组成，模型设计借鉴了NIN的一些思想。
+GoogleNet [12] won the ILSVRC championship in 2014. GoogleNet borrowed some ideas from the Network in Network(NIN) model [13] and is built on the Inception blocks. Let us first familiarize ourselves with these first.
+
+The two main characteristics of the NIN model are:
+
+1) A single-layer convolutional network is replaced with a Multi-Layer Perceptron Convolution (MLPconv). MLPconv is a tiny multi-layer convolutional network. It enhances non-linearity by adding several 1x1 convolutional layers after linear ones.
 
-NIN模型主要有两个特点：1) 引入了多层感知卷积网络(Multi-Layer Perceptron Convolution, MLPconv)代替一层线性卷积网络。MLPconv是一个微小的多层卷积网络，即在线性卷积后面增加若干层1x1的卷积，这样可以提取出高度非线性特征。2) 传统的CNN最后几层一般都是全连接层，参数较多。而NIN模型设计最后一层卷积层包含类别维度大小的特征图，然后采用全局均值池化(Avg-Pooling)替代全连接层，得到类别维度大小的向量，再进行分类。这种替代全连接层的方式有利于减少参数。
+2) In traditional CNNs, the last fewer layers are usually fully-connected with a large number of parameters. In contrast, NIN replaces all fully-connected layers with convolutional layers with feature maps of the same size as the category dimension and a global average pooling. This replacement of fully-connected layers significantly reduces the number of parameters.
 
-Inception模块如下图7所示，图(a)是最简单的设计，输出是3个卷积层和一个池化层的特征拼接。这种设计的缺点是池化层不会改变特征通道数，拼接后会导致特征的通道数较大，经过几层这样的模块堆积后，通道数会越来越大，导致参数和计算量也随之增大。为了改善这个缺点，图(b)引入3个1x1卷积层进行降维，所谓的降维就是减少通道数，同时如NIN模型中提到的1x1卷积也可以修正线性特征。
+Figure 7 depicts two Inception blocks. Figure 7(a) is the simplest design. The output is a concatenation of features from three convolutional layers and one pooling layer. The disadvantage of this design is that the pooling layer does not change the number of filters and leads to an increase in the number of outputs. After several of such blocks, the number of outputs and parameters become larger and larger and lead to higher computation complexity. To overcome this drawback, the Inception block in Figure 7(b) employs three 1x1 convolutional layers. These reduce dimensions or the number of channels but improve the non-linearity of the network.
 
 <p align="center">
-<img src="image/inception.png" width="800" ><br/>
-图7. Inception模块
+<img src="image/inception_en.png" width="800" ><br/>
+Figure 7. Inception block
 </p>
 
-GoogleNet由多组Inception模块堆积而成。另外，在网络最后也没有采用传统的多层全连接层，而是像NIN网络一样采用了均值池化层；但与NIN不同的是，池化层后面接了一层到类别数映射的全连接层。除了这两个特点之外，由于网络中间层特征也很有判别性，GoogleNet在中间层添加了两个辅助分类器，在后向传播中增强梯度并且增强正则化，而整个网络的损失函数是这个三个分类器的损失加权求和。
+GoogleNet consists of multiple stacked Inception blocks followed by an avg-pooling layer as in NIN instead of traditional fully connected layers. The difference between GoogleNet and NIN is that GoogleNet adds a fully connected layer after avg-pooling layer to output a vector of category size. Besides these two characteristics, the features from middle layers of a GoogleNet are also very discriminative. Therefore, GoogeleNet inserts two auxiliary classifiers in the model for enhancing gradient and regularization when doing backpropagation. The loss function of the whole network is the weighted sum of these three classifiers.
 
-GoogleNet整体网络结构如图8所示，总共22层网络：开始由3层普通的卷积组成；接下来由三组子网络组成，第一组子网络包含2个Inception模块，第二组包含5个Inception模块，第三组包含2个Inception模块；然后接均值池化层、全连接层。
+Figure 8 illustrates the neural architecture of a GoogleNet which consists of 22 layers: it starts with three regular convolutional layers followed by three groups of sub-networks -- the first group contains two Inception blocks, the second one five, and the third one two. It ends up with an average pooling and a fully-connected layer.
 
 <p align="center">
 <img src="image/googlenet.jpeg" ><br/>
-图8. GoogleNet[12]
+Figure 8. GoogleNet[12]
 </p>
 
-
-上面介绍的是GoogleNet第一版模型(称作GoogleNet-v1)。GoogleNet-v2 \[[14](#参考文献)\] 引入BN层；GoogleNet-v3 \[[16](#参考文献)\] 对一些卷积层做了分解，进一步提高网络非线性能力和加深网络；GoogleNet-v4 \[[17](#参考文献)\] 引入下面要讲的ResNet设计思路。从v1到v4每一版的改进都会带来准确度的提升，介于篇幅，这里不再详细介绍v2到v4的结构。
-
+The above model is the first version of GoogleNet or GoogelNet-v1. GoogleNet-v2 [14] introduced BN layer; GoogleNet-v3 [16] further split some convolutional layers, which increases non-linearity and network depth; GoogelNet-v4 [17] leads to the design idea of ResNet which will be introduced in the next section. The evolution from v1 to v4 improved the accuracy rate consistently. We will not go into details of the neural architectures of v2 to v4.
 
 ### ResNet
 
-ResNet(Residual Network) \[[15](#参考文献)\] 是2015年ImageNet图像分类、图像物体定位和图像物体检测比赛的冠军。针对训练卷积神经网络时加深网络导致准确度下降的问题，ResNet提出了采用残差学习。在已有设计思路(BN, 小卷积核，全卷积网络)的基础上，引入了残差模块。每个残差模块包含两条路径，其中一条路径是输入特征的直连通路，另一条路径对该特征做两到三次卷积操作得到该特征的残差，最后再将两条路径上的特征相加。
+Residual Network(ResNet)[15] won the 2015 championship on three ImageNet competitions -- image classification, object localization, and object detection. The main challenge in training deeper networks is that accuracy degrades with network depth. The authors of ResNet proposed a residual learning approach to ease the difficulty of training deeper networks. Based on the design ideas of BN, small convolutional kernels, full convolutional network, ResNets reformulate the layers as residual blocks, with each block containing two branches, one directly connecting input to the output, the other performing two to three convolutions and calculating the residual function with reference to the layer inputs. The outputs of these two branches are then added up.
 
-残差模块如图9所示，左边是基本模块连接方式，由两个输出通道数相同的3x3卷积组成。右边是瓶颈模块(Bottleneck)连接方式，之所以称为瓶颈，是因为上面的1x1卷积用来降维(图示例即256->64)，下面的1x1卷积用来升维(图示例即64->256)，这样中间3x3卷积的输入和输出通道数都较小(图示例即64->64)。
+Figure 9 illustrates the ResNet architecture. To the left is the basic building block, it consists of two 3x3 convolutional layers of the same channels. To the right is a Bottleneck block. The bottleneck is a 1x1 convolutional layer used to reduce dimension from 256 to 64. The other 1x1 convolutional layer is used to increase dimension from 64 to 256. Thus, the number of input and output channels of the middle 3x3 convolutional layer is 64, which is relatively small.
 
 <p align="center">
 <img src="image/resnet_block.jpg" width="400"><br/>
-图9. 残差模块
+Figure 9. Residual block
 </p>
 
-图10展示了50、101、152层网络连接示意图，使用的是瓶颈模块。这三个模型的区别在于每组中残差模块的重复次数不同(见图右上角)。ResNet训练收敛较快，成功的训练了上百乃至近千层的卷积神经网络。
+Figure 10 illustrates ResNets with 50, 101, 152 layers, respectively. All three networks use bottleneck blocks of different numbers of repetitions. ResNet converges very fast and can be trained with hundreds or thousands of layers.
 
 <p align="center">
 <img src="image/resnet.png"><br/>
-图10. 基于ImageNet的ResNet模型
+Figure 10. ResNet model for ImageNet
 </p>
 
 
-## 数据准备
+## Dataset
 
-通用图像分类公开的标准数据集常用的有[CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html)、[ImageNet](http://image-net.org/)、[COCO](http://mscoco.org/)等，常用的细粒度图像分类数据集包括[CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html)、[Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/)、[Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/)等。其中ImageNet数据集规模相对较大，如[模型概览](#模型概览)一章所讲，大量研究成果基于ImageNet。ImageNet数据从2010年来稍有变化，常用的是ImageNet-2012数据集，该数据集包含1000个类别：训练集包含1,281,167张图片，每个类别数据732至1300张不等，验证集包含50,000张图片，平均每个类别50张图片。
+Commonly used public datasets for image classification are [CIFAR](https://www.cs.toronto.edu/~kriz/cifar.html), [ImageNet](http://image-net.org/), [COCO](http://mscoco.org/), etc. Those used for fine-grained image classification are [CUB-200-2011](http://www.vision.caltech.edu/visipedia/CUB-200-2011.html), [Stanford Dog](http://vision.stanford.edu/aditya86/ImageNetDogs/), [Oxford-flowers](http://www.robots.ox.ac.uk/~vgg/data/flowers/), etc. Among these, the ImageNet dataset is the largest. Most research results are reported on ImageNet as mentioned in the Model Overview section. Since 2010, the ImageNet dataset has gone through some changes. The commonly used ImageNet-2012 dataset contains 1000 categories. There are 1,281,167 training images, ranging from 732 to 1200 images per category, and 50,000 validation images with 50 images per category in average.
 
-由于ImageNet数据集较大，下载和训练较慢，为了方便大家学习，我们使用[CIFAR10](<https://www.cs.toronto.edu/~kriz/cifar.html>)数据集。CIFAR10数据集包含60,000张32x32的彩色图片，10个类别，每个类包含6,000张。其中50,000张图片作为训练集，10000张作为测试集。图11从每个类别中随机抽取了10张图片，展示了所有的类别。
+Since ImageNet is too large to be downloaded and trained efficiently, we use [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) in this tutorial. The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. Figure 11 shows all the classes in CIFAR-10 as well as 10 images randomly sampled from each category.
 
 <p align="center">
 <img src="image/cifar.png" width="350"><br/>
-图11. CIFAR10数据集[21]
+Figure 11. CIFAR10 dataset[21]
 </p>
 
-Paddle API提供了自动加载cifar数据集模块 `paddle.dataset.cifar`。
+ `paddle.datasets` package encapsulates multiple public datasets, including `cifar`, `imdb`, `mnist`, `moivelens` and `wmt14`, etc. There's no need to manually download and preprocess CIFAR-10.
 
-通过输入`python train.py`，就可以开始训练模型了，以下小节将详细介绍`train.py`的相关内容。
+After issuing a command `python train.py`, training will start immediately. The following sections describe the details:
 
-### 模型结构
+## Model Structure
 
-#### Paddle 初始化
+### Initialize PaddlePaddle
 
-通过 `paddle.init`，初始化Paddle是否使用GPU，trainer的数目等等。
+We must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
 
 ```python
 import sys
@@ -206,30 +219,29 @@ from resnet import resnet_cifar10
 paddle.init(use_gpu=False, trainer_count=1)
 ```
 
-本教程中我们提供了VGG和ResNet两个模型的配置。
+As mentioned in section [Model Overview](#model-overview), here we provide the implementations of the VGG and ResNet models.
 
-#### VGG
+### VGG
 
-首先介绍VGG模型结构，由于CIFAR10图片大小和数量相比ImageNet数据小很多，因此这里的模型针对CIFAR10数据做了一定的适配。卷积部分引入了BN和Dropout操作。
+First, we use a VGG network. Since the image size and amount of CIFAR10 are relatively small comparing to ImageNet, we use a small version of VGG network for CIFAR10. Convolution groups incorporate BN and dropout operations.
 
-1. 定义数据输入及其维度
+1. Define input data and its dimension
 
-    网络输入定义为 `data_layer` (数据层)，在图像分类中即为图像像素信息。CIFRAR10是RGB 3通道32x32大小的彩色图，因此输入数据大小为3072(3x32x32)，类别大小为10，即10分类。
+    The input to the network is defined as `paddle.layer.data`, or image pixels in the context of image classification. The images in CIFAR10 are 32x32 color images of three channels. Therefore, the size of the input data is 3072 (3x32x32), and the number of categories is 10.
 
     ```python
     datadim = 3 * 32 * 32
     classdim = 10
-
     image = paddle.layer.data(
         name="image", type=paddle.data_type.dense_vector(datadim))
     ```
 
-2. 定义VGG网络核心模块
+2. Define VGG main module
 
     ```python
     net = vgg_bn_drop(image)
     ```
-    VGG核心模块的输入是数据层，`vgg_bn_drop` 定义了16层VGG结构，每层卷积后面引入BN层和Dropout层，详细的定义如下：
+    The input to VGG main module is from the data layer. `vgg_bn_drop` defines a 16-layer VGG network, with each convolutional layer followed by BN and dropout layers. Here is the definition in detail:
 
     ```python
     def vgg_bn_drop(input):
@@ -262,15 +274,15 @@ paddle.init(use_gpu=False, trainer_count=1)
         return fc2
     ```
 
-    2.1. 首先定义了一组卷积网络，即conv_block。卷积核大小为3x3，池化窗口大小为2x2，窗口滑动大小为2，groups决定每组VGG模块是几次连续的卷积操作，dropouts指定Dropout操作的概率。所使用的`img_conv_group`是在`paddle.networks`中预定义的模块，由若干组 Conv->BN->ReLu->Dropout 和 一组 Pooling 组成。
+    2.1. First, define a convolution block or conv_block. The default convolution kernel is 3x3, and the default pooling size is 2x2 with stride 2. Dropout specifies the probability in dropout operation. Function `img_conv_group` is defined in `paddle.networks` consisting of a series of `Conv->BN->ReLu->Dropout` and a `Pooling`.
 
-    2.2. 五组卷积操作，即 5个conv_block。 第一、二组采用两次连续的卷积操作。第三、四、五组采用三次连续的卷积操作。每组最后一个卷积后面Dropout概率为0，即不使用Dropout操作。
+    2.2. Five groups of convolutions. The first two groups perform two convolutions, while the last three groups perform three convolutions. The dropout rate of the last convolution in each group is set to 0, which means there is no dropout for this layer.
 
-    2.3. 最后接两层512维的全连接。
+    2.3. The last two layers are fully-connected layers of dimension 512.
 
-3. 定义分类器
+3. Define Classifier
 
-    通过上面VGG网络提取高层特征，然后经过全连接层映射到类别维度大小的向量，再通过Softmax归一化得到每个类别的概率，也可称作分类器。
+    The above VGG network extracts high-level features and maps them to a vector of the same size as the categories. Softmax function or classifier is then used for calculating the probability of the image belonging to each category.
 
     ```python
     out = paddle.layer.fc(input=net,
@@ -278,9 +290,9 @@ paddle.init(use_gpu=False, trainer_count=1)
                           act=paddle.activation.Softmax())
     ```
 
-4. 定义损失函数和网络输出
+4. Define Loss Function and Outputs
 
-    在有监督训练中需要输入图像对应的类别信息，同样通过`paddle.layer.data`来定义。训练中采用多类交叉熵作为损失函数，并作为网络的输出，预测阶段定义网络的输出为分类器得到的概率信息。
+    In the context of supervised learning, labels of training images are defined in `paddle.layer.data` as well. During training, the cross-entropy loss function is used and the loss is the output of the network. During testing, the outputs are the probabilities calculated in the classifier.
 
     ```python
     lbl = paddle.layer.data(
@@ -290,19 +302,20 @@ paddle.init(use_gpu=False, trainer_count=1)
 
 ### ResNet
 
-ResNet模型的第1、3、4步和VGG模型相同，这里不再介绍。主要介绍第2步即CIFAR10数据集上ResNet核心模块。
+The first, third and fourth steps of a ResNet are the same as a VGG. The second one is the main module.
 
 ```python
 net = resnet_cifar10(image, depth=56)
 ```
 
-先介绍`resnet_cifar10`中的一些基本函数，再介绍网络连接过程。
+Here are some basic functions used in `resnet_cifar10`:
+
+  - `conv_bn_layer` : convolutional layer followed by BN.
+  - `shortcut` : the shortcut branch in a residual block. There are two kinds of shortcuts: 1x1 convolution used when the number of channels between input and output is different; direct connection used otherwise.
 
-  - `conv_bn_layer` : 带BN的卷积层。
-  - `shortcut` : 残差模块的"直连"路径，"直连"实际分两种形式：残差模块输入和输出特征通道数不等时，采用1x1卷积的升维操作；残差模块输入和输出通道相等时，采用直连操作。
-  - `basicblock` : 一个基础残差模块，即图9左边所示，由两组3x3卷积组成的路径和一条"直连"路径组成。
-  - `bottleneck` : 一个瓶颈残差模块，即图9右边所示，由上下1x1卷积和中间3x3卷积组成的路径和一条"直连"路径组成。
-  - `layer_warp` : 一组残差模块，由若干个残差模块堆积而成。每组中第一个残差模块滑动窗口大小与其他可以不同，以用来减少特征图在垂直和水平方向的大小。
+  - `basicblock` : a basic residual module as shown in the left of Figure 9, it consists of two sequential 3x3 convolutions and one "shortcut" branch.
+  - `bottleneck` : a bottleneck module as shown in the right of Figure 9, it consists of two 1x1 convolutions with one 3x3 convolution in between branch and a "shortcut" branch.
+  - `layer_warp` : a group of residual modules consisting of several stacking blocks. In each group, the sliding window size of the first residual block could be different from the rest of blocks, in order to reduce the size of feature maps along horizontal and vertical directions.
 
 ```python
 def conv_bn_layer(input,
@@ -344,13 +357,13 @@ def layer_warp(block_func, ipt, features, count, stride):
     return tmp
 ```
 
-`resnet_cifar10` 的连接结构主要有以下几个过程。
+The following are the components of `resnet_cifar10`:
 
-1. 底层输入连接一层 `conv_bn_layer`，即带BN的卷积层。
-2. 然后连接3组残差模块即下面配置3组 `layer_warp` ，每组采用图 10 左边残差模块组成。
-3. 最后对网络做均值池化并返回该层。
+1. The lowest level is `conv_bn_layer`.
+2. The middle level consists of three `layer_warp`, each of which uses the left residual block in Figure 9.
+3. The last level is average pooling layer.
 
-注意：除过第一层卷积层和最后一层全连接层之外，要求三组 `layer_warp` 总的含参层数能够被6整除，即 `resnet_cifar10` 的 depth 要满足 $(depth - 2) % 6 == 0$ 。
+Note: besides the first convolutional layer and the last fully-connected layer, the total number of layers in three `layer_warp` should be dividable by 6, that is the depth of `resnet_cifar10` should satisfy $(depth - 2) % 6 == 0$.
 
 ```python
 def resnet_cifar10(ipt, depth=32):
@@ -368,26 +381,21 @@ def resnet_cifar10(ipt, depth=32):
     return pool
 ```
 
-## 训练模型
+## Model Training
 
-### 定义参数
+### Define Parameters
 
-首先依据模型配置的`cost`定义模型参数。
+First, we create the model parameters according to the previous model configuration `cost`.
 
 ```python
 # Create parameters
 parameters = paddle.parameters.create(cost)
 ```
 
-可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
-
-```python
-print parameters.keys()
-```
-
-### 构造训练(Trainer)
+### Create Trainer
 
-根据网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的Momentum方法，同时设定了学习率、正则等。
+Before creating a training module, it is necessary to set the algorithm.
+Here we specify `Momentum` optimization algorithm via `paddle.optimizer`.
 
 ```python
 # Create optimizer
@@ -397,8 +405,7 @@ momentum_optimizer = paddle.optimizer.Momentum(
     learning_rate=0.1 / 128.0,
     learning_rate_decay_a=0.1,
     learning_rate_decay_b=50000 * 100,
-    learning_rate_schedule='discexp',
-    batch_size=128)
+    learning_rate_schedule='discexp')
 
 # Create trainer
 trainer = paddle.trainer.SGD(cost=cost,
@@ -406,14 +413,13 @@ trainer = paddle.trainer.SGD(cost=cost,
                              update_equation=momentum_optimizer)
 ```
 
-通过 `learning_rate_decay_a` (简写$a$） 、`learning_rate_decay_b` (简写$b$) 和 `learning_rate_schedule` 指定学习率调整策略，这里采用离散指数的方式调节学习率，计算公式如下， $n$ 代表已经处理过的累计总样本数，$lr_{0}$ 即为 `settings` 里设置的 `learning_rate`。
-
+The learning rate adjustment policy can be defined with variables `learning_rate_decay_a`($a$), `learning_rate_decay_b`($b$) and `learning_rate_schedule`. In this example, discrete exponential method is used for adjusting learning rate. The formula is as follows,
 $$  lr = lr_{0} * a^ {\lfloor \frac{n}{ b}\rfloor} $$
+where $n$ is the number of processed samples, $lr_{0}$ is the learning_rate.
 
+### Training
 
-### 训练
-
-cifar.train10()每次产生一条样本，在完成shuffle和batch之后，作为训练的输入。
+`cifar.train10()` will yield records during each pass, after shuffling, a batch input is generated for training.
 
 ```python
 reader=paddle.batch(
@@ -422,17 +428,47 @@ reader=paddle.batch(
         batch_size=128)
 ```
 
-通过`feeding`来指定每一个数据和`paddle.layer.data`的对应关系。例如: `cifar.train10()`产生数据的第0列对应image层的特征。
+`feeding` is devoted to specifying the correspondence between each yield record and `paddle.layer.data`. For instance,
+ the first column of data generated by `cifar.train10()` corresponds to image layer's feature.
 
 ```python
 feeding={'image': 0,
          'label': 1}
 ```
 
-可以使用`event_handler`回调函数来观察训练过程，或进行测试等, 该回调函数是`trainer.train`函数里设定。
+Callback function `event_handler` will be called during training when a pre-defined event happens.
+
+`event_handler_plot`is used to plot a figure like below：
+
+![png](./image/train_and_test.png)
 
 ```python
-# End batch and end pass event handler
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+def event_handler_plot(event):
+    global step
+    if isinstance(event, paddle.event.EndIteration):
+        if step % 1 == 0:
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+        step += 1
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+            reader=paddle.batch(
+                paddle.dataset.cifar.test10(), batch_size=128),
+            feeding=feeding)
+        cost_ploter.append(test_title, step, result.cost)
+```
+
+`event_handler` is used to plot some text data when training.
+
+```python
+# event handler to track training and testing process
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -442,6 +478,10 @@ def event_handler(event):
             sys.stdout.write('.')
             sys.stdout.flush()
     if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
         result = trainer.test(
             reader=paddle.batch(
                 paddle.dataset.cifar.test10(), batch_size=128),
@@ -449,17 +489,17 @@ def event_handler(event):
         print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
-通过`trainer.train`函数训练:
+Finally, we can invoke `trainer.train` to start training:
 
 ```python
 trainer.train(
     reader=reader,
     num_passes=200,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding)
 ```
 
-一轮训练log示例如下所示，经过1个pass， 训练集上平均error为0.6875 ，测试集上平均error为0.8852 。
+Here is an example log after training for one pass. The average error rates are 0.6875 on the training set and 0.8852 on the validation set.
 
 ```text
 Pass 0, Batch 0, Cost 2.473182, {'classification_error_evaluator': 0.9140625}
@@ -473,20 +513,58 @@ Pass 0, Batch 300, Cost 1.668833, {'classification_error_evaluator': 0.6875}
 Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 ```
 
-图12是训练的分类错误率曲线图，运行到第200个pass后基本收敛，最终得到测试集上分类错误率为8.54%。
-
+Figure 12 shows the curve of training error rate, which indicates it converges at Pass 200 with error rate 8.54%.
 <p align="center">
-<img src="image/plot.png" width="400" ><br/>
-图12. CIFAR10数据集上VGG模型的分类错误率
+<img src="image/plot_en.png" width="400" ><br/>
+Figure 12. The error rate of VGG model on CIFAR10
 </p>
 
 
-## 总结
 
-传统图像分类方法由多个阶段构成，框架较为复杂，而端到端的CNN模型结构可一步到位，而且大幅度提升了分类准确率。本文我们首先介绍VGG、GoogleNet、ResNet三个经典的模型；然后基于CIFAR10数据集，介绍如何使用PaddlePaddle配置和训练CNN模型，尤其是VGG和ResNet模型；最后介绍如何使用PaddlePaddle的API接口对图片进行预测和特征提取。对于其他数据集比如ImageNet，配置和训练流程是同样的，大家可以自行进行实验。
+## Application
+
+After training is done, users can use the trained model to classify images. The following code shows how to infer through `paddle.infer` interface. You can remove the comments to change the model name.
+
+```python
+from PIL import Image
+import numpy as np
+import os
+def load_image(file):
+    im = Image.open(file)
+    im = im.resize((32, 32), Image.ANTIALIAS)
+    im = np.array(im).astype(np.float32)
+    # The storage order of the loaded image is W(widht),
+    # H(height), C(channel). PaddlePaddle requires
+    # the CHW order, so transpose them.
+    im = im.transpose((2, 0, 1)) # CHW
+    # In the training phase, the channel order of CIFAR
+    # image is B(Blue), G(green), R(Red). But PIL open
+    # image in RGB mode. It must swap the channel order.
+    im = im[(2, 1, 0),:,:] # BGR
+    im = im.flatten()
+    im = im / 255.0
+    return im
+test_data = []
+cur_dir = os.getcwd()
+test_data.append((load_image(cur_dir + '/image/dog.png'),))
+
+# users can remove the comments and change the model name
+# with open('params_pass_50.tar', 'r') as f:
+#    parameters = paddle.parameters.Parameters.from_tar(f)
+
+probs = paddle.infer(
+    output_layer=out, parameters=parameters, input=test_data)
+lab = np.argsort(-probs) # probs and lab are the results of one batch data
+print "Label of image/dog.png is: %d" % lab[0][0]
+```
+
+
+## Conclusion
+
+Traditional image classification methods have complicated frameworks that involve multiple stages of processing. In contrast, CNN models can be trained end-to-end with a significant increase in classification accuracy. In this chapter, we introduced three models -- VGG, GoogleNet, ResNet and provided PaddlePaddle config files for training VGG and ResNet on CIFAR10. We also explained how to perform prediction and feature extraction using the PaddlePaddle API. For other datasets such as ImageNet, the procedure for config and training are the same and you are welcome to give it a try.
 
 
-## 参考文献
+## Reference
 
 [1] D. G. Lowe, [Distinctive image features from scale-invariant keypoints](http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf). IJCV, 60(2):91-110, 2004.
 
@@ -533,7 +611,7 @@ Test with Pass 0, {'classification_error_evaluator': 0.885200023651123}
 [22] http://cs231n.github.io/classification/
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

03.image_classification/resnet.py

@@ -36,26 +36,25 @@ def conv_bn_layer(input,
     return paddle.layer.batch_norm(input=tmp, act=active_type)
 
 
-def shortcut(ipt, n_in, n_out, stride):
-    if n_in != n_out:
-        return conv_bn_layer(ipt, n_out, 1, stride, 0,
+def shortcut(ipt, ch_in, ch_out, stride):
+    if ch_in != ch_out:
+        return conv_bn_layer(ipt, ch_out, 1, stride, 0,
                              paddle.activation.Linear())
     else:
         return ipt
 
 
-def basicblock(ipt, ch_out, stride):
-    ch_in = ch_out * 2
+def basicblock(ipt, ch_in, ch_out, stride):
     tmp = conv_bn_layer(ipt, ch_out, 3, stride, 1)
     tmp = conv_bn_layer(tmp, ch_out, 3, 1, 1, paddle.activation.Linear())
     short = shortcut(ipt, ch_in, ch_out, stride)
     return paddle.layer.addto(input=[tmp, short], act=paddle.activation.Relu())
 
 
-def layer_warp(block_func, ipt, features, count, stride):
-    tmp = block_func(ipt, features, stride)
+def layer_warp(block_func, ipt, ch_in, ch_out, count, stride):
+    tmp = block_func(ipt, ch_in, ch_out, stride)
     for i in range(1, count):
-        tmp = block_func(tmp, features, 1)
+        tmp = block_func(tmp, ch_out, ch_out, 1)
     return tmp
 
 
@@ -66,9 +65,9 @@ def resnet_cifar10(ipt, depth=32):
     nStages = {16, 64, 128}
     conv1 = conv_bn_layer(
         ipt, ch_in=3, ch_out=16, filter_size=3, stride=1, padding=1)
-    res1 = layer_warp(basicblock, conv1, 16, n, 1)
-    res2 = layer_warp(basicblock, res1, 32, n, 2)
-    res3 = layer_warp(basicblock, res2, 64, n, 2)
+    res1 = layer_warp(basicblock, conv1, 16, 16, n, 1)
+    res2 = layer_warp(basicblock, res1, 16, 32, n, 2)
+    res3 = layer_warp(basicblock, res2, 32, 64, n, 2)
     pool = paddle.layer.img_pool(
         input=res3, pool_size=8, stride=1, pool_type=paddle.pooling.Avg())
     return pool

03.image_classification/train.py

@@ -53,8 +53,7 @@ def main():
         learning_rate=0.1 / 128.0,
         learning_rate_decay_a=0.1,
         learning_rate_decay_b=50000 * 100,
-        learning_rate_schedule='discexp',
-        batch_size=128)
+        learning_rate_schedule='discexp')
 
     # End batch and end pass event handler
     def event_handler(event):
@@ -66,6 +65,10 @@ def main():
                 sys.stdout.write('.')
                 sys.stdout.flush()
         if isinstance(event, paddle.event.EndPass):
+            # save parameters
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
+
             result = trainer.test(
                 reader=paddle.batch(
                     paddle.dataset.cifar.test10(), batch_size=128),
@@ -86,6 +89,40 @@ def main():
         feeding={'image': 0,
                  'label': 1})
 
+    # inference
+    from PIL import Image
+    import numpy as np
+    import os
+
+    def load_image(file):
+        im = Image.open(file)
+        im = im.resize((32, 32), Image.ANTIALIAS)
+        im = np.array(im).astype(np.float32)
+        # The storage order of the loaded image is W(widht),
+        # H(height), C(channel). PaddlePaddle requires
+        # the CHW order, so transpose them.
+        im = im.transpose((2, 0, 1))  # CHW
+        # In the training phase, the channel order of CIFAR
+        # image is B(Blue), G(green), R(Red). But PIL open
+        # image in RGB mode. It must swap the channel order.
+        im = im[(2, 1, 0), :, :]  # BGR
+        im = im.flatten()
+        im = im / 255.0
+        return im
+
+    test_data = []
+    cur_dir = os.path.dirname(os.path.realpath(__file__))
+    test_data.append((load_image(cur_dir + '/image/dog.png'), ))
+
+    # users can remove the comments and change the model name
+    # with open('params_pass_50.tar', 'r') as f:
+    #    parameters = paddle.parameters.Parameters.from_tar(f)
+
+    probs = paddle.infer(
+        output_layer=out, parameters=parameters, input=test_data)
+    lab = np.argsort(-probs)  # probs and lab are the results of one batch data
+    print "Label of image/dog.png is: %d" % lab[0][0]
+
 
 if __name__ == '__main__':
     main()

04.word2vec/README.cn.md

+
+# 词向量
+
+本教程源代码目录在[book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/175.html)。
+
+## 背景介绍
+
+本章我们介绍词的向量表征，也称为word embedding。词向量是自然语言处理中常见的一个操作，是搜索引擎、广告系统、推荐系统等互联网服务背后常见的基础技术。
+
+在这些互联网服务里，我们经常要比较两个词或者两段文本之间的相关性。为了做这样的比较，我们往往先要把词表示成计算机适合处理的方式。最自然的方式恐怕莫过于向量空间模型(vector space model)。
+在这种方式里，每个词被表示成一个实数向量（one-hot vector），其长度为字典大小，每个维度对应一个字典里的每个词，除了这个词对应维度上的值是1，其他元素都是0。
+
+One-hot vector虽然自然，但是用处有限。比如，在互联网广告系统里，如果用户输入的query是“母亲节”，而有一个广告的关键词是“康乃馨”。虽然按照常理，我们知道这两个词之间是有联系的——母亲节通常应该送给母亲一束康乃馨；但是这两个词对应的one-hot vectors之间的距离度量，无论是欧氏距离还是余弦相似度(cosine similarity)，由于其向量正交，都认为这两个词毫无相关性。 得出这种与我们相悖的结论的根本原因是：每个词本身的信息量都太小。所以，仅仅给定两个词，不足以让我们准确判别它们是否相关。要想精确计算相关性，我们还需要更多的信息——从大量数据里通过机器学习方法归纳出来的知识。
+
+在机器学习领域里，各种“知识”被各种模型表示，词向量模型(word embedding model)就是其中的一类。通过词向量模型可将一个 one-hot vector映射到一个维度更低的实数向量（embedding vector），如$embedding(母亲节) = [0.3, 4.2, -1.5, ...], embedding(康乃馨) = [0.2, 5.6, -2.3, ...]$。在这个映射到的实数向量表示中，希望两个语义（或用法）上相似的词对应的词向量“更像”，这样如“母亲节”和“康乃馨”的对应词向量的余弦相似度就不再为零了。
+
+词向量模型可以是概率模型、共生矩阵(co-occurrence matrix)模型或神经元网络模型。在用神经网络求词向量之前，传统做法是统计一个词语的共生矩阵$X$。$X$是一个$|V| \times |V|$ 大小的矩阵，$X_{ij}$表示在所有语料中，词汇表`V`(vocabulary)中第i个词和第j个词同时出现的词数，$|V|$为词汇表的大小。对$X$做矩阵分解（如奇异值分解，Singular Value Decomposition \[[5](#参考文献)\]），得到的$U$即视为所有词的词向量：
+
+$$X = USV^T$$
+
+但这样的传统做法有很多问题：<br/>
+1) 由于很多词没有出现，导致矩阵极其稀疏，因此需要对词频做额外处理来达到好的矩阵分解效果；<br/>
+2) 矩阵非常大，维度太高(通常达到$10^6*10^6$的数量级)；<br/>
+3) 需要手动去掉停用词（如although, a,...），不然这些频繁出现的词也会影响矩阵分解的效果。
+
+
+基于神经网络的模型不需要计算存储一个在全语料上统计的大表，而是通过学习语义信息得到词向量，因此能很好地解决以上问题。在本章里，我们将展示基于神经网络训练词向量的细节，以及如何用PaddlePaddle训练一个词向量模型。
+
+
+## 效果展示
+
+本章中，当词向量训练好后，我们可以用数据可视化算法t-SNE\[[4](#参考文献)\]画出词语特征在二维上的投影（如下图所示）。从图中可以看出，语义相关的词语（如a, the, these; big, huge）在投影上距离很近，语意无关的词（如say, business; decision, japan）在投影上的距离很远。
+
+<p align="center">
+    <img src = "image/2d_similarity.png" width=400><br/>
+    图1. 词向量的二维投影
+</p>
+
+另一方面，我们知道两个向量的余弦值在$[-1,1]$的区间内：两个完全相同的向量余弦值为1, 两个相互垂直的向量之间余弦值为0，两个方向完全相反的向量余弦值为-1，即相关性和余弦值大小成正比。因此我们还可以计算两个词向量的余弦相似度:
+
+```
+similarity: 0.899180685161
+please input two words: big huge
+
+please input two words: from company
+similarity: -0.0997506977351
+```
+
+以上结果可以通过运行`calculate_dis.py`, 加载字典里的单词和对应训练特征结果得到，我们将在[应用模型](#应用模型)中详细描述用法。
+
+
+## 模型概览
+
+在这里我们介绍三个训练词向量的模型：N-gram模型，CBOW模型和Skip-gram模型，它们的中心思想都是通过上下文得到一个词出现的概率。对于N-gram模型，我们会先介绍语言模型的概念，并在之后的[训练模型](#训练模型)中，带大家用PaddlePaddle实现它。而后两个模型，是近年来最有名的神经元词向量模型，由 Tomas Mikolov 在Google 研发\[[3](#参考文献)\]，虽然它们很浅很简单，但训练效果很好。
+
+### 语言模型
+
+在介绍词向量模型之前，我们先来引入一个概念：语言模型。
+语言模型旨在为语句的联合概率函数$P(w_1, ..., w_T)$建模, 其中$w_i$表示句子中的第i个词。语言模型的目标是，希望模型对有意义的句子赋予大概率，对没意义的句子赋予小概率。
+这样的模型可以应用于很多领域，如机器翻译、语音识别、信息检索、词性标注、手写识别等，它们都希望能得到一个连续序列的概率。 以信息检索为例，当你在搜索“how long is a football bame”时（bame是一个医学名词），搜索引擎会提示你是否希望搜索"how long is a football game", 这是因为根据语言模型计算出“how long is a football bame”的概率很低，而与bame近似的，可能引起错误的词中，game会使该句生成的概率最大。
+
+对语言模型的目标概率$P(w_1, ..., w_T)$，如果假设文本中每个词都是相互独立的，则整句话的联合概率可以表示为其中所有词语条件概率的乘积，即：
+
+$$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t)$$
+
+然而我们知道语句中的每个词出现的概率都与其前面的词紧密相关, 所以实际上通常用条件概率表示语言模型：
+
+$$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
+
+
+
+### N-gram neural model
+
+在计算语言学中，n-gram是一种重要的文本表示方法，表示一个文本中连续的n个项。基于具体的应用场景，每一项可以是一个字母、单词或者音节。 n-gram模型也是统计语言模型中的一种重要方法，用n-gram训练语言模型时，一般用每个n-gram的历史n-1个词语组成的内容来预测第n个词。
+
+Yoshua Bengio等科学家就于2003年在著名论文 Neural Probabilistic Language Models \[[1](#参考文献)\] 中介绍如何学习一个神经元网络表示的词向量模型。文中的神经概率语言模型（Neural Network Language Model，NNLM）通过一个线性映射和一个非线性隐层连接，同时学习了语言模型和词向量，即通过学习大量语料得到词语的向量表达，通过这些向量得到整个句子的概率。用这种方法学习语言模型可以克服维度灾难（curse of dimensionality）,即训练和测试数据不同导致的模型不准。注意：由于“神经概率语言模型”说法较为泛泛，我们在这里不用其NNLM的本名，考虑到其具体做法，本文中称该模型为N-gram neural model。
+
+我们在上文中已经讲到用条件概率建模语言模型，即一句话中第$t$个词的概率和该句话的前$t-1$个词相关。可实际上越远的词语其实对该词的影响越小，那么如果考虑一个n-gram, 每个词都只受其前面`n-1`个词的影响，则有：
+
+$$P(w_1, ..., w_T) = \prod_{t=n}^TP(w_t|w_{t-1}, w_{t-2}, ..., w_{t-n+1})$$
+
+给定一些真实语料，这些语料中都是有意义的句子，N-gram模型的优化目标则是最大化目标函数:
+
+$$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
+
+其中$f(w_t, w_{t-1}, ..., w_{t-n+1})$表示根据历史n-1个词得到当前词$w_t$的条件概率，$R(\theta)$表示参数正则项。
+
+<p align="center">
+       <img src="image/nnlm.png" width=500><br/>
+       图2. N-gram神经网络模型
+</p>
+
+图2展示了N-gram神经网络模型，从下往上看，该模型分为以下几个部分：
+ - 对于每个样本，模型输入$w_{t-n+1},...w_{t-1}$, 输出句子第t个词为字典中`|V|`个词的概率。
+
+   每个输入词$w_{t-n+1},...w_{t-1}$首先通过映射矩阵映射到词向量$C(w_{t-n+1}),...C(w_{t-1})$。
+
+ - 然后所有词语的词向量连接成一个大向量，并经过一个非线性映射得到历史词语的隐层表示：
+
+    $$g=Utanh(\theta^Tx + b_1) + Wx + b_2$$
+
+    其中，$x$为所有词语的词向量连接成的大向量，表示文本历史特征；$\theta$、$U$、$b_1$、$b_2$和$W$分别为词向量层到隐层连接的参数。$g$表示未经归一化的所有输出单词概率，$g_i$表示未经归一化的字典中第$i$个单词的输出概率。
+
+ - 根据softmax的定义，通过归一化$g_i$, 生成目标词$w_t$的概率为：
+
+  $$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
+
+ - 整个网络的损失值(cost)为多类分类交叉熵，用公式表示为
+
+   $$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$
+
+   其中$y_k^i$表示第$i$个样本第$k$类的真实标签(0或1)，$softmax(g_k^i)$表示第i个样本第k类softmax输出的概率。
+
+
+
+### Continuous Bag-of-Words model(CBOW)
+
+CBOW模型通过一个词的上下文（各N个词）预测当前词。当N=2时，模型如下图所示：
+
+<p align="center">
+    <img src="image/cbow.png" width=250><br/>
+    图3. CBOW模型
+</p>
+
+具体来说，不考虑上下文的词语输入顺序，CBOW是用上下文词语的词向量的均值来预测当前词。即：
+
+$$context = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
+
+其中$x_t$为第$t$个词的词向量，分类分数（score）向量 $z=U*context$，最终的分类$y$采用softmax，损失函数采用多类分类交叉熵。
+
+### Skip-gram model
+
+CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去掉了噪声，因此在小数据集上很有效。而Skip-gram的方法中，用一个词预测其上下文，得到了当前词上下文的很多样本，因此可用于更大的数据集。
+
+<p align="center">
+    <img src="image/skipgram.png" width=250><br/>
+    图4. Skip-gram模型
+</p>
+
+如上图所示，Skip-gram模型的具体做法是，将一个词的词向量映射到$2n$个词的词向量（$2n$表示当前输入词的前后各$n$个词），然后分别通过softmax得到这$2n$个词的分类损失值之和。
+
+
+## 数据准备
+
+### 数据介绍
+
+本教程使用Penn Treebank （PTB）（经Tomas Mikolov预处理过的版本）数据集。PTB数据集较小，训练速度快，应用于Mikolov的公开语言模型训练工具\[[2](#参考文献)\]中。其统计情况如下：
+
+<p align="center">
+<table>
+    <tr>
+        <td>训练数据</td>
+        <td>验证数据</td>
+        <td>测试数据</td>
+    </tr>
+    <tr>
+        <td>ptb.train.txt</td>
+        <td>ptb.valid.txt</td>
+        <td>ptb.test.txt</td>
+    </tr>
+    <tr>
+        <td>42068句</td>
+        <td>3370句</td>
+        <td>3761句</td>
+    </tr>
+</table>
+</p>
+
+
+### 数据预处理
+
+本章训练的是5-gram模型，表示在PaddlePaddle训练时，每条数据的前4个词用来预测第5个词。PaddlePaddle提供了对应PTB数据集的python包`paddle.dataset.imikolov`，自动做数据的下载与预处理，方便大家使用。
+
+预处理会把数据集中的每一句话前后加上开始符号`<s>`以及结束符号`<e>`。然后依据窗口大小（本教程中为5），从头到尾每次向右滑动窗口并生成一条数据。
+
+如"I have a dream that one day" 一句提供了5条数据：
+
+```text
+<s> I have a dream
+I have a dream that
+have a dream that one
+a dream that one day
+dream that one day <e>
+```
+
+最后，每个输入会按其单词次在字典里的位置，转化成整数的索引序列，作为PaddlePaddle的输入。
+## 编程实现
+
+本配置的模型结构如下图所示：
+
+<p align="center">
+    <img src="image/ngram.png" width=400><br/>
+    图5. 模型配置中的N-gram神经网络模型
+</p>
+
+首先，加载所需要的包：
+
+```python
+import math
+import paddle.v2 as paddle
+```
+
+然后，定义参数：
+```python
+embsize = 32 # 词向量维度
+hiddensize = 256 # 隐层维度
+N = 5 # 训练5-Gram
+```
+
+接着，定义网络结构：
+
+- 将$w_t$之前的$n-1$个词 $w_{t-n+1},...w_{t-1}$，通过$|V|\times D$的矩阵映射到D维词向量（本例中取D=32）。
+
+```python
+def wordemb(inlayer):
+    wordemb = paddle.layer.table_projection(
+        input=inlayer,
+        size=embsize,
+        param_attr=paddle.attr.Param(
+            name="_proj",
+            initial_std=0.001,
+            learning_rate=1,
+            l2_rate=0,
+            sparse_update=True))
+    return wordemb
+```
+
+- 定义输入层接受的数据类型以及名字。
+
+```python
+paddle.init(use_gpu=False, trainer_count=3) # 初始化PaddlePaddle
+word_dict = paddle.dataset.imikolov.build_dict()
+dict_size = len(word_dict)
+# 每个输入层都接受整形数据，这些数据的范围是[0, dict_size)
+firstword = paddle.layer.data(
+    name="firstw", type=paddle.data_type.integer_value(dict_size))
+secondword = paddle.layer.data(
+    name="secondw", type=paddle.data_type.integer_value(dict_size))
+thirdword = paddle.layer.data(
+    name="thirdw", type=paddle.data_type.integer_value(dict_size))
+fourthword = paddle.layer.data(
+    name="fourthw", type=paddle.data_type.integer_value(dict_size))
+nextword = paddle.layer.data(
+    name="fifthw", type=paddle.data_type.integer_value(dict_size))
+
+Efirst = wordemb(firstword)
+Esecond = wordemb(secondword)
+Ethird = wordemb(thirdword)
+Efourth = wordemb(fourthword)
+```
+
+- 将这n-1个词向量经过concat_layer连接成一个大向量作为历史文本特征。
+
+```python
+contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
+```
+
+- 将历史文本特征经过一个全连接得到文本隐层特征。
+
+```python
+hidden1 = paddle.layer.fc(input=contextemb,
+                          size=hiddensize,
+                          act=paddle.activation.Sigmoid(),
+                          layer_attr=paddle.attr.Extra(drop_rate=0.5),
+                          bias_attr=paddle.attr.Param(learning_rate=2),
+                          param_attr=paddle.attr.Param(
+                                initial_std=1. / math.sqrt(embsize * 8),
+                                learning_rate=1))
+```
+
+- 将文本隐层特征，再经过一个全连接，映射成一个$|V|$维向量，同时通过softmax归一化得到这`|V|`个词的生成概率。
+
+```python
+predictword = paddle.layer.fc(input=hidden1,
+                              size=dict_size,
+                              bias_attr=paddle.attr.Param(learning_rate=2),
+                              act=paddle.activation.Softmax())
+```
+
+- 网络的损失函数为多分类交叉熵，可直接调用`classification_cost`函数。
+
+```python
+cost = paddle.layer.classification_cost(input=predictword, label=nextword)
+```
+
+然后，指定训练相关的参数：
+
+- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器，本教程使用Adam优化器。
+- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
+- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+
+```python
+parameters = paddle.parameters.create(cost)
+adagrad = paddle.optimizer.AdaGrad(
+    learning_rate=3e-3,
+    regularization=paddle.optimizer.L2Regularization(8e-4))
+trainer = paddle.trainer.SGD(cost, parameters, adagrad)
+```
+
+下一步，我们开始训练过程。`paddle.dataset.imikolov.train()`和`paddle.dataset.imikolov.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python generator。
+
+`paddle.batch`的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minbatch。
+
+```python
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+
+    if isinstance(event, paddle.event.EndPass):
+        result = trainer.test(
+                    paddle.batch(
+                        paddle.dataset.imikolov.test(word_dict, N), 32))
+        print "Pass %d, Testing metrics %s" % (event.pass_id, result.metrics)
+        with open("model_%d.tar"%event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+trainer.train(
+    paddle.batch(paddle.dataset.imikolov.train(word_dict, N), 32),
+    num_passes=100,
+    event_handler=event_handler)
+```
+
+```text
+Pass 0, Batch 0, Cost 7.870579, {'classification_error_evaluator': 1.0}, Testing metrics {'classification_error_evaluator': 0.999591588973999}
+Pass 0, Batch 100, Cost 6.136420, {'classification_error_evaluator': 0.84375}, Testing metrics {'classification_error_evaluator': 0.8328699469566345}
+Pass 0, Batch 200, Cost 5.786797, {'classification_error_evaluator': 0.8125}, Testing metrics {'classification_error_evaluator': 0.8328542709350586}
+...
+```
+
+训练过程是完全自动的，event_handler里打印的日志类似如上所示：
+
+经过30个pass，我们将得到平均错误率为classification_error_evaluator=0.735611。
+
+
+## 应用模型
+训练模型后，我们可以加载模型参数，用训练出来的词向量初始化其他模型，也可以将模型查看参数用来做后续应用。
+
+
+### 查看词向量
+
+PaddlePaddle训练出来的参数可以直接使用`parameters.get()`获取出来。例如查看单词`apple`的词向量，即为
+
+
+```python
+embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
+
+print embeddings[word_dict['apple']]
+```
+
+```text
+[-0.38961065 -0.02392169 -0.00093231  0.36301503  0.13538605  0.16076435
+-0.0678709   0.1090285   0.42014077 -0.24119169 -0.31847557  0.20410083
+0.04910378  0.19021918 -0.0122014  -0.04099389 -0.16924137  0.1911236
+-0.10917275  0.13068172 -0.23079982  0.42699069 -0.27679482 -0.01472992
+0.2069038   0.09005053 -0.3282454   0.12717034 -0.24218646  0.25304323
+0.19072419 -0.24286366]
+```
+
+
+### 修改词向量
+
+获得到的embedding为一个标准的numpy矩阵。我们可以对这个numpy矩阵进行修改，然后赋值回去。
+
+
+```python
+def modify_embedding(emb):
+    # Add your modification here.
+    pass
+
+modify_embedding(embeddings)
+parameters.set("_proj", embeddings)
+```
+
+### 计算词语之间的余弦距离
+
+两个向量之间的距离可以用余弦值来表示，余弦值在$[-1,1]$的区间内，向量间余弦值越大，其距离越近。这里我们在`calculate_dis.py`中实现不同词语的距离度量。
+用法如下：
+
+
+```python
+from scipy import spatial
+
+emb_1 = embeddings[word_dict['world']]
+emb_2 = embeddings[word_dict['would']]
+
+print spatial.distance.cosine(emb_1, emb_2)
+```
+
+```text
+0.99375076448
+```
+
+## 总结
+本章中，我们介绍了词向量、语言模型和词向量的关系、以及如何通过训练神经网络模型获得词向量。在信息检索中，我们可以根据向量间的余弦夹角，来判断query和文档关键词这二者间的相关性。在句法分析和语义分析中，训练好的词向量可以用来初始化模型，以得到更好的效果。在文档分类中，有了词向量之后，可以用聚类的方法将文档中同义词进行分组。希望大家在本章后能够自行运用词向量进行相关领域的研究。
+
+
+## 参考文献
+1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
+2. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
+3. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space](https://arxiv.org/pdf/1301.3781.pdf)[J]. arXiv preprint arXiv:1301.3781, 2013.
+4. Maaten L, Hinton G. [Visualizing data using t-SNE](https://lvdmaaten.github.io/publications/papers/JMLR_2008.pdf)[J]. Journal of Machine Learning Research, 2008, 9(Nov): 2579-2605.
+5. https://en.wikipedia.org/wiki/Singular_value_decomposition
+
+<br/>
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

04.word2vec/README.en.md

-# Word2Vec
-
-This is intended as a reference tutorial. The source code of this tutorial lives on [book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec).
-
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
-
-## Background Introduction
-
-This section introduces the concept of **word embedding**, which is a vector representation of words. It is a popular technique used in natural language processing. Word embeddings support many Internet services, including search engines, advertising systems, and recommendation systems.
-
-### One-Hot Vectors
-
-Building these services requires us to quantify the similarity between two words or paragraphs. This calls for a new representation of all the words to make them more suitable for computation. An obvious way to achieve this is through the vector space model, where every word is represented as an **one-hot vector**.
-
-For each word, its vector representation has the corresponding entry in the vector as 1, and all other entries as 0. The lengths of one-hot vectors match the size of the dictionary. Each entry of a vector corresponds to the presence (or absence) of a word in the dictionary.
-
-One-hot vectors are intuitive, yet they have limited usefulness. Take the example of an Internet advertising system: Suppose a customer enters the query "Mother's Day", while an ad bids for the keyword carnations". Because the one-hot vectors of these two words are perpendicular, the metric distance (either Euclidean or cosine similarity) between them would indicate  little relevance. However, *we* know that these two queries are connected semantically, since people often gift their mothers bundles of carnation flowers on Mother's Day. This discrepancy is due to the low information capacity in each vector. That is, comparing the vector representations of two words does not assess their relevance sufficiently. To calculate their similarity accurately, we need more information, which could be learned from large amounts of data through machine learning methods.
-
-Like many machine learning models, word embeddings can represent knowledge in various ways. Another model may project an one-hot vector to an embedding vector of lower dimension e.g. $embedding(mother's day) = [0.3, 4.2, -1.5, ...], embedding(carnations) = [0.2, 5.6, -2.3, ...]$. Mapping one-hot vectors onto an embedded vector space has the potential to bring the embedding vectors of similar words (either semantically or usage-wise) closer to each other, so that the cosine similarity between the corresponding vectors for words like "Mother's Day" and "carnations" are no longer zero.
-
-A word embedding model could be a probabilistic model, a co-occurrence matrix model, or a neural network. Before people started using neural networks to generate word embedding, the traditional method was to calculate a co-occurrence matrix $X$ of words. Here, $X$ is a $|V| \times |V|$ matrix, where $X_{ij}$ represents the co-occurrence times of the $i$th and $j$th words in the vocabulary `V` within all corpus, and $|V|$ is the size of the vocabulary. By performing matrix decomposition on $X$ e.g. Singular Value Decomposition \[[5](#References)\]
-
-$$X = USV^T$$
-
-the resulting $U$ can be seen as the word embedding of all the words.
-
-However, this method suffers from many drawbacks:
-1) Since many pairs of words don't co-occur, the co-occurrence matrix is sparse. To achieve good performance of matrix factorization, further treatment on word frequency is needed;
-2) The matrix is large, frequently on the order of $10^6*10^6$;
-3) We need to manually filter out stop words (like "although", "a", ...), otherwise these frequent words will affect the performance of matrix factorization.
-
-The neural network based model does not require storing huge hash tables of statistics on all of the corpus. It obtains the word embedding by learning from semantic information, hence could avoid the aforementioned problems in the traditional method. In this chapter, we will introduce the details of neural network word embedding model and how to train such model in PaddlePaddle.
-
-## Results Demonstration
-
-In this section, after training the word embedding model, we could use the data visualization algorithm $t-$SNE\[[4](#reference)\] to draw the word embedding vectors after projecting them onto a two-dimensional space (see figure below). From the figure we could see that the semantically relevant words -- *a*, *the*, and *these* or *big* and *huge* -- are close to each other in the projected space, while irrelevant words -- *say* and *business* or *decision* and *japan* -- are far from each other.
-
-<p align="center">
-    <img src = "image/2d_similarity.png" width=400><br/>
-    Figure 1. Two dimension projection of word embeddings
-</p>
-
-### Cosine Similarity
-
-On the other hand, we know that the cosine similarity between two vectors falls between $[-1,1]$. Specifically, the cosine similarity is 1 when the vectors are identical, 0 when the vectors are perpendicular, -1 when the are of opposite directions. That is, the cosine similarity between two vectors scales with their relevance. So we can calculate the cosine similarity of two word embedding vectors to represent their relevance:
-
-```
-please input two words: big huge
-similarity: 0.899180685161
-
-please input two words: from company
-similarity: -0.0997506977351
-```
-
-The above results could be obtained by running `calculate_dis.py`, which loads the words in the dictionary and their corresponding trained word embeddings. For detailed instruction, see section [Model Application](#Model Application).
-
-
-## Model Overview
-
-In this section, we will introduce three word embedding models: N-gram model, CBOW, and Skip-gram, which all output the frequency of each word given its immediate context.
-
-For N-gram model, we will first introduce the concept of language model, and implement it using PaddlePaddle in section [Model Training](#Model Training).
-
-The latter two models, which became popular recently, are neural word embedding model developed by Tomas Mikolov at Google \[[3](#reference)\]. Despite their apparent simplicity, these models train very well.
-
-### Language Model
-
-Before diving into word embedding models, we will first introduce the concept of **language model**. Language models build the joint probability function $P(w_1, ..., w_T)$ of a sentence, where $w_i$ is the i-th word in the sentence. The goal is to give higher probabilities to meaningful sentences, and lower probabilities to meaningless constructions.
-
-In general, models that generate the probability of a sequence can be applied to many fields, like machine translation, speech recognition, information retrieval, part-of-speech tagging, and handwriting recognition. Take information retrieval, for example. If you were to search for "how long is a football bame" (where bame is a medical noun), the search engine would have asked if you had meant "how long is a football game" instead. This is because the probability of "how long is a football bame" is very low according to the language model; in addition, among all of the words easily confused with "bame", "game" would build the most probable sentence.
-
-#### Target Probability
-For language model's target probability $P(w_1, ..., w_T)$, if the words in the sentence were to be independent, the joint probability of the whole sentence would be the product of each word's probability:
-
-$$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t)$$
-
-However, the frequency of words in a sentence typically relates to the words before them, so canonical language models are constructed using conditional probability in its target probability:
-
-$$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
-
-
-### N-gram neural model
-
-In computational linguistics, n-gram is an important method to represent text. An n-gram represents a contiguous sequence of n consecutive items given a text. Based on the desired application scenario, each item could be a letter, a syllable or a word. The N-gram model is also an important method in statistical language modeling. When training language models with n-grams, the first (n-1) words of an n-gram are used to predict the *n*th word.
-
-Yoshua Bengio and other scientists describe how to train a word embedding model using neural network in the famous paper of Neural Probabilistic Language Models \[[1](#reference)\] published in 2003. The Neural Network Language Model (NNLM) described in the paper learns the language model and word embedding simultaneously through a linear transformation and a non-linear hidden connection. That is, after training on large amounts of corpus, the model learns the word embedding; then, it computes the probability of the whole sentence, using the embedding. This type of language model can overcome the **curse of dimensionality** i.e. model inaccuracy caused by the difference in dimensionality between training and testing data. Note that the term *neural network language model* is ill-defined, so we will not use the name NNLM but only refer to it as *N-gram neural model* in this section.
-
-We have previously described language model using conditional probability, where the probability of the *t*-th word in a sentence depends on all $t-1$ words before it. Furthermore, since words further prior have less impact on a word, and every word within an n-gram is only effected by its previous n-1 words, we have:
-
-$$P(w_1, ..., w_T) = \prod_{t=n}^TP(w_t|w_{t-1}, w_{t-2}, ..., w_{t-n+1})$$
-
-Given some real corpus in which all sentences are meaningful, the n-gram model should maximize the following objective function:
-
-$$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
-
-where $f(w_t, w_{t-1}, ..., w_{t-n+1})$ represents the conditional probability of the current word $w_t$ given its previous $n-1$ words, and $R(\theta)$ represents parameter regularization term.
-
-<p align="center">
-       <img src="image/nnlm_en.png" width=500><br/>
-       Figure 2. N-gram neural network model
-</p>
-
-
-Figure 2 shows the N-gram neural network model. From the bottom up, the model has the following components:
-
- - For each sample, the model gets input $w_{t-n+1},...w_{t-1}$, and outputs the probability that the t-th word is one of `|V|` in the dictionary.
-
- Every input word $w_{t-n+1},...w_{t-1}$ first gets transformed into word embedding $C(w_{t-n+1}),...C(w_{t-1})$ through a transformation matrix.
-
- - All the word embeddings concatenate into a single vector, which is mapped (nonlinearly) into the $t$-th word hidden representation:
-
-    $$g=Utanh(\theta^Tx + b_1) + Wx + b_2$$
-
-   where $x$ is the large vector concatenated from all the word embeddings representing the context; $\theta$, $U$, $b_1$, $b_2$ and $W$ are parameters connecting word embedding layers to the hidden layers. $g$ represents the unnormalized probability of the output word, $g_i$ represents the unnormalized probability of the output word being the i-th word in the dictionary.
-
- - Based on the definition of softmax, using normalized $g_i$, the probability that the output word is $w_t$ is represented as:
-
-  $$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
-
- - The cost of the entire network is a multi-class cross-entropy and can be described by the following loss function
-
-   $$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$
-
-   where $y_k^i$ represents the true label for the $k$-th class in the $i$-th sample ($0$ or $1$), $softmax(g_k^i)$ represents the softmax probability for the $k$-th class in the $i$-th sample.
-
-### Continuous Bag-of-Words model(CBOW)
-
-CBOW model predicts the current word based on the N words both before and after it. When $N=2$, the model is as the figure below:
-
-<p align="center">
-    <img src="image/cbow_en.png" width=250><br/>
-    Figure 3. CBOW model
-</p>
-
-Specifically, by ignoring the order of words in the sequence, CBOW uses the average value of the word embedding of the context to predict the current word:
-
-$$\text{context} = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
-
-where $x_t$ is the word embedding of the t-th word, classification score vector is $z=U*\text{context}$, the final classification $y$ uses softmax and the loss function uses multi-class cross-entropy.
-
-### Skip-gram model
-
-The advantages of CBOW is that it smooths over the word embeddings of the context and reduces noise, so it is very effective on small dataset. Skip-gram uses a word to predict its context and get multiple context for the given word, so it can be used in larger datasets.
-
-<p align="center">
-    <img src="image/skipgram_en.png" width=250><br/>
-    Figure 4. Skip-gram model
-</p>
-
-As illustrated in the figure above, skip-gram model maps the word embedding of the given word onto $2n$ word embeddings (including $n$ words before and $n$ words after the given word), and then combine the classification loss of all those $2n$ words by softmax.
-
-## Dataset
-
-We will use Peen Treebank (PTB) (Tomas Mikolov's pre-processed version) dataset. PTB is a small dataset, used in Recurrent Neural Network Language Modeling Toolkit\[[2](#reference)\]. Its statistics are as follows:
-
-<p align="center">
-<table>
-    <tr>
-        <td>training set</td>
-        <td>validation set</td>
-        <td>test set</td>
-    </tr>
-    <tr>
-        <td>ptb.train.txt</td>
-        <td>ptb.valid.txt</td>
-        <td>ptb.test.txt</td>
-    </tr>
-    <tr>
-        <td>42068 lines</td>
-        <td>3370 lines</td>
-        <td>3761 lines</td>
-    </tr>
-</table>
-</p>
-
-### Python Dataset Module
-
-We encapsulated the PTB Data Set in our Python module `paddle.dataset.imikolov`. This module can
-
-1. download the dataset to `~/.cache/paddle/dataset/imikolov`, if not yet, and
-2. [preprocesses](#preprocessing) the dataset.
-
-### Preprocessing
-
-We will be training a 5-gram model. Given five words in a window, we will predict the fifth word given the first four words.
-
-Beginning and end of a sentence have a special meaning, so we will add begin token `<s>` in the front of the sentence. And end token `<e>` in the end of the sentence. By moving the five word window in the sentence, data instances are generated.
-
-For example, the sentence "I have a dream that one day" generates five data instances:
-
-```text
-<s> I have a dream
-I have a dream that
-have a dream that one
-a dream that one day
-dream that one day <e>
-```
-
-At last, each data instance will be converted into an integer sequence according it's words' index inside the dictionary.
-
-## Training
-
-The neural network that we will be using is illustrated in the graph below:
-
-<p align="center">
-    <img src="image/ngram.en.png" width=400><br/>
-    Figure 5. N-gram neural network model in model configuration
-</p>
-
-`word2vec/train.py` demonstrates training word2vec using PaddlePaddle:
-
-- Import packages.
-
-```python
-import math
-import paddle.v2 as paddle
-```
-
-- Configure parameter.
-
-```python
-embsize = 32 # word vector dimension
-hiddensize = 256 # hidden layer dimension
-N = 5 # train 5-gram
-```
-
-- Map the $n-1$ words $w_{t-n+1},...w_{t-1}$ before $w_t$ to a D-dimensional vector though matrix of dimention $|V|\times D$ (D=32 in this example).
-
-```python
-def wordemb(inlayer):
-    wordemb = paddle.layer.table_projection(
-        input=inlayer,
-        size=embsize,
-        param_attr=paddle.attr.Param(
-            name="_proj",
-            initial_std=0.001,
-            learning_rate=1,
-            l2_rate=0, ))
-    return wordemb
-```
-
-- Define name and type for input to data layer.
-
-```python
-paddle.init(use_gpu=False, trainer_count=3)
-word_dict = paddle.dataset.imikolov.build_dict()
-dict_size = len(word_dict)
-# Every layer takes integer value of range [0, dict_size)
-firstword = paddle.layer.data(
-    name="firstw", type=paddle.data_type.integer_value(dict_size))
-secondword = paddle.layer.data(
-    name="secondw", type=paddle.data_type.integer_value(dict_size))
-thirdword = paddle.layer.data(
-    name="thirdw", type=paddle.data_type.integer_value(dict_size))
-fourthword = paddle.layer.data(
-    name="fourthw", type=paddle.data_type.integer_value(dict_size))
-nextword = paddle.layer.data(
-    name="fifthw", type=paddle.data_type.integer_value(dict_size))
-
-Efirst = wordemb(firstword)
-Esecond = wordemb(secondword)
-Ethird = wordemb(thirdword)
-Efourth = wordemb(fourthword)
-```
-
-- Concatenate n-1 word embedding vectors into a single feature vector.
-
-```python
-contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
-```
-
-- Feature vector will go through a fully connected layer which outputs a hidden feature vector.
-
-```python
-hidden1 = paddle.layer.fc(input=contextemb,
-                          size=hiddensize,
-                          act=paddle.activation.Sigmoid(),
-                          layer_attr=paddle.attr.Extra(drop_rate=0.5),
-                          bias_attr=paddle.attr.Param(learning_rate=2),
-                          param_attr=paddle.attr.Param(
-                                initial_std=1. / math.sqrt(embsize * 8),
-                                learning_rate=1))
-```
-
-- Hidden feature vector will go through another fully conected layer, turn into a $|V|$ dimensional vector. At the same time softmax will be applied to get the probability of each word being generated.
-
-```python
-predictword = paddle.layer.fc(input=hidden1,
-                              size=dict_size,
-                              bias_attr=paddle.attr.Param(learning_rate=2),
-                              act=paddle.activation.Softmax())
-```
-
-- We will use cross-entropy cost function.
-
-```python
-cost = paddle.layer.classification_cost(input=predictword, label=nextword)
-```
-
-- Create parameter, optimizer and trainer.
-
-```python
-parameters = paddle.parameters.create(cost)
-adam_optimizer = paddle.optimizer.Adam(
-    learning_rate=3e-3,
-    regularization=paddle.optimizer.L2Regularization(8e-4))
-trainer = paddle.trainer.SGD(cost, parameters, adam_optimizer)
-```
-
-Next, we will begin the training process. `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()` is our training set and test set. Both of the function will return a **reader**: In PaddlePaddle, reader is a python function which returns a Python iterator which output a single data instance at a time.
-
-`paddle.batch` takes reader as input, outputs a **batched reader**: In PaddlePaddle, a reader outputs a single data instance at a time but batched reader outputs a minibatch of data instances.
-
-```python
-import gzip
-
-def event_handler(event):
-    if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f, %s" % (
-                event.pass_id, event.batch_id, event.cost, event.metrics)
-
-    if isinstance(event, paddle.event.EndPass):
-        result = trainer.test(
-                    paddle.batch(
-                        paddle.dataset.imikolov.test(word_dict, N), 32))
-        print "Pass %d, Testing metrics %s" % (event.pass_id, result.metrics)
-        with gzip.open("model_%d.tar.gz"%event.pass_id, 'w') as f:
-            parameters.to_tar(f)
-
-trainer.train(
-    paddle.batch(paddle.dataset.imikolov.train(word_dict, N), 32),
-    num_passes=100,
-    event_handler=event_handler)
-```
-
-`trainer.train` will start training, the output of `event_handler` will be similar to following:
-```text
-Pass 0, Batch 0, Cost 7.870579, {'classification_error_evaluator': 1.0}, Testing metrics {'classification_error_evaluator': 0.999591588973999}
-Pass 0, Batch 100, Cost 6.136420, {'classification_error_evaluator': 0.84375}, Testing metrics {'classification_error_evaluator': 0.8328699469566345}
-Pass 0, Batch 200, Cost 5.786797, {'classification_error_evaluator': 0.8125}, Testing metrics {'classification_error_evaluator': 0.8328542709350586}
-...
-```
-
-After 30 passes, we can get average error rate around 0.735611.
-
-
-## Model Application
-
-After the model is trained, we can load saved model parameters and uses it for other models. We can also use the parameters in applications.
-
-### Viewing Word Vector
-
-Parameters trained by PaddlePaddle can be viewed by `parameters.get()`. For example, we can check the word vector for word `apple`.
-
-```python
-embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
-
-print embeddings[word_dict['apple']]
-```
-
-```text
-[-0.38961065 -0.02392169 -0.00093231  0.36301503  0.13538605  0.16076435
--0.0678709   0.1090285   0.42014077 -0.24119169 -0.31847557  0.20410083
-0.04910378  0.19021918 -0.0122014  -0.04099389 -0.16924137  0.1911236
--0.10917275  0.13068172 -0.23079982  0.42699069 -0.27679482 -0.01472992
-0.2069038   0.09005053 -0.3282454   0.12717034 -0.24218646  0.25304323
-0.19072419 -0.24286366]
-```
-
-### Modifying Word Vector
-
-Word vectors (`embeddings`) that we get is a numpy array. We can modify this array and set it back to `parameters`.
-
-
-```python
-def modify_embedding(emb):
-    # Add your modification here.
-    pass
-
-modify_embedding(embeddings)
-parameters.set("_proj", embeddings)
-```
-
-### Calculating Cosine Similarity
-
-Cosine similarity is one way of quantifying the similarity between two vectors. The range of result is $[-1, 1]$. The bigger the value, the similar two vectors are:
-
-
-```python
-from scipy import spatial
-
-emb_1 = embeddings[word_dict['world']]
-emb_2 = embeddings[word_dict['would']]
-
-print spatial.distance.cosine(emb_1, emb_2)
-```
-
-```text
-0.99375076448
-```
-
-## Conclusion
-
-This chapter introduces word embedding, the relationship between language model and word embedding, and how to train neural networks to learn word embedding.
-
-In information retrieval, the relevance between the query and document keyword can be computed through the cosine similarity of their word embeddings. In grammar analysis and semantic analysis, a previously trained word embedding can initialize models for better performance. In document classification, clustering the word embedding can group synonyms in the documents. We hope that readers can use word embedding models in their work after reading this chapter.
-
-
-## Referenes
-1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
-2. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
-3. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space](https://arxiv.org/pdf/1301.3781.pdf)[J]. arXiv preprint arXiv:1301.3781, 2013.
-4. Maaten L, Hinton G. [Visualizing data using t-SNE](https://lvdmaaten.github.io/publications/papers/JMLR_2008.pdf)[J]. Journal of Machine Learning Research, 2008, 9(Nov): 2579-2605.
-5. https://en.wikipedia.org/wiki/Singular_value_decomposition
-
-<br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.

04.word2vec/README.md

+# Word2Vec
 
-# 词向量
+This is intended as a reference tutorial. The source code of this tutorial lives on [book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec).
 
-本教程源代码目录在[book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
+## Background Introduction
 
-本章我们介绍词的向量表征，也称为word embedding。词向量是自然语言处理中常见的一个操作，是搜索引擎、广告系统、推荐系统等互联网服务背后常见的基础技术。
+This section introduces the concept of **word embedding**, which is a vector representation of words. It is a popular technique used in natural language processing. Word embeddings support many Internet services, including search engines, advertising systems, and recommendation systems.
 
-在这些互联网服务里，我们经常要比较两个词或者两段文本之间的相关性。为了做这样的比较，我们往往先要把词表示成计算机适合处理的方式。最自然的方式恐怕莫过于向量空间模型(vector space model)。
-在这种方式里，每个词被表示成一个实数向量（one-hot vector），其长度为字典大小，每个维度对应一个字典里的每个词，除了这个词对应维度上的值是1，其他元素都是0。
+### One-Hot Vectors
 
-One-hot vector虽然自然，但是用处有限。比如，在互联网广告系统里，如果用户输入的query是“母亲节”，而有一个广告的关键词是“康乃馨”。虽然按照常理，我们知道这两个词之间是有联系的——母亲节通常应该送给母亲一束康乃馨；但是这两个词对应的one-hot vectors之间的距离度量，无论是欧氏距离还是余弦相似度(cosine similarity)，由于其向量正交，都认为这两个词毫无相关性。 得出这种与我们相悖的结论的根本原因是：每个词本身的信息量都太小。所以，仅仅给定两个词，不足以让我们准确判别它们是否相关。要想精确计算相关性，我们还需要更多的信息——从大量数据里通过机器学习方法归纳出来的知识。
+Building these services requires us to quantify the similarity between two words or paragraphs. This calls for a new representation of all the words to make them more suitable for computation. An obvious way to achieve this is through the vector space model, where every word is represented as an **one-hot vector**.
 
-在机器学习领域里，各种“知识”被各种模型表示，词向量模型(word embedding model)就是其中的一类。通过词向量模型可将一个 one-hot vector映射到一个维度更低的实数向量（embedding vector），如$embedding(母亲节) = [0.3, 4.2, -1.5, ...], embedding(康乃馨) = [0.2, 5.6, -2.3, ...]$。在这个映射到的实数向量表示中，希望两个语义（或用法）上相似的词对应的词向量“更像”，这样如“母亲节”和“康乃馨”的对应词向量的余弦相似度就不再为零了。
+For each word, its vector representation has the corresponding entry in the vector as 1, and all other entries as 0. The lengths of one-hot vectors match the size of the dictionary. Each entry of a vector corresponds to the presence (or absence) of a word in the dictionary.
 
-词向量模型可以是概率模型、共生矩阵(co-occurrence matrix)模型或神经元网络模型。在用神经网络求词向量之前，传统做法是统计一个词语的共生矩阵$X$。$X$是一个$|V| \times |V|$ 大小的矩阵，$X_{ij}$表示在所有语料中，词汇表`V`(vocabulary)中第i个词和第j个词同时出现的词数，$|V|$为词汇表的大小。对$X$做矩阵分解（如奇异值分解，Singular Value Decomposition \[[5](#参考文献)\]），得到的$U$即视为所有词的词向量：
+One-hot vectors are intuitive, yet they have limited usefulness. Take the example of an Internet advertising system: Suppose a customer enters the query "Mother's Day", while an ad bids for the keyword carnations". Because the one-hot vectors of these two words are perpendicular, the metric distance (either Euclidean or cosine similarity) between them would indicate  little relevance. However, *we* know that these two queries are connected semantically, since people often gift their mothers bundles of carnation flowers on Mother's Day. This discrepancy is due to the low information capacity in each vector. That is, comparing the vector representations of two words does not assess their relevance sufficiently. To calculate their similarity accurately, we need more information, which could be learned from large amounts of data through machine learning methods.
 
-$$X = USV^T$$
+Like many machine learning models, word embeddings can represent knowledge in various ways. Another model may project an one-hot vector to an embedding vector of lower dimension e.g. $embedding(mother's day) = [0.3, 4.2, -1.5, ...], embedding(carnations) = [0.2, 5.6, -2.3, ...]$. Mapping one-hot vectors onto an embedded vector space has the potential to bring the embedding vectors of similar words (either semantically or usage-wise) closer to each other, so that the cosine similarity between the corresponding vectors for words like "Mother's Day" and "carnations" are no longer zero.
+
+A word embedding model could be a probabilistic model, a co-occurrence matrix model, or a neural network. Before people started using neural networks to generate word embedding, the traditional method was to calculate a co-occurrence matrix $X$ of words. Here, $X$ is a $|V| \times |V|$ matrix, where $X_{ij}$ represents the co-occurrence times of the $i$th and $j$th words in the vocabulary `V` within all corpus, and $|V|$ is the size of the vocabulary. By performing matrix decomposition on $X$ e.g. Singular Value Decomposition \[[5](#References)\]
 
-但这样的传统做法有很多问题：<br/>
-1) 由于很多词没有出现，导致矩阵极其稀疏，因此需要对词频做额外处理来达到好的矩阵分解效果；<br/>
-2) 矩阵非常大，维度太高(通常达到$10^6*10^6$的数量级)；<br/>
-3) 需要手动去掉停用词（如although, a,...），不然这些频繁出现的词也会影响矩阵分解的效果。
+$$X = USV^T$$
 
+the resulting $U$ can be seen as the word embedding of all the words.
 
-基于神经网络的模型不需要计算存储一个在全语料上统计的大表，而是通过学习语义信息得到词向量，因此能很好地解决以上问题。在本章里，我们将展示基于神经网络训练词向量的细节，以及如何用PaddlePaddle训练一个词向量模型。
+However, this method suffers from many drawbacks:
+1) Since many pairs of words don't co-occur, the co-occurrence matrix is sparse. To achieve good performance of matrix factorization, further treatment on word frequency is needed;
+2) The matrix is large, frequently on the order of $10^6*10^6$;
+3) We need to manually filter out stop words (like "although", "a", ...), otherwise these frequent words will affect the performance of matrix factorization.
 
+The neural network based model does not require storing huge hash tables of statistics on all of the corpus. It obtains the word embedding by learning from semantic information, hence could avoid the aforementioned problems in the traditional method. In this chapter, we will introduce the details of neural network word embedding model and how to train such model in PaddlePaddle.
 
-## 效果展示
+## Results Demonstration
 
-本章中，当词向量训练好后，我们可以用数据可视化算法t-SNE\[[4](#参考文献)\]画出词语特征在二维上的投影（如下图所示）。从图中可以看出，语义相关的词语（如a, the, these; big, huge）在投影上距离很近，语意无关的词（如say, business; decision, japan）在投影上的距离很远。
+In this section, after training the word embedding model, we could use the data visualization algorithm $t-$SNE\[[4](#reference)\] to draw the word embedding vectors after projecting them onto a two-dimensional space (see figure below). From the figure we could see that the semantically relevant words -- *a*, *the*, and *these* or *big* and *huge* -- are close to each other in the projected space, while irrelevant words -- *say* and *business* or *decision* and *japan* -- are far from each other.
 
 <p align="center">
     <img src = "image/2d_similarity.png" width=400><br/>
-    图1. 词向量的二维投影
+    Figure 1. Two dimension projection of word embeddings
 </p>
 
-另一方面，我们知道两个向量的余弦值在$[-1,1]$的区间内：两个完全相同的向量余弦值为1, 两个相互垂直的向量之间余弦值为0，两个方向完全相反的向量余弦值为-1，即相关性和余弦值大小成正比。因此我们还可以计算两个词向量的余弦相似度:
+### Cosine Similarity
+
+On the other hand, we know that the cosine similarity between two vectors falls between $[-1,1]$. Specifically, the cosine similarity is 1 when the vectors are identical, 0 when the vectors are perpendicular, -1 when the are of opposite directions. That is, the cosine similarity between two vectors scales with their relevance. So we can calculate the cosine similarity of two word embedding vectors to represent their relevance:
 
 ```
-similarity: 0.899180685161
 please input two words: big huge
+similarity: 0.899180685161
 
 please input two words: from company
 similarity: -0.0997506977351
 ```
 
-以上结果可以通过运行`calculate_dis.py`, 加载字典里的单词和对应训练特征结果得到，我们将在[应用模型](#应用模型)中详细描述用法。
+The above results could be obtained by running `calculate_dis.py`, which loads the words in the dictionary and their corresponding trained word embeddings. For detailed instruction, see section [Model Application](#Model Application).
+
+
+## Model Overview
 
+In this section, we will introduce three word embedding models: N-gram model, CBOW, and Skip-gram, which all output the frequency of each word given its immediate context.
 
-## 模型概览
+For N-gram model, we will first introduce the concept of language model, and implement it using PaddlePaddle in section [Model Training](#Model Training).
 
-在这里我们介绍三个训练词向量的模型：N-gram模型，CBOW模型和Skip-gram模型，它们的中心思想都是通过上下文得到一个词出现的概率。对于N-gram模型，我们会先介绍语言模型的概念，并在之后的[训练模型](#训练模型)中，带大家用PaddlePaddle实现它。而后两个模型，是近年来最有名的神经元词向量模型，由 Tomas Mikolov 在Google 研发\[[3](#参考文献)\]，虽然它们很浅很简单，但训练效果很好。
+The latter two models, which became popular recently, are neural word embedding model developed by Tomas Mikolov at Google \[[3](#reference)\]. Despite their apparent simplicity, these models train very well.
 
-### 语言模型
+### Language Model
 
-在介绍词向量模型之前，我们先来引入一个概念：语言模型。
-语言模型旨在为语句的联合概率函数$P(w_1, ..., w_T)$建模, 其中$w_i$表示句子中的第i个词。语言模型的目标是，希望模型对有意义的句子赋予大概率，对没意义的句子赋予小概率。
-这样的模型可以应用于很多领域，如机器翻译、语音识别、信息检索、词性标注、手写识别等，它们都希望能得到一个连续序列的概率。 以信息检索为例，当你在搜索“how long is a football bame”时（bame是一个医学名词），搜索引擎会提示你是否希望搜索"how long is a football game", 这是因为根据语言模型计算出“how long is a football bame”的概率很低，而与bame近似的，可能引起错误的词中，game会使该句生成的概率最大。
+Before diving into word embedding models, we will first introduce the concept of **language model**. Language models build the joint probability function $P(w_1, ..., w_T)$ of a sentence, where $w_i$ is the i-th word in the sentence. The goal is to give higher probabilities to meaningful sentences, and lower probabilities to meaningless constructions.
 
-对语言模型的目标概率$P(w_1, ..., w_T)$，如果假设文本中每个词都是相互独立的，则整句话的联合概率可以表示为其中所有词语条件概率的乘积，即：
+In general, models that generate the probability of a sequence can be applied to many fields, like machine translation, speech recognition, information retrieval, part-of-speech tagging, and handwriting recognition. Take information retrieval, for example. If you were to search for "how long is a football bame" (where bame is a medical noun), the search engine would have asked if you had meant "how long is a football game" instead. This is because the probability of "how long is a football bame" is very low according to the language model; in addition, among all of the words easily confused with "bame", "game" would build the most probable sentence.
+
+#### Target Probability
+For language model's target probability $P(w_1, ..., w_T)$, if the words in the sentence were to be independent, the joint probability of the whole sentence would be the product of each word's probability:
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t)$$
 
-然而我们知道语句中的每个词出现的概率都与其前面的词紧密相关, 所以实际上通常用条件概率表示语言模型：
+However, the frequency of words in a sentence typically relates to the words before them, so canonical language models are constructed using conditional probability in its target probability:
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
 
 
-
 ### N-gram neural model
 
-在计算语言学中，n-gram是一种重要的文本表示方法，表示一个文本中连续的n个项。基于具体的应用场景，每一项可以是一个字母、单词或者音节。 n-gram模型也是统计语言模型中的一种重要方法，用n-gram训练语言模型时，一般用每个n-gram的历史n-1个词语组成的内容来预测第n个词。
+In computational linguistics, n-gram is an important method to represent text. An n-gram represents a contiguous sequence of n consecutive items given a text. Based on the desired application scenario, each item could be a letter, a syllable or a word. The N-gram model is also an important method in statistical language modeling. When training language models with n-grams, the first (n-1) words of an n-gram are used to predict the *n*th word.
 
-Yoshua Bengio等科学家就于2003年在著名论文 Neural Probabilistic Language Models \[[1](#参考文献)\] 中介绍如何学习一个神经元网络表示的词向量模型。文中的神经概率语言模型（Neural Network Language Model，NNLM）通过一个线性映射和一个非线性隐层连接，同时学习了语言模型和词向量，即通过学习大量语料得到词语的向量表达，通过这些向量得到整个句子的概率。用这种方法学习语言模型可以克服维度灾难（curse of dimensionality）,即训练和测试数据不同导致的模型不准。注意：由于“神经概率语言模型”说法较为泛泛，我们在这里不用其NNLM的本名，考虑到其具体做法，本文中称该模型为N-gram neural model。
+Yoshua Bengio and other scientists describe how to train a word embedding model using neural network in the famous paper of Neural Probabilistic Language Models \[[1](#reference)\] published in 2003. The Neural Network Language Model (NNLM) described in the paper learns the language model and word embedding simultaneously through a linear transformation and a non-linear hidden connection. That is, after training on large amounts of corpus, the model learns the word embedding; then, it computes the probability of the whole sentence, using the embedding. This type of language model can overcome the **curse of dimensionality** i.e. model inaccuracy caused by the difference in dimensionality between training and testing data. Note that the term *neural network language model* is ill-defined, so we will not use the name NNLM but only refer to it as *N-gram neural model* in this section.
 
-我们在上文中已经讲到用条件概率建模语言模型，即一句话中第$t$个词的概率和该句话的前$t-1$个词相关。可实际上越远的词语其实对该词的影响越小，那么如果考虑一个n-gram, 每个词都只受其前面`n-1`个词的影响，则有：
+We have previously described language model using conditional probability, where the probability of the *t*-th word in a sentence depends on all $t-1$ words before it. Furthermore, since words further prior have less impact on a word, and every word within an n-gram is only effected by its previous n-1 words, we have:
 
 $$P(w_1, ..., w_T) = \prod_{t=n}^TP(w_t|w_{t-1}, w_{t-2}, ..., w_{t-n+1})$$
 
-给定一些真实语料，这些语料中都是有意义的句子，N-gram模型的优化目标则是最大化目标函数:
+Given some real corpus in which all sentences are meaningful, the n-gram model should maximize the following objective function:
 
 $$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
 
-其中$f(w_t, w_{t-1}, ..., w_{t-n+1})$表示根据历史n-1个词得到当前词$w_t$的条件概率，$R(\theta)$表示参数正则项。
+where $f(w_t, w_{t-1}, ..., w_{t-n+1})$ represents the conditional probability of the current word $w_t$ given its previous $n-1$ words, and $R(\theta)$ represents parameter regularization term.
 
 <p align="center">
-       <img src="image/nnlm.png" width=500><br/>
-       图2. N-gram神经网络模型
+       <img src="image/nnlm_en.png" width=500><br/>
+       Figure 2. N-gram neural network model
 </p>
 
-图2展示了N-gram神经网络模型，从下往上看，该模型分为以下几个部分：
- - 对于每个样本，模型输入$w_{t-n+1},...w_{t-1}$, 输出句子第t个词为字典中`|V|`个词的概率。
 
-   每个输入词$w_{t-n+1},...w_{t-1}$首先通过映射矩阵映射到词向量$C(w_{t-n+1}),...C(w_{t-1})$。
+Figure 2 shows the N-gram neural network model. From the bottom up, the model has the following components:
+
+ - For each sample, the model gets input $w_{t-n+1},...w_{t-1}$, and outputs the probability that the t-th word is one of `|V|` in the dictionary.
+
+ Every input word $w_{t-n+1},...w_{t-1}$ first gets transformed into word embedding $C(w_{t-n+1}),...C(w_{t-1})$ through a transformation matrix.
 
- - 然后所有词语的词向量连接成一个大向量，并经过一个非线性映射得到历史词语的隐层表示：
+ - All the word embeddings concatenate into a single vector, which is mapped (nonlinearly) into the $t$-th word hidden representation:
 
     $$g=Utanh(\theta^Tx + b_1) + Wx + b_2$$
 
-    其中，$x$为所有词语的词向量连接成的大向量，表示文本历史特征；$\theta$、$U$、$b_1$、$b_2$和$W$分别为词向量层到隐层连接的参数。$g$表示未经归一化的所有输出单词概率，$g_i$表示未经归一化的字典中第$i$个单词的输出概率。
+   where $x$ is the large vector concatenated from all the word embeddings representing the context; $\theta$, $U$, $b_1$, $b_2$ and $W$ are parameters connecting word embedding layers to the hidden layers. $g$ represents the unnormalized probability of the output word, $g_i$ represents the unnormalized probability of the output word being the i-th word in the dictionary.
 
- - 根据softmax的定义，通过归一化$g_i$, 生成目标词$w_t$的概率为：
+ - Based on the definition of softmax, using normalized $g_i$, the probability that the output word is $w_t$ is represented as:
 
   $$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
 
- - 整个网络的损失值(cost)为多类分类交叉熵，用公式表示为
+ - The cost of the entire network is a multi-class cross-entropy and can be described by the following loss function
 
    $$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$
 
-   其中$y_k^i$表示第$i$个样本第$k$类的真实标签(0或1)，$softmax(g_k^i)$表示第i个样本第k类softmax输出的概率。
-
-
+   where $y_k^i$ represents the true label for the $k$-th class in the $i$-th sample ($0$ or $1$), $softmax(g_k^i)$ represents the softmax probability for the $k$-th class in the $i$-th sample.
 
 ### Continuous Bag-of-Words model(CBOW)
 
-CBOW模型通过一个词的上下文（各N个词）预测当前词。当N=2时，模型如下图所示：
+CBOW model predicts the current word based on the N words both before and after it. When $N=2$, the model is as the figure below:
 
 <p align="center">
-    <img src="image/cbow.png" width=250><br/>
-    图3. CBOW模型
+    <img src="image/cbow_en.png" width=250><br/>
+    Figure 3. CBOW model
 </p>
 
-具体来说，不考虑上下文的词语输入顺序，CBOW是用上下文词语的词向量的均值来预测当前词。即：
+Specifically, by ignoring the order of words in the sequence, CBOW uses the average value of the word embedding of the context to predict the current word:
 
-$$context = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
+$$\text{context} = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
 
-其中$x_t$为第$t$个词的词向量，分类分数（score）向量 $z=U*context$，最终的分类$y$采用softmax，损失函数采用多类分类交叉熵。
+where $x_t$ is the word embedding of the t-th word, classification score vector is $z=U*\text{context}$, the final classification $y$ uses softmax and the loss function uses multi-class cross-entropy.
 
 ### Skip-gram model
 
-CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去掉了噪声，因此在小数据集上很有效。而Skip-gram的方法中，用一个词预测其上下文，得到了当前词上下文的很多样本，因此可用于更大的数据集。
+The advantages of CBOW is that it smooths over the word embeddings of the context and reduces noise, so it is very effective on small dataset. Skip-gram uses a word to predict its context and get multiple context for the given word, so it can be used in larger datasets.
 
 <p align="center">
-    <img src="image/skipgram.png" width=250><br/>
-    图4. Skip-gram模型
+    <img src="image/skipgram_en.png" width=250><br/>
+    Figure 4. Skip-gram model
 </p>
 
-如上图所示，Skip-gram模型的具体做法是，将一个词的词向量映射到$2n$个词的词向量（$2n$表示当前输入词的前后各$n$个词），然后分别通过softmax得到这$2n$个词的分类损失值之和。
-
+As illustrated in the figure above, skip-gram model maps the word embedding of the given word onto $2n$ word embeddings (including $n$ words before and $n$ words after the given word), and then combine the classification loss of all those $2n$ words by softmax.
 
-## 数据准备
+## Dataset
 
-### 数据介绍
-
-本教程使用Penn Treebank （PTB）（经Tomas Mikolov预处理过的版本）数据集。PTB数据集较小，训练速度快，应用于Mikolov的公开语言模型训练工具\[[2](#参考文献)\]中。其统计情况如下：
+We will use Peen Treebank (PTB) (Tomas Mikolov's pre-processed version) dataset. PTB is a small dataset, used in Recurrent Neural Network Language Modeling Toolkit\[[2](#reference)\]. Its statistics are as follows:
 
 <p align="center">
 <table>
     <tr>
-        <td>训练数据</td>
-        <td>验证数据</td>
-        <td>测试数据</td>
+        <td>training set</td>
+        <td>validation set</td>
+        <td>test set</td>
     </tr>
     <tr>
         <td>ptb.train.txt</td>
@@ -159,21 +166,27 @@ CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去
         <td>ptb.test.txt</td>
     </tr>
     <tr>
-        <td>42068句</td>
-        <td>3370句</td>
-        <td>3761句</td>
+        <td>42068 lines</td>
+        <td>3370 lines</td>
+        <td>3761 lines</td>
     </tr>
 </table>
 </p>
 
+### Python Dataset Module
+
+We encapsulated the PTB Data Set in our Python module `paddle.dataset.imikolov`. This module can
 
-### 数据预处理
+1. download the dataset to `~/.cache/paddle/dataset/imikolov`, if not yet, and
+2. [preprocesses](#preprocessing) the dataset.
 
-本章训练的是5-gram模型，表示在PaddlePaddle训练时，每条数据的前4个词用来预测第5个词。PaddlePaddle提供了对应PTB数据集的python包`paddle.dataset.imikolov`，自动做数据的下载与预处理，方便大家使用。
+### Preprocessing
 
-预处理会把数据集中的每一句话前后加上开始符号`<s>`以及结束符号`<e>`。然后依据窗口大小（本教程中为5），从头到尾每次向右滑动窗口并生成一条数据。
+We will be training a 5-gram model. Given five words in a window, we will predict the fifth word given the first four words.
 
-如"I have a dream that one day" 一句提供了5条数据：
+Beginning and end of a sentence have a special meaning, so we will add begin token `<s>` in the front of the sentence. And end token `<e>` in the end of the sentence. By moving the five word window in the sentence, data instances are generated.
+
+For example, the sentence "I have a dream that one day" generates five data instances:
 
 ```text
 <s> I have a dream
@@ -183,33 +196,35 @@ a dream that one day
 dream that one day <e>
 ```
 
-最后，每个输入会按其单词次在字典里的位置，转化成整数的索引序列，作为PaddlePaddle的输入。
-## 编程实现
+At last, each data instance will be converted into an integer sequence according it's words' index inside the dictionary.
+
+## Training
 
-本配置的模型结构如下图所示：
+The neural network that we will be using is illustrated in the graph below:
 
 <p align="center">
-    <img src="image/ngram.png" width=400><br/>
-    图5. 模型配置中的N-gram神经网络模型
+    <img src="image/ngram.en.png" width=400><br/>
+    Figure 5. N-gram neural network model in model configuration
 </p>
 
-首先，加载所需要的包：
+`word2vec/train.py` demonstrates training word2vec using PaddlePaddle:
+
+- Import packages.
 
 ```python
 import math
 import paddle.v2 as paddle
 ```
 
-然后，定义参数：
+- Configure parameter.
+
 ```python
-embsize = 32 # 词向量维度
-hiddensize = 256 # 隐层维度
-N = 5 # 训练5-Gram
+embsize = 32 # word vector dimension
+hiddensize = 256 # hidden layer dimension
+N = 5 # train 5-gram
 ```
 
-接着，定义网络结构：
-
-- 将$w_t$之前的$n-1$个词 $w_{t-n+1},...w_{t-1}$，通过$|V|\times D$的矩阵映射到D维词向量（本例中取D=32）。
+- Map the $n-1$ words $w_{t-n+1},...w_{t-1}$ before $w_t$ to a D-dimensional vector though matrix of dimention $|V|\times D$ (D=32 in this example).
 
 ```python
 def wordemb(inlayer):
@@ -220,17 +235,18 @@ def wordemb(inlayer):
             name="_proj",
             initial_std=0.001,
             learning_rate=1,
-            l2_rate=0, ))
+            l2_rate=0,
+            sparse_update=True))
     return wordemb
 ```
 
-- 定义输入层接受的数据类型以及名字。
+- Define name and type for input to data layer.
 
 ```python
-paddle.init(use_gpu=False, trainer_count=3) # 初始化PaddlePaddle
+paddle.init(use_gpu=False, trainer_count=3)
 word_dict = paddle.dataset.imikolov.build_dict()
 dict_size = len(word_dict)
-# 每个输入层都接受整形数据，这些数据的范围是[0, dict_size)
+# Every layer takes integer value of range [0, dict_size)
 firstword = paddle.layer.data(
     name="firstw", type=paddle.data_type.integer_value(dict_size))
 secondword = paddle.layer.data(
@@ -248,13 +264,13 @@ Ethird = wordemb(thirdword)
 Efourth = wordemb(fourthword)
 ```
 
-- 将这n-1个词向量经过concat_layer连接成一个大向量作为历史文本特征。
+- Concatenate n-1 word embedding vectors into a single feature vector.
 
 ```python
 contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
 ```
 
-- 将历史文本特征经过一个全连接得到文本隐层特征。
+- Feature vector will go through a fully connected layer which outputs a hidden feature vector.
 
 ```python
 hidden1 = paddle.layer.fc(input=contextemb,
@@ -267,7 +283,7 @@ hidden1 = paddle.layer.fc(input=contextemb,
                                 learning_rate=1))
 ```
 
-- 将文本隐层特征，再经过一个全连接，映射成一个$|V|$维向量，同时通过softmax归一化得到这`|V|`个词的生成概率。
+- Hidden feature vector will go through another fully conected layer, turn into a $|V|$ dimensional vector. At the same time softmax will be applied to get the probability of each word being generated.
 
 ```python
 predictword = paddle.layer.fc(input=hidden1,
@@ -276,33 +292,27 @@ predictword = paddle.layer.fc(input=hidden1,
                               act=paddle.activation.Softmax())
 ```
 
-- 网络的损失函数为多分类交叉熵，可直接调用`classification_cost`函数。
+- We will use cross-entropy cost function.
 
 ```python
 cost = paddle.layer.classification_cost(input=predictword, label=nextword)
 ```
 
-然后，指定训练相关的参数：
-
-- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器，本教程使用Adam优化器。
-- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
-- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+- Create parameter, optimizer and trainer.
 
 ```python
 parameters = paddle.parameters.create(cost)
-adam_optimizer = paddle.optimizer.Adam(
+adagrad = paddle.optimizer.AdaGrad(
     learning_rate=3e-3,
     regularization=paddle.optimizer.L2Regularization(8e-4))
-trainer = paddle.trainer.SGD(cost, parameters, adam_optimizer)
+trainer = paddle.trainer.SGD(cost, parameters, adagrad)
 ```
 
-下一步，我们开始训练过程。`paddle.dataset.imikolov.train()`和`paddle.dataset.imikolov.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python generator。
+Next, we will begin the training process. `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()` is our training set and test set. Both of the function will return a **reader**: In PaddlePaddle, reader is a python function which returns a Python iterator which output a single data instance at a time.
 
-`paddle.batch`的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minbatch。
+`paddle.batch` takes reader as input, outputs a **batched reader**: In PaddlePaddle, a reader outputs a single data instance at a time but batched reader outputs a minibatch of data instances.
 
 ```python
-import gzip
-
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -314,7 +324,7 @@ def event_handler(event):
                     paddle.batch(
                         paddle.dataset.imikolov.test(word_dict, N), 32))
         print "Pass %d, Testing metrics %s" % (event.pass_id, result.metrics)
-        with gzip.open("model_%d.tar.gz"%event.pass_id, 'w') as f:
+        with open("model_%d.tar"%event.pass_id, 'w') as f:
             parameters.to_tar(f)
 
 trainer.train(
@@ -323,6 +333,7 @@ trainer.train(
     event_handler=event_handler)
 ```
 
+`trainer.train` will start training, the output of `event_handler` will be similar to following:
 ```text
 Pass 0, Batch 0, Cost 7.870579, {'classification_error_evaluator': 1.0}, Testing metrics {'classification_error_evaluator': 0.999591588973999}
 Pass 0, Batch 100, Cost 6.136420, {'classification_error_evaluator': 0.84375}, Testing metrics {'classification_error_evaluator': 0.8328699469566345}
@@ -330,19 +341,16 @@ Pass 0, Batch 200, Cost 5.786797, {'classification_error_evaluator': 0.8125}, Te
 ...
 ```
 
-训练过程是完全自动的，event_handler里打印的日志类似如上所示：
-
-经过30个pass，我们将得到平均错误率为classification_error_evaluator=0.735611。
-
+After 30 passes, we can get average error rate around 0.735611.
 
-## 应用模型
-训练模型后，我们可以加载模型参数，用训练出来的词向量初始化其他模型，也可以将模型查看参数用来做后续应用。
 
+## Model Application
 
-### 查看词向量
+After the model is trained, we can load saved model parameters and uses it for other models. We can also use the parameters in applications.
 
-PaddlePaddle训练出来的参数可以直接使用`parameters.get()`获取出来。例如查看单词`apple`的词向量，即为
+### Viewing Word Vector
 
+Parameters trained by PaddlePaddle can be viewed by `parameters.get()`. For example, we can check the word vector for word `apple`.
 
 ```python
 embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
@@ -359,10 +367,9 @@ print embeddings[word_dict['apple']]
 0.19072419 -0.24286366]
 ```
 
+### Modifying Word Vector
 
-### 修改词向量
-
-获得到的embedding为一个标准的numpy矩阵。我们可以对这个numpy矩阵进行修改，然后赋值回去。
+Word vectors (`embeddings`) that we get is a numpy array. We can modify this array and set it back to `parameters`.
 
 
 ```python
@@ -374,10 +381,9 @@ modify_embedding(embeddings)
 parameters.set("_proj", embeddings)
 ```
 
-### 计算词语之间的余弦距离
+### Calculating Cosine Similarity
 
-两个向量之间的距离可以用余弦值来表示，余弦值在$[-1,1]$的区间内，向量间余弦值越大，其距离越近。这里我们在`calculate_dis.py`中实现不同词语的距离度量。
-用法如下：
+Cosine similarity is one way of quantifying the similarity between two vectors. The range of result is $[-1, 1]$. The bigger the value, the similar two vectors are:
 
 
 ```python
@@ -393,11 +399,14 @@ print spatial.distance.cosine(emb_1, emb_2)
 0.99375076448
 ```
 
-## 总结
-本章中，我们介绍了词向量、语言模型和词向量的关系、以及如何通过训练神经网络模型获得词向量。在信息检索中，我们可以根据向量间的余弦夹角，来判断query和文档关键词这二者间的相关性。在句法分析和语义分析中，训练好的词向量可以用来初始化模型，以得到更好的效果。在文档分类中，有了词向量之后，可以用聚类的方法将文档中同义词进行分组。希望大家在本章后能够自行运用词向量进行相关领域的研究。
+## Conclusion
+
+This chapter introduces word embedding, the relationship between language model and word embedding, and how to train neural networks to learn word embedding.
+
+In information retrieval, the relevance between the query and document keyword can be computed through the cosine similarity of their word embeddings. In grammar analysis and semantic analysis, a previously trained word embedding can initialize models for better performance. In document classification, clustering the word embedding can group synonyms in the documents. We hope that readers can use word embedding models in their work after reading this chapter.
 
 
-## 参考文献
+## Referenes
 1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
 2. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
 3. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space](https://arxiv.org/pdf/1301.3781.pdf)[J]. arXiv preprint arXiv:1301.3781, 2013.
@@ -405,4 +414,4 @@ print spatial.distance.cosine(emb_1, emb_2)
 5. https://en.wikipedia.org/wiki/Singular_value_decomposition
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

04.word2vec/index.en.html → 04.word2vec/index.cn.html

@@ -40,167 +40,160 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
-# Word2Vec
 
-This is intended as a reference tutorial. The source code of this tutorial lives on [book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec).
+# 词向量
 
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+本教程源代码目录在[book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/175.html)。
 
-## Background Introduction
+## 背景介绍
 
-This section introduces the concept of **word embedding**, which is a vector representation of words. It is a popular technique used in natural language processing. Word embeddings support many Internet services, including search engines, advertising systems, and recommendation systems.
+本章我们介绍词的向量表征，也称为word embedding。词向量是自然语言处理中常见的一个操作，是搜索引擎、广告系统、推荐系统等互联网服务背后常见的基础技术。
 
-### One-Hot Vectors
+在这些互联网服务里，我们经常要比较两个词或者两段文本之间的相关性。为了做这样的比较，我们往往先要把词表示成计算机适合处理的方式。最自然的方式恐怕莫过于向量空间模型(vector space model)。
+在这种方式里，每个词被表示成一个实数向量（one-hot vector），其长度为字典大小，每个维度对应一个字典里的每个词，除了这个词对应维度上的值是1，其他元素都是0。
 
-Building these services requires us to quantify the similarity between two words or paragraphs. This calls for a new representation of all the words to make them more suitable for computation. An obvious way to achieve this is through the vector space model, where every word is represented as an **one-hot vector**.
+One-hot vector虽然自然，但是用处有限。比如，在互联网广告系统里，如果用户输入的query是“母亲节”，而有一个广告的关键词是“康乃馨”。虽然按照常理，我们知道这两个词之间是有联系的——母亲节通常应该送给母亲一束康乃馨；但是这两个词对应的one-hot vectors之间的距离度量，无论是欧氏距离还是余弦相似度(cosine similarity)，由于其向量正交，都认为这两个词毫无相关性。 得出这种与我们相悖的结论的根本原因是：每个词本身的信息量都太小。所以，仅仅给定两个词，不足以让我们准确判别它们是否相关。要想精确计算相关性，我们还需要更多的信息——从大量数据里通过机器学习方法归纳出来的知识。
 
-For each word, its vector representation has the corresponding entry in the vector as 1, and all other entries as 0. The lengths of one-hot vectors match the size of the dictionary. Each entry of a vector corresponds to the presence (or absence) of a word in the dictionary.
+在机器学习领域里，各种“知识”被各种模型表示，词向量模型(word embedding model)就是其中的一类。通过词向量模型可将一个 one-hot vector映射到一个维度更低的实数向量（embedding vector），如$embedding(母亲节) = [0.3, 4.2, -1.5, ...], embedding(康乃馨) = [0.2, 5.6, -2.3, ...]$。在这个映射到的实数向量表示中，希望两个语义（或用法）上相似的词对应的词向量“更像”，这样如“母亲节”和“康乃馨”的对应词向量的余弦相似度就不再为零了。
 
-One-hot vectors are intuitive, yet they have limited usefulness. Take the example of an Internet advertising system: Suppose a customer enters the query "Mother's Day", while an ad bids for the keyword carnations". Because the one-hot vectors of these two words are perpendicular, the metric distance (either Euclidean or cosine similarity) between them would indicate  little relevance. However, *we* know that these two queries are connected semantically, since people often gift their mothers bundles of carnation flowers on Mother's Day. This discrepancy is due to the low information capacity in each vector. That is, comparing the vector representations of two words does not assess their relevance sufficiently. To calculate their similarity accurately, we need more information, which could be learned from large amounts of data through machine learning methods.
-
-Like many machine learning models, word embeddings can represent knowledge in various ways. Another model may project an one-hot vector to an embedding vector of lower dimension e.g. $embedding(mother's day) = [0.3, 4.2, -1.5, ...], embedding(carnations) = [0.2, 5.6, -2.3, ...]$. Mapping one-hot vectors onto an embedded vector space has the potential to bring the embedding vectors of similar words (either semantically or usage-wise) closer to each other, so that the cosine similarity between the corresponding vectors for words like "Mother's Day" and "carnations" are no longer zero.
-
-A word embedding model could be a probabilistic model, a co-occurrence matrix model, or a neural network. Before people started using neural networks to generate word embedding, the traditional method was to calculate a co-occurrence matrix $X$ of words. Here, $X$ is a $|V| \times |V|$ matrix, where $X_{ij}$ represents the co-occurrence times of the $i$th and $j$th words in the vocabulary `V` within all corpus, and $|V|$ is the size of the vocabulary. By performing matrix decomposition on $X$ e.g. Singular Value Decomposition \[[5](#References)\]
+词向量模型可以是概率模型、共生矩阵(co-occurrence matrix)模型或神经元网络模型。在用神经网络求词向量之前，传统做法是统计一个词语的共生矩阵$X$。$X$是一个$|V| \times |V|$ 大小的矩阵，$X_{ij}$表示在所有语料中，词汇表`V`(vocabulary)中第i个词和第j个词同时出现的词数，$|V|$为词汇表的大小。对$X$做矩阵分解（如奇异值分解，Singular Value Decomposition \[[5](#参考文献)\]），得到的$U$即视为所有词的词向量：
 
 $$X = USV^T$$
 
-the resulting $U$ can be seen as the word embedding of all the words.
+但这样的传统做法有很多问题：<br/>
+1) 由于很多词没有出现，导致矩阵极其稀疏，因此需要对词频做额外处理来达到好的矩阵分解效果；<br/>
+2) 矩阵非常大，维度太高(通常达到$10^6*10^6$的数量级)；<br/>
+3) 需要手动去掉停用词（如although, a,...），不然这些频繁出现的词也会影响矩阵分解的效果。
 
-However, this method suffers from many drawbacks:
-1) Since many pairs of words don't co-occur, the co-occurrence matrix is sparse. To achieve good performance of matrix factorization, further treatment on word frequency is needed;
-2) The matrix is large, frequently on the order of $10^6*10^6$;
-3) We need to manually filter out stop words (like "although", "a", ...), otherwise these frequent words will affect the performance of matrix factorization.
 
-The neural network based model does not require storing huge hash tables of statistics on all of the corpus. It obtains the word embedding by learning from semantic information, hence could avoid the aforementioned problems in the traditional method. In this chapter, we will introduce the details of neural network word embedding model and how to train such model in PaddlePaddle.
+基于神经网络的模型不需要计算存储一个在全语料上统计的大表，而是通过学习语义信息得到词向量，因此能很好地解决以上问题。在本章里，我们将展示基于神经网络训练词向量的细节，以及如何用PaddlePaddle训练一个词向量模型。
 
-## Results Demonstration
 
-In this section, after training the word embedding model, we could use the data visualization algorithm $t-$SNE\[[4](#reference)\] to draw the word embedding vectors after projecting them onto a two-dimensional space (see figure below). From the figure we could see that the semantically relevant words -- *a*, *the*, and *these* or *big* and *huge* -- are close to each other in the projected space, while irrelevant words -- *say* and *business* or *decision* and *japan* -- are far from each other.
+## 效果展示
+
+本章中，当词向量训练好后，我们可以用数据可视化算法t-SNE\[[4](#参考文献)\]画出词语特征在二维上的投影（如下图所示）。从图中可以看出，语义相关的词语（如a, the, these; big, huge）在投影上距离很近，语意无关的词（如say, business; decision, japan）在投影上的距离很远。
 
 <p align="center">
     <img src = "image/2d_similarity.png" width=400><br/>
-    Figure 1. Two dimension projection of word embeddings
+    图1. 词向量的二维投影
 </p>
 
-### Cosine Similarity
-
-On the other hand, we know that the cosine similarity between two vectors falls between $[-1,1]$. Specifically, the cosine similarity is 1 when the vectors are identical, 0 when the vectors are perpendicular, -1 when the are of opposite directions. That is, the cosine similarity between two vectors scales with their relevance. So we can calculate the cosine similarity of two word embedding vectors to represent their relevance:
+另一方面，我们知道两个向量的余弦值在$[-1,1]$的区间内：两个完全相同的向量余弦值为1, 两个相互垂直的向量之间余弦值为0，两个方向完全相反的向量余弦值为-1，即相关性和余弦值大小成正比。因此我们还可以计算两个词向量的余弦相似度:
 
 ```
-please input two words: big huge
 similarity: 0.899180685161
+please input two words: big huge
 
 please input two words: from company
 similarity: -0.0997506977351
 ```
 
-The above results could be obtained by running `calculate_dis.py`, which loads the words in the dictionary and their corresponding trained word embeddings. For detailed instruction, see section [Model Application](#Model Application).
-
-
-## Model Overview
+以上结果可以通过运行`calculate_dis.py`, 加载字典里的单词和对应训练特征结果得到，我们将在[应用模型](#应用模型)中详细描述用法。
 
-In this section, we will introduce three word embedding models: N-gram model, CBOW, and Skip-gram, which all output the frequency of each word given its immediate context.
 
-For N-gram model, we will first introduce the concept of language model, and implement it using PaddlePaddle in section [Model Training](#Model Training).
+## 模型概览
 
-The latter two models, which became popular recently, are neural word embedding model developed by Tomas Mikolov at Google \[[3](#reference)\]. Despite their apparent simplicity, these models train very well.
+在这里我们介绍三个训练词向量的模型：N-gram模型，CBOW模型和Skip-gram模型，它们的中心思想都是通过上下文得到一个词出现的概率。对于N-gram模型，我们会先介绍语言模型的概念，并在之后的[训练模型](#训练模型)中，带大家用PaddlePaddle实现它。而后两个模型，是近年来最有名的神经元词向量模型，由 Tomas Mikolov 在Google 研发\[[3](#参考文献)\]，虽然它们很浅很简单，但训练效果很好。
 
-### Language Model
+### 语言模型
 
-Before diving into word embedding models, we will first introduce the concept of **language model**. Language models build the joint probability function $P(w_1, ..., w_T)$ of a sentence, where $w_i$ is the i-th word in the sentence. The goal is to give higher probabilities to meaningful sentences, and lower probabilities to meaningless constructions.
+在介绍词向量模型之前，我们先来引入一个概念：语言模型。
+语言模型旨在为语句的联合概率函数$P(w_1, ..., w_T)$建模, 其中$w_i$表示句子中的第i个词。语言模型的目标是，希望模型对有意义的句子赋予大概率，对没意义的句子赋予小概率。
+这样的模型可以应用于很多领域，如机器翻译、语音识别、信息检索、词性标注、手写识别等，它们都希望能得到一个连续序列的概率。 以信息检索为例，当你在搜索“how long is a football bame”时（bame是一个医学名词），搜索引擎会提示你是否希望搜索"how long is a football game", 这是因为根据语言模型计算出“how long is a football bame”的概率很低，而与bame近似的，可能引起错误的词中，game会使该句生成的概率最大。
 
-In general, models that generate the probability of a sequence can be applied to many fields, like machine translation, speech recognition, information retrieval, part-of-speech tagging, and handwriting recognition. Take information retrieval, for example. If you were to search for "how long is a football bame" (where bame is a medical noun), the search engine would have asked if you had meant "how long is a football game" instead. This is because the probability of "how long is a football bame" is very low according to the language model; in addition, among all of the words easily confused with "bame", "game" would build the most probable sentence.
-
-#### Target Probability
-For language model's target probability $P(w_1, ..., w_T)$, if the words in the sentence were to be independent, the joint probability of the whole sentence would be the product of each word's probability:
+对语言模型的目标概率$P(w_1, ..., w_T)$，如果假设文本中每个词都是相互独立的，则整句话的联合概率可以表示为其中所有词语条件概率的乘积，即：
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t)$$
 
-However, the frequency of words in a sentence typically relates to the words before them, so canonical language models are constructed using conditional probability in its target probability:
+然而我们知道语句中的每个词出现的概率都与其前面的词紧密相关, 所以实际上通常用条件概率表示语言模型：
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
 
 
+
 ### N-gram neural model
 
-In computational linguistics, n-gram is an important method to represent text. An n-gram represents a contiguous sequence of n consecutive items given a text. Based on the desired application scenario, each item could be a letter, a syllable or a word. The N-gram model is also an important method in statistical language modeling. When training language models with n-grams, the first (n-1) words of an n-gram are used to predict the *n*th word.
+在计算语言学中，n-gram是一种重要的文本表示方法，表示一个文本中连续的n个项。基于具体的应用场景，每一项可以是一个字母、单词或者音节。 n-gram模型也是统计语言模型中的一种重要方法，用n-gram训练语言模型时，一般用每个n-gram的历史n-1个词语组成的内容来预测第n个词。
 
-Yoshua Bengio and other scientists describe how to train a word embedding model using neural network in the famous paper of Neural Probabilistic Language Models \[[1](#reference)\] published in 2003. The Neural Network Language Model (NNLM) described in the paper learns the language model and word embedding simultaneously through a linear transformation and a non-linear hidden connection. That is, after training on large amounts of corpus, the model learns the word embedding; then, it computes the probability of the whole sentence, using the embedding. This type of language model can overcome the **curse of dimensionality** i.e. model inaccuracy caused by the difference in dimensionality between training and testing data. Note that the term *neural network language model* is ill-defined, so we will not use the name NNLM but only refer to it as *N-gram neural model* in this section.
+Yoshua Bengio等科学家就于2003年在著名论文 Neural Probabilistic Language Models \[[1](#参考文献)\] 中介绍如何学习一个神经元网络表示的词向量模型。文中的神经概率语言模型（Neural Network Language Model，NNLM）通过一个线性映射和一个非线性隐层连接，同时学习了语言模型和词向量，即通过学习大量语料得到词语的向量表达，通过这些向量得到整个句子的概率。用这种方法学习语言模型可以克服维度灾难（curse of dimensionality）,即训练和测试数据不同导致的模型不准。注意：由于“神经概率语言模型”说法较为泛泛，我们在这里不用其NNLM的本名，考虑到其具体做法，本文中称该模型为N-gram neural model。
 
-We have previously described language model using conditional probability, where the probability of the *t*-th word in a sentence depends on all $t-1$ words before it. Furthermore, since words further prior have less impact on a word, and every word within an n-gram is only effected by its previous n-1 words, we have:
+我们在上文中已经讲到用条件概率建模语言模型，即一句话中第$t$个词的概率和该句话的前$t-1$个词相关。可实际上越远的词语其实对该词的影响越小，那么如果考虑一个n-gram, 每个词都只受其前面`n-1`个词的影响，则有：
 
 $$P(w_1, ..., w_T) = \prod_{t=n}^TP(w_t|w_{t-1}, w_{t-2}, ..., w_{t-n+1})$$
 
-Given some real corpus in which all sentences are meaningful, the n-gram model should maximize the following objective function:
+给定一些真实语料，这些语料中都是有意义的句子，N-gram模型的优化目标则是最大化目标函数:
 
 $$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
 
-where $f(w_t, w_{t-1}, ..., w_{t-n+1})$ represents the conditional probability of the current word $w_t$ given its previous $n-1$ words, and $R(\theta)$ represents parameter regularization term.
+其中$f(w_t, w_{t-1}, ..., w_{t-n+1})$表示根据历史n-1个词得到当前词$w_t$的条件概率，$R(\theta)$表示参数正则项。
 
 <p align="center">
-       <img src="image/nnlm_en.png" width=500><br/>
-       Figure 2. N-gram neural network model
+       <img src="image/nnlm.png" width=500><br/>
+       图2. N-gram神经网络模型
 </p>
 
+图2展示了N-gram神经网络模型，从下往上看，该模型分为以下几个部分：
+ - 对于每个样本，模型输入$w_{t-n+1},...w_{t-1}$, 输出句子第t个词为字典中`|V|`个词的概率。
 
-Figure 2 shows the N-gram neural network model. From the bottom up, the model has the following components:
-
- - For each sample, the model gets input $w_{t-n+1},...w_{t-1}$, and outputs the probability that the t-th word is one of `|V|` in the dictionary.
-
- Every input word $w_{t-n+1},...w_{t-1}$ first gets transformed into word embedding $C(w_{t-n+1}),...C(w_{t-1})$ through a transformation matrix.
+   每个输入词$w_{t-n+1},...w_{t-1}$首先通过映射矩阵映射到词向量$C(w_{t-n+1}),...C(w_{t-1})$。
 
- - All the word embeddings concatenate into a single vector, which is mapped (nonlinearly) into the $t$-th word hidden representation:
+ - 然后所有词语的词向量连接成一个大向量，并经过一个非线性映射得到历史词语的隐层表示：
 
     $$g=Utanh(\theta^Tx + b_1) + Wx + b_2$$
 
-   where $x$ is the large vector concatenated from all the word embeddings representing the context; $\theta$, $U$, $b_1$, $b_2$ and $W$ are parameters connecting word embedding layers to the hidden layers. $g$ represents the unnormalized probability of the output word, $g_i$ represents the unnormalized probability of the output word being the i-th word in the dictionary.
+    其中，$x$为所有词语的词向量连接成的大向量，表示文本历史特征；$\theta$、$U$、$b_1$、$b_2$和$W$分别为词向量层到隐层连接的参数。$g$表示未经归一化的所有输出单词概率，$g_i$表示未经归一化的字典中第$i$个单词的输出概率。
 
- - Based on the definition of softmax, using normalized $g_i$, the probability that the output word is $w_t$ is represented as:
+ - 根据softmax的定义，通过归一化$g_i$, 生成目标词$w_t$的概率为：
 
   $$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
 
- - The cost of the entire network is a multi-class cross-entropy and can be described by the following loss function
+ - 整个网络的损失值(cost)为多类分类交叉熵，用公式表示为
 
    $$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$
 
-   where $y_k^i$ represents the true label for the $k$-th class in the $i$-th sample ($0$ or $1$), $softmax(g_k^i)$ represents the softmax probability for the $k$-th class in the $i$-th sample.
+   其中$y_k^i$表示第$i$个样本第$k$类的真实标签(0或1)，$softmax(g_k^i)$表示第i个样本第k类softmax输出的概率。
+
+
 
 ### Continuous Bag-of-Words model(CBOW)
 
-CBOW model predicts the current word based on the N words both before and after it. When $N=2$, the model is as the figure below:
+CBOW模型通过一个词的上下文（各N个词）预测当前词。当N=2时，模型如下图所示：
 
 <p align="center">
-    <img src="image/cbow_en.png" width=250><br/>
-    Figure 3. CBOW model
+    <img src="image/cbow.png" width=250><br/>
+    图3. CBOW模型
 </p>
 
-Specifically, by ignoring the order of words in the sequence, CBOW uses the average value of the word embedding of the context to predict the current word:
+具体来说，不考虑上下文的词语输入顺序，CBOW是用上下文词语的词向量的均值来预测当前词。即：
 
-$$\text{context} = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
+$$context = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
 
-where $x_t$ is the word embedding of the t-th word, classification score vector is $z=U*\text{context}$, the final classification $y$ uses softmax and the loss function uses multi-class cross-entropy.
+其中$x_t$为第$t$个词的词向量，分类分数（score）向量 $z=U*context$，最终的分类$y$采用softmax，损失函数采用多类分类交叉熵。
 
 ### Skip-gram model
 
-The advantages of CBOW is that it smooths over the word embeddings of the context and reduces noise, so it is very effective on small dataset. Skip-gram uses a word to predict its context and get multiple context for the given word, so it can be used in larger datasets.
+CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去掉了噪声，因此在小数据集上很有效。而Skip-gram的方法中，用一个词预测其上下文，得到了当前词上下文的很多样本，因此可用于更大的数据集。
 
 <p align="center">
-    <img src="image/skipgram_en.png" width=250><br/>
-    Figure 4. Skip-gram model
+    <img src="image/skipgram.png" width=250><br/>
+    图4. Skip-gram模型
 </p>
 
-As illustrated in the figure above, skip-gram model maps the word embedding of the given word onto $2n$ word embeddings (including $n$ words before and $n$ words after the given word), and then combine the classification loss of all those $2n$ words by softmax.
+如上图所示，Skip-gram模型的具体做法是，将一个词的词向量映射到$2n$个词的词向量（$2n$表示当前输入词的前后各$n$个词），然后分别通过softmax得到这$2n$个词的分类损失值之和。
+
 
-## Dataset
+## 数据准备
 
-We will use Peen Treebank (PTB) (Tomas Mikolov's pre-processed version) dataset. PTB is a small dataset, used in Recurrent Neural Network Language Modeling Toolkit\[[2](#reference)\]. Its statistics are as follows:
+### 数据介绍
+
+本教程使用Penn Treebank （PTB）（经Tomas Mikolov预处理过的版本）数据集。PTB数据集较小，训练速度快，应用于Mikolov的公开语言模型训练工具\[[2](#参考文献)\]中。其统计情况如下：
 
 <p align="center">
 <table>
     <tr>
-        <td>training set</td>
-        <td>validation set</td>
-        <td>test set</td>
+        <td>训练数据</td>
+        <td>验证数据</td>
+        <td>测试数据</td>
     </tr>
     <tr>
         <td>ptb.train.txt</td>
@@ -208,27 +201,21 @@ We will use Peen Treebank (PTB) (Tomas Mikolov's pre-processed version) dataset.
         <td>ptb.test.txt</td>
     </tr>
     <tr>
-        <td>42068 lines</td>
-        <td>3370 lines</td>
-        <td>3761 lines</td>
+        <td>42068句</td>
+        <td>3370句</td>
+        <td>3761句</td>
     </tr>
 </table>
 </p>
 
-### Python Dataset Module
-
-We encapsulated the PTB Data Set in our Python module `paddle.dataset.imikolov`. This module can
 
-1. download the dataset to `~/.cache/paddle/dataset/imikolov`, if not yet, and
-2. [preprocesses](#preprocessing) the dataset.
+### 数据预处理
 
-### Preprocessing
+本章训练的是5-gram模型，表示在PaddlePaddle训练时，每条数据的前4个词用来预测第5个词。PaddlePaddle提供了对应PTB数据集的python包`paddle.dataset.imikolov`，自动做数据的下载与预处理，方便大家使用。
 
-We will be training a 5-gram model. Given five words in a window, we will predict the fifth word given the first four words.
+预处理会把数据集中的每一句话前后加上开始符号`<s>`以及结束符号`<e>`。然后依据窗口大小（本教程中为5），从头到尾每次向右滑动窗口并生成一条数据。
 
-Beginning and end of a sentence have a special meaning, so we will add begin token `<s>` in the front of the sentence. And end token `<e>` in the end of the sentence. By moving the five word window in the sentence, data instances are generated.
-
-For example, the sentence "I have a dream that one day" generates five data instances:
+如"I have a dream that one day" 一句提供了5条数据：
 
 ```text
 <s> I have a dream
@@ -238,35 +225,33 @@ a dream that one day
 dream that one day <e>
 ```
 
-At last, each data instance will be converted into an integer sequence according it's words' index inside the dictionary.
-
-## Training
+最后，每个输入会按其单词次在字典里的位置，转化成整数的索引序列，作为PaddlePaddle的输入。
+## 编程实现
 
-The neural network that we will be using is illustrated in the graph below:
+本配置的模型结构如下图所示：
 
 <p align="center">
-    <img src="image/ngram.en.png" width=400><br/>
-    Figure 5. N-gram neural network model in model configuration
+    <img src="image/ngram.png" width=400><br/>
+    图5. 模型配置中的N-gram神经网络模型
 </p>
 
-`word2vec/train.py` demonstrates training word2vec using PaddlePaddle:
-
-- Import packages.
+首先，加载所需要的包：
 
 ```python
 import math
 import paddle.v2 as paddle
 ```
 
-- Configure parameter.
-
+然后，定义参数：
 ```python
-embsize = 32 # word vector dimension
-hiddensize = 256 # hidden layer dimension
-N = 5 # train 5-gram
+embsize = 32 # 词向量维度
+hiddensize = 256 # 隐层维度
+N = 5 # 训练5-Gram
 ```
 
-- Map the $n-1$ words $w_{t-n+1},...w_{t-1}$ before $w_t$ to a D-dimensional vector though matrix of dimention $|V|\times D$ (D=32 in this example).
+接着，定义网络结构：
+
+- 将$w_t$之前的$n-1$个词 $w_{t-n+1},...w_{t-1}$，通过$|V|\times D$的矩阵映射到D维词向量（本例中取D=32）。
 
 ```python
 def wordemb(inlayer):
@@ -277,17 +262,18 @@ def wordemb(inlayer):
             name="_proj",
             initial_std=0.001,
             learning_rate=1,
-            l2_rate=0, ))
+            l2_rate=0,
+            sparse_update=True))
     return wordemb
 ```
 
-- Define name and type for input to data layer.
+- 定义输入层接受的数据类型以及名字。
 
 ```python
-paddle.init(use_gpu=False, trainer_count=3)
+paddle.init(use_gpu=False, trainer_count=3) # 初始化PaddlePaddle
 word_dict = paddle.dataset.imikolov.build_dict()
 dict_size = len(word_dict)
-# Every layer takes integer value of range [0, dict_size)
+# 每个输入层都接受整形数据，这些数据的范围是[0, dict_size)
 firstword = paddle.layer.data(
     name="firstw", type=paddle.data_type.integer_value(dict_size))
 secondword = paddle.layer.data(
@@ -305,13 +291,13 @@ Ethird = wordemb(thirdword)
 Efourth = wordemb(fourthword)
 ```
 
-- Concatenate n-1 word embedding vectors into a single feature vector.
+- 将这n-1个词向量经过concat_layer连接成一个大向量作为历史文本特征。
 
 ```python
 contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
 ```
 
-- Feature vector will go through a fully connected layer which outputs a hidden feature vector.
+- 将历史文本特征经过一个全连接得到文本隐层特征。
 
 ```python
 hidden1 = paddle.layer.fc(input=contextemb,
@@ -324,7 +310,7 @@ hidden1 = paddle.layer.fc(input=contextemb,
                                 learning_rate=1))
 ```
 
-- Hidden feature vector will go through another fully conected layer, turn into a $|V|$ dimensional vector. At the same time softmax will be applied to get the probability of each word being generated.
+- 将文本隐层特征，再经过一个全连接，映射成一个$|V|$维向量，同时通过softmax归一化得到这`|V|`个词的生成概率。
 
 ```python
 predictword = paddle.layer.fc(input=hidden1,
@@ -333,29 +319,31 @@ predictword = paddle.layer.fc(input=hidden1,
                               act=paddle.activation.Softmax())
 ```
 
-- We will use cross-entropy cost function.
+- 网络的损失函数为多分类交叉熵，可直接调用`classification_cost`函数。
 
 ```python
 cost = paddle.layer.classification_cost(input=predictword, label=nextword)
 ```
 
-- Create parameter, optimizer and trainer.
+然后，指定训练相关的参数：
+
+- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器，本教程使用Adam优化器。
+- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
+- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
 
 ```python
 parameters = paddle.parameters.create(cost)
-adam_optimizer = paddle.optimizer.Adam(
+adagrad = paddle.optimizer.AdaGrad(
     learning_rate=3e-3,
     regularization=paddle.optimizer.L2Regularization(8e-4))
-trainer = paddle.trainer.SGD(cost, parameters, adam_optimizer)
+trainer = paddle.trainer.SGD(cost, parameters, adagrad)
 ```
 
-Next, we will begin the training process. `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()` is our training set and test set. Both of the function will return a **reader**: In PaddlePaddle, reader is a python function which returns a Python iterator which output a single data instance at a time.
+下一步，我们开始训练过程。`paddle.dataset.imikolov.train()`和`paddle.dataset.imikolov.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python generator。
 
-`paddle.batch` takes reader as input, outputs a **batched reader**: In PaddlePaddle, a reader outputs a single data instance at a time but batched reader outputs a minibatch of data instances.
+`paddle.batch`的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minbatch。
 
 ```python
-import gzip
-
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -367,7 +355,7 @@ def event_handler(event):
                     paddle.batch(
                         paddle.dataset.imikolov.test(word_dict, N), 32))
         print "Pass %d, Testing metrics %s" % (event.pass_id, result.metrics)
-        with gzip.open("model_%d.tar.gz"%event.pass_id, 'w') as f:
+        with open("model_%d.tar"%event.pass_id, 'w') as f:
             parameters.to_tar(f)
 
 trainer.train(
@@ -376,7 +364,6 @@ trainer.train(
     event_handler=event_handler)
 ```
 
-`trainer.train` will start training, the output of `event_handler` will be similar to following:
 ```text
 Pass 0, Batch 0, Cost 7.870579, {'classification_error_evaluator': 1.0}, Testing metrics {'classification_error_evaluator': 0.999591588973999}
 Pass 0, Batch 100, Cost 6.136420, {'classification_error_evaluator': 0.84375}, Testing metrics {'classification_error_evaluator': 0.8328699469566345}
@@ -384,16 +371,19 @@ Pass 0, Batch 200, Cost 5.786797, {'classification_error_evaluator': 0.8125}, Te
 ...
 ```
 
-After 30 passes, we can get average error rate around 0.735611.
+训练过程是完全自动的，event_handler里打印的日志类似如上所示：
+
+经过30个pass，我们将得到平均错误率为classification_error_evaluator=0.735611。
+
 
+## 应用模型
+训练模型后，我们可以加载模型参数，用训练出来的词向量初始化其他模型，也可以将模型查看参数用来做后续应用。
 
-## Model Application
 
-After the model is trained, we can load saved model parameters and uses it for other models. We can also use the parameters in applications.
+### 查看词向量
 
-### Viewing Word Vector
+PaddlePaddle训练出来的参数可以直接使用`parameters.get()`获取出来。例如查看单词`apple`的词向量，即为
 
-Parameters trained by PaddlePaddle can be viewed by `parameters.get()`. For example, we can check the word vector for word `apple`.
 
 ```python
 embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
@@ -410,9 +400,10 @@ print embeddings[word_dict['apple']]
 0.19072419 -0.24286366]
 ```
 
-### Modifying Word Vector
 
-Word vectors (`embeddings`) that we get is a numpy array. We can modify this array and set it back to `parameters`.
+### 修改词向量
+
+获得到的embedding为一个标准的numpy矩阵。我们可以对这个numpy矩阵进行修改，然后赋值回去。
 
 
 ```python
@@ -424,9 +415,10 @@ modify_embedding(embeddings)
 parameters.set("_proj", embeddings)
 ```
 
-### Calculating Cosine Similarity
+### 计算词语之间的余弦距离
 
-Cosine similarity is one way of quantifying the similarity between two vectors. The range of result is $[-1, 1]$. The bigger the value, the similar two vectors are:
+两个向量之间的距离可以用余弦值来表示，余弦值在$[-1,1]$的区间内，向量间余弦值越大，其距离越近。这里我们在`calculate_dis.py`中实现不同词语的距离度量。
+用法如下：
 
 
 ```python
@@ -442,14 +434,11 @@ print spatial.distance.cosine(emb_1, emb_2)
 0.99375076448
 ```
 
-## Conclusion
-
-This chapter introduces word embedding, the relationship between language model and word embedding, and how to train neural networks to learn word embedding.
-
-In information retrieval, the relevance between the query and document keyword can be computed through the cosine similarity of their word embeddings. In grammar analysis and semantic analysis, a previously trained word embedding can initialize models for better performance. In document classification, clustering the word embedding can group synonyms in the documents. We hope that readers can use word embedding models in their work after reading this chapter.
+## 总结
+本章中，我们介绍了词向量、语言模型和词向量的关系、以及如何通过训练神经网络模型获得词向量。在信息检索中，我们可以根据向量间的余弦夹角，来判断query和文档关键词这二者间的相关性。在句法分析和语义分析中，训练好的词向量可以用来初始化模型，以得到更好的效果。在文档分类中，有了词向量之后，可以用聚类的方法将文档中同义词进行分组。希望大家在本章后能够自行运用词向量进行相关领域的研究。
 
 
-## Referenes
+## 参考文献
 1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
 2. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
 3. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space](https://arxiv.org/pdf/1301.3781.pdf)[J]. arXiv preprint arXiv:1301.3781, 2013.
@@ -457,7 +446,7 @@ In information retrieval, the relevance between the query and document keyword c
 5. https://en.wikipedia.org/wiki/Singular_value_decomposition
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

04.word2vec/index.html

@@ -40,160 +40,167 @@
 
 <!-- This block will be replaced by each markdown file content. Please do not change lines below.-->
 <div id="markdown" style='display:none'>
+# Word2Vec
 
-# 词向量
+This is intended as a reference tutorial. The source code of this tutorial lives on [book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec).
 
-本教程源代码目录在[book/word2vec](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
-## 背景介绍
+## Background Introduction
 
-本章我们介绍词的向量表征，也称为word embedding。词向量是自然语言处理中常见的一个操作，是搜索引擎、广告系统、推荐系统等互联网服务背后常见的基础技术。
+This section introduces the concept of **word embedding**, which is a vector representation of words. It is a popular technique used in natural language processing. Word embeddings support many Internet services, including search engines, advertising systems, and recommendation systems.
 
-在这些互联网服务里，我们经常要比较两个词或者两段文本之间的相关性。为了做这样的比较，我们往往先要把词表示成计算机适合处理的方式。最自然的方式恐怕莫过于向量空间模型(vector space model)。
-在这种方式里，每个词被表示成一个实数向量（one-hot vector），其长度为字典大小，每个维度对应一个字典里的每个词，除了这个词对应维度上的值是1，其他元素都是0。
+### One-Hot Vectors
 
-One-hot vector虽然自然，但是用处有限。比如，在互联网广告系统里，如果用户输入的query是“母亲节”，而有一个广告的关键词是“康乃馨”。虽然按照常理，我们知道这两个词之间是有联系的——母亲节通常应该送给母亲一束康乃馨；但是这两个词对应的one-hot vectors之间的距离度量，无论是欧氏距离还是余弦相似度(cosine similarity)，由于其向量正交，都认为这两个词毫无相关性。 得出这种与我们相悖的结论的根本原因是：每个词本身的信息量都太小。所以，仅仅给定两个词，不足以让我们准确判别它们是否相关。要想精确计算相关性，我们还需要更多的信息——从大量数据里通过机器学习方法归纳出来的知识。
+Building these services requires us to quantify the similarity between two words or paragraphs. This calls for a new representation of all the words to make them more suitable for computation. An obvious way to achieve this is through the vector space model, where every word is represented as an **one-hot vector**.
 
-在机器学习领域里，各种“知识”被各种模型表示，词向量模型(word embedding model)就是其中的一类。通过词向量模型可将一个 one-hot vector映射到一个维度更低的实数向量（embedding vector），如$embedding(母亲节) = [0.3, 4.2, -1.5, ...], embedding(康乃馨) = [0.2, 5.6, -2.3, ...]$。在这个映射到的实数向量表示中，希望两个语义（或用法）上相似的词对应的词向量“更像”，这样如“母亲节”和“康乃馨”的对应词向量的余弦相似度就不再为零了。
+For each word, its vector representation has the corresponding entry in the vector as 1, and all other entries as 0. The lengths of one-hot vectors match the size of the dictionary. Each entry of a vector corresponds to the presence (or absence) of a word in the dictionary.
 
-词向量模型可以是概率模型、共生矩阵(co-occurrence matrix)模型或神经元网络模型。在用神经网络求词向量之前，传统做法是统计一个词语的共生矩阵$X$。$X$是一个$|V| \times |V|$ 大小的矩阵，$X_{ij}$表示在所有语料中，词汇表`V`(vocabulary)中第i个词和第j个词同时出现的词数，$|V|$为词汇表的大小。对$X$做矩阵分解（如奇异值分解，Singular Value Decomposition \[[5](#参考文献)\]），得到的$U$即视为所有词的词向量：
+One-hot vectors are intuitive, yet they have limited usefulness. Take the example of an Internet advertising system: Suppose a customer enters the query "Mother's Day", while an ad bids for the keyword carnations". Because the one-hot vectors of these two words are perpendicular, the metric distance (either Euclidean or cosine similarity) between them would indicate  little relevance. However, *we* know that these two queries are connected semantically, since people often gift their mothers bundles of carnation flowers on Mother's Day. This discrepancy is due to the low information capacity in each vector. That is, comparing the vector representations of two words does not assess their relevance sufficiently. To calculate their similarity accurately, we need more information, which could be learned from large amounts of data through machine learning methods.
 
-$$X = USV^T$$
+Like many machine learning models, word embeddings can represent knowledge in various ways. Another model may project an one-hot vector to an embedding vector of lower dimension e.g. $embedding(mother's day) = [0.3, 4.2, -1.5, ...], embedding(carnations) = [0.2, 5.6, -2.3, ...]$. Mapping one-hot vectors onto an embedded vector space has the potential to bring the embedding vectors of similar words (either semantically or usage-wise) closer to each other, so that the cosine similarity between the corresponding vectors for words like "Mother's Day" and "carnations" are no longer zero.
+
+A word embedding model could be a probabilistic model, a co-occurrence matrix model, or a neural network. Before people started using neural networks to generate word embedding, the traditional method was to calculate a co-occurrence matrix $X$ of words. Here, $X$ is a $|V| \times |V|$ matrix, where $X_{ij}$ represents the co-occurrence times of the $i$th and $j$th words in the vocabulary `V` within all corpus, and $|V|$ is the size of the vocabulary. By performing matrix decomposition on $X$ e.g. Singular Value Decomposition \[[5](#References)\]
 
-但这样的传统做法有很多问题：<br/>
-1) 由于很多词没有出现，导致矩阵极其稀疏，因此需要对词频做额外处理来达到好的矩阵分解效果；<br/>
-2) 矩阵非常大，维度太高(通常达到$10^6*10^6$的数量级)；<br/>
-3) 需要手动去掉停用词（如although, a,...），不然这些频繁出现的词也会影响矩阵分解的效果。
+$$X = USV^T$$
 
+the resulting $U$ can be seen as the word embedding of all the words.
 
-基于神经网络的模型不需要计算存储一个在全语料上统计的大表，而是通过学习语义信息得到词向量，因此能很好地解决以上问题。在本章里，我们将展示基于神经网络训练词向量的细节，以及如何用PaddlePaddle训练一个词向量模型。
+However, this method suffers from many drawbacks:
+1) Since many pairs of words don't co-occur, the co-occurrence matrix is sparse. To achieve good performance of matrix factorization, further treatment on word frequency is needed;
+2) The matrix is large, frequently on the order of $10^6*10^6$;
+3) We need to manually filter out stop words (like "although", "a", ...), otherwise these frequent words will affect the performance of matrix factorization.
 
+The neural network based model does not require storing huge hash tables of statistics on all of the corpus. It obtains the word embedding by learning from semantic information, hence could avoid the aforementioned problems in the traditional method. In this chapter, we will introduce the details of neural network word embedding model and how to train such model in PaddlePaddle.
 
-## 效果展示
+## Results Demonstration
 
-本章中，当词向量训练好后，我们可以用数据可视化算法t-SNE\[[4](#参考文献)\]画出词语特征在二维上的投影（如下图所示）。从图中可以看出，语义相关的词语（如a, the, these; big, huge）在投影上距离很近，语意无关的词（如say, business; decision, japan）在投影上的距离很远。
+In this section, after training the word embedding model, we could use the data visualization algorithm $t-$SNE\[[4](#reference)\] to draw the word embedding vectors after projecting them onto a two-dimensional space (see figure below). From the figure we could see that the semantically relevant words -- *a*, *the*, and *these* or *big* and *huge* -- are close to each other in the projected space, while irrelevant words -- *say* and *business* or *decision* and *japan* -- are far from each other.
 
 <p align="center">
     <img src = "image/2d_similarity.png" width=400><br/>
-    图1. 词向量的二维投影
+    Figure 1. Two dimension projection of word embeddings
 </p>
 
-另一方面，我们知道两个向量的余弦值在$[-1,1]$的区间内：两个完全相同的向量余弦值为1, 两个相互垂直的向量之间余弦值为0，两个方向完全相反的向量余弦值为-1，即相关性和余弦值大小成正比。因此我们还可以计算两个词向量的余弦相似度:
+### Cosine Similarity
+
+On the other hand, we know that the cosine similarity between two vectors falls between $[-1,1]$. Specifically, the cosine similarity is 1 when the vectors are identical, 0 when the vectors are perpendicular, -1 when the are of opposite directions. That is, the cosine similarity between two vectors scales with their relevance. So we can calculate the cosine similarity of two word embedding vectors to represent their relevance:
 
 ```
-similarity: 0.899180685161
 please input two words: big huge
+similarity: 0.899180685161
 
 please input two words: from company
 similarity: -0.0997506977351
 ```
 
-以上结果可以通过运行`calculate_dis.py`, 加载字典里的单词和对应训练特征结果得到，我们将在[应用模型](#应用模型)中详细描述用法。
+The above results could be obtained by running `calculate_dis.py`, which loads the words in the dictionary and their corresponding trained word embeddings. For detailed instruction, see section [Model Application](#Model Application).
+
+
+## Model Overview
 
+In this section, we will introduce three word embedding models: N-gram model, CBOW, and Skip-gram, which all output the frequency of each word given its immediate context.
 
-## 模型概览
+For N-gram model, we will first introduce the concept of language model, and implement it using PaddlePaddle in section [Model Training](#Model Training).
 
-在这里我们介绍三个训练词向量的模型：N-gram模型，CBOW模型和Skip-gram模型，它们的中心思想都是通过上下文得到一个词出现的概率。对于N-gram模型，我们会先介绍语言模型的概念，并在之后的[训练模型](#训练模型)中，带大家用PaddlePaddle实现它。而后两个模型，是近年来最有名的神经元词向量模型，由 Tomas Mikolov 在Google 研发\[[3](#参考文献)\]，虽然它们很浅很简单，但训练效果很好。
+The latter two models, which became popular recently, are neural word embedding model developed by Tomas Mikolov at Google \[[3](#reference)\]. Despite their apparent simplicity, these models train very well.
 
-### 语言模型
+### Language Model
 
-在介绍词向量模型之前，我们先来引入一个概念：语言模型。
-语言模型旨在为语句的联合概率函数$P(w_1, ..., w_T)$建模, 其中$w_i$表示句子中的第i个词。语言模型的目标是，希望模型对有意义的句子赋予大概率，对没意义的句子赋予小概率。
-这样的模型可以应用于很多领域，如机器翻译、语音识别、信息检索、词性标注、手写识别等，它们都希望能得到一个连续序列的概率。 以信息检索为例，当你在搜索“how long is a football bame”时（bame是一个医学名词），搜索引擎会提示你是否希望搜索"how long is a football game", 这是因为根据语言模型计算出“how long is a football bame”的概率很低，而与bame近似的，可能引起错误的词中，game会使该句生成的概率最大。
+Before diving into word embedding models, we will first introduce the concept of **language model**. Language models build the joint probability function $P(w_1, ..., w_T)$ of a sentence, where $w_i$ is the i-th word in the sentence. The goal is to give higher probabilities to meaningful sentences, and lower probabilities to meaningless constructions.
 
-对语言模型的目标概率$P(w_1, ..., w_T)$，如果假设文本中每个词都是相互独立的，则整句话的联合概率可以表示为其中所有词语条件概率的乘积，即：
+In general, models that generate the probability of a sequence can be applied to many fields, like machine translation, speech recognition, information retrieval, part-of-speech tagging, and handwriting recognition. Take information retrieval, for example. If you were to search for "how long is a football bame" (where bame is a medical noun), the search engine would have asked if you had meant "how long is a football game" instead. This is because the probability of "how long is a football bame" is very low according to the language model; in addition, among all of the words easily confused with "bame", "game" would build the most probable sentence.
+
+#### Target Probability
+For language model's target probability $P(w_1, ..., w_T)$, if the words in the sentence were to be independent, the joint probability of the whole sentence would be the product of each word's probability:
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t)$$
 
-然而我们知道语句中的每个词出现的概率都与其前面的词紧密相关, 所以实际上通常用条件概率表示语言模型：
+However, the frequency of words in a sentence typically relates to the words before them, so canonical language models are constructed using conditional probability in its target probability:
 
 $$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
 
 
-
 ### N-gram neural model
 
-在计算语言学中，n-gram是一种重要的文本表示方法，表示一个文本中连续的n个项。基于具体的应用场景，每一项可以是一个字母、单词或者音节。 n-gram模型也是统计语言模型中的一种重要方法，用n-gram训练语言模型时，一般用每个n-gram的历史n-1个词语组成的内容来预测第n个词。
+In computational linguistics, n-gram is an important method to represent text. An n-gram represents a contiguous sequence of n consecutive items given a text. Based on the desired application scenario, each item could be a letter, a syllable or a word. The N-gram model is also an important method in statistical language modeling. When training language models with n-grams, the first (n-1) words of an n-gram are used to predict the *n*th word.
 
-Yoshua Bengio等科学家就于2003年在著名论文 Neural Probabilistic Language Models \[[1](#参考文献)\] 中介绍如何学习一个神经元网络表示的词向量模型。文中的神经概率语言模型（Neural Network Language Model，NNLM）通过一个线性映射和一个非线性隐层连接，同时学习了语言模型和词向量，即通过学习大量语料得到词语的向量表达，通过这些向量得到整个句子的概率。用这种方法学习语言模型可以克服维度灾难（curse of dimensionality）,即训练和测试数据不同导致的模型不准。注意：由于“神经概率语言模型”说法较为泛泛，我们在这里不用其NNLM的本名，考虑到其具体做法，本文中称该模型为N-gram neural model。
+Yoshua Bengio and other scientists describe how to train a word embedding model using neural network in the famous paper of Neural Probabilistic Language Models \[[1](#reference)\] published in 2003. The Neural Network Language Model (NNLM) described in the paper learns the language model and word embedding simultaneously through a linear transformation and a non-linear hidden connection. That is, after training on large amounts of corpus, the model learns the word embedding; then, it computes the probability of the whole sentence, using the embedding. This type of language model can overcome the **curse of dimensionality** i.e. model inaccuracy caused by the difference in dimensionality between training and testing data. Note that the term *neural network language model* is ill-defined, so we will not use the name NNLM but only refer to it as *N-gram neural model* in this section.
 
-我们在上文中已经讲到用条件概率建模语言模型，即一句话中第$t$个词的概率和该句话的前$t-1$个词相关。可实际上越远的词语其实对该词的影响越小，那么如果考虑一个n-gram, 每个词都只受其前面`n-1`个词的影响，则有：
+We have previously described language model using conditional probability, where the probability of the *t*-th word in a sentence depends on all $t-1$ words before it. Furthermore, since words further prior have less impact on a word, and every word within an n-gram is only effected by its previous n-1 words, we have:
 
 $$P(w_1, ..., w_T) = \prod_{t=n}^TP(w_t|w_{t-1}, w_{t-2}, ..., w_{t-n+1})$$
 
-给定一些真实语料，这些语料中都是有意义的句子，N-gram模型的优化目标则是最大化目标函数:
+Given some real corpus in which all sentences are meaningful, the n-gram model should maximize the following objective function:
 
 $$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
 
-其中$f(w_t, w_{t-1}, ..., w_{t-n+1})$表示根据历史n-1个词得到当前词$w_t$的条件概率，$R(\theta)$表示参数正则项。
+where $f(w_t, w_{t-1}, ..., w_{t-n+1})$ represents the conditional probability of the current word $w_t$ given its previous $n-1$ words, and $R(\theta)$ represents parameter regularization term.
 
 <p align="center">
-       <img src="image/nnlm.png" width=500><br/>
-       图2. N-gram神经网络模型
+       <img src="image/nnlm_en.png" width=500><br/>
+       Figure 2. N-gram neural network model
 </p>
 
-图2展示了N-gram神经网络模型，从下往上看，该模型分为以下几个部分：
- - 对于每个样本，模型输入$w_{t-n+1},...w_{t-1}$, 输出句子第t个词为字典中`|V|`个词的概率。
 
-   每个输入词$w_{t-n+1},...w_{t-1}$首先通过映射矩阵映射到词向量$C(w_{t-n+1}),...C(w_{t-1})$。
+Figure 2 shows the N-gram neural network model. From the bottom up, the model has the following components:
+
+ - For each sample, the model gets input $w_{t-n+1},...w_{t-1}$, and outputs the probability that the t-th word is one of `|V|` in the dictionary.
+
+ Every input word $w_{t-n+1},...w_{t-1}$ first gets transformed into word embedding $C(w_{t-n+1}),...C(w_{t-1})$ through a transformation matrix.
 
- - 然后所有词语的词向量连接成一个大向量，并经过一个非线性映射得到历史词语的隐层表示：
+ - All the word embeddings concatenate into a single vector, which is mapped (nonlinearly) into the $t$-th word hidden representation:
 
     $$g=Utanh(\theta^Tx + b_1) + Wx + b_2$$
 
-    其中，$x$为所有词语的词向量连接成的大向量，表示文本历史特征；$\theta$、$U$、$b_1$、$b_2$和$W$分别为词向量层到隐层连接的参数。$g$表示未经归一化的所有输出单词概率，$g_i$表示未经归一化的字典中第$i$个单词的输出概率。
+   where $x$ is the large vector concatenated from all the word embeddings representing the context; $\theta$, $U$, $b_1$, $b_2$ and $W$ are parameters connecting word embedding layers to the hidden layers. $g$ represents the unnormalized probability of the output word, $g_i$ represents the unnormalized probability of the output word being the i-th word in the dictionary.
 
- - 根据softmax的定义，通过归一化$g_i$, 生成目标词$w_t$的概率为：
+ - Based on the definition of softmax, using normalized $g_i$, the probability that the output word is $w_t$ is represented as:
 
   $$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
 
- - 整个网络的损失值(cost)为多类分类交叉熵，用公式表示为
+ - The cost of the entire network is a multi-class cross-entropy and can be described by the following loss function
 
    $$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$
 
-   其中$y_k^i$表示第$i$个样本第$k$类的真实标签(0或1)，$softmax(g_k^i)$表示第i个样本第k类softmax输出的概率。
-
-
+   where $y_k^i$ represents the true label for the $k$-th class in the $i$-th sample ($0$ or $1$), $softmax(g_k^i)$ represents the softmax probability for the $k$-th class in the $i$-th sample.
 
 ### Continuous Bag-of-Words model(CBOW)
 
-CBOW模型通过一个词的上下文（各N个词）预测当前词。当N=2时，模型如下图所示：
+CBOW model predicts the current word based on the N words both before and after it. When $N=2$, the model is as the figure below:
 
 <p align="center">
-    <img src="image/cbow.png" width=250><br/>
-    图3. CBOW模型
+    <img src="image/cbow_en.png" width=250><br/>
+    Figure 3. CBOW model
 </p>
 
-具体来说，不考虑上下文的词语输入顺序，CBOW是用上下文词语的词向量的均值来预测当前词。即：
+Specifically, by ignoring the order of words in the sequence, CBOW uses the average value of the word embedding of the context to predict the current word:
 
-$$context = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
+$$\text{context} = \frac{x_{t-1} + x_{t-2} + x_{t+1} + x_{t+2}}{4}$$
 
-其中$x_t$为第$t$个词的词向量，分类分数（score）向量 $z=U*context$，最终的分类$y$采用softmax，损失函数采用多类分类交叉熵。
+where $x_t$ is the word embedding of the t-th word, classification score vector is $z=U*\text{context}$, the final classification $y$ uses softmax and the loss function uses multi-class cross-entropy.
 
 ### Skip-gram model
 
-CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去掉了噪声，因此在小数据集上很有效。而Skip-gram的方法中，用一个词预测其上下文，得到了当前词上下文的很多样本，因此可用于更大的数据集。
+The advantages of CBOW is that it smooths over the word embeddings of the context and reduces noise, so it is very effective on small dataset. Skip-gram uses a word to predict its context and get multiple context for the given word, so it can be used in larger datasets.
 
 <p align="center">
-    <img src="image/skipgram.png" width=250><br/>
-    图4. Skip-gram模型
+    <img src="image/skipgram_en.png" width=250><br/>
+    Figure 4. Skip-gram model
 </p>
 
-如上图所示，Skip-gram模型的具体做法是，将一个词的词向量映射到$2n$个词的词向量（$2n$表示当前输入词的前后各$n$个词），然后分别通过softmax得到这$2n$个词的分类损失值之和。
-
+As illustrated in the figure above, skip-gram model maps the word embedding of the given word onto $2n$ word embeddings (including $n$ words before and $n$ words after the given word), and then combine the classification loss of all those $2n$ words by softmax.
 
-## 数据准备
+## Dataset
 
-### 数据介绍
-
-本教程使用Penn Treebank （PTB）（经Tomas Mikolov预处理过的版本）数据集。PTB数据集较小，训练速度快，应用于Mikolov的公开语言模型训练工具\[[2](#参考文献)\]中。其统计情况如下：
+We will use Peen Treebank (PTB) (Tomas Mikolov's pre-processed version) dataset. PTB is a small dataset, used in Recurrent Neural Network Language Modeling Toolkit\[[2](#reference)\]. Its statistics are as follows:
 
 <p align="center">
 <table>
     <tr>
-        <td>训练数据</td>
-        <td>验证数据</td>
-        <td>测试数据</td>
+        <td>training set</td>
+        <td>validation set</td>
+        <td>test set</td>
     </tr>
     <tr>
         <td>ptb.train.txt</td>
@@ -201,21 +208,27 @@ CBOW的好处是对上下文词语的分布在词向量上进行了平滑，去
         <td>ptb.test.txt</td>
     </tr>
     <tr>
-        <td>42068句</td>
-        <td>3370句</td>
-        <td>3761句</td>
+        <td>42068 lines</td>
+        <td>3370 lines</td>
+        <td>3761 lines</td>
     </tr>
 </table>
 </p>
 
+### Python Dataset Module
+
+We encapsulated the PTB Data Set in our Python module `paddle.dataset.imikolov`. This module can
 
-### 数据预处理
+1. download the dataset to `~/.cache/paddle/dataset/imikolov`, if not yet, and
+2. [preprocesses](#preprocessing) the dataset.
 
-本章训练的是5-gram模型，表示在PaddlePaddle训练时，每条数据的前4个词用来预测第5个词。PaddlePaddle提供了对应PTB数据集的python包`paddle.dataset.imikolov`，自动做数据的下载与预处理，方便大家使用。
+### Preprocessing
 
-预处理会把数据集中的每一句话前后加上开始符号`<s>`以及结束符号`<e>`。然后依据窗口大小（本教程中为5），从头到尾每次向右滑动窗口并生成一条数据。
+We will be training a 5-gram model. Given five words in a window, we will predict the fifth word given the first four words.
 
-如"I have a dream that one day" 一句提供了5条数据：
+Beginning and end of a sentence have a special meaning, so we will add begin token `<s>` in the front of the sentence. And end token `<e>` in the end of the sentence. By moving the five word window in the sentence, data instances are generated.
+
+For example, the sentence "I have a dream that one day" generates five data instances:
 
 ```text
 <s> I have a dream
@@ -225,33 +238,35 @@ a dream that one day
 dream that one day <e>
 ```
 
-最后，每个输入会按其单词次在字典里的位置，转化成整数的索引序列，作为PaddlePaddle的输入。
-## 编程实现
+At last, each data instance will be converted into an integer sequence according it's words' index inside the dictionary.
+
+## Training
 
-本配置的模型结构如下图所示：
+The neural network that we will be using is illustrated in the graph below:
 
 <p align="center">
-    <img src="image/ngram.png" width=400><br/>
-    图5. 模型配置中的N-gram神经网络模型
+    <img src="image/ngram.en.png" width=400><br/>
+    Figure 5. N-gram neural network model in model configuration
 </p>
 
-首先，加载所需要的包：
+`word2vec/train.py` demonstrates training word2vec using PaddlePaddle:
+
+- Import packages.
 
 ```python
 import math
 import paddle.v2 as paddle
 ```
 
-然后，定义参数：
+- Configure parameter.
+
 ```python
-embsize = 32 # 词向量维度
-hiddensize = 256 # 隐层维度
-N = 5 # 训练5-Gram
+embsize = 32 # word vector dimension
+hiddensize = 256 # hidden layer dimension
+N = 5 # train 5-gram
 ```
 
-接着，定义网络结构：
-
-- 将$w_t$之前的$n-1$个词 $w_{t-n+1},...w_{t-1}$，通过$|V|\times D$的矩阵映射到D维词向量（本例中取D=32）。
+- Map the $n-1$ words $w_{t-n+1},...w_{t-1}$ before $w_t$ to a D-dimensional vector though matrix of dimention $|V|\times D$ (D=32 in this example).
 
 ```python
 def wordemb(inlayer):
@@ -262,17 +277,18 @@ def wordemb(inlayer):
             name="_proj",
             initial_std=0.001,
             learning_rate=1,
-            l2_rate=0, ))
+            l2_rate=0,
+            sparse_update=True))
     return wordemb
 ```
 
-- 定义输入层接受的数据类型以及名字。
+- Define name and type for input to data layer.
 
 ```python
-paddle.init(use_gpu=False, trainer_count=3) # 初始化PaddlePaddle
+paddle.init(use_gpu=False, trainer_count=3)
 word_dict = paddle.dataset.imikolov.build_dict()
 dict_size = len(word_dict)
-# 每个输入层都接受整形数据，这些数据的范围是[0, dict_size)
+# Every layer takes integer value of range [0, dict_size)
 firstword = paddle.layer.data(
     name="firstw", type=paddle.data_type.integer_value(dict_size))
 secondword = paddle.layer.data(
@@ -290,13 +306,13 @@ Ethird = wordemb(thirdword)
 Efourth = wordemb(fourthword)
 ```
 
-- 将这n-1个词向量经过concat_layer连接成一个大向量作为历史文本特征。
+- Concatenate n-1 word embedding vectors into a single feature vector.
 
 ```python
 contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
 ```
 
-- 将历史文本特征经过一个全连接得到文本隐层特征。
+- Feature vector will go through a fully connected layer which outputs a hidden feature vector.
 
 ```python
 hidden1 = paddle.layer.fc(input=contextemb,
@@ -309,7 +325,7 @@ hidden1 = paddle.layer.fc(input=contextemb,
                                 learning_rate=1))
 ```
 
-- 将文本隐层特征，再经过一个全连接，映射成一个$|V|$维向量，同时通过softmax归一化得到这`|V|`个词的生成概率。
+- Hidden feature vector will go through another fully conected layer, turn into a $|V|$ dimensional vector. At the same time softmax will be applied to get the probability of each word being generated.
 
 ```python
 predictword = paddle.layer.fc(input=hidden1,
@@ -318,33 +334,27 @@ predictword = paddle.layer.fc(input=hidden1,
                               act=paddle.activation.Softmax())
 ```
 
-- 网络的损失函数为多分类交叉熵，可直接调用`classification_cost`函数。
+- We will use cross-entropy cost function.
 
 ```python
 cost = paddle.layer.classification_cost(input=predictword, label=nextword)
 ```
 
-然后，指定训练相关的参数：
-
-- 训练方法（optimizer)： 代表训练过程在更新权重时采用动量优化器，本教程使用Adam优化器。
-- 训练速度（learning_rate）： 迭代的速度，与网络的训练收敛速度有关系。
-- 正则化（regularization）： 是防止网络过拟合的一种手段，此处采用L2正则化。
+- Create parameter, optimizer and trainer.
 
 ```python
 parameters = paddle.parameters.create(cost)
-adam_optimizer = paddle.optimizer.Adam(
+adagrad = paddle.optimizer.AdaGrad(
     learning_rate=3e-3,
     regularization=paddle.optimizer.L2Regularization(8e-4))
-trainer = paddle.trainer.SGD(cost, parameters, adam_optimizer)
+trainer = paddle.trainer.SGD(cost, parameters, adagrad)
 ```
 
-下一步，我们开始训练过程。`paddle.dataset.imikolov.train()`和`paddle.dataset.imikolov.test()`分别做训练和测试数据集。这两个函数各自返回一个reader——PaddlePaddle中的reader是一个Python函数，每次调用的时候返回一个Python generator。
+Next, we will begin the training process. `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()` is our training set and test set. Both of the function will return a **reader**: In PaddlePaddle, reader is a python function which returns a Python iterator which output a single data instance at a time.
 
-`paddle.batch`的输入是一个reader，输出是一个batched reader —— 在PaddlePaddle里，一个reader每次yield一条训练数据，而一个batched reader每次yield一个minbatch。
+`paddle.batch` takes reader as input, outputs a **batched reader**: In PaddlePaddle, a reader outputs a single data instance at a time but batched reader outputs a minibatch of data instances.
 
 ```python
-import gzip
-
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
         if event.batch_id % 100 == 0:
@@ -356,7 +366,7 @@ def event_handler(event):
                     paddle.batch(
                         paddle.dataset.imikolov.test(word_dict, N), 32))
         print "Pass %d, Testing metrics %s" % (event.pass_id, result.metrics)
-        with gzip.open("model_%d.tar.gz"%event.pass_id, 'w') as f:
+        with open("model_%d.tar"%event.pass_id, 'w') as f:
             parameters.to_tar(f)
 
 trainer.train(
@@ -365,6 +375,7 @@ trainer.train(
     event_handler=event_handler)
 ```
 
+`trainer.train` will start training, the output of `event_handler` will be similar to following:
 ```text
 Pass 0, Batch 0, Cost 7.870579, {'classification_error_evaluator': 1.0}, Testing metrics {'classification_error_evaluator': 0.999591588973999}
 Pass 0, Batch 100, Cost 6.136420, {'classification_error_evaluator': 0.84375}, Testing metrics {'classification_error_evaluator': 0.8328699469566345}
@@ -372,19 +383,16 @@ Pass 0, Batch 200, Cost 5.786797, {'classification_error_evaluator': 0.8125}, Te
 ...
 ```
 
-训练过程是完全自动的，event_handler里打印的日志类似如上所示：
-
-经过30个pass，我们将得到平均错误率为classification_error_evaluator=0.735611。
-
+After 30 passes, we can get average error rate around 0.735611.
 
-## 应用模型
-训练模型后，我们可以加载模型参数，用训练出来的词向量初始化其他模型，也可以将模型查看参数用来做后续应用。
 
+## Model Application
 
-### 查看词向量
+After the model is trained, we can load saved model parameters and uses it for other models. We can also use the parameters in applications.
 
-PaddlePaddle训练出来的参数可以直接使用`parameters.get()`获取出来。例如查看单词`apple`的词向量，即为
+### Viewing Word Vector
 
+Parameters trained by PaddlePaddle can be viewed by `parameters.get()`. For example, we can check the word vector for word `apple`.
 
 ```python
 embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
@@ -401,10 +409,9 @@ print embeddings[word_dict['apple']]
 0.19072419 -0.24286366]
 ```
 
+### Modifying Word Vector
 
-### 修改词向量
-
-获得到的embedding为一个标准的numpy矩阵。我们可以对这个numpy矩阵进行修改，然后赋值回去。
+Word vectors (`embeddings`) that we get is a numpy array. We can modify this array and set it back to `parameters`.
 
 
 ```python
@@ -416,10 +423,9 @@ modify_embedding(embeddings)
 parameters.set("_proj", embeddings)
 ```
 
-### 计算词语之间的余弦距离
+### Calculating Cosine Similarity
 
-两个向量之间的距离可以用余弦值来表示，余弦值在$[-1,1]$的区间内，向量间余弦值越大，其距离越近。这里我们在`calculate_dis.py`中实现不同词语的距离度量。
-用法如下：
+Cosine similarity is one way of quantifying the similarity between two vectors. The range of result is $[-1, 1]$. The bigger the value, the similar two vectors are:
 
 
 ```python
@@ -435,11 +441,14 @@ print spatial.distance.cosine(emb_1, emb_2)
 0.99375076448
 ```
 
-## 总结
-本章中，我们介绍了词向量、语言模型和词向量的关系、以及如何通过训练神经网络模型获得词向量。在信息检索中，我们可以根据向量间的余弦夹角，来判断query和文档关键词这二者间的相关性。在句法分析和语义分析中，训练好的词向量可以用来初始化模型，以得到更好的效果。在文档分类中，有了词向量之后，可以用聚类的方法将文档中同义词进行分组。希望大家在本章后能够自行运用词向量进行相关领域的研究。
+## Conclusion
+
+This chapter introduces word embedding, the relationship between language model and word embedding, and how to train neural networks to learn word embedding.
+
+In information retrieval, the relevance between the query and document keyword can be computed through the cosine similarity of their word embeddings. In grammar analysis and semantic analysis, a previously trained word embedding can initialize models for better performance. In document classification, clustering the word embedding can group synonyms in the documents. We hope that readers can use word embedding models in their work after reading this chapter.
 
 
-## 参考文献
+## Referenes
 1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
 2. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
 3. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space](https://arxiv.org/pdf/1301.3781.pdf)[J]. arXiv preprint arXiv:1301.3781, 2013.
@@ -447,7 +456,7 @@ print spatial.distance.cosine(emb_1, emb_2)
 5. https://en.wikipedia.org/wiki/Singular_value_decomposition
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

04.word2vec/train.py

@@ -12,17 +12,15 @@ def wordemb(inlayer):
         input=inlayer,
         size=embsize,
         param_attr=paddle.attr.Param(
-            name="_proj",
-            initial_std=0.001,
-            learning_rate=1,
-            l2_rate=0, ))
+            name="_proj", initial_std=0.001, learning_rate=1, l2_rate=0))
     return wordemb
 
 
 def main():
-    paddle.init(use_gpu=False, trainer_count=1)
+    paddle.init(use_gpu=False, trainer_count=3)
     word_dict = paddle.dataset.imikolov.build_dict()
     dict_size = len(word_dict)
+    # Every layer takes integer value of range [0, dict_size)
     firstword = paddle.layer.data(
         name="firstw", type=paddle.data_type.integer_value(dict_size))
     secondword = paddle.layer.data(
@@ -57,22 +55,26 @@ def main():
     def event_handler(event):
         if isinstance(event, paddle.event.EndIteration):
             if event.batch_id % 100 == 0:
+                print "Pass %d, Batch %d, Cost %f, %s" % (
+                    event.pass_id, event.batch_id, event.cost, event.metrics)
+
+        if isinstance(event, paddle.event.EndPass):
             result = trainer.test(
-                    paddle.batch(
-                        paddle.dataset.imikolov.test(word_dict, N), 32))
-                print "Pass %d, Batch %d, Cost %f, %s, Testing metrics %s" % (
-                    event.pass_id, event.batch_id, event.cost, event.metrics,
+                paddle.batch(paddle.dataset.imikolov.test(word_dict, N), 32))
+            print "Pass %d, Testing metrics %s" % (event.pass_id,
                                                    result.metrics)
+            with open("model_%d.tar" % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
 
     cost = paddle.layer.classification_cost(input=predictword, label=nextword)
     parameters = paddle.parameters.create(cost)
-    adam_optimizer = paddle.optimizer.Adam(
+    adagrad = paddle.optimizer.AdaGrad(
         learning_rate=3e-3,
         regularization=paddle.optimizer.L2Regularization(8e-4))
-    trainer = paddle.trainer.SGD(cost, parameters, adam_optimizer)
+    trainer = paddle.trainer.SGD(cost, parameters, adagrad)
     trainer.train(
         paddle.batch(paddle.dataset.imikolov.train(word_dict, N), 32),
-        num_passes=30,
+        num_passes=100,
         event_handler=event_handler)
 
 

08.recommender_system/README.md → 05.recommender_system/README.cn.md

 # 个性化推荐
 
-本教程源代码目录在[book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/08.recommender_system)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/176.html)。
 
 ## 背景介绍
 
@@ -20,7 +20,7 @@
 
 ## 效果展示
 
-我们使用包含用户信息、电影信息与电影评分的数据集作为个性化推荐的应用场景。当我们训练好模型后，只需要输入对应的用户ID和电影ID，就可以得出一个匹配的分数（范围[1,5]，分数越高视为兴趣越大），然后根据所有电影的推荐得分排序，推荐给用户可能感兴趣的电影。
+我们使用包含用户信息、电影信息与电影评分的数据集作为个性化推荐的应用场景。当我们训练好模型后，只需要输入对应的用户ID和电影ID，就可以得出一个匹配的分数（范围[0,5]，分数越高视为兴趣越大），然后根据所有电影的推荐得分排序，推荐给用户可能感兴趣的电影。
 
 ```
 Input movie_id: 1962
@@ -45,7 +45,7 @@ YouTube是世界上最大的视频上传、分享和发现网站，YouTube推荐
 
 候选生成网络将推荐问题建模为一个类别数极大的多类分类问题：对于一个Youtube用户，使用其观看历史（视频ID）、搜索词记录（search tokens）、人口学信息（如地理位置、用户登录设备）、二值特征（如性别，是否登录）和连续特征（如用户年龄）等，对视频库中所有视频进行多分类，得到每一类别的分类结果（即每一个视频的推荐概率），最终输出概率较高的几百个视频。
 
-首先，将观看历史及搜索词记录这类历史信息，映射为向量后取平均值得到定长表示；同时，输入人口学特征以优化新用户的推荐效果，并将二值特征和连续特征归一化处理到[0, 1]范围。接下来，将所有特征表示拼接为一个向量，并输入给非线形多层感知器（MLP，详见[识别数字](https://github.com/PaddlePaddle/book/blob/develop/recognize_digits/README.md)教程）处理。最后，训练时将MLP的输出给softmax做分类，预测时计算用户的综合特征（MLP的输出）与所有视频的相似度，取得分最高的$k$个作为候选生成网络的筛选结果。图2显示了候选生成网络结构。
+首先，将观看历史及搜索词记录这类历史信息，映射为向量后取平均值得到定长表示；同时，输入人口学特征以优化新用户的推荐效果，并将二值特征和连续特征归一化处理到[0, 1]范围。接下来，将所有特征表示拼接为一个向量，并输入给非线形多层感知器（MLP，详见[识别数字](https://github.com/PaddlePaddle/book/blob/develop/02.recognize_digits/README.cn.md)教程）处理。最后，训练时将MLP的输出给softmax做分类，预测时计算用户的综合特征（MLP的输出）与所有视频的相似度，取得分最高的$k$个作为候选生成网络的筛选结果。图2显示了候选生成网络结构。
 
 <p align="center">
 <img src="image/Deep_candidate_generation_model_architecture.png" width="70%" ><br/>
@@ -64,8 +64,34 @@ $$P(\omega=i|u)=\frac{e^{v_{i}u}}{\sum_{j \in V}e^{v_{j}u}}$$
 排序网络的结构类似于候选生成网络，但是它的目标是对候选进行更细致的打分排序。和传统广告排序中的特征抽取方法类似，这里也构造了大量的用于视频排序的相关特征（如视频 ID、上次观看时间等）。这些特征的处理方式和候选生成网络类似，不同之处是排序网络的顶部是一个加权逻辑回归（weighted logistic regression），它对所有候选视频进行打分，从高到底排序后将分数较高的一些视频返回给用户。
 
 ### 融合推荐模型
+本节会使卷积神经网络（Convolutional Neural Networks）来学习电影名称的表示。下面会依次介绍文本卷积神经网络以及融合推荐模型。
 
-在下文的电影推荐系统中：
+#### 文本卷积神经网络（CNN）
+
+卷积神经网络经常用来处理具有类似网格拓扑结构（grid-like topology）的数据。例如，图像可以视为二维网格的像素点，自然语言可以视为一维的词序列。卷积神经网络可以提取多种局部特征，并对其进行组合抽象得到更高级的特征表示。实验表明，卷积神经网络能高效地对图像及文本问题进行建模处理。  
+
+卷积神经网络主要由卷积（convolution）和池化（pooling）操作构成，其应用及组合方式灵活多变，种类繁多。本小结我们以如图3所示的网络进行讲解：
+
+<p align="center">
+<img src="image/text_cnn.png" width = "80%" align="center"/><br/>
+图3. 卷积神经网络文本分类模型
+</p>
+
+假设待处理句子的长度为$n$，其中第$i$个词的词向量（word embedding）为$x_i\in\mathbb{R}^k$，$k$为维度大小。  
+
+首先，进行词向量的拼接操作：将每$h$个词拼接起来形成一个大小为$h$的词窗口，记为$x_{i:i+h-1}$，它表示词序列$x_{i},x_{i+1},\ldots,x_{i+h-1}$的拼接，其中，$i$表示词窗口中第一个词在整个句子中的位置，取值范围从$1$到$n-h+1$，$x_{i:i+h-1}\in\mathbb{R}^{hk}$。  
+
+其次，进行卷积操作：把卷积核(kernel)$w\in\mathbb{R}^{hk}$应用于包含$h$个词的窗口$x_{i:i+h-1}$，得到特征$c_i=f(w\cdot x_{i:i+h-1}+b)$，其中$b\in\mathbb{R}$为偏置项（bias），$f$为非线性激活函数，如$sigmoid$。将卷积核应用于句子中所有的词窗口${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$，产生一个特征图（feature map）：
+
+$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+
+接下来，对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征$\hat c$，它是特征图中所有元素的最大值：
+
+$$\hat c=max(c)$$  
+
+#### 模型概览
+
+在融合推荐模型的电影推荐系统中：
 
 1. 首先，使用用户特征和电影特征作为神经网络的输入，其中：
 
@@ -75,14 +101,14 @@ $$P(\omega=i|u)=\frac{e^{v_{i}u}}{\sum_{j \in V}e^{v_{j}u}}$$
 
 2. 对用户特征，将用户ID映射为维度大小为256的向量表示，输入全连接层，并对其他三个属性也做类似的处理。然后将四个属性的特征表示分别全连接并相加。
 
-3. 对电影特征，将电影ID以类似用户ID的方式进行处理，电影类型ID以向量的形式直接输入全连接层，电影名称用文本卷积神经网络（详见[第5章](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)）得到其定长向量表示。然后将三个属性的特征表示分别全连接并相加。
+3. 对电影特征，将电影ID以类似用户ID的方式进行处理，电影类型ID以向量的形式直接输入全连接层，电影名称用文本卷积神经网络得到其定长向量表示。然后将三个属性的特征表示分别全连接并相加。
 
 4. 得到用户和电影的向量表示后，计算二者的余弦相似度作为推荐系统的打分。最后，用该相似度打分和用户真实打分的差异的平方作为该回归模型的损失函数。
 
 <p align="center">
 
 <img src="image/rec_regression_network.png" width="90%" ><br/>
-图3. 融合推荐模型
+图4. 融合推荐模型
 </p>
 
 ## 数据准备
@@ -193,22 +219,26 @@ uid = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_user_id() + 1))
 usr_emb = paddle.layer.embedding(input=uid, size=32)
+usr_fc = paddle.layer.fc(input=usr_emb, size=32)
 
 usr_gender_id = paddle.layer.data(
     name='gender_id', type=paddle.data_type.integer_value(2))
 usr_gender_emb = paddle.layer.embedding(input=usr_gender_id, size=16)
+usr_gender_fc = paddle.layer.fc(input=usr_gender_emb, size=16)
 
 usr_age_id = paddle.layer.data(
     name='age_id',
     type=paddle.data_type.integer_value(
         len(paddle.dataset.movielens.age_table)))
 usr_age_emb = paddle.layer.embedding(input=usr_age_id, size=16)
+usr_age_fc = paddle.layer.fc(input=usr_age_emb, size=16)
 
 usr_job_id = paddle.layer.data(
     name='job_id',
-        type=paddle.data_type.integer_value(paddle.dataset.movielens.max_job_id(
-        ) + 1))
+    type=paddle.data_type.integer_value(
+        paddle.dataset.movielens.max_job_id() + 1))
 usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
+usr_job_fc = paddle.layer.fc(input=usr_job_emb, size=16)
 ```
 
 如上述代码所示，对于每个用户，我们输入4维特征。其中包括`user_id`,`gender_id`,`age_id`,`job_id`。这几维特征均是简单的整数值。为了后续神经网络处理这些特征方便，我们借鉴NLP中的语言模型，将这几维离散的整数值，变换成embedding取出。分别形成`usr_emb`, `usr_gender_emb`, `usr_age_emb`, `usr_job_emb`。
@@ -216,7 +246,7 @@ usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
 
 ```python
 usr_combined_features = paddle.layer.fc(
-        input=[usr_emb, usr_gender_emb, usr_age_emb, usr_job_emb],
+        input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
         size=200,
         act=paddle.activation.Tanh())
 ```
@@ -232,15 +262,14 @@ mov_id = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_movie_id() + 1))
 mov_emb = paddle.layer.embedding(input=mov_id, size=32)
+mov_fc = paddle.layer.fc(input=mov_emb, size=32)
 
 mov_categories = paddle.layer.data(
     name='category_id',
     type=paddle.data_type.sparse_binary_vector(
         len(paddle.dataset.movielens.movie_categories())))
-
 mov_categories_hidden = paddle.layer.fc(input=mov_categories, size=32)
 
-
 movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
 mov_title_id = paddle.layer.data(
     name='movie_title',
@@ -250,7 +279,7 @@ mov_title_conv = paddle.networks.sequence_conv_pool(
     input=mov_title_emb, hidden_size=32, context_len=3)
 
 mov_combined_features = paddle.layer.fc(
-    input=[mov_emb, mov_categories_hidden, mov_title_conv],
+    input=[mov_fc, mov_categories_hidden, mov_title_conv],
     size=200,
     act=paddle.activation.Tanh())
 ```
@@ -268,7 +297,7 @@ inference = paddle.layer.cos_sim(a=usr_combined_features, b=mov_combined_feature
 
 
 ```python
-cost = paddle.layer.mse_cost(
+cost = paddle.layer.square_error_cost(
         input=inference,
         label=paddle.layer.data(
             name='score', type=paddle.data_type.dense_vector(1)))
@@ -287,7 +316,7 @@ parameters = paddle.parameters.create(cost)
 ```
 
     [INFO 2017-03-06 17:12:13,284 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-    [INFO 2017-03-06 17:12:13,287 networks.py:1478] The output order is [__mse_cost_0__]
+    [INFO 2017-03-06 17:12:13,287 networks.py:1478] The output order is [__square_error_cost_0__]
 
 
 `parameters`是模型的所有参数集合。他是一个python的dict。我们可以查看到这个网络中的所有参数名称。因为之前定义模型的时候，我们没有指定参数名称，这里参数名称是自动生成的。当然，我们也可以指定每一个参数名称，方便日后维护。
@@ -311,27 +340,18 @@ trainer = paddle.trainer.SGD(cost=cost, parameters=parameters,
 ```
 
     [INFO 2017-03-06 17:12:13,378 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-    [INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__mse_cost_0__]
+    [INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__square_error_cost_0__]
 
 
 ### 训练
 
 下面我们开始训练过程。
 
-我们直接使用Paddle提供的数据集读取程序。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和预测数据集。并且通过`reader_dict`来指定每一个数据和data_layer的对应关系。
-
-例如，这里的reader_dict表示的是，对于数据层 `user_id`，使用了reader中每一条数据的第0个元素。`gender_id`数据层使用了第1个元素。以此类推。
-
-训练过程是完全自动的。我们可以使用event_handler来观察训练过程，或进行测试等。这里我们在event_handler里面绘制了训练误差曲线和测试误差曲线。并且保存了模型。
+我们直接使用Paddle提供的数据集读取程序。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和预测数据集。并且通过`feeding`来指定每一个数据和data_layer的对应关系。
 
+例如，这里的feeding表示的是，对于数据层 `user_id`，使用了reader中每一条数据的第0个元素。`gender_id`数据层使用了第1个元素。以此类推。
 
 ```python
-%matplotlib inline
-
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
 feeding = {
     'user_id': 0,
     'gender_id': 1,
@@ -342,43 +362,53 @@ feeding = {
     'movie_title': 6,
     'score': 7
 }
+```
 
-step=0
-
-train_costs=[],[]
-test_costs=[],[]
+训练过程是完全自动的。我们可以使用event_handler与event_handler_plot来观察训练过程，或进行测试等。这里我们在event_handler_plot里面绘制了训练误差曲线和测试误差曲线。并且保存了模型。
 
+```python
 def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d Batch %d Cost %.2f" % (
+                event.pass_id, event.batch_id, event.cost)
+```
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            cost_ploter.append(train_title, step, event.cost)
 
         if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(reader=paddle.batch(
-                  paddle.dataset.movielens.test(), batch_size=256))
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            result = trainer.test(
+                reader=paddle.batch(
+                    paddle.dataset.movielens.test(), batch_size=256),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            cost_ploter.plot()
+
         step += 1
+```
 
+```python
 trainer.train(
     reader=paddle.batch(
             paddle.reader.shuffle(
             paddle.dataset.movielens.train(), buf_size=8192),
                             batch_size=256),
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding,
     num_passes=2)
 ```
@@ -430,5 +460,6 @@ print "[Predict] User %d Rating Movie %d With Score %.2f"%(user_id, movie_id, sc
 6. Yuan, Jianbo, et al. ["Solving Cold-Start Problem in Large-scale Recommendation Engines: A Deep Learning Approach."](https://arxiv.org/pdf/1611.05480v1.pdf) *arXiv preprint arXiv:1611.05480* (2016).
 7. Covington P, Adams J, Sargin E. [Deep neural networks for youtube recommendations](https://static.googleusercontent.com/media/research.google.com/zh-CN//pubs/archive/45530.pdf)[C]//Proceedings of the 10th ACM Conference on Recommender Systems. ACM, 2016: 191-198.
 
+
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

08.recommender_system/README.en.md → 05.recommender_system/README.md

 # Personalized Recommendation
 
-The source code of this tutorial is in [book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/08.recommender_system).
+The source code of this tutorial is in [book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system).
 
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 
 ## Background
@@ -24,7 +24,7 @@ This tutorial explains a deep learning based approach and how to implement it us
 
 ## Model Overview
 
-To know more about deep learning based recommendation, let us start from going over the Youtube recommender system[[7](#参考文献)] before introducing our hybrid model.
+To know more about deep learning based recommendation, let us start from going over the Youtube recommender system[[7](#reference)] before introducing our hybrid model.
 
 
 ### YouTube's Deep Learning Recommendation Model
@@ -42,7 +42,7 @@ Youtube models candidate generation as a multiclass classification problem with
 
 <p align="center">
 <img src="image/Deep_candidate_generation_model_architecture.en.png" width="70%" ><br/>
-Figure. Deep candidate geeration model.
+Figure 2. Deep candidate generation model.
 </p>
 
 The first stage of this model maps watching history and search queries into fixed-length representative features.  Then, an MLP (multi-layer perceptron, as described in the [Recognize Digits](https://github.com/PaddlePaddle/book/blob/develop/recognize_digits/README.md) tutorial) takes the concatenation of all representative vectors.  The output of the MLP represents the user' *intrinsic interests*.  At training time, it is used together with a softmax output layer for minimizing the classification error.   At serving time, it is used to compute the relevance of the user with all movies.
@@ -55,26 +55,49 @@ where $u$ is the representative vector of user $U$, $V$ is the corpus of all vid
 
 This model could have a performance issue as the softmax output covers millions of classification labels.  To optimize performance, at the training time, the authors down-sample negative samples, so the actual number of classes is reduced to thousands.  At serving time, the authors ignore the normalization of the softmax outputs, because the results are just for ranking.
 
-
 #### Ranking Network
 
 The architecture of the ranking network is similar to that of the candidate generation network.  Similar to ranking models widely used in online advertising, it uses rich features like video ID, last watching time, etc.  The output layer of the ranking network is a weighted logistic regression, which rates all candidate videos.
 
-
 ### Hybrid Model
 
-In the section, let us introduce our movie recommendation system.
+In the section, let us introduce our movie recommendation system. Especially, we feed moives titles into a text convolution network to get a fixed-length representative feature vector. Accordingly we will introduce the convolutional neural network for texts and the hybrid recommendation model respectively.
+
+#### Convolutional Neural Networks for Texts (CNN)
+
+**Convolutional Neural Networks** are frequently applied to data with grid-like topology such as two-dimensional images and one-dimensional texts. A CNN can extract multiple local features, combine them, and produce high-level abstractions, which correspond to semantic understanding. Empirically, CNN is shown to be efficient for image and text modeling.
+
+CNN mainly contains convolution and pooling operation, with versatile combinations in various applications. Here, we briefly describe a CNN as shown in Figure 3.
+
+
+<p align="center">
+<img src="image/text_cnn_en.png" width = "80%" align="center"/><br/>
+Figure 3. CNN for text modeling.
+</p>
+
+Let $n$ be the length of the sentence to process, and the $i$-th word has embedding as $x_i\in\mathbb{R}^k$，where $k$ is the embedding dimensionality.
+
+First, we concatenate the words by piecing together every $h$ words, each as a window of length $h$. This window is denoted as $x_{i:i+h-1}$, consisting of $x_{i},x_{i+1},\ldots,x_{i+h-1}$, where $x_i$ is the first word in the window and $i$ takes value ranging from $1$ to $n-h+1$: $x_{i:i+h-1}\in\mathbb{R}^{hk}$.
+
+Next, we apply the convolution operation: we apply the kernel $w\in\mathbb{R}^{hk}$ in each window, extracting features $c_i=f(w\cdot x_{i:i+h-1}+b)$, where $b\in\mathbb{R}$ is the bias and $f$ is a non-linear activation function such as $sigmoid$. Convolving by the kernel at every window ${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$ produces a feature map in the following form:
+
+$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+
+Next, we apply *max pooling* over time to represent the whole sentence $\hat c$, which is the maximum element across the feature map:
+
+$$\hat c=max(c)$$
+
+#### Model Structure Of The Hybrid Model
 
 In our network, the input includes features of users and movies.  The user feature includes four properties: user ID, gender, occupation, and age.  Movie features include their IDs, genres, and titles.
 
-We use fully-connected layers to map user features into representative feature vectors and concatenate them.  The process of movie features is similar, except that for movie titles -- we feed titles into a text convolution network as described in the [sentiment analysis tutorial](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)）to get a fixed-length representative feature vector.
+We use fully-connected layers to map user features into representative feature vectors and concatenate them.  The process of movie features is similar, except that for movie titles -- we feed titles into a text convolution network as described in the above section to get a fixed-length representative feature vector.
 
 Given the feature vectors of users and movies, we compute the relevance using cosine similarity.  We minimize the squared error at training time.
 
 <p align="center">
-
 <img src="image/rec_regression_network_en.png" width="90%" ><br/>
-Figure 3. A hybrid recommendation model.
+Figure 4. A hybrid recommendation model.
 </p>
 
 ## Dataset
@@ -171,12 +194,6 @@ After issuing a command `python train.py`, training will start immediately. The
 First, we must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
 
 ```python
-%matplotlib inline
-
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
 import paddle.v2 as paddle
 paddle.init(use_gpu=False)
 ```
@@ -189,29 +206,33 @@ uid = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_user_id() + 1))
 usr_emb = paddle.layer.embedding(input=uid, size=32)
+usr_fc = paddle.layer.fc(input=usr_emb, size=32)
 
 usr_gender_id = paddle.layer.data(
     name='gender_id', type=paddle.data_type.integer_value(2))
 usr_gender_emb = paddle.layer.embedding(input=usr_gender_id, size=16)
+usr_gender_fc = paddle.layer.fc(input=usr_gender_emb, size=16)
 
 usr_age_id = paddle.layer.data(
     name='age_id',
     type=paddle.data_type.integer_value(
         len(paddle.dataset.movielens.age_table)))
 usr_age_emb = paddle.layer.embedding(input=usr_age_id, size=16)
+usr_age_fc = paddle.layer.fc(input=usr_age_emb, size=16)
 
 usr_job_id = paddle.layer.data(
     name='job_id',
-        type=paddle.data_type.integer_value(paddle.dataset.movielens.max_job_id(
-        ) + 1))
+    type=paddle.data_type.integer_value(
+        paddle.dataset.movielens.max_job_id() + 1))
 usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
+usr_job_fc = paddle.layer.fc(input=usr_job_emb, size=16)
 ```
 
 As shown in the above code, the input is four dimension integers for each user, that is,  `user_id`,`gender_id`, `age_id` and `job_id`. In order to deal with these features conveniently, we use the language model in NLP to transform these discrete values into embedding vaules `usr_emb`, `usr_gender_emb`, `usr_age_emb` and `usr_job_emb`.
 
 ```python
 usr_combined_features = paddle.layer.fc(
-        input=[usr_emb, usr_gender_emb, usr_age_emb, usr_job_emb],
+        input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
         size=200,
         act=paddle.activation.Tanh())
 ```
@@ -226,15 +247,14 @@ mov_id = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_movie_id() + 1))
 mov_emb = paddle.layer.embedding(input=mov_id, size=32)
+mov_fc = paddle.layer.fc(input=mov_emb, size=32)
 
 mov_categories = paddle.layer.data(
     name='category_id',
     type=paddle.data_type.sparse_binary_vector(
         len(paddle.dataset.movielens.movie_categories())))
-
 mov_categories_hidden = paddle.layer.fc(input=mov_categories, size=32)
 
-
 movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
 mov_title_id = paddle.layer.data(
     name='movie_title',
@@ -244,7 +264,7 @@ mov_title_conv = paddle.networks.sequence_conv_pool(
     input=mov_title_emb, hidden_size=32, context_len=3)
 
 mov_combined_features = paddle.layer.fc(
-    input=[mov_emb, mov_categories_hidden, mov_title_conv],
+    input=[mov_fc, mov_categories_hidden, mov_title_conv],
     size=200,
     act=paddle.activation.Tanh())
 ```
@@ -255,7 +275,7 @@ Finally, we can use cosine similarity to calculate the similarity between user c
 
 ```python
 inference = paddle.layer.cos_sim(a=usr_combined_features, b=mov_combined_features, size=1, scale=5)
-cost = paddle.layer.mse_cost(
+cost = paddle.layer.square_error_cost(
         input=inference,
         label=paddle.layer.data(
         name='score', type=paddle.data_type.dense_vector(1)))
@@ -283,7 +303,7 @@ trainer = paddle.trainer.SGD(cost=cost, parameters=parameters,
 
 ```text
 [INFO 2017-03-06 17:12:13,378 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-[INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__mse_cost_0__]
+[INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__square_error_cost_0__]
 ```
 
 ### Training
@@ -312,37 +332,41 @@ feeding = {
 }
 ```
 
-Callback function `event_handler` will be called during training when a pre-defined event happens.
+Callback function `event_handler` and  `event_handler_plot` will be called during training when a pre-defined event happens.
 
 ```python
-step=0
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d Batch %d Cost %.2f" % (
+                event.pass_id, event.batch_id, event.cost)
+```
 
-train_costs=[],[]
-test_costs=[],[]
+```python
+from paddle.v2.plot import Ploter
 
-def event_handler(event):
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            cost_ploter.append(train_title, step, event.cost)
 
         if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(reader=paddle.batch(
-                  paddle.dataset.movielens.test(), batch_size=256))
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            result = trainer.test(
+                reader=paddle.batch(
+                    paddle.dataset.movielens.test(), batch_size=256),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            cost_ploter.plot()
+
         step += 1
 ```
 
@@ -351,9 +375,9 @@ Finally, we can invoke `trainer.train` to start training:
 ```python
 trainer.train(
     reader=reader,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding,
-    num_passes=200)
+    num_passes=2)
 ```
 
 ## Conclusion
@@ -371,4 +395,4 @@ This tutorial goes over traditional approaches in recommender system and a deep
 7. Covington P, Adams J, Sargin E. [Deep neural networks for youtube recommendations](https://static.googleusercontent.com/media/research.google.com/zh-CN//pubs/archive/45530.pdf)[C]//Proceedings of the 10th ACM Conference on Recommender Systems. ACM, 2016: 191-198.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

08.recommender_system/index.html → 05.recommender_system/index.cn.html

@@ -42,7 +42,7 @@
 <div id="markdown" style='display:none'>
 # 个性化推荐
 
-本教程源代码目录在[book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/08.recommender_system)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/176.html)。
 
 ## 背景介绍
 
@@ -62,7 +62,7 @@
 
 ## 效果展示
 
-我们使用包含用户信息、电影信息与电影评分的数据集作为个性化推荐的应用场景。当我们训练好模型后，只需要输入对应的用户ID和电影ID，就可以得出一个匹配的分数（范围[1,5]，分数越高视为兴趣越大），然后根据所有电影的推荐得分排序，推荐给用户可能感兴趣的电影。
+我们使用包含用户信息、电影信息与电影评分的数据集作为个性化推荐的应用场景。当我们训练好模型后，只需要输入对应的用户ID和电影ID，就可以得出一个匹配的分数（范围[0,5]，分数越高视为兴趣越大），然后根据所有电影的推荐得分排序，推荐给用户可能感兴趣的电影。
 
 ```
 Input movie_id: 1962
@@ -87,7 +87,7 @@ YouTube是世界上最大的视频上传、分享和发现网站，YouTube推荐
 
 候选生成网络将推荐问题建模为一个类别数极大的多类分类问题：对于一个Youtube用户，使用其观看历史（视频ID）、搜索词记录（search tokens）、人口学信息（如地理位置、用户登录设备）、二值特征（如性别，是否登录）和连续特征（如用户年龄）等，对视频库中所有视频进行多分类，得到每一类别的分类结果（即每一个视频的推荐概率），最终输出概率较高的几百个视频。
 
-首先，将观看历史及搜索词记录这类历史信息，映射为向量后取平均值得到定长表示；同时，输入人口学特征以优化新用户的推荐效果，并将二值特征和连续特征归一化处理到[0, 1]范围。接下来，将所有特征表示拼接为一个向量，并输入给非线形多层感知器（MLP，详见[识别数字](https://github.com/PaddlePaddle/book/blob/develop/recognize_digits/README.md)教程）处理。最后，训练时将MLP的输出给softmax做分类，预测时计算用户的综合特征（MLP的输出）与所有视频的相似度，取得分最高的$k$个作为候选生成网络的筛选结果。图2显示了候选生成网络结构。
+首先，将观看历史及搜索词记录这类历史信息，映射为向量后取平均值得到定长表示；同时，输入人口学特征以优化新用户的推荐效果，并将二值特征和连续特征归一化处理到[0, 1]范围。接下来，将所有特征表示拼接为一个向量，并输入给非线形多层感知器（MLP，详见[识别数字](https://github.com/PaddlePaddle/book/blob/develop/02.recognize_digits/README.cn.md)教程）处理。最后，训练时将MLP的输出给softmax做分类，预测时计算用户的综合特征（MLP的输出）与所有视频的相似度，取得分最高的$k$个作为候选生成网络的筛选结果。图2显示了候选生成网络结构。
 
 <p align="center">
 <img src="image/Deep_candidate_generation_model_architecture.png" width="70%" ><br/>
@@ -106,8 +106,34 @@ $$P(\omega=i|u)=\frac{e^{v_{i}u}}{\sum_{j \in V}e^{v_{j}u}}$$
 排序网络的结构类似于候选生成网络，但是它的目标是对候选进行更细致的打分排序。和传统广告排序中的特征抽取方法类似，这里也构造了大量的用于视频排序的相关特征（如视频 ID、上次观看时间等）。这些特征的处理方式和候选生成网络类似，不同之处是排序网络的顶部是一个加权逻辑回归（weighted logistic regression），它对所有候选视频进行打分，从高到底排序后将分数较高的一些视频返回给用户。
 
 ### 融合推荐模型
+本节会使卷积神经网络（Convolutional Neural Networks）来学习电影名称的表示。下面会依次介绍文本卷积神经网络以及融合推荐模型。
 
-在下文的电影推荐系统中：
+#### 文本卷积神经网络（CNN）
+
+卷积神经网络经常用来处理具有类似网格拓扑结构（grid-like topology）的数据。例如，图像可以视为二维网格的像素点，自然语言可以视为一维的词序列。卷积神经网络可以提取多种局部特征，并对其进行组合抽象得到更高级的特征表示。实验表明，卷积神经网络能高效地对图像及文本问题进行建模处理。  
+
+卷积神经网络主要由卷积（convolution）和池化（pooling）操作构成，其应用及组合方式灵活多变，种类繁多。本小结我们以如图3所示的网络进行讲解：
+
+<p align="center">
+<img src="image/text_cnn.png" width = "80%" align="center"/><br/>
+图3. 卷积神经网络文本分类模型
+</p>
+
+假设待处理句子的长度为$n$，其中第$i$个词的词向量（word embedding）为$x_i\in\mathbb{R}^k$，$k$为维度大小。  
+
+首先，进行词向量的拼接操作：将每$h$个词拼接起来形成一个大小为$h$的词窗口，记为$x_{i:i+h-1}$，它表示词序列$x_{i},x_{i+1},\ldots,x_{i+h-1}$的拼接，其中，$i$表示词窗口中第一个词在整个句子中的位置，取值范围从$1$到$n-h+1$，$x_{i:i+h-1}\in\mathbb{R}^{hk}$。  
+
+其次，进行卷积操作：把卷积核(kernel)$w\in\mathbb{R}^{hk}$应用于包含$h$个词的窗口$x_{i:i+h-1}$，得到特征$c_i=f(w\cdot x_{i:i+h-1}+b)$，其中$b\in\mathbb{R}$为偏置项（bias），$f$为非线性激活函数，如$sigmoid$。将卷积核应用于句子中所有的词窗口${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$，产生一个特征图（feature map）：
+
+$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+
+接下来，对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征$\hat c$，它是特征图中所有元素的最大值：
+
+$$\hat c=max(c)$$  
+
+#### 模型概览
+
+在融合推荐模型的电影推荐系统中：
 
 1. 首先，使用用户特征和电影特征作为神经网络的输入，其中：
 
@@ -117,14 +143,14 @@ $$P(\omega=i|u)=\frac{e^{v_{i}u}}{\sum_{j \in V}e^{v_{j}u}}$$
 
 2. 对用户特征，将用户ID映射为维度大小为256的向量表示，输入全连接层，并对其他三个属性也做类似的处理。然后将四个属性的特征表示分别全连接并相加。
 
-3. 对电影特征，将电影ID以类似用户ID的方式进行处理，电影类型ID以向量的形式直接输入全连接层，电影名称用文本卷积神经网络（详见[第5章](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)）得到其定长向量表示。然后将三个属性的特征表示分别全连接并相加。
+3. 对电影特征，将电影ID以类似用户ID的方式进行处理，电影类型ID以向量的形式直接输入全连接层，电影名称用文本卷积神经网络得到其定长向量表示。然后将三个属性的特征表示分别全连接并相加。
 
 4. 得到用户和电影的向量表示后，计算二者的余弦相似度作为推荐系统的打分。最后，用该相似度打分和用户真实打分的差异的平方作为该回归模型的损失函数。
 
 <p align="center">
 
 <img src="image/rec_regression_network.png" width="90%" ><br/>
-图3. 融合推荐模型
+图4. 融合推荐模型
 </p>
 
 ## 数据准备
@@ -235,22 +261,26 @@ uid = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_user_id() + 1))
 usr_emb = paddle.layer.embedding(input=uid, size=32)
+usr_fc = paddle.layer.fc(input=usr_emb, size=32)
 
 usr_gender_id = paddle.layer.data(
     name='gender_id', type=paddle.data_type.integer_value(2))
 usr_gender_emb = paddle.layer.embedding(input=usr_gender_id, size=16)
+usr_gender_fc = paddle.layer.fc(input=usr_gender_emb, size=16)
 
 usr_age_id = paddle.layer.data(
     name='age_id',
     type=paddle.data_type.integer_value(
         len(paddle.dataset.movielens.age_table)))
 usr_age_emb = paddle.layer.embedding(input=usr_age_id, size=16)
+usr_age_fc = paddle.layer.fc(input=usr_age_emb, size=16)
 
 usr_job_id = paddle.layer.data(
     name='job_id',
-        type=paddle.data_type.integer_value(paddle.dataset.movielens.max_job_id(
-        ) + 1))
+    type=paddle.data_type.integer_value(
+        paddle.dataset.movielens.max_job_id() + 1))
 usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
+usr_job_fc = paddle.layer.fc(input=usr_job_emb, size=16)
 ```
 
 如上述代码所示，对于每个用户，我们输入4维特征。其中包括`user_id`,`gender_id`,`age_id`,`job_id`。这几维特征均是简单的整数值。为了后续神经网络处理这些特征方便，我们借鉴NLP中的语言模型，将这几维离散的整数值，变换成embedding取出。分别形成`usr_emb`, `usr_gender_emb`, `usr_age_emb`, `usr_job_emb`。
@@ -258,7 +288,7 @@ usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
 
 ```python
 usr_combined_features = paddle.layer.fc(
-        input=[usr_emb, usr_gender_emb, usr_age_emb, usr_job_emb],
+        input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
         size=200,
         act=paddle.activation.Tanh())
 ```
@@ -274,15 +304,14 @@ mov_id = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_movie_id() + 1))
 mov_emb = paddle.layer.embedding(input=mov_id, size=32)
+mov_fc = paddle.layer.fc(input=mov_emb, size=32)
 
 mov_categories = paddle.layer.data(
     name='category_id',
     type=paddle.data_type.sparse_binary_vector(
         len(paddle.dataset.movielens.movie_categories())))
-
 mov_categories_hidden = paddle.layer.fc(input=mov_categories, size=32)
 
-
 movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
 mov_title_id = paddle.layer.data(
     name='movie_title',
@@ -292,7 +321,7 @@ mov_title_conv = paddle.networks.sequence_conv_pool(
     input=mov_title_emb, hidden_size=32, context_len=3)
 
 mov_combined_features = paddle.layer.fc(
-    input=[mov_emb, mov_categories_hidden, mov_title_conv],
+    input=[mov_fc, mov_categories_hidden, mov_title_conv],
     size=200,
     act=paddle.activation.Tanh())
 ```
@@ -310,7 +339,7 @@ inference = paddle.layer.cos_sim(a=usr_combined_features, b=mov_combined_feature
 
 
 ```python
-cost = paddle.layer.mse_cost(
+cost = paddle.layer.square_error_cost(
         input=inference,
         label=paddle.layer.data(
             name='score', type=paddle.data_type.dense_vector(1)))
@@ -329,7 +358,7 @@ parameters = paddle.parameters.create(cost)
 ```
 
     [INFO 2017-03-06 17:12:13,284 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-    [INFO 2017-03-06 17:12:13,287 networks.py:1478] The output order is [__mse_cost_0__]
+    [INFO 2017-03-06 17:12:13,287 networks.py:1478] The output order is [__square_error_cost_0__]
 
 
 `parameters`是模型的所有参数集合。他是一个python的dict。我们可以查看到这个网络中的所有参数名称。因为之前定义模型的时候，我们没有指定参数名称，这里参数名称是自动生成的。当然，我们也可以指定每一个参数名称，方便日后维护。
@@ -353,27 +382,18 @@ trainer = paddle.trainer.SGD(cost=cost, parameters=parameters,
 ```
 
     [INFO 2017-03-06 17:12:13,378 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-    [INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__mse_cost_0__]
+    [INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__square_error_cost_0__]
 
 
 ### 训练
 
 下面我们开始训练过程。
 
-我们直接使用Paddle提供的数据集读取程序。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和预测数据集。并且通过`reader_dict`来指定每一个数据和data_layer的对应关系。
-
-例如，这里的reader_dict表示的是，对于数据层 `user_id`，使用了reader中每一条数据的第0个元素。`gender_id`数据层使用了第1个元素。以此类推。
-
-训练过程是完全自动的。我们可以使用event_handler来观察训练过程，或进行测试等。这里我们在event_handler里面绘制了训练误差曲线和测试误差曲线。并且保存了模型。
+我们直接使用Paddle提供的数据集读取程序。`paddle.dataset.movielens.train()`和`paddle.dataset.movielens.test()`分别做训练和预测数据集。并且通过`feeding`来指定每一个数据和data_layer的对应关系。
 
+例如，这里的feeding表示的是，对于数据层 `user_id`，使用了reader中每一条数据的第0个元素。`gender_id`数据层使用了第1个元素。以此类推。
 
 ```python
-%matplotlib inline
-
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
 feeding = {
     'user_id': 0,
     'gender_id': 1,
@@ -384,43 +404,53 @@ feeding = {
     'movie_title': 6,
     'score': 7
 }
+```
 
-step=0
-
-train_costs=[],[]
-test_costs=[],[]
+训练过程是完全自动的。我们可以使用event_handler与event_handler_plot来观察训练过程，或进行测试等。这里我们在event_handler_plot里面绘制了训练误差曲线和测试误差曲线。并且保存了模型。
 
+```python
 def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d Batch %d Cost %.2f" % (
+                event.pass_id, event.batch_id, event.cost)
+```
+
+```python
+from paddle.v2.plot import Ploter
+
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            cost_ploter.append(train_title, step, event.cost)
 
         if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(reader=paddle.batch(
-                  paddle.dataset.movielens.test(), batch_size=256))
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            result = trainer.test(
+                reader=paddle.batch(
+                    paddle.dataset.movielens.test(), batch_size=256),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            cost_ploter.plot()
+
         step += 1
+```
 
+```python
 trainer.train(
     reader=paddle.batch(
             paddle.reader.shuffle(
             paddle.dataset.movielens.train(), buf_size=8192),
                             batch_size=256),
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding,
     num_passes=2)
 ```
@@ -472,8 +502,9 @@ print "[Predict] User %d Rating Movie %d With Score %.2f"%(user_id, movie_id, sc
 6. Yuan, Jianbo, et al. ["Solving Cold-Start Problem in Large-scale Recommendation Engines: A Deep Learning Approach."](https://arxiv.org/pdf/1611.05480v1.pdf) *arXiv preprint arXiv:1611.05480* (2016).
 7. Covington P, Adams J, Sargin E. [Deep neural networks for youtube recommendations](https://static.googleusercontent.com/media/research.google.com/zh-CN//pubs/archive/45530.pdf)[C]//Proceedings of the 10th ACM Conference on Recommender Systems. ACM, 2016: 191-198.
 
+
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

08.recommender_system/index.en.html → 05.recommender_system/index.html

@@ -42,9 +42,9 @@
 <div id="markdown" style='display:none'>
 # Personalized Recommendation
 
-The source code of this tutorial is in [book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/08.recommender_system).
+The source code of this tutorial is in [book/recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system).
 
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 
 ## Background
@@ -66,7 +66,7 @@ This tutorial explains a deep learning based approach and how to implement it us
 
 ## Model Overview
 
-To know more about deep learning based recommendation, let us start from going over the Youtube recommender system[[7](#参考文献)] before introducing our hybrid model.
+To know more about deep learning based recommendation, let us start from going over the Youtube recommender system[[7](#reference)] before introducing our hybrid model.
 
 
 ### YouTube's Deep Learning Recommendation Model
@@ -84,7 +84,7 @@ Youtube models candidate generation as a multiclass classification problem with
 
 <p align="center">
 <img src="image/Deep_candidate_generation_model_architecture.en.png" width="70%" ><br/>
-Figure. Deep candidate geeration model.
+Figure 2. Deep candidate generation model.
 </p>
 
 The first stage of this model maps watching history and search queries into fixed-length representative features.  Then, an MLP (multi-layer perceptron, as described in the [Recognize Digits](https://github.com/PaddlePaddle/book/blob/develop/recognize_digits/README.md) tutorial) takes the concatenation of all representative vectors.  The output of the MLP represents the user' *intrinsic interests*.  At training time, it is used together with a softmax output layer for minimizing the classification error.   At serving time, it is used to compute the relevance of the user with all movies.
@@ -97,26 +97,49 @@ where $u$ is the representative vector of user $U$, $V$ is the corpus of all vid
 
 This model could have a performance issue as the softmax output covers millions of classification labels.  To optimize performance, at the training time, the authors down-sample negative samples, so the actual number of classes is reduced to thousands.  At serving time, the authors ignore the normalization of the softmax outputs, because the results are just for ranking.
 
-
 #### Ranking Network
 
 The architecture of the ranking network is similar to that of the candidate generation network.  Similar to ranking models widely used in online advertising, it uses rich features like video ID, last watching time, etc.  The output layer of the ranking network is a weighted logistic regression, which rates all candidate videos.
 
-
 ### Hybrid Model
 
-In the section, let us introduce our movie recommendation system.
+In the section, let us introduce our movie recommendation system. Especially, we feed moives titles into a text convolution network to get a fixed-length representative feature vector. Accordingly we will introduce the convolutional neural network for texts and the hybrid recommendation model respectively.
+
+#### Convolutional Neural Networks for Texts (CNN)
+
+**Convolutional Neural Networks** are frequently applied to data with grid-like topology such as two-dimensional images and one-dimensional texts. A CNN can extract multiple local features, combine them, and produce high-level abstractions, which correspond to semantic understanding. Empirically, CNN is shown to be efficient for image and text modeling.
+
+CNN mainly contains convolution and pooling operation, with versatile combinations in various applications. Here, we briefly describe a CNN as shown in Figure 3.
+
+
+<p align="center">
+<img src="image/text_cnn_en.png" width = "80%" align="center"/><br/>
+Figure 3. CNN for text modeling.
+</p>
+
+Let $n$ be the length of the sentence to process, and the $i$-th word has embedding as $x_i\in\mathbb{R}^k$，where $k$ is the embedding dimensionality.
+
+First, we concatenate the words by piecing together every $h$ words, each as a window of length $h$. This window is denoted as $x_{i:i+h-1}$, consisting of $x_{i},x_{i+1},\ldots,x_{i+h-1}$, where $x_i$ is the first word in the window and $i$ takes value ranging from $1$ to $n-h+1$: $x_{i:i+h-1}\in\mathbb{R}^{hk}$.
+
+Next, we apply the convolution operation: we apply the kernel $w\in\mathbb{R}^{hk}$ in each window, extracting features $c_i=f(w\cdot x_{i:i+h-1}+b)$, where $b\in\mathbb{R}$ is the bias and $f$ is a non-linear activation function such as $sigmoid$. Convolving by the kernel at every window ${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$ produces a feature map in the following form:
+
+$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+
+Next, we apply *max pooling* over time to represent the whole sentence $\hat c$, which is the maximum element across the feature map:
+
+$$\hat c=max(c)$$
+
+#### Model Structure Of The Hybrid Model
 
 In our network, the input includes features of users and movies.  The user feature includes four properties: user ID, gender, occupation, and age.  Movie features include their IDs, genres, and titles.
 
-We use fully-connected layers to map user features into representative feature vectors and concatenate them.  The process of movie features is similar, except that for movie titles -- we feed titles into a text convolution network as described in the [sentiment analysis tutorial](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)）to get a fixed-length representative feature vector.
+We use fully-connected layers to map user features into representative feature vectors and concatenate them.  The process of movie features is similar, except that for movie titles -- we feed titles into a text convolution network as described in the above section to get a fixed-length representative feature vector.
 
 Given the feature vectors of users and movies, we compute the relevance using cosine similarity.  We minimize the squared error at training time.
 
 <p align="center">
-
 <img src="image/rec_regression_network_en.png" width="90%" ><br/>
-Figure 3. A hybrid recommendation model.
+Figure 4. A hybrid recommendation model.
 </p>
 
 ## Dataset
@@ -213,12 +236,6 @@ After issuing a command `python train.py`, training will start immediately. The
 First, we must import and initialize PaddlePaddle (enable/disable GPU, set the number of trainers, etc).
 
 ```python
-%matplotlib inline
-
-import matplotlib.pyplot as plt
-from IPython import display
-import cPickle
-
 import paddle.v2 as paddle
 paddle.init(use_gpu=False)
 ```
@@ -231,29 +248,33 @@ uid = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_user_id() + 1))
 usr_emb = paddle.layer.embedding(input=uid, size=32)
+usr_fc = paddle.layer.fc(input=usr_emb, size=32)
 
 usr_gender_id = paddle.layer.data(
     name='gender_id', type=paddle.data_type.integer_value(2))
 usr_gender_emb = paddle.layer.embedding(input=usr_gender_id, size=16)
+usr_gender_fc = paddle.layer.fc(input=usr_gender_emb, size=16)
 
 usr_age_id = paddle.layer.data(
     name='age_id',
     type=paddle.data_type.integer_value(
         len(paddle.dataset.movielens.age_table)))
 usr_age_emb = paddle.layer.embedding(input=usr_age_id, size=16)
+usr_age_fc = paddle.layer.fc(input=usr_age_emb, size=16)
 
 usr_job_id = paddle.layer.data(
     name='job_id',
-        type=paddle.data_type.integer_value(paddle.dataset.movielens.max_job_id(
-        ) + 1))
+    type=paddle.data_type.integer_value(
+        paddle.dataset.movielens.max_job_id() + 1))
 usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
+usr_job_fc = paddle.layer.fc(input=usr_job_emb, size=16)
 ```
 
 As shown in the above code, the input is four dimension integers for each user, that is,  `user_id`,`gender_id`, `age_id` and `job_id`. In order to deal with these features conveniently, we use the language model in NLP to transform these discrete values into embedding vaules `usr_emb`, `usr_gender_emb`, `usr_age_emb` and `usr_job_emb`.
 
 ```python
 usr_combined_features = paddle.layer.fc(
-        input=[usr_emb, usr_gender_emb, usr_age_emb, usr_job_emb],
+        input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
         size=200,
         act=paddle.activation.Tanh())
 ```
@@ -268,15 +289,14 @@ mov_id = paddle.layer.data(
     type=paddle.data_type.integer_value(
         paddle.dataset.movielens.max_movie_id() + 1))
 mov_emb = paddle.layer.embedding(input=mov_id, size=32)
+mov_fc = paddle.layer.fc(input=mov_emb, size=32)
 
 mov_categories = paddle.layer.data(
     name='category_id',
     type=paddle.data_type.sparse_binary_vector(
         len(paddle.dataset.movielens.movie_categories())))
-
 mov_categories_hidden = paddle.layer.fc(input=mov_categories, size=32)
 
-
 movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
 mov_title_id = paddle.layer.data(
     name='movie_title',
@@ -286,7 +306,7 @@ mov_title_conv = paddle.networks.sequence_conv_pool(
     input=mov_title_emb, hidden_size=32, context_len=3)
 
 mov_combined_features = paddle.layer.fc(
-    input=[mov_emb, mov_categories_hidden, mov_title_conv],
+    input=[mov_fc, mov_categories_hidden, mov_title_conv],
     size=200,
     act=paddle.activation.Tanh())
 ```
@@ -297,7 +317,7 @@ Finally, we can use cosine similarity to calculate the similarity between user c
 
 ```python
 inference = paddle.layer.cos_sim(a=usr_combined_features, b=mov_combined_features, size=1, scale=5)
-cost = paddle.layer.mse_cost(
+cost = paddle.layer.square_error_cost(
         input=inference,
         label=paddle.layer.data(
         name='score', type=paddle.data_type.dense_vector(1)))
@@ -325,7 +345,7 @@ trainer = paddle.trainer.SGD(cost=cost, parameters=parameters,
 
 ```text
 [INFO 2017-03-06 17:12:13,378 networks.py:1472] The input order is [user_id, gender_id, age_id, job_id, movie_id, category_id, movie_title, score]
-[INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__mse_cost_0__]
+[INFO 2017-03-06 17:12:13,379 networks.py:1478] The output order is [__square_error_cost_0__]
 ```
 
 ### Training
@@ -354,37 +374,41 @@ feeding = {
 }
 ```
 
-Callback function `event_handler` will be called during training when a pre-defined event happens.
+Callback function `event_handler` and  `event_handler_plot` will be called during training when a pre-defined event happens.
 
 ```python
-step=0
+def event_handler(event):
+    if isinstance(event, paddle.event.EndIteration):
+        if event.batch_id % 100 == 0:
+            print "Pass %d Batch %d Cost %.2f" % (
+                event.pass_id, event.batch_id, event.cost)
+```
 
-train_costs=[],[]
-test_costs=[],[]
+```python
+from paddle.v2.plot import Ploter
 
-def event_handler(event):
+train_title = "Train cost"
+test_title = "Test cost"
+cost_ploter = Ploter(train_title, test_title)
+
+step = 0
+
+def event_handler_plot(event):
     global step
-    global train_costs
-    global test_costs
     if isinstance(event, paddle.event.EndIteration):
-        need_plot = False
         if step % 10 == 0:  # every 10 batches, record a train cost
-            train_costs[0].append(step)
-            train_costs[1].append(event.cost)
+            cost_ploter.append(train_title, step, event.cost)
 
         if step % 1000 == 0: # every 1000 batches, record a test cost
-            result = trainer.test(reader=paddle.batch(
-                  paddle.dataset.movielens.test(), batch_size=256))
-            test_costs[0].append(step)
-            test_costs[1].append(result.cost)
+            result = trainer.test(
+                reader=paddle.batch(
+                    paddle.dataset.movielens.test(), batch_size=256),
+                feeding=feeding)
+            cost_ploter.append(test_title, step, result.cost)
 
         if step % 100 == 0: # every 100 batches, update cost plot
-            plt.plot(*train_costs)
-            plt.plot(*test_costs)
-            plt.legend(['Train Cost', 'Test Cost'], loc='upper left')
-            display.clear_output(wait=True)
-            display.display(plt.gcf())
-            plt.gcf().clear()
+            cost_ploter.plot()
+
         step += 1
 ```
 
@@ -393,9 +417,9 @@ Finally, we can invoke `trainer.train` to start training:
 ```python
 trainer.train(
     reader=reader,
-    event_handler=event_handler,
+    event_handler=event_handler_plot,
     feeding=feeding,
-    num_passes=200)
+    num_passes=2)
 ```
 
 ## Conclusion
@@ -413,7 +437,7 @@ This tutorial goes over traditional approaches in recommender system and a deep
 7. Covington P, Adams J, Sargin E. [Deep neural networks for youtube recommendations](https://static.googleusercontent.com/media/research.google.com/zh-CN//pubs/archive/45530.pdf)[C]//Proceedings of the 10th ACM Conference on Recommender Systems. ACM, 2016: 191-198.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

08.recommender_system/train.py → 05.recommender_system/train.py

@@ -3,48 +3,53 @@ import cPickle
 import copy
 
 
-def main():
-    paddle.init(use_gpu=False)
-    movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
+def get_usr_combined_features():
     uid = paddle.layer.data(
         name='user_id',
         type=paddle.data_type.integer_value(
             paddle.dataset.movielens.max_user_id() + 1))
     usr_emb = paddle.layer.embedding(input=uid, size=32)
+    usr_fc = paddle.layer.fc(input=usr_emb, size=32)
 
     usr_gender_id = paddle.layer.data(
         name='gender_id', type=paddle.data_type.integer_value(2))
     usr_gender_emb = paddle.layer.embedding(input=usr_gender_id, size=16)
+    usr_gender_fc = paddle.layer.fc(input=usr_gender_emb, size=16)
 
     usr_age_id = paddle.layer.data(
         name='age_id',
         type=paddle.data_type.integer_value(
             len(paddle.dataset.movielens.age_table)))
     usr_age_emb = paddle.layer.embedding(input=usr_age_id, size=16)
+    usr_age_fc = paddle.layer.fc(input=usr_age_emb, size=16)
 
     usr_job_id = paddle.layer.data(
         name='job_id',
         type=paddle.data_type.integer_value(
             paddle.dataset.movielens.max_job_id() + 1))
-
     usr_job_emb = paddle.layer.embedding(input=usr_job_id, size=16)
+    usr_job_fc = paddle.layer.fc(input=usr_job_emb, size=16)
 
     usr_combined_features = paddle.layer.fc(
-        input=[usr_emb, usr_gender_emb, usr_age_emb, usr_job_emb],
+        input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
         size=200,
         act=paddle.activation.Tanh())
+    return usr_combined_features
+
 
+def get_mov_combined_features():
+    movie_title_dict = paddle.dataset.movielens.get_movie_title_dict()
     mov_id = paddle.layer.data(
         name='movie_id',
         type=paddle.data_type.integer_value(
             paddle.dataset.movielens.max_movie_id() + 1))
     mov_emb = paddle.layer.embedding(input=mov_id, size=32)
+    mov_fc = paddle.layer.fc(input=mov_emb, size=32)
 
     mov_categories = paddle.layer.data(
         name='category_id',
         type=paddle.data_type.sparse_binary_vector(
             len(paddle.dataset.movielens.movie_categories())))
-
     mov_categories_hidden = paddle.layer.fc(input=mov_categories, size=32)
 
     mov_title_id = paddle.layer.data(
@@ -55,13 +60,19 @@ def main():
         input=mov_title_emb, hidden_size=32, context_len=3)
 
     mov_combined_features = paddle.layer.fc(
-        input=[mov_emb, mov_categories_hidden, mov_title_conv],
+        input=[mov_fc, mov_categories_hidden, mov_title_conv],
         size=200,
         act=paddle.activation.Tanh())
+    return mov_combined_features
+
 
+def main():
+    paddle.init(use_gpu=False)
+    usr_combined_features = get_usr_combined_features()
+    mov_combined_features = get_mov_combined_features()
     inference = paddle.layer.cos_sim(
         a=usr_combined_features, b=mov_combined_features, size=1, scale=5)
-    cost = paddle.layer.mse_cost(
+    cost = paddle.layer.square_error_cost(
         input=inference,
         label=paddle.layer.data(
             name='score', type=paddle.data_type.dense_vector(1)))
@@ -106,9 +117,6 @@ def main():
 
     feature = user.value() + movie.value()
 
-    def reader():
-        yield feature
-
     infer_dict = copy.copy(feeding)
     del infer_dict['score']
 

05.understand_sentiment/README.md → 06.understand_sentiment/README.cn.md

 # 情感分析
 
-本教程源代码目录在[book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/05.understand_sentiment)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/06.understand_sentiment)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/177.html)。
 
 ## 背景介绍
 
@@ -22,35 +22,13 @@
 本章我们所要介绍的深度学习模型克服了BOW表示的上述缺陷，它在考虑词顺序的基础上把文本映射到低维度的语义空间，并且以端对端（end to end）的方式进行文本表示及分类，其性能相对于传统方法有显著的提升\[[1](#参考文献)\]。
 
 ## 模型概览
-
 本章所使用的文本表示模型为卷积神经网络（Convolutional Neural Networks）和循环神经网络(Recurrent Neural Networks)及其扩展。下面依次介绍这几个模型。
 
-### 文本卷积神经网络（CNN）
-
-卷积神经网络经常用来处理具有类似网格拓扑结构（grid-like topology）的数据。例如，图像可以视为二维网格的像素点，自然语言可以视为一维的词序列。卷积神经网络可以提取多种局部特征，并对其进行组合抽象得到更高级的特征表示。实验表明，卷积神经网络能高效地对图像及文本问题进行建模处理。  
-
-卷积神经网络主要由卷积（convolution）和池化（pooling）操作构成，其应用及组合方式灵活多变，种类繁多。本小结我们以一种简单的文本分类卷积神经网络为例进行讲解\[[1](#参考文献)\]，如图1所示：
-
-<p align="center">
-<img src="image/text_cnn.png" width = "80%" align="center"/><br/>
-图1. 卷积神经网络文本分类模型
-</p>
-
-假设待处理句子的长度为$n$，其中第$i$个词的词向量（word embedding）为$x_i\in\mathbb{R}^k$，$k$为维度大小。  
+### 文本卷积神经网络简介（CNN）
 
-首先，进行词向量的拼接操作：将每$h$个词拼接起来形成一个大小为$h$的词窗口，记为$x_{i:i+h-1}$，它表示词序列$x_{i},x_{i+1},\ldots,x_{i+h-1}$的拼接，其中，$i$表示词窗口中第一个词在整个句子中的位置，取值范围从$1$到$n-h+1$，$x_{i:i+h-1}\in\mathbb{R}^{hk}$。  
+我们在[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节介绍过应用于文本数据的卷积神经网络模型的计算过程，这里进行一个简单的回顾。
 
-其次，进行卷积操作：把卷积核(kernel)$w\in\mathbb{R}^{hk}$应用于包含$h$个词的窗口$x_{i:i+h-1}$，得到特征$c_i=f(w\cdot x_{i:i+h-1}+b)$，其中$b\in\mathbb{R}$为偏置项（bias），$f$为非线性激活函数，如$sigmoid$。将卷积核应用于句子中所有的词窗口${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$，产生一个特征图（feature map）：
-
-$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
-
-接下来，对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征$\hat c$，它是特征图中所有元素的最大值：
-
-$$\hat c=max(c)$$
-
-在实际应用中，我们会使用多个卷积核来处理句子，窗口大小相同的卷积核堆叠起来形成一个矩阵（上文中的单个卷积核参数$w$相当于矩阵的某一行），这样可以更高效的完成运算。另外，我们也可使用窗口大小不同的卷积核来处理句子（图1作为示意画了四个卷积核，不同颜色表示不同大小的卷积核操作）。  
-
-最后，将所有卷积核得到的特征拼接起来即为文本的定长向量表示，对于文本分类问题，将其连接至softmax即构建出完整的模型。
+对卷积神经网络来说，首先使用卷积处理输入的词向量序列，产生一个特征图（feature map），对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征，最后，将所有卷积核得到的特征拼接起来即为文本的定长向量表示，对于文本分类问题，将其连接至softmax即构建出完整的模型。在实际应用中，我们会使用多个卷积核来处理句子，窗口大小相同的卷积核堆叠起来形成一个矩阵，这样可以更高效的完成运算。另外，我们也可使用窗口大小不同的卷积核来处理句子，[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节的图3作为示意画了四个卷积核，不同颜色表示不同大小的卷积核操作。
 
 对于一般的短文本分类问题，上文所述的简单的文本卷积网络即可达到很高的正确率\[[1](#参考文献)\]。若想得到更抽象更高级的文本特征表示，可以构建深层文本卷积神经网络\[[2](#参考文献),[3](#参考文献)\]。
 
@@ -60,10 +38,10 @@ $$\hat c=max(c)$$
 
 <p align="center">
 <img src="image/rnn.png" width = "60%" align="center"/><br/>
-图2. 循环神经网络按时间展开的示意图
+图1. 循环神经网络按时间展开的示意图
 </p>
 
-循环神经网络按时间展开后如图2所示：在第$t$时刻，网络读入第$t$个输入$x_t$（向量表示）及前一时刻隐层的状态值$h_{t-1}$（向量表示，$h_0$一般初始化为$0$向量），计算得出本时刻隐层的状态值$h_t$，重复这一步骤直至读完所有输入。如果将循环神经网络所表示的函数记为$f$，则其公式可表示为：
+循环神经网络按时间展开后如图1所示：在第$t$时刻，网络读入第$t$个输入$x_t$（向量表示）及前一时刻隐层的状态值$h_{t-1}$（向量表示，$h_0$一般初始化为$0$向量），计算得出本时刻隐层的状态值$h_t$，重复这一步骤直至读完所有输入。如果将循环神经网络所表示的函数记为$f$，则其公式可表示为：
 
 $$h_t=f(x_t,h_{t-1})=\sigma(W_{xh}x_t+W_{hh}h_{h-1}+b_h)$$
 
@@ -87,11 +65,11 @@ c_t & = f_t\odot c_{t-1}+i_t\odot tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
 o_t & = \sigma(W_{xo}x_t+W_{ho}h_{h-1}+W_{co}c_{t}+b_o)\\\\
 h_t & = o_t\odot tanh(c_t)\\\\
 \end{align}
-其中，$i_t, f_t, c_t, o_t$分别表示输入门，遗忘门，记忆单元及输出门的向量值，带角标的$W$及$b$为模型参数，$tanh$为双曲正切函数，$\odot$表示逐元素（elementwise）的乘法操作。输入门控制着新输入进入记忆单元$c$的强度，遗忘门控制着记忆单元维持上一时刻值的强度，输出门控制着输出记忆单元的强度。三种门的计算方式类似，但有着完全不同的参数，它们各自以不同的方式控制着记忆单元$c$，如图3所示：
+其中，$i_t, f_t, c_t, o_t$分别表示输入门，遗忘门，记忆单元及输出门的向量值，带角标的$W$及$b$为模型参数，$tanh$为双曲正切函数，$\odot$表示逐元素（elementwise）的乘法操作。输入门控制着新输入进入记忆单元$c$的强度，遗忘门控制着记忆单元维持上一时刻值的强度，输出门控制着输出记忆单元的强度。三种门的计算方式类似，但有着完全不同的参数，它们各自以不同的方式控制着记忆单元$c$，如图2所示：
 
 <p align="center">
 <img src="image/lstm.png" width = "65%" align="center"/><br/>
-图3. 时刻$t$的LSTM [7]
+图2. 时刻$t$的LSTM [7]
 </p>
 
 LSTM通过给简单的循环神经网络增加记忆及控制门的方式，增强了其处理远距离依赖问题的能力。类似原理的改进还有Gated Recurrent Unit (GRU)\[[8](#参考文献)\]，其设计更为简洁一些。**这些改进虽然各有不同，但是它们的宏观描述却与简单的循环神经网络一样（如图2所示），即隐状态依据当前输入及前一时刻的隐状态来改变，不断地循环这一过程直至输入处理完毕：**
@@ -104,11 +82,11 @@ $$ h_t=Recrurent(x_t,h_{t-1})$$
 
 对于正常顺序的循环神经网络，$h_t$包含了$t$时刻之前的输入信息，也就是上文信息。同样，为了得到下文信息，我们可以使用反方向（将输入逆序处理）的循环神经网络。结合构建深层循环神经网络的方法（深层神经网络往往能得到更抽象和高级的特征表示），我们可以通过构建更加强有力的基于LSTM的栈式双向循环神经网络\[[9](#参考文献)\]，来对时序数据进行建模。  
 
-如图4所示（以三层为例），奇数层LSTM正向，偶数层LSTM反向，高一层的LSTM使用低一层LSTM及之前所有层的信息作为输入，对最高层LSTM序列使用时间维度上的最大池化即可得到文本的定长向量表示（这一表示充分融合了文本的上下文信息，并且对文本进行了深层次抽象），最后我们将文本表示连接至softmax构建分类模型。
+如图3所示（以三层为例），奇数层LSTM正向，偶数层LSTM反向，高一层的LSTM使用低一层LSTM及之前所有层的信息作为输入，对最高层LSTM序列使用时间维度上的最大池化即可得到文本的定长向量表示（这一表示充分融合了文本的上下文信息，并且对文本进行了深层次抽象），最后我们将文本表示连接至softmax构建分类模型。
 
 <p align="center">
 <img src="image/stacked_lstm.jpg" width=450><br/>
-图4. 栈式双向LSTM用于文本分类
+图3. 栈式双向LSTM用于文本分类
 </p>
 
 ## 示例程序
@@ -133,7 +111,7 @@ import paddle.v2 as paddle
 ```
 ## 配置模型
 
-在该示例中，我们实现了两种文本分类算法，分别基于上文所述的[文本卷积神经网络](#文本卷积神经网络（CNN）)和[栈式双向LSTM](#栈式双向LSTM（Stacked Bidirectional LSTM）)。
+在该示例中，我们实现了两种文本分类算法，分别基于[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节介绍过的文本卷积神经网络，以及[栈式双向LSTM](#栈式双向LSTM（Stacked Bidirectional LSTM）)。
 
 ### 文本卷积神经网络
 
@@ -141,7 +119,8 @@ import paddle.v2 as paddle
 def convolution_net(input_dim,
                     class_dim=2,
                     emb_dim=128,
-                    hid_dim=128):
+                    hid_dim=128,
+                    is_predict=False):
     data = paddle.layer.data("word",
                              paddle.data_type.integer_value_sequence(input_dim))
     emb = paddle.layer.embedding(input=data, size=emb_dim)
@@ -152,9 +131,12 @@ def convolution_net(input_dim,
     output = paddle.layer.fc(input=[conv_3, conv_4],
                              size=class_dim,
                              act=paddle.activation.Softmax())
+    if not is_predict:
         lbl = paddle.layer.data("label", paddle.data_type.integer_value(2))
         cost = paddle.layer.classification_cost(input=output, label=lbl)
         return cost
+    else:
+        return output
 ```
 网络的输入`input_dim`表示的是词典的大小，`class_dim`表示类别数。这里，我们使用[`sequence_conv_pool`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/trainer_config_helpers/networks.py) API实现了卷积和池化操作。
 
@@ -165,7 +147,8 @@ def stacked_lstm_net(input_dim,
                      class_dim=2,
                      emb_dim=128,
                      hid_dim=512,
-                     stacked_num=3):
+                     stacked_num=3,
+                     is_predict=False):
     """
     A Wrapper for sentiment classification task.
     This network uses bi-directional recurrent network,
@@ -223,9 +206,12 @@ def stacked_lstm_net(input_dim,
                              bias_attr=bias_attr,
                              param_attr=para_attr)
 
+    if not is_predict:
         lbl = paddle.layer.data("label", paddle.data_type.integer_value(2))
         cost = paddle.layer.classification_cost(input=output, label=lbl)
         return cost
+    else:
+        return output
 ```
 网络的输入`stacked_num`表示的是LSTM的层数，需要是奇数，确保最高层LSTM正向。Paddle里面是通过一个fc和一个lstmemory来实现基于LSTM的循环神经网络。
 
@@ -294,7 +280,7 @@ Paddle中提供了一系列优化算法的API，这里使用Adam优化算法。
 
 ### 训练
 
-可以通过`paddle.trainer.SGD`构造一个sgd trainer，并调用`trainer.train`来训练模型。
+可以通过`paddle.trainer.SGD`构造一个sgd trainer，并调用`trainer.train`来训练模型。另外，通过给train函数传递一个`event_handler`来获取每个batch和每个pass结束的状态。
 ```python
     # End batch and end pass event handler
     def event_handler(event):
@@ -309,7 +295,21 @@ Paddle中提供了一系列优化算法的API，这里使用Adam优化算法。
             result = trainer.test(reader=test_reader, feeding=feeding)
             print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
-可以通过给train函数传递一个`event_handler`来获取每个batch和每个pass结束的状态。比如构造如下一个`event_handler`可以在每100个batch结束后输出cost和error；在每个pass结束后调用`trainer.test`计算一遍测试集并获得当前模型在测试集上的error。
+比如，构造如下一个`event_handler`可以在每100个batch结束后输出cost和error；在每个pass结束后调用`trainer.test`计算一遍测试集并获得当前模型在测试集上的error。
+```python
+    from paddle.v2.plot import Ploter
+
+    train_title = "Train cost"
+    cost_ploter = Ploter(train_title)
+    step = 0
+    def event_handler_plot(event):
+        global step
+        if isinstance(event, paddle.event.EndIteration):
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+            step += 1
+```
+或者构造一个`event_handler_plot`画出cost曲线。
 ```python
     # create trainer
     trainer = paddle.trainer.SGD(cost=cost,
@@ -331,12 +331,43 @@ Pass 0, Batch 100, Cost 0.294321, {'classification_error_evaluator': 0.1015625}
 Test with Pass 0, {'classification_error_evaluator': 0.11432000249624252}
 ```
 
+## 应用模型
+
+可以使用训练好的模型对电影评论进行分类，下面程序展示了如何使用`paddle.infer`接口进行推断。
+```python
+    import numpy as np
+
+    # Movie Reviews, from imdb test
+    reviews = [
+        'Read the book, forget the movie!',
+        'This is a great movie.'
+    ]
+    reviews = [c.split() for c in reviews]
+
+    UNK = word_dict['<unk>']
+    input = []
+    for c in reviews:
+        input.append([[word_dict.get(words, UNK) for words in c]])
+
+    # 0 stands for positive sample, 1 stands for negative sample
+    label = {0:'pos', 1:'neg'}
+    # Use the network used by trainer
+    out = convolution_net(dict_dim, class_dim=class_dim, is_predict=True)
+    # out = stacked_lstm_net(dict_dim, class_dim=class_dim, stacked_num=3, is_predict=True)
+    probs = paddle.infer(output_layer=out, parameters=parameters, input=input)
+
+    labs = np.argsort(-probs)
+    for idx, lab in enumerate(labs):
+        print idx, "predicting probability is", probs[idx], "label is", label[lab[0]]
+```
+
+
 ## 总结
 
 本章我们以情感分析为例，介绍了使用深度学习的方法进行端对端的短文本分类，并且使用PaddlePaddle完成了全部相关实验。同时，我们简要介绍了两种文本处理模型：卷积神经网络和循环神经网络。在后续的章节中我们会看到这两种基本的深度学习模型在其它任务上的应用。
 
-## 参考文献
 
+## 参考文献
 1. Kim Y. [Convolutional neural networks for sentence classification](http://arxiv.org/pdf/1408.5882)[J]. arXiv preprint arXiv:1408.5882, 2014.
 2. Kalchbrenner N, Grefenstette E, Blunsom P. [A convolutional neural network for modelling sentences](http://arxiv.org/pdf/1404.2188.pdf?utm_medium=App.net&utm_source=PourOver)[J]. arXiv preprint arXiv:1404.2188, 2014.
 3. Yann N. Dauphin, et al. [Language Modeling with Gated Convolutional Networks](https://arxiv.org/pdf/1612.08083v1.pdf)[J] arXiv preprint arXiv:1612.08083, 2016.
@@ -348,4 +379,4 @@ Test with Pass 0, {'classification_error_evaluator': 0.11432000249624252}
 9. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

05.understand_sentiment/README.en.md → 06.understand_sentiment/README.md

 # Sentiment Analysis
 
-The source codes of this section can be located at [book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/05.understand_sentiment). First-time users may refer to PaddlePaddle for [Installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+The source codes of this section can be located at [book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/06.understand_sentiment). First-time users may refer to PaddlePaddle for [Installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 ## Background
 
-In natural language processing, sentiment analysis refers to describing emotion status in texts. The texts may refer to a sentence, a paragraph or a document. Emotion status can be a binary classification problem (positive/negative or happy/sad), or a three-class problem (positive/neutral/negative). Sentiment analysis can be applied widely in various situations, such as online shopping (Amazon, Taobao), travel and movie websites. It can be used to grasp from the reviews how the customers feel about the product. Table 1 is an example of sentiment analysis in movie reviews:
+In natural language processing, sentiment analysis refers to determining the emotion expressed in a piece of text. The text can be a sentence, a paragraph, or a document. Emotion categorization can be binary -- positive/negative or happy/sad -- or in three classes -- positive/neutral/negative. Sentiment analysis is applicable in a wide range of services, such as e-commerce sites like Amazon and Taobao, hospitality services like Airbnb and hotels.com, and movie rating sites like Rotten Tomatoes and IMDB. It can be used to gauge from the reviews how the customers feel about the product. Table 1 illustrates an example of sentiment analysis in movie reviews:
 
 | Movie Review       | Category  |
 | --------     | -----  |
@@ -15,71 +15,47 @@ In natural language processing, sentiment analysis refers to describing emotion
 
 <p align="center">Table 1 Sentiment Analysis in Movie Reviews</p>
 
-In natural language processing, sentiment analysis can be categorized as a **Text Classification problem**, i.e., to categorize a piece of text to a specific class. It involves two related tasks: text representation and classification. Before deep learning becomes heated, the main-stream methods for the former include BOW (bag of words) and topic modeling, while the latter contain SVM(support vector machine), LR(logistic regression).
+In natural language processing, sentiment analysis can be categorized as a **Text Classification problem**, i.e., to categorize a piece of text to a specific class. It involves two related tasks: text representation and classification. Before the emergence of deep learning techniques, the mainstream methods for text representation include BOW (*bag of words*) and topic modeling, while the latter contain SVM (*support vector machine*) and LR (*logistic regression*).
 
-For a piece of text, BOW model ignores its word order, grammar and syntax, and regard it as a set of words, so BOW does not capture all the information in the text. For example, “this movie is extremely bad“ and “boring, dull and empty work” describe very similar semantic with low similarity in sense of BOW. Also, “the movie is bad“ and “the movie is not bad“ have high similarity with BOW feature, but they express completely opposite semantics.
+The BOW model does not capture all the information in a piece of text, as it ignores syntax and grammar and just treats the text as a set of words. For example, “this movie is extremely bad“ and “boring, dull, and empty work” describe very similar semantic meaning, yet their BOW representations have with little similarity. Furthermore, “the movie is bad“ and “the movie is not bad“ have high similarity with BOW features, but they express completely opposite semantics.
 
-
-In this chapter, we introduce our deep learning model which handles these issues in BOW. Our model embeds texts into a low-dimensional space and takes word order into consideration. It is an end-to-end framework, and has large performance improvement over traditional methods \[[1](#Reference)\].
+This chapter introduces a deep learning model that handles these issues in BOW. Our model embeds texts into a low-dimensional space and takes word order into consideration. It is an end-to-end framework and it has large performance improvement over traditional methods \[[1](#Reference)\].
 
 ## Model Overview
 
-The model we used in this chapter is the CNN (Convolutional Neural Networks) and RNN (Recurrent Neural Networks) with some specific extension.
-
-
-### Convolutional Neural Networks for Texts (CNN)
-
-Convolutional Neural Networks are always applied in data with grid-like topology, such as 2-d images and 1-d texts. CNN can combine extracted multiple local features to produce higher-level abstract semantics. Experimentally, CNN is very efficient for image and text modeling.
-
-CNN mainly contains convolution and pooling operation, with various extensions. We briefly describe CNN here with an example \[[1](#Refernce)\]. As shown in Figure 1：
-
-
-<p align="center">
-<img src="image/text_cnn_en.png" width = "80%" align="center"/><br/>
-Figure 1. CNN for text modeling.
-</p>
-
-Assuming the length of the sentence is $n$, where the $i$-th word has embedding as $x_i\in\mathbb{R}^k$，where $k$ is the embedding dimensionality.
-
-First, we concatenate the words together: we piece every $h$ words as a window of length $h$: $x_{i:i+h-1}$. It refers to $x_{i},x_{i+1},\ldots,x_{i+h-1}$, where $i$ is the first word in the window, ranging from $1$ to $n-h+1$: $x_{i:i+h-1}\in\mathbb{R}^{hk}$.
-
-Next, we apply the convolution operation: we apply the kernel $w\in\mathbb{R}^{hk}$ in each window, extracting features $c_i=f(w\cdot x_{i:i+h-1}+b)$，
-where $b\in\mathbb{R}$ is the bias and $f$ is a non-linear activation function such as $sigmoid$. Applying CNN on every window ${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$ produces a feature map as:
-
-$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+The model we used in this chapter uses **Convolutional Neural Networks** (**CNNs**) and **Recurrent Neural Networks** (**RNNs**) with some specific extensions.
 
-Next, we apply max pooling over time to represent the whole sentence $\hat c$, which is the maximum element across the feature map:
 
-$$\hat c=max(c)$$
+### Revisit to the Convolutional Neural Networks for Texts (CNN)
 
-In real applications, we will apply multiple CNN kernels on the sentences. It can be implemented efficiently by concatenating the kernels together as a matrix. Also, we can use CNN kernels with different kernel size (as shown in Figure 1 in different colors).
+The convolutional neural network for texts is introduced in chapter [recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system), here we make a brief overview.
 
-Finally, the CNN features are concatenated together to produce a fixed-length representation, which can be combined with a softmax for sentiment analysis problem.
+CNN mainly contains convolution and pooling operation, with versatile combinations in various applications. We first apply the convolution operation: we apply the kernel in each window, extracting features. Convolving by the kernel at every window produces a feature map. Next, we apply *max pooling* over time to represent the whole sentence, which is the maximum element across the feature map. In real applications, we will apply multiple CNN kernels on the sentences. It can be implemented efficiently by concatenating the kernels together as a matrix. Also, we can use CNN kernels with different kernel size. Finally, concatenating the resulting features produces a fixed-length representation, which can be combined with a softmax to form the model for the sentiment analysis problem.
 
-For short texts, above CNN model can achieve high accuracy \[[1](#Reference)\]. If we want to extract more abstract representation, we may apply a deeper CNN model \[[2](#Reference),[3](#Reference)\].
+For short texts, the aforementioned CNN model can achieve very high accuracy \[[1](#Reference)\]. If we want to extract more abstract representations, we may apply a deeper CNN model \[[2](#Reference),[3](#Reference)\].
 
 ### Recurrent Neural Network (RNN)
 
-RNN is an effective model for sequential data. Theoretical, the  computational ability of RNN is Turing-complete \[[4](#Reference)\]. NLP is a classical sequential data, and RNN (especially its variant LSTM\[[5](#Reference)\]) achieves State-of-the-Art performance on various tasks in NLP, such as language modeling, syntax parsing, POS-tagging, image captioning, dialog, machine translation and so forth.
+RNN is an effective model for sequential data. In terms of computability, the RNN is Turing-complete \[[4](#Reference)\]. Since NLP is a classical problem on sequential data, the RNN, especially its variant LSTM\[[5](#Reference)\]), achieves state-of-the-art performance on various NLP tasks, such as language modeling, syntax parsing, POS-tagging, image captioning, dialog, machine translation, and so forth.
 
 <p align="center">
 <img src="image/rnn.png" width = "60%" align="center"/><br/>
-Figure 2. An illustration of an unrolled RNN across “time”.
+Figure 1. An illustration of an unfolded RNN in time.
 </p>
 
-As shown in Figure 2, we unroll an RNN: at $t$-th time step, the network takes the $t$-th input vector and the latent state from last time-step $h_{t-1}$ as inputs and compute the latent state of current step. The whole process is repeated until all inputs are consumed. If we regard the RNN as a function $f$, it can be formulated as:
+As shown in Figure 1, we unfold an RNN: at the $t$-th time step, the network takes two inputs: the $t$-th input vector $\vec{x_t}$ and the latent state from the last time-step $\vec{h_{t-1}}$. From those, it computes the latent state of the current step $\vec{h_t}$. This process is repeated until all inputs are consumed. Denoting the RNN as function $f$, it can be formulated as follows:
 
-$$h_t=f(x_t,h_{t-1})=\sigma(W_{xh}x_t+W_{hh}h_{h-1}+b_h)$$
+$$\vec{h_t}=f(\vec{x_t},\vec{h_{t-1}})=\sigma(W_{xh}\vec{x_t}+W_{hh}\vec{h_{h-1}}+\vec{b_h})$$
 
-where $W_{xh}$ is the weight matrix from input to latent; $W_{hh}$ is the latent-to-latent matrix; $b_h$ is the latent bias and $\sigma$ refers to the $sigmoid$function.
+where $W_{xh}$ is the weight matrix to feed into the latent layer; $W_{hh}$ is the latent-to-latent matrix; $b_h$ is the latent bias and $\sigma$ refers to the $sigmoid$ function.
 
-In NLP, words are first represented as a one-hot vector and then mapped to an embedding. The embedded feature goes through an RNN as input $x_t$ at every time step. Moreover, we can add other layers on top of RNN. e.g., a deep or stacked RNN. Also, the last latent state can be used as a feature for sentence classification.
+In NLP, words are often represented as a one-hot vectors and then mapped to an embedding. The embedded feature goes through an RNN as input $x_t$ at every time step. Moreover, we can add other layers on top of RNN, such as a deep or stacked RNN. Finally, the last latent state may be used as a feature for sentence classification.
 
 ### Long-Short Term Memory (LSTM)
 
-For data of long sequence, training RNN sometimes has gradient vanishing and explosion problem \[[6](#)\]. To solve this problem Hochreiter S, Schmidhuber J. (1997) proposed the LSTM(long short term memory\[[5](#Reference)\]).  
+Training an RNN on long sequential data sometimes leads to the gradient vanishing or exploding\[[6](#)\]. To solve this problem Hochreiter S, Schmidhuber J. (1997) proposed **Long Short Term Memory** (LSTM)\[[5](#Reference)\]).
 
-Compared with simple RNN, the structrue of LSTM has included memory cell $c$, input gate $i$, forget gate $f$ and output gate $o$. These gates and memory cells largely improves the ability of handling long sequences. We can formulate LSTM-RNN as a function $F$ as：
+Compared to the structure of a simple RNN, an LSTM includes memory cell $c$, input gate $i$, forget gate $f$ and output gate $o$. These gates and memory cells dramatically improve the ability for the network to handle long sequences. We can formulate the **LSTM-RNN**, denoted as a function $F$, as follows：
 
 $$ h_t=F(x_t,h_{t-1})$$
 
@@ -87,16 +63,16 @@ $F$ contains following formulations\[[7](#Reference)\]：
 \begin{align}
 i_t & = \sigma(W_{xi}x_t+W_{hi}h_{h-1}+W_{ci}c_{t-1}+b_i)\\\\
 f_t & = \sigma(W_{xf}x_t+W_{hf}h_{h-1}+W_{cf}c_{t-1}+b_f)\\\\
-c_t & = f_t\odot c_{t-1}+i_t\odot tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
+c_t & = f_t\odot c_{t-1}+i_t\odot \tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
 o_t & = \sigma(W_{xo}x_t+W_{ho}h_{h-1}+W_{co}c_{t}+b_o)\\\\
-h_t & = o_t\odot tanh(c_t)\\\\
+h_t & = o_t\odot \tanh(c_t)\\\\
 \end{align}
 
-In the equation，$i_t, f_t, c_t, o_t$ stand for input gate, forget gate, memory cell and output gate separately; $W$ and $b$ are model parameters. The $tanh$ is a hyperbolic tangent， and $\odot$ denotes an element-wise product operation. Input gate controls the magnitude of new input into the memory cell $c$; forget gate controls memory propagated from the last time step; output gate controls output magnitude. The three gates are computed similarly with different parameters, and they influence memory cell $c$ separately, as shown in Figure 3:
+In the equation，$i_t, f_t, c_t, o_t$ stand for input gate, forget gate, memory cell and output gate, respectively. $W$ and $b$ are model parameters, $\tanh$ is a hyperbolic tangent, and $\odot$ denotes an element-wise product operation. The input gate controls the magnitude of the new input into the memory cell $c$; the forget gate controls the memory propagated from the last time step; the output gate controls the magnitutde of the output. The three gates are computed similarly with different parameters, and they influence memory cell $c$ separately, as shown in Figure 2:
 
 <p align="center">
 <img src="image/lstm_en.png" width = "65%" align="center"/><br/>
-Figure 3. LSTM at time step $t$ [7].
+Figure 2. LSTM at time step $t$ [7].
 </p>
 
 LSTM enhances the ability of considering long-term reliance, with the help of memory cell and gate. Similar structures are also proposed in Gated Recurrent Unit (GRU)\[[8](Reference)\] with simpler design. **The structures are still similar to RNN, though with some modifications (As shown in Figure 2), i.e., latent status depends on input as well as the latent status of last time-step, and the process goes on recurrently until all input are consumed:**
@@ -108,11 +84,11 @@ where $Recrurent$ is a simple RNN, GRU or LSTM.
 
 For vanilla LSTM, $h_t$ contains input information from previous time-step $1..t-1$ context. We can also apply an RNN with reverse-direction to take successive context $t+1…n$ into consideration. Combining constructing deep RNN (deeper RNN can contain more abstract and higher level semantic), we can design structures with deep stacked bidirectional LSTM to model sequential data\[[9](#Reference)\].
 
-As shown in Figure 4 (3-layer RNN), odd/even layers are forward/reverse LSTM. Higher layers of LSTM take lower-layers LSTM as input, and the top-layer LSTM produces a fixed length vector by max-pooling (this representation considers contexts from previous and successive words for higher-level abstractions). Finally, we concatenate the output to a softmax layer for classification.
+As shown in Figure 3 (3-layer RNN), odd/even layers are forward/reverse LSTM. Higher layers of LSTM take lower-layers LSTM as input, and the top-layer LSTM produces a fixed length vector by max-pooling (this representation considers contexts from previous and successive words for higher-level abstractions). Finally, we concatenate the output to a softmax layer for classification.
 
 <p align="center">
 <img src="image/stacked_lstm_en.png" width=450><br/>
-Figure 4. Stacked Bidirectional LSTM for NLP modeling.
+Figure 3. Stacked Bidirectional LSTM for NLP modeling.
 </p>
 
 ## Dataset
@@ -367,4 +343,4 @@ In this chapter, we use sentiment analysis as an example to introduce applying d
 9. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

05.understand_sentiment/index.html → 06.understand_sentiment/index.cn.html

@@ -42,7 +42,7 @@
 <div id="markdown" style='display:none'>
 # 情感分析
 
-本教程源代码目录在[book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/05.understand_sentiment)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/06.understand_sentiment)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/177.html)。
 
 ## 背景介绍
 
@@ -64,35 +64,13 @@
 本章我们所要介绍的深度学习模型克服了BOW表示的上述缺陷，它在考虑词顺序的基础上把文本映射到低维度的语义空间，并且以端对端（end to end）的方式进行文本表示及分类，其性能相对于传统方法有显著的提升\[[1](#参考文献)\]。
 
 ## 模型概览
-
 本章所使用的文本表示模型为卷积神经网络（Convolutional Neural Networks）和循环神经网络(Recurrent Neural Networks)及其扩展。下面依次介绍这几个模型。
 
-### 文本卷积神经网络（CNN）
-
-卷积神经网络经常用来处理具有类似网格拓扑结构（grid-like topology）的数据。例如，图像可以视为二维网格的像素点，自然语言可以视为一维的词序列。卷积神经网络可以提取多种局部特征，并对其进行组合抽象得到更高级的特征表示。实验表明，卷积神经网络能高效地对图像及文本问题进行建模处理。  
-
-卷积神经网络主要由卷积（convolution）和池化（pooling）操作构成，其应用及组合方式灵活多变，种类繁多。本小结我们以一种简单的文本分类卷积神经网络为例进行讲解\[[1](#参考文献)\]，如图1所示：
-
-<p align="center">
-<img src="image/text_cnn.png" width = "80%" align="center"/><br/>
-图1. 卷积神经网络文本分类模型
-</p>
-
-假设待处理句子的长度为$n$，其中第$i$个词的词向量（word embedding）为$x_i\in\mathbb{R}^k$，$k$为维度大小。  
+### 文本卷积神经网络简介（CNN）
 
-首先，进行词向量的拼接操作：将每$h$个词拼接起来形成一个大小为$h$的词窗口，记为$x_{i:i+h-1}$，它表示词序列$x_{i},x_{i+1},\ldots,x_{i+h-1}$的拼接，其中，$i$表示词窗口中第一个词在整个句子中的位置，取值范围从$1$到$n-h+1$，$x_{i:i+h-1}\in\mathbb{R}^{hk}$。  
+我们在[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节介绍过应用于文本数据的卷积神经网络模型的计算过程，这里进行一个简单的回顾。
 
-其次，进行卷积操作：把卷积核(kernel)$w\in\mathbb{R}^{hk}$应用于包含$h$个词的窗口$x_{i:i+h-1}$，得到特征$c_i=f(w\cdot x_{i:i+h-1}+b)$，其中$b\in\mathbb{R}$为偏置项（bias），$f$为非线性激活函数，如$sigmoid$。将卷积核应用于句子中所有的词窗口${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$，产生一个特征图（feature map）：
-
-$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
-
-接下来，对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征$\hat c$，它是特征图中所有元素的最大值：
-
-$$\hat c=max(c)$$
-
-在实际应用中，我们会使用多个卷积核来处理句子，窗口大小相同的卷积核堆叠起来形成一个矩阵（上文中的单个卷积核参数$w$相当于矩阵的某一行），这样可以更高效的完成运算。另外，我们也可使用窗口大小不同的卷积核来处理句子（图1作为示意画了四个卷积核，不同颜色表示不同大小的卷积核操作）。  
-
-最后，将所有卷积核得到的特征拼接起来即为文本的定长向量表示，对于文本分类问题，将其连接至softmax即构建出完整的模型。
+对卷积神经网络来说，首先使用卷积处理输入的词向量序列，产生一个特征图（feature map），对特征图采用时间维度上的最大池化（max pooling over time）操作得到此卷积核对应的整句话的特征，最后，将所有卷积核得到的特征拼接起来即为文本的定长向量表示，对于文本分类问题，将其连接至softmax即构建出完整的模型。在实际应用中，我们会使用多个卷积核来处理句子，窗口大小相同的卷积核堆叠起来形成一个矩阵，这样可以更高效的完成运算。另外，我们也可使用窗口大小不同的卷积核来处理句子，[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节的图3作为示意画了四个卷积核，不同颜色表示不同大小的卷积核操作。
 
 对于一般的短文本分类问题，上文所述的简单的文本卷积网络即可达到很高的正确率\[[1](#参考文献)\]。若想得到更抽象更高级的文本特征表示，可以构建深层文本卷积神经网络\[[2](#参考文献),[3](#参考文献)\]。
 
@@ -102,10 +80,10 @@ $$\hat c=max(c)$$
 
 <p align="center">
 <img src="image/rnn.png" width = "60%" align="center"/><br/>
-图2. 循环神经网络按时间展开的示意图
+图1. 循环神经网络按时间展开的示意图
 </p>
 
-循环神经网络按时间展开后如图2所示：在第$t$时刻，网络读入第$t$个输入$x_t$（向量表示）及前一时刻隐层的状态值$h_{t-1}$（向量表示，$h_0$一般初始化为$0$向量），计算得出本时刻隐层的状态值$h_t$，重复这一步骤直至读完所有输入。如果将循环神经网络所表示的函数记为$f$，则其公式可表示为：
+循环神经网络按时间展开后如图1所示：在第$t$时刻，网络读入第$t$个输入$x_t$（向量表示）及前一时刻隐层的状态值$h_{t-1}$（向量表示，$h_0$一般初始化为$0$向量），计算得出本时刻隐层的状态值$h_t$，重复这一步骤直至读完所有输入。如果将循环神经网络所表示的函数记为$f$，则其公式可表示为：
 
 $$h_t=f(x_t,h_{t-1})=\sigma(W_{xh}x_t+W_{hh}h_{h-1}+b_h)$$
 
@@ -129,11 +107,11 @@ c_t & = f_t\odot c_{t-1}+i_t\odot tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
 o_t & = \sigma(W_{xo}x_t+W_{ho}h_{h-1}+W_{co}c_{t}+b_o)\\\\
 h_t & = o_t\odot tanh(c_t)\\\\
 \end{align}
-其中，$i_t, f_t, c_t, o_t$分别表示输入门，遗忘门，记忆单元及输出门的向量值，带角标的$W$及$b$为模型参数，$tanh$为双曲正切函数，$\odot$表示逐元素（elementwise）的乘法操作。输入门控制着新输入进入记忆单元$c$的强度，遗忘门控制着记忆单元维持上一时刻值的强度，输出门控制着输出记忆单元的强度。三种门的计算方式类似，但有着完全不同的参数，它们各自以不同的方式控制着记忆单元$c$，如图3所示：
+其中，$i_t, f_t, c_t, o_t$分别表示输入门，遗忘门，记忆单元及输出门的向量值，带角标的$W$及$b$为模型参数，$tanh$为双曲正切函数，$\odot$表示逐元素（elementwise）的乘法操作。输入门控制着新输入进入记忆单元$c$的强度，遗忘门控制着记忆单元维持上一时刻值的强度，输出门控制着输出记忆单元的强度。三种门的计算方式类似，但有着完全不同的参数，它们各自以不同的方式控制着记忆单元$c$，如图2所示：
 
 <p align="center">
 <img src="image/lstm.png" width = "65%" align="center"/><br/>
-图3. 时刻$t$的LSTM [7]
+图2. 时刻$t$的LSTM [7]
 </p>
 
 LSTM通过给简单的循环神经网络增加记忆及控制门的方式，增强了其处理远距离依赖问题的能力。类似原理的改进还有Gated Recurrent Unit (GRU)\[[8](#参考文献)\]，其设计更为简洁一些。**这些改进虽然各有不同，但是它们的宏观描述却与简单的循环神经网络一样（如图2所示），即隐状态依据当前输入及前一时刻的隐状态来改变，不断地循环这一过程直至输入处理完毕：**
@@ -146,11 +124,11 @@ $$ h_t=Recrurent(x_t,h_{t-1})$$
 
 对于正常顺序的循环神经网络，$h_t$包含了$t$时刻之前的输入信息，也就是上文信息。同样，为了得到下文信息，我们可以使用反方向（将输入逆序处理）的循环神经网络。结合构建深层循环神经网络的方法（深层神经网络往往能得到更抽象和高级的特征表示），我们可以通过构建更加强有力的基于LSTM的栈式双向循环神经网络\[[9](#参考文献)\]，来对时序数据进行建模。  
 
-如图4所示（以三层为例），奇数层LSTM正向，偶数层LSTM反向，高一层的LSTM使用低一层LSTM及之前所有层的信息作为输入，对最高层LSTM序列使用时间维度上的最大池化即可得到文本的定长向量表示（这一表示充分融合了文本的上下文信息，并且对文本进行了深层次抽象），最后我们将文本表示连接至softmax构建分类模型。
+如图3所示（以三层为例），奇数层LSTM正向，偶数层LSTM反向，高一层的LSTM使用低一层LSTM及之前所有层的信息作为输入，对最高层LSTM序列使用时间维度上的最大池化即可得到文本的定长向量表示（这一表示充分融合了文本的上下文信息，并且对文本进行了深层次抽象），最后我们将文本表示连接至softmax构建分类模型。
 
 <p align="center">
 <img src="image/stacked_lstm.jpg" width=450><br/>
-图4. 栈式双向LSTM用于文本分类
+图3. 栈式双向LSTM用于文本分类
 </p>
 
 ## 示例程序
@@ -175,7 +153,7 @@ import paddle.v2 as paddle
 ```
 ## 配置模型
 
-在该示例中，我们实现了两种文本分类算法，分别基于上文所述的[文本卷积神经网络](#文本卷积神经网络（CNN）)和[栈式双向LSTM](#栈式双向LSTM（Stacked Bidirectional LSTM）)。
+在该示例中，我们实现了两种文本分类算法，分别基于[推荐系统](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system)一节介绍过的文本卷积神经网络，以及[栈式双向LSTM](#栈式双向LSTM（Stacked Bidirectional LSTM）)。
 
 ### 文本卷积神经网络
 
@@ -183,7 +161,8 @@ import paddle.v2 as paddle
 def convolution_net(input_dim,
                     class_dim=2,
                     emb_dim=128,
-                    hid_dim=128):
+                    hid_dim=128,
+                    is_predict=False):
     data = paddle.layer.data("word",
                              paddle.data_type.integer_value_sequence(input_dim))
     emb = paddle.layer.embedding(input=data, size=emb_dim)
@@ -194,9 +173,12 @@ def convolution_net(input_dim,
     output = paddle.layer.fc(input=[conv_3, conv_4],
                              size=class_dim,
                              act=paddle.activation.Softmax())
+    if not is_predict:
         lbl = paddle.layer.data("label", paddle.data_type.integer_value(2))
         cost = paddle.layer.classification_cost(input=output, label=lbl)
         return cost
+    else:
+        return output
 ```
 网络的输入`input_dim`表示的是词典的大小，`class_dim`表示类别数。这里，我们使用[`sequence_conv_pool`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/trainer_config_helpers/networks.py) API实现了卷积和池化操作。
 
@@ -207,7 +189,8 @@ def stacked_lstm_net(input_dim,
                      class_dim=2,
                      emb_dim=128,
                      hid_dim=512,
-                     stacked_num=3):
+                     stacked_num=3,
+                     is_predict=False):
     """
     A Wrapper for sentiment classification task.
     This network uses bi-directional recurrent network,
@@ -265,9 +248,12 @@ def stacked_lstm_net(input_dim,
                              bias_attr=bias_attr,
                              param_attr=para_attr)
 
+    if not is_predict:
         lbl = paddle.layer.data("label", paddle.data_type.integer_value(2))
         cost = paddle.layer.classification_cost(input=output, label=lbl)
         return cost
+    else:
+        return output
 ```
 网络的输入`stacked_num`表示的是LSTM的层数，需要是奇数，确保最高层LSTM正向。Paddle里面是通过一个fc和一个lstmemory来实现基于LSTM的循环神经网络。
 
@@ -336,7 +322,7 @@ Paddle中提供了一系列优化算法的API，这里使用Adam优化算法。
 
 ### 训练
 
-可以通过`paddle.trainer.SGD`构造一个sgd trainer，并调用`trainer.train`来训练模型。
+可以通过`paddle.trainer.SGD`构造一个sgd trainer，并调用`trainer.train`来训练模型。另外，通过给train函数传递一个`event_handler`来获取每个batch和每个pass结束的状态。
 ```python
     # End batch and end pass event handler
     def event_handler(event):
@@ -351,7 +337,21 @@ Paddle中提供了一系列优化算法的API，这里使用Adam优化算法。
             result = trainer.test(reader=test_reader, feeding=feeding)
             print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
-可以通过给train函数传递一个`event_handler`来获取每个batch和每个pass结束的状态。比如构造如下一个`event_handler`可以在每100个batch结束后输出cost和error；在每个pass结束后调用`trainer.test`计算一遍测试集并获得当前模型在测试集上的error。
+比如，构造如下一个`event_handler`可以在每100个batch结束后输出cost和error；在每个pass结束后调用`trainer.test`计算一遍测试集并获得当前模型在测试集上的error。
+```python
+    from paddle.v2.plot import Ploter
+
+    train_title = "Train cost"
+    cost_ploter = Ploter(train_title)
+    step = 0
+    def event_handler_plot(event):
+        global step
+        if isinstance(event, paddle.event.EndIteration):
+            cost_ploter.append(train_title, step, event.cost)
+            cost_ploter.plot()
+            step += 1
+```
+或者构造一个`event_handler_plot`画出cost曲线。
 ```python
     # create trainer
     trainer = paddle.trainer.SGD(cost=cost,
@@ -373,12 +373,43 @@ Pass 0, Batch 100, Cost 0.294321, {'classification_error_evaluator': 0.1015625}
 Test with Pass 0, {'classification_error_evaluator': 0.11432000249624252}
 ```
 
+## 应用模型
+
+可以使用训练好的模型对电影评论进行分类，下面程序展示了如何使用`paddle.infer`接口进行推断。
+```python
+    import numpy as np
+
+    # Movie Reviews, from imdb test
+    reviews = [
+        'Read the book, forget the movie!',
+        'This is a great movie.'
+    ]
+    reviews = [c.split() for c in reviews]
+
+    UNK = word_dict['<unk>']
+    input = []
+    for c in reviews:
+        input.append([[word_dict.get(words, UNK) for words in c]])
+
+    # 0 stands for positive sample, 1 stands for negative sample
+    label = {0:'pos', 1:'neg'}
+    # Use the network used by trainer
+    out = convolution_net(dict_dim, class_dim=class_dim, is_predict=True)
+    # out = stacked_lstm_net(dict_dim, class_dim=class_dim, stacked_num=3, is_predict=True)
+    probs = paddle.infer(output_layer=out, parameters=parameters, input=input)
+
+    labs = np.argsort(-probs)
+    for idx, lab in enumerate(labs):
+        print idx, "predicting probability is", probs[idx], "label is", label[lab[0]]
+```
+
+
 ## 总结
 
 本章我们以情感分析为例，介绍了使用深度学习的方法进行端对端的短文本分类，并且使用PaddlePaddle完成了全部相关实验。同时，我们简要介绍了两种文本处理模型：卷积神经网络和循环神经网络。在后续的章节中我们会看到这两种基本的深度学习模型在其它任务上的应用。
 
-## 参考文献
 
+## 参考文献
 1. Kim Y. [Convolutional neural networks for sentence classification](http://arxiv.org/pdf/1408.5882)[J]. arXiv preprint arXiv:1408.5882, 2014.
 2. Kalchbrenner N, Grefenstette E, Blunsom P. [A convolutional neural network for modelling sentences](http://arxiv.org/pdf/1404.2188.pdf?utm_medium=App.net&utm_source=PourOver)[J]. arXiv preprint arXiv:1404.2188, 2014.
 3. Yann N. Dauphin, et al. [Language Modeling with Gated Convolutional Networks](https://arxiv.org/pdf/1612.08083v1.pdf)[J] arXiv preprint arXiv:1612.08083, 2016.
@@ -390,7 +421,7 @@ Test with Pass 0, {'classification_error_evaluator': 0.11432000249624252}
 9. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

05.understand_sentiment/index.en.html → 06.understand_sentiment/index.html

@@ -42,11 +42,11 @@
 <div id="markdown" style='display:none'>
 # Sentiment Analysis
 
-The source codes of this section can be located at [book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/05.understand_sentiment). First-time users may refer to PaddlePaddle for [Installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+The source codes of this section can be located at [book/understand_sentiment](https://github.com/PaddlePaddle/book/tree/develop/06.understand_sentiment). First-time users may refer to PaddlePaddle for [Installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 ## Background
 
-In natural language processing, sentiment analysis refers to describing emotion status in texts. The texts may refer to a sentence, a paragraph or a document. Emotion status can be a binary classification problem (positive/negative or happy/sad), or a three-class problem (positive/neutral/negative). Sentiment analysis can be applied widely in various situations, such as online shopping (Amazon, Taobao), travel and movie websites. It can be used to grasp from the reviews how the customers feel about the product. Table 1 is an example of sentiment analysis in movie reviews:
+In natural language processing, sentiment analysis refers to determining the emotion expressed in a piece of text. The text can be a sentence, a paragraph, or a document. Emotion categorization can be binary -- positive/negative or happy/sad -- or in three classes -- positive/neutral/negative. Sentiment analysis is applicable in a wide range of services, such as e-commerce sites like Amazon and Taobao, hospitality services like Airbnb and hotels.com, and movie rating sites like Rotten Tomatoes and IMDB. It can be used to gauge from the reviews how the customers feel about the product. Table 1 illustrates an example of sentiment analysis in movie reviews:
 
 | Movie Review       | Category  |
 | --------     | -----  |
@@ -57,71 +57,47 @@ In natural language processing, sentiment analysis refers to describing emotion
 
 <p align="center">Table 1 Sentiment Analysis in Movie Reviews</p>
 
-In natural language processing, sentiment analysis can be categorized as a **Text Classification problem**, i.e., to categorize a piece of text to a specific class. It involves two related tasks: text representation and classification. Before deep learning becomes heated, the main-stream methods for the former include BOW (bag of words) and topic modeling, while the latter contain SVM(support vector machine), LR(logistic regression).
+In natural language processing, sentiment analysis can be categorized as a **Text Classification problem**, i.e., to categorize a piece of text to a specific class. It involves two related tasks: text representation and classification. Before the emergence of deep learning techniques, the mainstream methods for text representation include BOW (*bag of words*) and topic modeling, while the latter contain SVM (*support vector machine*) and LR (*logistic regression*).
 
-For a piece of text, BOW model ignores its word order, grammar and syntax, and regard it as a set of words, so BOW does not capture all the information in the text. For example, “this movie is extremely bad“ and “boring, dull and empty work” describe very similar semantic with low similarity in sense of BOW. Also, “the movie is bad“ and “the movie is not bad“ have high similarity with BOW feature, but they express completely opposite semantics.
+The BOW model does not capture all the information in a piece of text, as it ignores syntax and grammar and just treats the text as a set of words. For example, “this movie is extremely bad“ and “boring, dull, and empty work” describe very similar semantic meaning, yet their BOW representations have with little similarity. Furthermore, “the movie is bad“ and “the movie is not bad“ have high similarity with BOW features, but they express completely opposite semantics.
 
-
-In this chapter, we introduce our deep learning model which handles these issues in BOW. Our model embeds texts into a low-dimensional space and takes word order into consideration. It is an end-to-end framework, and has large performance improvement over traditional methods \[[1](#Reference)\].
+This chapter introduces a deep learning model that handles these issues in BOW. Our model embeds texts into a low-dimensional space and takes word order into consideration. It is an end-to-end framework and it has large performance improvement over traditional methods \[[1](#Reference)\].
 
 ## Model Overview
 
-The model we used in this chapter is the CNN (Convolutional Neural Networks) and RNN (Recurrent Neural Networks) with some specific extension.
-
-
-### Convolutional Neural Networks for Texts (CNN)
-
-Convolutional Neural Networks are always applied in data with grid-like topology, such as 2-d images and 1-d texts. CNN can combine extracted multiple local features to produce higher-level abstract semantics. Experimentally, CNN is very efficient for image and text modeling.
-
-CNN mainly contains convolution and pooling operation, with various extensions. We briefly describe CNN here with an example \[[1](#Refernce)\]. As shown in Figure 1：
-
-
-<p align="center">
-<img src="image/text_cnn_en.png" width = "80%" align="center"/><br/>
-Figure 1. CNN for text modeling.
-</p>
-
-Assuming the length of the sentence is $n$, where the $i$-th word has embedding as $x_i\in\mathbb{R}^k$，where $k$ is the embedding dimensionality.
-
-First, we concatenate the words together: we piece every $h$ words as a window of length $h$: $x_{i:i+h-1}$. It refers to $x_{i},x_{i+1},\ldots,x_{i+h-1}$, where $i$ is the first word in the window, ranging from $1$ to $n-h+1$: $x_{i:i+h-1}\in\mathbb{R}^{hk}$.
-
-Next, we apply the convolution operation: we apply the kernel $w\in\mathbb{R}^{hk}$ in each window, extracting features $c_i=f(w\cdot x_{i:i+h-1}+b)$，
-where $b\in\mathbb{R}$ is the bias and $f$ is a non-linear activation function such as $sigmoid$. Applying CNN on every window ${x_{1:h},x_{2:h+1},\ldots,x_{n-h+1:n}}$ produces a feature map as:
-
-$$c=[c_1,c_2,\ldots,c_{n-h+1}], c \in \mathbb{R}^{n-h+1}$$
+The model we used in this chapter uses **Convolutional Neural Networks** (**CNNs**) and **Recurrent Neural Networks** (**RNNs**) with some specific extensions.
 
-Next, we apply max pooling over time to represent the whole sentence $\hat c$, which is the maximum element across the feature map:
 
-$$\hat c=max(c)$$
+### Revisit to the Convolutional Neural Networks for Texts (CNN)
 
-In real applications, we will apply multiple CNN kernels on the sentences. It can be implemented efficiently by concatenating the kernels together as a matrix. Also, we can use CNN kernels with different kernel size (as shown in Figure 1 in different colors).
+The convolutional neural network for texts is introduced in chapter [recommender_system](https://github.com/PaddlePaddle/book/tree/develop/05.recommender_system), here we make a brief overview.
 
-Finally, the CNN features are concatenated together to produce a fixed-length representation, which can be combined with a softmax for sentiment analysis problem.
+CNN mainly contains convolution and pooling operation, with versatile combinations in various applications. We first apply the convolution operation: we apply the kernel in each window, extracting features. Convolving by the kernel at every window produces a feature map. Next, we apply *max pooling* over time to represent the whole sentence, which is the maximum element across the feature map. In real applications, we will apply multiple CNN kernels on the sentences. It can be implemented efficiently by concatenating the kernels together as a matrix. Also, we can use CNN kernels with different kernel size. Finally, concatenating the resulting features produces a fixed-length representation, which can be combined with a softmax to form the model for the sentiment analysis problem.
 
-For short texts, above CNN model can achieve high accuracy \[[1](#Reference)\]. If we want to extract more abstract representation, we may apply a deeper CNN model \[[2](#Reference),[3](#Reference)\].
+For short texts, the aforementioned CNN model can achieve very high accuracy \[[1](#Reference)\]. If we want to extract more abstract representations, we may apply a deeper CNN model \[[2](#Reference),[3](#Reference)\].
 
 ### Recurrent Neural Network (RNN)
 
-RNN is an effective model for sequential data. Theoretical, the  computational ability of RNN is Turing-complete \[[4](#Reference)\]. NLP is a classical sequential data, and RNN (especially its variant LSTM\[[5](#Reference)\]) achieves State-of-the-Art performance on various tasks in NLP, such as language modeling, syntax parsing, POS-tagging, image captioning, dialog, machine translation and so forth.
+RNN is an effective model for sequential data. In terms of computability, the RNN is Turing-complete \[[4](#Reference)\]. Since NLP is a classical problem on sequential data, the RNN, especially its variant LSTM\[[5](#Reference)\]), achieves state-of-the-art performance on various NLP tasks, such as language modeling, syntax parsing, POS-tagging, image captioning, dialog, machine translation, and so forth.
 
 <p align="center">
 <img src="image/rnn.png" width = "60%" align="center"/><br/>
-Figure 2. An illustration of an unrolled RNN across “time”.
+Figure 1. An illustration of an unfolded RNN in time.
 </p>
 
-As shown in Figure 2, we unroll an RNN: at $t$-th time step, the network takes the $t$-th input vector and the latent state from last time-step $h_{t-1}$ as inputs and compute the latent state of current step. The whole process is repeated until all inputs are consumed. If we regard the RNN as a function $f$, it can be formulated as:
+As shown in Figure 1, we unfold an RNN: at the $t$-th time step, the network takes two inputs: the $t$-th input vector $\vec{x_t}$ and the latent state from the last time-step $\vec{h_{t-1}}$. From those, it computes the latent state of the current step $\vec{h_t}$. This process is repeated until all inputs are consumed. Denoting the RNN as function $f$, it can be formulated as follows:
 
-$$h_t=f(x_t,h_{t-1})=\sigma(W_{xh}x_t+W_{hh}h_{h-1}+b_h)$$
+$$\vec{h_t}=f(\vec{x_t},\vec{h_{t-1}})=\sigma(W_{xh}\vec{x_t}+W_{hh}\vec{h_{h-1}}+\vec{b_h})$$
 
-where $W_{xh}$ is the weight matrix from input to latent; $W_{hh}$ is the latent-to-latent matrix; $b_h$ is the latent bias and $\sigma$ refers to the $sigmoid$function.
+where $W_{xh}$ is the weight matrix to feed into the latent layer; $W_{hh}$ is the latent-to-latent matrix; $b_h$ is the latent bias and $\sigma$ refers to the $sigmoid$ function.
 
-In NLP, words are first represented as a one-hot vector and then mapped to an embedding. The embedded feature goes through an RNN as input $x_t$ at every time step. Moreover, we can add other layers on top of RNN. e.g., a deep or stacked RNN. Also, the last latent state can be used as a feature for sentence classification.
+In NLP, words are often represented as a one-hot vectors and then mapped to an embedding. The embedded feature goes through an RNN as input $x_t$ at every time step. Moreover, we can add other layers on top of RNN, such as a deep or stacked RNN. Finally, the last latent state may be used as a feature for sentence classification.
 
 ### Long-Short Term Memory (LSTM)
 
-For data of long sequence, training RNN sometimes has gradient vanishing and explosion problem \[[6](#)\]. To solve this problem Hochreiter S, Schmidhuber J. (1997) proposed the LSTM(long short term memory\[[5](#Reference)\]).  
+Training an RNN on long sequential data sometimes leads to the gradient vanishing or exploding\[[6](#)\]. To solve this problem Hochreiter S, Schmidhuber J. (1997) proposed **Long Short Term Memory** (LSTM)\[[5](#Reference)\]).
 
-Compared with simple RNN, the structrue of LSTM has included memory cell $c$, input gate $i$, forget gate $f$ and output gate $o$. These gates and memory cells largely improves the ability of handling long sequences. We can formulate LSTM-RNN as a function $F$ as：
+Compared to the structure of a simple RNN, an LSTM includes memory cell $c$, input gate $i$, forget gate $f$ and output gate $o$. These gates and memory cells dramatically improve the ability for the network to handle long sequences. We can formulate the **LSTM-RNN**, denoted as a function $F$, as follows：
 
 $$ h_t=F(x_t,h_{t-1})$$
 
@@ -129,16 +105,16 @@ $F$ contains following formulations\[[7](#Reference)\]：
 \begin{align}
 i_t & = \sigma(W_{xi}x_t+W_{hi}h_{h-1}+W_{ci}c_{t-1}+b_i)\\\\
 f_t & = \sigma(W_{xf}x_t+W_{hf}h_{h-1}+W_{cf}c_{t-1}+b_f)\\\\
-c_t & = f_t\odot c_{t-1}+i_t\odot tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
+c_t & = f_t\odot c_{t-1}+i_t\odot \tanh(W_{xc}x_t+W_{hc}h_{h-1}+b_c)\\\\
 o_t & = \sigma(W_{xo}x_t+W_{ho}h_{h-1}+W_{co}c_{t}+b_o)\\\\
-h_t & = o_t\odot tanh(c_t)\\\\
+h_t & = o_t\odot \tanh(c_t)\\\\
 \end{align}
 
-In the equation，$i_t, f_t, c_t, o_t$ stand for input gate, forget gate, memory cell and output gate separately; $W$ and $b$ are model parameters. The $tanh$ is a hyperbolic tangent， and $\odot$ denotes an element-wise product operation. Input gate controls the magnitude of new input into the memory cell $c$; forget gate controls memory propagated from the last time step; output gate controls output magnitude. The three gates are computed similarly with different parameters, and they influence memory cell $c$ separately, as shown in Figure 3:
+In the equation，$i_t, f_t, c_t, o_t$ stand for input gate, forget gate, memory cell and output gate, respectively. $W$ and $b$ are model parameters, $\tanh$ is a hyperbolic tangent, and $\odot$ denotes an element-wise product operation. The input gate controls the magnitude of the new input into the memory cell $c$; the forget gate controls the memory propagated from the last time step; the output gate controls the magnitutde of the output. The three gates are computed similarly with different parameters, and they influence memory cell $c$ separately, as shown in Figure 2:
 
 <p align="center">
 <img src="image/lstm_en.png" width = "65%" align="center"/><br/>
-Figure 3. LSTM at time step $t$ [7].
+Figure 2. LSTM at time step $t$ [7].
 </p>
 
 LSTM enhances the ability of considering long-term reliance, with the help of memory cell and gate. Similar structures are also proposed in Gated Recurrent Unit (GRU)\[[8](Reference)\] with simpler design. **The structures are still similar to RNN, though with some modifications (As shown in Figure 2), i.e., latent status depends on input as well as the latent status of last time-step, and the process goes on recurrently until all input are consumed:**
@@ -150,11 +126,11 @@ where $Recrurent$ is a simple RNN, GRU or LSTM.
 
 For vanilla LSTM, $h_t$ contains input information from previous time-step $1..t-1$ context. We can also apply an RNN with reverse-direction to take successive context $t+1…n$ into consideration. Combining constructing deep RNN (deeper RNN can contain more abstract and higher level semantic), we can design structures with deep stacked bidirectional LSTM to model sequential data\[[9](#Reference)\].
 
-As shown in Figure 4 (3-layer RNN), odd/even layers are forward/reverse LSTM. Higher layers of LSTM take lower-layers LSTM as input, and the top-layer LSTM produces a fixed length vector by max-pooling (this representation considers contexts from previous and successive words for higher-level abstractions). Finally, we concatenate the output to a softmax layer for classification.
+As shown in Figure 3 (3-layer RNN), odd/even layers are forward/reverse LSTM. Higher layers of LSTM take lower-layers LSTM as input, and the top-layer LSTM produces a fixed length vector by max-pooling (this representation considers contexts from previous and successive words for higher-level abstractions). Finally, we concatenate the output to a softmax layer for classification.
 
 <p align="center">
 <img src="image/stacked_lstm_en.png" width=450><br/>
-Figure 4. Stacked Bidirectional LSTM for NLP modeling.
+Figure 3. Stacked Bidirectional LSTM for NLP modeling.
 </p>
 
 ## Dataset
@@ -409,7 +385,7 @@ In this chapter, we use sentiment analysis as an example to introduce applying d
 9. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

06.label_semantic_roles/README.md → 07.label_semantic_roles/README.cn.md

 # 语义角色标注
 
-本教程源代码目录在[book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/06.label_semantic_roles)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/07.label_semantic_roles)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/178.html)。
 
 ## 背景介绍
 
@@ -68,7 +68,7 @@ $$\mbox{[小明]}_{\mbox{Agent}}\mbox{[昨天]}_{\mbox{Time}}\mbox{[晚上]}_\mb
 图4. 基于LSTM的双向循环神经网络结构示意图
 </p>
 
-需要说明的是，这种双向RNN结构和Bengio等人在机器翻译任务中使用的双向RNN结构\[[3](#参考文献), [4](#参考文献)\] 并不相同，我们会在后续[机器翻译](https://github.com/PaddlePaddle/book/blob/develop/machine_translation/README.md)任务中，介绍另一种双向循环神经网络。
+需要说明的是，这种双向RNN结构和Bengio等人在机器翻译任务中使用的双向RNN结构\[[3](#参考文献), [4](#参考文献)\] 并不相同，我们会在后续[机器翻译](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.cn.md)任务中，介绍另一种双向循环神经网络。
 
 ### 条件随机场 (Conditional Random Field)
 
@@ -182,7 +182,7 @@ conll05st-release/
 | predicate_dict | 谓词的词典，共计3162个词 |
 | emb | 一个训练好的词表，32维 |
 
-我们在英文维基百科上训练语言模型得到了一份词向量用来初始化SRL模型。在SRL模型训练过程中，词向量不再被更新。关于语言模型和词向量可以参考[词向量](https://github.com/PaddlePaddle/book/blob/develop/04.word2vec/README.md) 这篇教程。我们训练语言模型的语料共有995,000,000个token，词典大小控制为4900,000词。CoNLL 2005训练语料中有5%的词不在这4900,000个词中，我们将它们全部看作未登录词，用`<unk>`表示。
+我们在英文维基百科上训练语言模型得到了一份词向量用来初始化SRL模型。在SRL模型训练过程中，词向量不再被更新。关于语言模型和词向量可以参考[词向量](https://github.com/PaddlePaddle/book/blob/develop/04.word2vec/README.cn.md) 这篇教程。我们训练语言模型的语料共有995,000,000个token，词典大小控制为4900,000词。CoNLL 2005训练语料中有5%的词不在这4900,000个词中，我们将它们全部看作未登录词，用`<unk>`表示。
 
 获取词典，打印词典大小：
 
@@ -191,6 +191,7 @@ import math
 import numpy as np
 import paddle.v2 as paddle
 import paddle.v2.dataset.conll05 as conll05
+import paddle.v2.evaluator as evaluator
 
 paddle.init(use_gpu=False, trainer_count=1)
 
@@ -273,12 +274,12 @@ emb_layers.append(mark_embedding)
 
 ```python  
 hidden_0 = paddle.layer.mixed(
-size=hidden_dim,
-bias_attr=std_default,
-input=[
+    size=hidden_dim,
+    bias_attr=std_default,
+    input=[
         paddle.layer.full_matrix_projection(
             input=emb, param_attr=std_default) for emb in emb_layers
-])
+    ])
 
 mix_hidden_lr = 1e-3
 lstm_para_attr = paddle.attr.Param(initial_std=0.0, learning_rate=1.0)
@@ -319,23 +320,24 @@ for i in range(1, depth):
     input_tmp = [mix_hidden, lstm]
 ```
 
-- 取最后一个栈式LSTM的输出和这个LSTM单元的输入到隐层映射，经过一个全连接层映射到标记字典的维度，得到最终的特征向量表示。
+- 在PaddlePaddle中，CRF的状态特征和转移特征分别由一个全连接层和一个PaddlePaddle中的CRF层分别学习。在这个例子中，我们用线性激活的paddle.layer.mixed 来学习CRF的状态特征（也可以使用paddle.layer.fc），而 paddle.layer.crf只学习转移特征。paddle.layer.crf层是一个 cost 层，处于整个网络的末端，输出给定输入序列下，标记序列的log probability作为代价。训练阶段，该层需要输入正确的标记序列作为学习目标。
 
 ```python
+
+# 取最后一个栈式LSTM的输出和这个LSTM单元的输入到隐层映射，
+# 经过一个全连接层映射到标记字典的维度，来学习 CRF 的状态特征
+
 feature_out = paddle.layer.mixed(
-size=label_dict_len,
-bias_attr=std_default,
-input=[
+    size=label_dict_len,
+    bias_attr=std_default,
+    input=[
         paddle.layer.full_matrix_projection(
             input=input_tmp[0], param_attr=hidden_para_attr),
         paddle.layer.full_matrix_projection(
             input=input_tmp[1], param_attr=lstm_para_attr)
-], )
-```
-
-- 网络的末端定义CRF层计算损失(cost)，指定参数名字为 `crfw`，该层需要输入正确的数据标签(target)。
+    ], )
 
-```python
+# 学习 CRF 的转移特征
 crf_cost = paddle.layer.crf(
     size=label_dict_len,
     input=feature_out,
@@ -346,15 +348,15 @@ crf_cost = paddle.layer.crf(
         learning_rate=mix_hidden_lr))
 ```
 
-- CRF译码层和CRF层参数名字相同，即共享权重。如果输入了正确的数据标签(target)，会统计错误标签的个数，可以用来评估模型。如果没有输入正确的数据标签，该层可以推到出最优解，可以用来预测模型。
+- CRF解码和CRF层参数名字相同，即：加载了`paddle.layer.crf`层学习到的参数。在训练阶段，为`paddle.layer.crf_decoding` 输入了正确的标记序列(target)，这一层会输出是否正确标记，`evaluator.sum` 用来计算序列上的标记错误率，可以用来评估模型。解码阶段，没有输入正确的数据标签，该层通过寻找概率最高的标记序列，解码出标记结果。
 
 ```python
 crf_dec = paddle.layer.crf_decoding(
-   name='crf_dec_l',
    size=label_dict_len,
    input=feature_out,
    label=target,
    param_attr=paddle.attr.Param(name='crfw'))
+evaluator.sum(input=crf_dec)
 ```
 
 ## 训练模型
@@ -365,7 +367,7 @@ crf_dec = paddle.layer.crf_decoding(
 
 ```python
 # create parameters
-parameters = paddle.parameters.create([crf_cost, crf_dec])
+parameters = paddle.parameters.create(crf_cost)
 ```
 
 可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
@@ -393,14 +395,15 @@ parameters.set('emb', load_parameter(conll05.get_embedding(), 44068, 32))
 # create optimizer
 optimizer = paddle.optimizer.Momentum(
     momentum=0,
-    learning_rate=2e-2,
+    learning_rate=1e-3,
     regularization=paddle.optimizer.L2Regularization(rate=8e-4),
     model_average=paddle.optimizer.ModelAverage(
         average_window=0.5, max_average_window=10000), )
 
 trainer = paddle.trainer.SGD(cost=crf_cost,
                              parameters=parameters,
-                             update_equation=optimizer)
+                             update_equation=optimizer,
+                             extra_layers=crf_dec)
 ```
 
 ### 训练
@@ -410,7 +413,7 @@ trainer = paddle.trainer.SGD(cost=crf_cost,
 ```python
 reader = paddle.batch(
     paddle.reader.shuffle(
-        conll05.test(), buf_size=8192), batch_size=20)
+        conll05.test(), buf_size=8192), batch_size=2)
 ```
 
 通过`feeding`来指定每一个数据和data_layer的对应关系。 例如 下面`feeding`表示: `conll05.test()`产生数据的第0列对应`word_data`层的特征。
@@ -435,9 +438,20 @@ feeding = {
 ```python
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f" % (
-                event.pass_id, event.batch_id, event.cost)
+        if event.batch_id and event.batch_id % 10 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+        if event.batch_id % 400 == 0:
+            result = trainer.test(reader=reader, feeding=feeding)
+            print "\nTest with Pass %d, Batch %d, %s" % (event.pass_id, event.batch_id, result.metrics)
+
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=reader, feeding=feeding)
+        print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
 通过`trainer.train`函数训练：
@@ -446,10 +460,45 @@ def event_handler(event):
 trainer.train(
     reader=reader,
     event_handler=event_handler,
-    num_passes=10000,
+    num_passes=1,
     feeding=feeding)
 ```
 
+### 应用模型
+
+训练完成之后，需要依据某个我们关心的性能指标选择最优的模型进行预测，可以简单的选择测试集上标记错误最少的那个模型。预测时使用 `paddle.layer.crf_decoding`，和训练不同的是，该层没有正确的标签层作为输入。如下所示：
+
+```python
+predict = paddle.layer.crf_decoding(
+    size=label_dict_len,
+    input=feature_out,
+    param_attr=paddle.attr.Param(name='crfw'))
+```
+
+这里选用测试集的一条数据作为示例。
+
+```python
+test_creator = paddle.dataset.conll05.test()
+test_data = []
+for item in test_creator():
+    test_data.append(item[0:8])
+    if len(test_data) == 1:
+        break
+```
+
+推断接口`paddle.infer`返回标签的索引，并查询词典`labels_reverse`，打印出标记的结果。
+
+```python
+labs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data, field='id')
+assert len(labs) == len(test_data[0][0])
+labels_reverse={}
+for (k,v) in label_dict.items():
+    labels_reverse[v]=k
+pre_lab = [labels_reverse[i] for i in labs]
+print pre_lab
+```
+
 ## 总结
 
 语义角色标注是许多自然语言理解任务的重要中间步骤。这篇教程中我们以语义角色标注任务为例，介绍如何利用PaddlePaddle进行序列标注任务。教程中所介绍的模型来自我们发表的论文\[[10](#参考文献)\]。由于 CoNLL 2005 SRL任务的训练数据目前并非完全开放，教程中只使用测试数据作为示例。在这个过程中，我们希望减少对其它自然语言处理工具的依赖，利用神经网络数据驱动、端到端学习的能力，得到一个和传统方法可比、甚至更好的模型。在论文中我们证实了这种可能性。关于模型更多的信息和讨论可以在论文中找到。
@@ -467,4 +516,4 @@ trainer.train(
 10. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

06.label_semantic_roles/README.en.md → 07.label_semantic_roles/README.md

 # Semantic Role Labeling
 
-The source code of this chapter is live on [book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/06.label_semantic_roles).
+The source code of this chapter is live on [book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/07.label_semantic_roles).
 
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 ## Background
 
@@ -87,7 +87,7 @@ To address, we can design a bidirectional recurrent neural network by making a m
 Fig 4. Bidirectional LSTMs
 </p>
 
-Note that, this bidirectional RNNs is different with the one proposed by Bengio et al. in machine translation tasks \[[3](#Reference), [4](#Reference)\]. We will introduce another bidirectional RNNs in the following tasks [machine translation](https://github.com/PaddlePaddle/book/blob/develop/machine_translation/README.en.md)
+Note that, this bidirectional RNNs is different with the one proposed by Bengio et al. in machine translation tasks \[[3](#Reference), [4](#Reference)\]. We will introduce another bidirectional RNNs in the following tasks [machine translation](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md)
 
 ### Conditional Random Field (CRF)
 
@@ -118,7 +118,7 @@ where $\omega$ are the weights to the feature function that the CRF learns. Whil
 $$\DeclareMathOperator*{\argmax}{arg\,max} L(\lambda, D) = - \text{log}\left(\prod_{m=1}^{N}p(Y_m|X_m, W)\right) + C \frac{1}{2}\lVert W\rVert^{2}$$
 
 
-This objective function can be solved via back-propagation in an end-to-end manner. While decoding, given input sequences $X$, search for sequence $\bar{Y}$ to maximize the conditional probability $\bar{P}(Y|X)$ via decoding methods (such as *Viterbi*, or [Beam Search Algorithm](https://github.com/PaddlePaddle/book/blob/develop/07.machine_translation/README.en.md#Beam%20Search%20Algorithm)).
+This objective function can be solved via back-propagation in an end-to-end manner. While decoding, given input sequences $X$, search for sequence $\bar{Y}$ to maximize the conditional probability $\bar{P}(Y|X)$ via decoding methods (such as *Viterbi*, or [Beam Search Algorithm](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md#beam-search-algorithm)).
 
 ### Deep Bidirectional LSTM (DB-LSTM) SRL model
 
@@ -213,6 +213,7 @@ import math
 import numpy as np
 import paddle.v2 as paddle
 import paddle.v2.dataset.conll05 as conll05
+import paddle.v2.evaluator as evaluator
 
 paddle.init(use_gpu=False, trainer_count=1)
 
@@ -342,23 +343,23 @@ for i in range(1, depth):
     input_tmp = [mix_hidden, lstm]
 ```
 
-- We will concatenate the output of the top LSTM unit with its input, and project the result into a hidden layer. Then, we put a fully connected layer on top to get the final feature vector representation.
+- In PaddlePaddle, state features and transition features of a CRF are implemented by a fully connected layer and a CRF layer seperately. The fully connected layer with linear activation learns the state features, here we use paddle.layer.mixed (paddle.layer.fc can be uesed as well), and the CRF layer in PaddlePaddle: paddle.layer.crf only learns the transition features, which is a cost layer and is the last layer of the network. paddle.layer.crf outputs the log probability of true tag sequence as the cost by given the input sequence and it requires the true tag sequence as target in the learning process.
 
- ```python
- feature_out = paddle.layer.mixed(
+```python
+
+# The output of the top LSTM unit and its input are feed into a fully connected layer,
+# size of which equals to size of tag labels.
+# The fully connected layer learns the state features
+
+feature_out = paddle.layer.mixed(
     size=label_dict_len,
     bias_attr=std_default,
     input=[
         paddle.layer.full_matrix_projection(
             input=input_tmp[0], param_attr=hidden_para_attr),
         paddle.layer.full_matrix_projection(
-         input=input_tmp[1], param_attr=lstm_para_attr)
- ], )
- ```
-
-- At the end of the network, we use CRF as the cost function; the parameter of CRF cost will be named `crfw`.
+            input=input_tmp[1], param_attr=lstm_para_attr)], )
 
-```python
 crf_cost = paddle.layer.crf(
     size=label_dict_len,
     input=feature_out,
@@ -369,15 +370,15 @@ crf_cost = paddle.layer.crf(
         learning_rate=mix_hidden_lr))
 ```
 
-- The CRF decoding layer is used for evaluation and inference. It shares weights with CRF layer.  The sharing of parameters among multiple layers is specified by using the same parameter name in these layers.
+- The CRF decoding layer is used for evaluation and inference. It shares weights with CRF layer.  The sharing of parameters among multiple layers is specified by using the same parameter name in these layers. If true tag sequence is provided in training process, `paddle.layer.crf_decoding` calculates labelling error for each input token and `evaluator.sum` sum the error over the entire sequence. Otherwise, `paddle.layer.crf_decoding`  generates the labelling tags.
 
 ```python
 crf_dec = paddle.layer.crf_decoding(
-   name='crf_dec_l',
    size=label_dict_len,
    input=feature_out,
    label=target,
    param_attr=paddle.attr.Param(name='crfw'))
+evaluator.sum(input=crf_dec)
 ```
 
 ## Train model
@@ -387,7 +388,7 @@ crf_dec = paddle.layer.crf_decoding(
 All necessary parameters will be traced created given output layers that we need to use.
 
 ```python
-parameters = paddle.parameters.create([crf_cost, crf_dec])
+parameters = paddle.parameters.create(crf_cost)
 ```
 
 We can print out parameter name. It will be generated if not specified.
@@ -413,14 +414,15 @@ We will create trainer given model topology, parameters, and optimization method
 ```python
 optimizer = paddle.optimizer.Momentum(
     momentum=0,
-    learning_rate=2e-2,
+    learning_rate=1e-3,
     regularization=paddle.optimizer.L2Regularization(rate=8e-4),
     model_average=paddle.optimizer.ModelAverage(
         average_window=0.5, max_average_window=10000), )
 
 trainer = paddle.trainer.SGD(cost=crf_cost,
                              parameters=parameters,
-                             update_equation=optimizer)
+                             update_equation=optimizer,
+                             extra_layers=crf_dec)
 ```
 
 ### Trainer
@@ -430,7 +432,7 @@ As mentioned in data preparation section, we will use CoNLL 2005 test corpus as
 ```python
 reader = paddle.batch(
     paddle.reader.shuffle(
-        conll05.test(), buf_size=8192), batch_size=20)
+        conll05.test(), buf_size=8192), batch_size=2)
 ```
 
 `feeding` is used to specify the correspondence between data instance and data layer. For example, according to following `feeding`, the 0th column of data instance produced by`conll05.test()` is matched to the data layer named `word_data`.
@@ -454,9 +456,20 @@ feeding = {
 ```python
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f" % (
-                event.pass_id, event.batch_id, event.cost)
+        if event.batch_id and event.batch_id % 10 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+        if event.batch_id % 400 == 0:
+            result = trainer.test(reader=reader, feeding=feeding)
+            print "\nTest with Pass %d, Batch %d, %s" % (event.pass_id, event.batch_id, result.metrics)
+
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=reader, feeding=feeding)
+        print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
 `trainer.train` will train the model.
@@ -469,6 +482,42 @@ trainer.train(
     feeding=feeding)
 ```
 
+### Application
+
+Aftern training is done, we need to select an optimal model based one performance index to do inference. In this task, one can simply select the model with the least number of marks on the test set. The `paddle.layer.crf_decoding` layer is used in the inference, but its inputs does not include the ground truth label.
+
+```python
+predict = paddle.layer.crf_decoding(
+    size=label_dict_len,
+    input=feature_out,
+    param_attr=paddle.attr.Param(name='crfw'))
+```
+
+Here, using one testing sample as an example.
+
+```python
+test_creator = paddle.dataset.conll05.test()
+test_data = []
+for item in test_creator():
+    test_data.append(item[0:8])
+    if len(test_data) == 1:
+        break
+```
+
+The inference interface `paddle.infer` returns the index of predicting labels. Then printing the tagging results based dictionary `labels_reverse`.
+
+
+```python
+labs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data, field='id')
+assert len(labs) == len(test_data[0][0])
+labels_reverse={}
+for (k,v) in label_dict.items():
+    labels_reverse[v]=k
+pre_lab = [labels_reverse[i] for i in labs]
+print pre_lab
+```
+
 ## Conclusion
 
 Semantic Role Labeling is an important intermediate step in a wide range of natural language processing tasks. In this tutorial, we use SRL as an example to illustrate using PaddlePaddle to do sequence tagging tasks. The models proposed are from our published paper\[[10](#Reference)\]. We only use test data for illustration since the training data on the CoNLL 2005 dataset is not completely public. This aims to propose an end-to-end neural network model with fewer dependencies on natural language processing tools but is comparable, or even better than traditional models in terms of performance. Please check out our paper for more information and discussions.
@@ -486,4 +535,4 @@ Semantic Role Labeling is an important intermediate step in a wide range of natu
 10. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

06.label_semantic_roles/index.html → 07.label_semantic_roles/index.cn.html

@@ -42,7 +42,7 @@
 <div id="markdown" style='display:none'>
 # 语义角色标注
 
-本教程源代码目录在[book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/06.label_semantic_roles)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/07.label_semantic_roles)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/178.html)。
 
 ## 背景介绍
 
@@ -110,7 +110,7 @@ $$\mbox{[小明]}_{\mbox{Agent}}\mbox{[昨天]}_{\mbox{Time}}\mbox{[晚上]}_\mb
 图4. 基于LSTM的双向循环神经网络结构示意图
 </p>
 
-需要说明的是，这种双向RNN结构和Bengio等人在机器翻译任务中使用的双向RNN结构\[[3](#参考文献), [4](#参考文献)\] 并不相同，我们会在后续[机器翻译](https://github.com/PaddlePaddle/book/blob/develop/machine_translation/README.md)任务中，介绍另一种双向循环神经网络。
+需要说明的是，这种双向RNN结构和Bengio等人在机器翻译任务中使用的双向RNN结构\[[3](#参考文献), [4](#参考文献)\] 并不相同，我们会在后续[机器翻译](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.cn.md)任务中，介绍另一种双向循环神经网络。
 
 ### 条件随机场 (Conditional Random Field)
 
@@ -224,7 +224,7 @@ conll05st-release/
 | predicate_dict | 谓词的词典，共计3162个词 |
 | emb | 一个训练好的词表，32维 |
 
-我们在英文维基百科上训练语言模型得到了一份词向量用来初始化SRL模型。在SRL模型训练过程中，词向量不再被更新。关于语言模型和词向量可以参考[词向量](https://github.com/PaddlePaddle/book/blob/develop/04.word2vec/README.md) 这篇教程。我们训练语言模型的语料共有995,000,000个token，词典大小控制为4900,000词。CoNLL 2005训练语料中有5%的词不在这4900,000个词中，我们将它们全部看作未登录词，用`<unk>`表示。
+我们在英文维基百科上训练语言模型得到了一份词向量用来初始化SRL模型。在SRL模型训练过程中，词向量不再被更新。关于语言模型和词向量可以参考[词向量](https://github.com/PaddlePaddle/book/blob/develop/04.word2vec/README.cn.md) 这篇教程。我们训练语言模型的语料共有995,000,000个token，词典大小控制为4900,000词。CoNLL 2005训练语料中有5%的词不在这4900,000个词中，我们将它们全部看作未登录词，用`<unk>`表示。
 
 获取词典，打印词典大小：
 
@@ -233,6 +233,7 @@ import math
 import numpy as np
 import paddle.v2 as paddle
 import paddle.v2.dataset.conll05 as conll05
+import paddle.v2.evaluator as evaluator
 
 paddle.init(use_gpu=False, trainer_count=1)
 
@@ -315,12 +316,12 @@ emb_layers.append(mark_embedding)
 
 ```python  
 hidden_0 = paddle.layer.mixed(
-size=hidden_dim,
-bias_attr=std_default,
-input=[
+    size=hidden_dim,
+    bias_attr=std_default,
+    input=[
         paddle.layer.full_matrix_projection(
             input=emb, param_attr=std_default) for emb in emb_layers
-])
+    ])
 
 mix_hidden_lr = 1e-3
 lstm_para_attr = paddle.attr.Param(initial_std=0.0, learning_rate=1.0)
@@ -361,23 +362,24 @@ for i in range(1, depth):
     input_tmp = [mix_hidden, lstm]
 ```
 
-- 取最后一个栈式LSTM的输出和这个LSTM单元的输入到隐层映射，经过一个全连接层映射到标记字典的维度，得到最终的特征向量表示。
+- 在PaddlePaddle中，CRF的状态特征和转移特征分别由一个全连接层和一个PaddlePaddle中的CRF层分别学习。在这个例子中，我们用线性激活的paddle.layer.mixed 来学习CRF的状态特征（也可以使用paddle.layer.fc），而 paddle.layer.crf只学习转移特征。paddle.layer.crf层是一个 cost 层，处于整个网络的末端，输出给定输入序列下，标记序列的log probability作为代价。训练阶段，该层需要输入正确的标记序列作为学习目标。
 
 ```python
+
+# 取最后一个栈式LSTM的输出和这个LSTM单元的输入到隐层映射，
+# 经过一个全连接层映射到标记字典的维度，来学习 CRF 的状态特征
+
 feature_out = paddle.layer.mixed(
-size=label_dict_len,
-bias_attr=std_default,
-input=[
+    size=label_dict_len,
+    bias_attr=std_default,
+    input=[
         paddle.layer.full_matrix_projection(
             input=input_tmp[0], param_attr=hidden_para_attr),
         paddle.layer.full_matrix_projection(
             input=input_tmp[1], param_attr=lstm_para_attr)
-], )
-```
-
-- 网络的末端定义CRF层计算损失(cost)，指定参数名字为 `crfw`，该层需要输入正确的数据标签(target)。
+    ], )
 
-```python
+# 学习 CRF 的转移特征
 crf_cost = paddle.layer.crf(
     size=label_dict_len,
     input=feature_out,
@@ -388,15 +390,15 @@ crf_cost = paddle.layer.crf(
         learning_rate=mix_hidden_lr))
 ```
 
-- CRF译码层和CRF层参数名字相同，即共享权重。如果输入了正确的数据标签(target)，会统计错误标签的个数，可以用来评估模型。如果没有输入正确的数据标签，该层可以推到出最优解，可以用来预测模型。
+- CRF解码和CRF层参数名字相同，即：加载了`paddle.layer.crf`层学习到的参数。在训练阶段，为`paddle.layer.crf_decoding` 输入了正确的标记序列(target)，这一层会输出是否正确标记，`evaluator.sum` 用来计算序列上的标记错误率，可以用来评估模型。解码阶段，没有输入正确的数据标签，该层通过寻找概率最高的标记序列，解码出标记结果。
 
 ```python
 crf_dec = paddle.layer.crf_decoding(
-   name='crf_dec_l',
    size=label_dict_len,
    input=feature_out,
    label=target,
    param_attr=paddle.attr.Param(name='crfw'))
+evaluator.sum(input=crf_dec)
 ```
 
 ## 训练模型
@@ -407,7 +409,7 @@ crf_dec = paddle.layer.crf_decoding(
 
 ```python
 # create parameters
-parameters = paddle.parameters.create([crf_cost, crf_dec])
+parameters = paddle.parameters.create(crf_cost)
 ```
 
 可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
@@ -435,14 +437,15 @@ parameters.set('emb', load_parameter(conll05.get_embedding(), 44068, 32))
 # create optimizer
 optimizer = paddle.optimizer.Momentum(
     momentum=0,
-    learning_rate=2e-2,
+    learning_rate=1e-3,
     regularization=paddle.optimizer.L2Regularization(rate=8e-4),
     model_average=paddle.optimizer.ModelAverage(
         average_window=0.5, max_average_window=10000), )
 
 trainer = paddle.trainer.SGD(cost=crf_cost,
                              parameters=parameters,
-                             update_equation=optimizer)
+                             update_equation=optimizer,
+                             extra_layers=crf_dec)
 ```
 
 ### 训练
@@ -452,7 +455,7 @@ trainer = paddle.trainer.SGD(cost=crf_cost,
 ```python
 reader = paddle.batch(
     paddle.reader.shuffle(
-        conll05.test(), buf_size=8192), batch_size=20)
+        conll05.test(), buf_size=8192), batch_size=2)
 ```
 
 通过`feeding`来指定每一个数据和data_layer的对应关系。 例如 下面`feeding`表示: `conll05.test()`产生数据的第0列对应`word_data`层的特征。
@@ -477,9 +480,20 @@ feeding = {
 ```python
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f" % (
-                event.pass_id, event.batch_id, event.cost)
+        if event.batch_id and event.batch_id % 10 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+        if event.batch_id % 400 == 0:
+            result = trainer.test(reader=reader, feeding=feeding)
+            print "\nTest with Pass %d, Batch %d, %s" % (event.pass_id, event.batch_id, result.metrics)
+
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=reader, feeding=feeding)
+        print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
 通过`trainer.train`函数训练：
@@ -488,10 +502,45 @@ def event_handler(event):
 trainer.train(
     reader=reader,
     event_handler=event_handler,
-    num_passes=10000,
+    num_passes=1,
     feeding=feeding)
 ```
 
+### 应用模型
+
+训练完成之后，需要依据某个我们关心的性能指标选择最优的模型进行预测，可以简单的选择测试集上标记错误最少的那个模型。预测时使用 `paddle.layer.crf_decoding`，和训练不同的是，该层没有正确的标签层作为输入。如下所示：
+
+```python
+predict = paddle.layer.crf_decoding(
+    size=label_dict_len,
+    input=feature_out,
+    param_attr=paddle.attr.Param(name='crfw'))
+```
+
+这里选用测试集的一条数据作为示例。
+
+```python
+test_creator = paddle.dataset.conll05.test()
+test_data = []
+for item in test_creator():
+    test_data.append(item[0:8])
+    if len(test_data) == 1:
+        break
+```
+
+推断接口`paddle.infer`返回标签的索引，并查询词典`labels_reverse`，打印出标记的结果。
+
+```python
+labs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data, field='id')
+assert len(labs) == len(test_data[0][0])
+labels_reverse={}
+for (k,v) in label_dict.items():
+    labels_reverse[v]=k
+pre_lab = [labels_reverse[i] for i in labs]
+print pre_lab
+```
+
 ## 总结
 
 语义角色标注是许多自然语言理解任务的重要中间步骤。这篇教程中我们以语义角色标注任务为例，介绍如何利用PaddlePaddle进行序列标注任务。教程中所介绍的模型来自我们发表的论文\[[10](#参考文献)\]。由于 CoNLL 2005 SRL任务的训练数据目前并非完全开放，教程中只使用测试数据作为示例。在这个过程中，我们希望减少对其它自然语言处理工具的依赖，利用神经网络数据驱动、端到端学习的能力，得到一个和传统方法可比、甚至更好的模型。在论文中我们证实了这种可能性。关于模型更多的信息和讨论可以在论文中找到。
@@ -509,7 +558,7 @@ trainer.train(
 10. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

06.label_semantic_roles/index.en.html → 07.label_semantic_roles/index.html

@@ -42,9 +42,9 @@
 <div id="markdown" style='display:none'>
 # Semantic Role Labeling
 
-The source code of this chapter is live on [book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/06.label_semantic_roles).
+The source code of this chapter is live on [book/label_semantic_roles](https://github.com/PaddlePaddle/book/tree/develop/07.label_semantic_roles).
 
-For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst).
+For instructions on getting started with PaddlePaddle, see [PaddlePaddle installation guide](https://github.com/PaddlePaddle/book/blob/develop/README.md#running-the-book).
 
 ## Background
 
@@ -129,7 +129,7 @@ To address, we can design a bidirectional recurrent neural network by making a m
 Fig 4. Bidirectional LSTMs
 </p>
 
-Note that, this bidirectional RNNs is different with the one proposed by Bengio et al. in machine translation tasks \[[3](#Reference), [4](#Reference)\]. We will introduce another bidirectional RNNs in the following tasks [machine translation](https://github.com/PaddlePaddle/book/blob/develop/machine_translation/README.en.md)
+Note that, this bidirectional RNNs is different with the one proposed by Bengio et al. in machine translation tasks \[[3](#Reference), [4](#Reference)\]. We will introduce another bidirectional RNNs in the following tasks [machine translation](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md)
 
 ### Conditional Random Field (CRF)
 
@@ -160,7 +160,7 @@ where $\omega$ are the weights to the feature function that the CRF learns. Whil
 $$\DeclareMathOperator*{\argmax}{arg\,max} L(\lambda, D) = - \text{log}\left(\prod_{m=1}^{N}p(Y_m|X_m, W)\right) + C \frac{1}{2}\lVert W\rVert^{2}$$
 
 
-This objective function can be solved via back-propagation in an end-to-end manner. While decoding, given input sequences $X$, search for sequence $\bar{Y}$ to maximize the conditional probability $\bar{P}(Y|X)$ via decoding methods (such as *Viterbi*, or [Beam Search Algorithm](https://github.com/PaddlePaddle/book/blob/develop/07.machine_translation/README.en.md#Beam%20Search%20Algorithm)).
+This objective function can be solved via back-propagation in an end-to-end manner. While decoding, given input sequences $X$, search for sequence $\bar{Y}$ to maximize the conditional probability $\bar{P}(Y|X)$ via decoding methods (such as *Viterbi*, or [Beam Search Algorithm](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md#beam-search-algorithm)).
 
 ### Deep Bidirectional LSTM (DB-LSTM) SRL model
 
@@ -255,6 +255,7 @@ import math
 import numpy as np
 import paddle.v2 as paddle
 import paddle.v2.dataset.conll05 as conll05
+import paddle.v2.evaluator as evaluator
 
 paddle.init(use_gpu=False, trainer_count=1)
 
@@ -384,23 +385,23 @@ for i in range(1, depth):
     input_tmp = [mix_hidden, lstm]
 ```
 
-- We will concatenate the output of the top LSTM unit with its input, and project the result into a hidden layer. Then, we put a fully connected layer on top to get the final feature vector representation.
+- In PaddlePaddle, state features and transition features of a CRF are implemented by a fully connected layer and a CRF layer seperately. The fully connected layer with linear activation learns the state features, here we use paddle.layer.mixed (paddle.layer.fc can be uesed as well), and the CRF layer in PaddlePaddle: paddle.layer.crf only learns the transition features, which is a cost layer and is the last layer of the network. paddle.layer.crf outputs the log probability of true tag sequence as the cost by given the input sequence and it requires the true tag sequence as target in the learning process.
 
- ```python
- feature_out = paddle.layer.mixed(
+```python
+
+# The output of the top LSTM unit and its input are feed into a fully connected layer,
+# size of which equals to size of tag labels.
+# The fully connected layer learns the state features
+
+feature_out = paddle.layer.mixed(
     size=label_dict_len,
     bias_attr=std_default,
     input=[
         paddle.layer.full_matrix_projection(
             input=input_tmp[0], param_attr=hidden_para_attr),
         paddle.layer.full_matrix_projection(
-         input=input_tmp[1], param_attr=lstm_para_attr)
- ], )
- ```
-
-- At the end of the network, we use CRF as the cost function; the parameter of CRF cost will be named `crfw`.
+            input=input_tmp[1], param_attr=lstm_para_attr)], )
 
-```python
 crf_cost = paddle.layer.crf(
     size=label_dict_len,
     input=feature_out,
@@ -411,15 +412,15 @@ crf_cost = paddle.layer.crf(
         learning_rate=mix_hidden_lr))
 ```
 
-- The CRF decoding layer is used for evaluation and inference. It shares weights with CRF layer.  The sharing of parameters among multiple layers is specified by using the same parameter name in these layers.
+- The CRF decoding layer is used for evaluation and inference. It shares weights with CRF layer.  The sharing of parameters among multiple layers is specified by using the same parameter name in these layers. If true tag sequence is provided in training process, `paddle.layer.crf_decoding` calculates labelling error for each input token and `evaluator.sum` sum the error over the entire sequence. Otherwise, `paddle.layer.crf_decoding`  generates the labelling tags.
 
 ```python
 crf_dec = paddle.layer.crf_decoding(
-   name='crf_dec_l',
    size=label_dict_len,
    input=feature_out,
    label=target,
    param_attr=paddle.attr.Param(name='crfw'))
+evaluator.sum(input=crf_dec)
 ```
 
 ## Train model
@@ -429,7 +430,7 @@ crf_dec = paddle.layer.crf_decoding(
 All necessary parameters will be traced created given output layers that we need to use.
 
 ```python
-parameters = paddle.parameters.create([crf_cost, crf_dec])
+parameters = paddle.parameters.create(crf_cost)
 ```
 
 We can print out parameter name. It will be generated if not specified.
@@ -455,14 +456,15 @@ We will create trainer given model topology, parameters, and optimization method
 ```python
 optimizer = paddle.optimizer.Momentum(
     momentum=0,
-    learning_rate=2e-2,
+    learning_rate=1e-3,
     regularization=paddle.optimizer.L2Regularization(rate=8e-4),
     model_average=paddle.optimizer.ModelAverage(
         average_window=0.5, max_average_window=10000), )
 
 trainer = paddle.trainer.SGD(cost=crf_cost,
                              parameters=parameters,
-                             update_equation=optimizer)
+                             update_equation=optimizer,
+                             extra_layers=crf_dec)
 ```
 
 ### Trainer
@@ -472,7 +474,7 @@ As mentioned in data preparation section, we will use CoNLL 2005 test corpus as
 ```python
 reader = paddle.batch(
     paddle.reader.shuffle(
-        conll05.test(), buf_size=8192), batch_size=20)
+        conll05.test(), buf_size=8192), batch_size=2)
 ```
 
 `feeding` is used to specify the correspondence between data instance and data layer. For example, according to following `feeding`, the 0th column of data instance produced by`conll05.test()` is matched to the data layer named `word_data`.
@@ -496,9 +498,20 @@ feeding = {
 ```python
 def event_handler(event):
     if isinstance(event, paddle.event.EndIteration):
-        if event.batch_id % 100 == 0:
-            print "Pass %d, Batch %d, Cost %f" % (
-                event.pass_id, event.batch_id, event.cost)
+        if event.batch_id and event.batch_id % 10 == 0:
+            print "Pass %d, Batch %d, Cost %f, %s" % (
+                event.pass_id, event.batch_id, event.cost, event.metrics)
+        if event.batch_id % 400 == 0:
+            result = trainer.test(reader=reader, feeding=feeding)
+            print "\nTest with Pass %d, Batch %d, %s" % (event.pass_id, event.batch_id, result.metrics)
+
+    if isinstance(event, paddle.event.EndPass):
+        # save parameters
+        with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+            parameters.to_tar(f)
+
+        result = trainer.test(reader=reader, feeding=feeding)
+        print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 ```
 
 `trainer.train` will train the model.
@@ -511,6 +524,42 @@ trainer.train(
     feeding=feeding)
 ```
 
+### Application
+
+Aftern training is done, we need to select an optimal model based one performance index to do inference. In this task, one can simply select the model with the least number of marks on the test set. The `paddle.layer.crf_decoding` layer is used in the inference, but its inputs does not include the ground truth label.
+
+```python
+predict = paddle.layer.crf_decoding(
+    size=label_dict_len,
+    input=feature_out,
+    param_attr=paddle.attr.Param(name='crfw'))
+```
+
+Here, using one testing sample as an example.
+
+```python
+test_creator = paddle.dataset.conll05.test()
+test_data = []
+for item in test_creator():
+    test_data.append(item[0:8])
+    if len(test_data) == 1:
+        break
+```
+
+The inference interface `paddle.infer` returns the index of predicting labels. Then printing the tagging results based dictionary `labels_reverse`.
+
+
+```python
+labs = paddle.infer(
+    output_layer=predict, parameters=parameters, input=test_data, field='id')
+assert len(labs) == len(test_data[0][0])
+labels_reverse={}
+for (k,v) in label_dict.items():
+    labels_reverse[v]=k
+pre_lab = [labels_reverse[i] for i in labs]
+print pre_lab
+```
+
 ## Conclusion
 
 Semantic Role Labeling is an important intermediate step in a wide range of natural language processing tasks. In this tutorial, we use SRL as an example to illustrate using PaddlePaddle to do sequence tagging tasks. The models proposed are from our published paper\[[10](#Reference)\]. We only use test data for illustration since the training data on the CoNLL 2005 dataset is not completely public. This aims to propose an end-to-end neural network model with fewer dependencies on natural language processing tools but is comparable, or even better than traditional models in terms of performance. Please check out our paper for more information and discussions.
@@ -528,7 +577,7 @@ Semantic Role Labeling is an important intermediate step in a wide range of natu
 10. Zhou J, Xu W. [End-to-end learning of semantic role labeling using recurrent neural networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf)[C]//Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

06.label_semantic_roles/train.py → 07.label_semantic_roles/train.py

@@ -2,24 +2,28 @@ import math
 import numpy as np
 import paddle.v2 as paddle
 import paddle.v2.dataset.conll05 as conll05
+import paddle.v2.evaluator as evaluator
 
+word_dict, verb_dict, label_dict = conll05.get_dict()
+word_dict_len = len(word_dict)
+label_dict_len = len(label_dict)
+pred_len = len(verb_dict)
 
-def db_lstm():
-    word_dict, verb_dict, label_dict = conll05.get_dict()
-    word_dict_len = len(word_dict)
-    label_dict_len = len(label_dict)
-    pred_len = len(verb_dict)
+mark_dict_len = 2
+word_dim = 32
+mark_dim = 5
+hidden_dim = 512
+depth = 8
+default_std = 1 / math.sqrt(hidden_dim) / 3.0
+mix_hidden_lr = 1e-3
 
-    mark_dict_len = 2
-    word_dim = 32
-    mark_dim = 5
-    hidden_dim = 512
-    depth = 8
 
-    #8 features
-    def d_type(size):
+def d_type(size):
     return paddle.data_type.integer_value_sequence(size)
 
+
+def db_lstm():
+    #8 features
     word = paddle.layer.data(name='word_data', type=d_type(word_dict_len))
     predicate = paddle.layer.data(name='verb_data', type=d_type(pred_len))
 
@@ -30,11 +34,8 @@ def db_lstm():
     ctx_p2 = paddle.layer.data(name='ctx_p2_data', type=d_type(word_dict_len))
     mark = paddle.layer.data(name='mark_data', type=d_type(mark_dict_len))
 
-    target = paddle.layer.data(name='target', type=d_type(label_dict_len))
-
     emb_para = paddle.attr.Param(name='emb', initial_std=0., is_static=True)
     std_0 = paddle.attr.Param(initial_std=0.)
-    default_std = 1 / math.sqrt(hidden_dim) / 3.0
     std_default = paddle.attr.Param(initial_std=default_std)
 
     predicate_embedding = paddle.layer.embedding(
@@ -60,7 +61,6 @@ def db_lstm():
                 input=emb, param_attr=std_default) for emb in emb_layers
         ])
 
-    mix_hidden_lr = 1e-3
     lstm_para_attr = paddle.attr.Param(initial_std=0.0, learning_rate=1.0)
     hidden_para_attr = paddle.attr.Param(
         initial_std=default_std, learning_rate=mix_hidden_lr)
@@ -108,21 +108,7 @@ def db_lstm():
                 input=input_tmp[1], param_attr=lstm_para_attr)
         ], )
 
-    crf_cost = paddle.layer.crf(
-        size=label_dict_len,
-        input=feature_out,
-        label=target,
-        param_attr=paddle.attr.Param(
-            name='crfw', initial_std=default_std, learning_rate=mix_hidden_lr))
-
-    crf_dec = paddle.layer.crf_decoding(
-        name='crf_dec_l',
-        size=label_dict_len,
-        input=feature_out,
-        label=target,
-        param_attr=paddle.attr.Param(name='crfw'))
-
-    return crf_cost, crf_dec
+    return feature_out
 
 
 def load_parameter(file_name, h, w):
@@ -135,10 +121,24 @@ def main():
     paddle.init(use_gpu=False, trainer_count=1)
 
     # define network topology
-    crf_cost, crf_dec = db_lstm()
+    feature_out = db_lstm()
+    target = paddle.layer.data(name='target', type=d_type(label_dict_len))
+    crf_cost = paddle.layer.crf(
+        size=label_dict_len,
+        input=feature_out,
+        label=target,
+        param_attr=paddle.attr.Param(
+            name='crfw', initial_std=default_std, learning_rate=mix_hidden_lr))
+
+    crf_dec = paddle.layer.crf_decoding(
+        size=label_dict_len,
+        input=feature_out,
+        label=target,
+        param_attr=paddle.attr.Param(name='crfw'))
+    evaluator.sum(input=crf_dec)
 
     # create parameters
-    parameters = paddle.parameters.create([crf_cost, crf_dec])
+    parameters = paddle.parameters.create(crf_cost)
     parameters.set('emb', load_parameter(conll05.get_embedding(), 44068, 32))
 
     # create optimizer
@@ -150,7 +150,10 @@ def main():
             average_window=0.5, max_average_window=10000), )
 
     trainer = paddle.trainer.SGD(
-        cost=crf_cost, parameters=parameters, update_equation=optimizer)
+        cost=crf_cost,
+        parameters=parameters,
+        update_equation=optimizer,
+        extra_layers=crf_dec)
 
     reader = paddle.batch(
         paddle.reader.shuffle(conll05.test(), buf_size=8192), batch_size=10)
@@ -170,15 +173,50 @@ def main():
     def event_handler(event):
         if isinstance(event, paddle.event.EndIteration):
             if event.batch_id % 100 == 0:
-                print "Pass %d, Batch %d, Cost %f" % (
-                    event.pass_id, event.batch_id, event.cost)
+                print "Pass %d, Batch %d, Cost %f, %s" % (
+                    event.pass_id, event.batch_id, event.cost, event.metrics)
+            if event.batch_id % 1000 == 0:
+                result = trainer.test(reader=reader, feeding=feeding)
+                print "\nTest with Pass %d, Batch %d, %s" % (
+                    event.pass_id, event.batch_id, result.metrics)
+
+        if isinstance(event, paddle.event.EndPass):
+            # save parameters
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                parameters.to_tar(f)
+
+            result = trainer.test(reader=reader, feeding=feeding)
+            print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
 
     trainer.train(
         reader=reader,
         event_handler=event_handler,
-        num_passes=10000,
+        num_passes=1,
         feeding=feeding)
 
+    test_creator = paddle.dataset.conll05.test()
+    test_data = []
+    for item in test_creator():
+        test_data.append(item[0:8])
+        if len(test_data) == 1:
+            break
+
+    predict = paddle.layer.crf_decoding(
+        size=label_dict_len,
+        input=feature_out,
+        param_attr=paddle.attr.Param(name='crfw'))
+    probs = paddle.infer(
+        output_layer=predict,
+        parameters=parameters,
+        input=test_data,
+        field='id')
+    assert len(probs) == len(test_data[0][0])
+    labels_reverse = {}
+    for (k, v) in label_dict.items():
+        labels_reverse[v] = k
+    pre_lab = [labels_reverse[i] for i in probs]
+    print pre_lab
+
 
 if __name__ == '__main__':
     main()

07.machine_translation/data/wmt14_data.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-set -x
-mkdir wmt14
-cd wmt14
-
-# download the dataset
-wget http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/bitexts.tgz
-wget http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz
-
-# untar the dataset
-tar -zxvf bitexts.tgz
-tar -zxvf dev+test.tgz
-gunzip bitexts.selected/*
-mv bitexts.selected train
-rm bitexts.tgz
-rm dev+test.tgz
-
-# separate the dev and test dataset
-mkdir test gen
-mv dev/ntst1213.* test
-mv dev/ntst14.* gen
-rm -rf dev
-
-set +x
-# rename the suffix, .fr->.src, .en->.trg
-for dir in train test gen
-do
-  filelist=`ls $dir`
-  cd $dir
-  for file in $filelist
-  do
-    if [ ${file##*.} = "fr" ]; then
-      mv $file ${file/%fr/src}
-    elif [ ${file##*.} = 'en' ]; then
-      mv $file ${file/%en/trg}
-    fi
-  done
-  cd ..
-done

07.machine_translation/eval_bleu.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-gen_file=$1
-beam_size=$2
-
-# find top1 generating result
-top1=$(printf '%s_top1.txt' `basename $gen_file .txt`)
-if [ $beam_size -eq 1 ]; then
-    awk -F "\t" '{sub(" <e>","",$2);sub(" ","",$2);print $2}' $gen_file >$top1
-else
-    awk 'BEGIN{
-        FS="\t";
-        OFS="\t";
-        read_pos = 2} {
-        if (NR == read_pos){
-            sub(" <e>","",$3);
-            sub(" ","",$3);
-            print $3;
-            read_pos += (2 + res_num);
-      }}' res_num=$beam_size $gen_file >$top1
-fi
-
-# evalute bleu value
-bleu_script=multi-bleu.perl
-standard_res=data/wmt14/gen/ntst14.trg
-bleu_res=`perl $bleu_script $standard_res <$top1`
-
-echo $bleu_res | cut -d, -f 1
-rm $top1

07.machine_translation/moses_bleu.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-set -x
-echo "Downloading multi-bleu.perl"
-wget https://raw.githubusercontent.com/moses-smt/mosesdecoder/master/scripts/generic/multi-bleu.perl --no-check-certificate

07.machine_translation/pretrained/wmt14_model.sh

-#!/bin/bash
-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-set -x
-
-# download the pretrained model
-wget http://paddlepaddle.bj.bcebos.com/model_zoo/wmt14_model.tar.gz
-
-# untar the model
-tar -zxvf wmt14_model.tar.gz
-rm wmt14_model.tar.gz

07.machine_translation/train.py

-import sys
-import paddle.v2 as paddle
-
-
-def seqToseq_net(source_dict_dim, target_dict_dim):
-    ### Network Architecture
-    word_vector_dim = 512  # dimension of word vector
-    decoder_size = 512  # dimension of hidden unit in GRU Decoder network
-    encoder_size = 512  # dimension of hidden unit in GRU Encoder network
-
-    #### Encoder
-    src_word_id = paddle.layer.data(
-        name='source_language_word',
-        type=paddle.data_type.integer_value_sequence(source_dict_dim))
-    src_embedding = paddle.layer.embedding(
-        input=src_word_id,
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_source_language_embedding'))
-    src_forward = paddle.networks.simple_gru(
-        input=src_embedding, size=encoder_size)
-    src_backward = paddle.networks.simple_gru(
-        input=src_embedding, size=encoder_size, reverse=True)
-    encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
-
-    #### Decoder
-    with paddle.layer.mixed(size=decoder_size) as encoded_proj:
-        encoded_proj += paddle.layer.full_matrix_projection(
-            input=encoded_vector)
-
-    backward_first = paddle.layer.first_seq(input=src_backward)
-
-    with paddle.layer.mixed(
-            size=decoder_size, act=paddle.activation.Tanh()) as decoder_boot:
-        decoder_boot += paddle.layer.full_matrix_projection(
-            input=backward_first)
-
-    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
-
-        decoder_mem = paddle.layer.memory(
-            name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
-
-        context = paddle.networks.simple_attention(
-            encoded_sequence=enc_vec,
-            encoded_proj=enc_proj,
-            decoder_state=decoder_mem)
-
-        with paddle.layer.mixed(size=decoder_size * 3) as decoder_inputs:
-            decoder_inputs += paddle.layer.full_matrix_projection(input=context)
-            decoder_inputs += paddle.layer.full_matrix_projection(
-                input=current_word)
-
-        gru_step = paddle.layer.gru_step(
-            name='gru_decoder',
-            input=decoder_inputs,
-            output_mem=decoder_mem,
-            size=decoder_size)
-
-        with paddle.layer.mixed(
-                size=target_dict_dim,
-                bias_attr=True,
-                act=paddle.activation.Softmax()) as out:
-            out += paddle.layer.full_matrix_projection(input=gru_step)
-        return out
-
-    decoder_group_name = "decoder_group"
-    group_input1 = paddle.layer.StaticInputV2(input=encoded_vector, is_seq=True)
-    group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
-    group_inputs = [group_input1, group_input2]
-
-    trg_embedding = paddle.layer.embedding(
-        input=paddle.layer.data(
-            name='target_language_word',
-            type=paddle.data_type.integer_value_sequence(target_dict_dim)),
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_target_language_embedding'))
-    group_inputs.append(trg_embedding)
-
-    # For decoder equipped with attention mechanism, in training,
-    # target embeding (the groudtruth) is the data input,
-    # while encoded source sequence is accessed to as an unbounded memory.
-    # Here, the StaticInput defines a read-only memory
-    # for the recurrent_group.
-    decoder = paddle.layer.recurrent_group(
-        name=decoder_group_name,
-        step=gru_decoder_with_attention,
-        input=group_inputs)
-
-    lbl = paddle.layer.data(
-        name='target_language_next_word',
-        type=paddle.data_type.integer_value_sequence(target_dict_dim))
-    cost = paddle.layer.classification_cost(input=decoder, label=lbl)
-
-    return cost
-
-
-def main():
-    paddle.init(use_gpu=False, trainer_count=1)
-
-    # source and target dict dim.
-    dict_size = 30000
-    source_dict_dim = target_dict_dim = dict_size
-
-    # define network topology
-    cost = seqToseq_net(source_dict_dim, target_dict_dim)
-    parameters = paddle.parameters.create(cost)
-
-    # define optimize method and trainer
-    optimizer = paddle.optimizer.Adam(
-        learning_rate=5e-5,
-        regularization=paddle.optimizer.L2Regularization(rate=8e-4))
-    trainer = paddle.trainer.SGD(
-        cost=cost, parameters=parameters, update_equation=optimizer)
-
-    # define data reader
-    feeding = {
-        'source_language_word': 0,
-        'target_language_word': 1,
-        'target_language_next_word': 2
-    }
-
-    wmt14_reader = paddle.batch(
-        paddle.reader.shuffle(
-            paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
-        batch_size=5)
-
-    # define event_handler callback
-    def event_handler(event):
-        if isinstance(event, paddle.event.EndIteration):
-            if event.batch_id % 10 == 0:
-                print "\nPass %d, Batch %d, Cost %f, %s" % (
-                    event.pass_id, event.batch_id, event.cost, event.metrics)
-            else:
-                sys.stdout.write('.')
-                sys.stdout.flush()
-
-    # start to train
-    trainer.train(
-        reader=wmt14_reader,
-        event_handler=event_handler,
-        num_passes=2,
-        feeding=feeding)
-
-
-if __name__ == '__main__':
-    main()

07.machine_translation/README.md → 08.machine_translation/README.cn.md

 # 机器翻译
 
-本教程源代码目录在[book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/07.machine_translation)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/179.html)。
 
 ## 背景介绍
 
@@ -26,9 +26,9 @@
 ```
 如果设定显示翻译结果的条数（即[柱搜索算法](#柱搜索算法)的宽度）为3，生成的英语句子如下：
 ```text
-0 -5.36816   these are signs of hope and relief . <e>
-1 -6.23177   these are the light of hope and relief . <e>
-2 -7.7914  these are the light of hope and the relief of hope . <e>
+0 -5.36816   These are signs of hope and relief . <e>
+1 -6.23177   These are the light of hope and relief . <e>
+2 -7.7914  These are the light of hope and the relief of hope . <e>
 ```
 - 左起第一列是生成句子的序号；左起第二列是该条句子的得分（从大到小），分值越高越好；左起第三列是生成的英语句子。
 - 另外有两个特殊标志：`<e>`表示句子的结尾，`<unk>`表示未登录词（unknown word），即未在训练字典中出现的词。
@@ -39,7 +39,7 @@
 
 ### GRU
 
-我们已经在[情感分析](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)一章中介绍了循环神经网络（RNN）及长短时间记忆网络（LSTM）。相比于简单的RNN，LSTM增加了记忆单元（memory cell）、输入门（input gate）、遗忘门（forget gate）及输出门（output gate），这些门及记忆单元组合起来大大提升了RNN处理远距离依赖问题的能力。
+我们已经在[情感分析](https://github.com/PaddlePaddle/book/blob/develop/06.understand_sentiment/README.cn.md)一章中介绍了循环神经网络（RNN）及长短时间记忆网络（LSTM）。相比于简单的RNN，LSTM增加了记忆单元（memory cell）、输入门（input gate）、遗忘门（forget gate）及输出门（output gate），这些门及记忆单元组合起来大大提升了RNN处理远距离依赖问题的能力。
 
 GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN的一种扩展，如下图所示。GRU单元只有两个门：
 - 重置门（reset gate）：如果重置门关闭，会忽略掉历史信息，即历史不相干的信息不会影响未来的输出。
@@ -53,7 +53,7 @@ GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN
 
 ### 双向循环神经网络
 
-我们已经在[语义角色标注](https://github.com/PaddlePaddle/book/blob/develop/label_semantic_roles/README.md)一章中介绍了一种双向循环神经网络，这里介绍Bengio团队在论文\[[2](#参考文献),[4](#参考文献)\]中提出的另一种结构。该结构的目的是输入一个序列，得到其在每个时刻的特征表示，即输出的每个时刻都用定长向量表示到该时刻的上下文语义信息。
+我们已经在[语义角色标注](https://github.com/PaddlePaddle/book/blob/develop/07.label_semantic_roles/README.cn.md)一章中介绍了一种双向循环神经网络，这里介绍Bengio团队在论文\[[2](#参考文献),[4](#参考文献)\]中提出的另一种结构。该结构的目的是输入一个序列，得到其在每个时刻的特征表示，即输出的每个时刻都用定长向量表示到该时刻的上下文语义信息。
 
 具体来说，该双向循环神经网络分别在时间维以顺序和逆序——即前向（forward）和后向（backward）——依次处理输入序列，并将每个时间步RNN的输出拼接成为最终的输出层。这样每个时间步的输出节点，都包含了输入序列中当前时刻完整的过去和未来的上下文信息。下图展示的是一个按时间步展开的双向循环神经网络。该网络包含一个前向和一个后向RNN，其中有六个权重矩阵：输入到前向隐层和后向隐层的权重矩阵（$W_1, W_3$），隐层到隐层自己的权重矩阵（$W_2,W_5$），前向隐层和后向隐层到输出层的权重矩阵（$W_4, W_6$）。注意，该网络的前向隐层和后向隐层之间没有连接。
 
@@ -74,7 +74,7 @@ GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN
 
 编码阶段分为三步：
 
-1. one-hot vector表示：将源语言句子$x=\left \{ x_1,x_2,...,x_T \right \}$的每个词$x_i$表示成一个列向量$w_i\epsilon R^{\left | V \right |},i=1,2,...,T$。这个向量$w_i$的维度与词汇表大小$\left | V \right |$ 相同，并且只有一个维度上有值1（该位置对应该词在词汇表中的位置），其余全是0。
+1. one-hot vector表示：将源语言句子$x=\left \{ x_1,x_2,...,x_T \right \}$的每个词$x_i$表示成一个列向量$w_i\epsilon \left \{ 0,1 \right \}^{\left | V \right |},i=1,2,...,T$。这个向量$w_i$的维度与词汇表大小$\left | V \right |$ 相同，并且只有一个维度上有值1（该位置对应该词在词汇表中的位置），其余全是0。
 
 2. 映射到低维语义空间的词向量：one-hot vector表示存在两个问题，1）生成的向量维度往往很大，容易造成维数灾难；2）难以刻画词与词之间的关系（如语义相似性，也就是无法很好地表达语义）。因此，需再one-hot vector映射到低维的语义空间，由一个固定维度的稠密向量（称为词向量）表示。记映射矩阵为$C\epsilon R^{K\times \left | V \right |}$，用$s_i=Cw_i$表示第$i$个词的词向量，$K$为向量维度。
 
@@ -167,7 +167,7 @@ e_{ij}&=align(z_i,h_j)\\\\
 
 该数据集有193319条训练数据，6003条测试数据，词典长度为30000。因为数据规模限制，使用该数据集训练出来的模型效果无法保证。
 
-## 训练流程说明
+## 流程说明
 
 ### paddle初始化
 
@@ -178,56 +178,40 @@ import paddle.v2 as paddle
 
 # 配置只使用cpu，并且使用一个cpu进行训练
 paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### 数据定义
-
-首先要定义词典大小，数据生成和网络配置都需要用到。然后获取wmt14的dataset reader。
-
-```python
-# source and target dict dim.
-dict_size = 30000
-
-feeding = {
-    'source_language_word': 0,
-    'target_language_word': 1,
-    'target_language_next_word': 2
-}
-wmt14_reader = paddle.batch(
-    paddle.reader.shuffle(
-        paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
-    batch_size=5)
+# 训练模式False，生成模式True
+is_generating = False
 ```
 
 ### 模型结构
 1. 首先，定义了一些全局变量。
 
     ```python
+    dict_size = 30000 # 字典维度
     source_dict_dim = dict_size # 源语言字典维度
     target_dict_dim = dict_size # 目标语言字典维度
     word_vector_dim = 512 # 词向量维度
     encoder_size = 512 # 编码器中的GRU隐层大小
     decoder_size = 512 # 解码器中的GRU隐层大小
+    beam_size = 3 # 柱宽度
+    max_length = 250 # 生成句子的最大长度
     ```
 
-1. 其次，实现编码器框架。分为三步：
+2. 其次，实现编码器框架。分为三步：
 
-   1 输入是一个文字序列，被表示成整型的序列。序列中每个元素是文字在字典中的索引。所以，我们定义数据层的数据类型为`integer_value_sequence`（整型序列），序列中每个元素的范围是`[0, source_dict_dim)`。
+   - 输入是一个文字序列，被表示成整型的序列。序列中每个元素是文字在字典中的索引。所以，我们定义数据层的数据类型为`integer_value_sequence`（整型序列），序列中每个元素的范围是`[0, source_dict_dim)`。
 
    ```python
     src_word_id = paddle.layer.data(
         name='source_language_word',
         type=paddle.data_type.integer_value_sequence(source_dict_dim))
    ```
-   1. 将上述编码映射到低维语言空间的词向量$\mathbf{s}$。
+   - 将上述编码映射到低维语言空间的词向量$\mathbf{s}$。
 
    ```python
     src_embedding = paddle.layer.embedding(
-        input=src_word_id,
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_source_language_embedding'))
+        input=src_word_id, size=word_vector_dim)
    ```
-   1. 用双向GRU编码源语言序列，拼接两个GRU的编码结果得到$\mathbf{h}$。
+   - 用双向GRU编码源语言序列，拼接两个GRU的编码结果得到$\mathbf{h}$。
 
    ```python
     src_forward = paddle.networks.simple_gru(
@@ -237,27 +221,30 @@ wmt14_reader = paddle.batch(
     encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
    ```
 
-1. 接着，定义基于注意力机制的解码器框架。分为三步：
+3. 接着，定义基于注意力机制的解码器框架。分为三步：
 
-   1. 对源语言序列编码后的结果（见2.3），过一个前馈神经网络（Feed Forward Neural Network），得到其映射。
+   - 对源语言序列编码后的结果（见2的最后一步），过一个前馈神经网络（Feed Forward Neural Network），得到其映射。
 
    ```python
-    with paddle.layer.mixed(size=decoder_size) as encoded_proj:
-        encoded_proj += paddle.layer.full_matrix_projection(
+   encoded_proj = paddle.layer.fc(
+         act=paddle.activation.Linear(),
+         size=decoder_size,
+         bias_attr=False,
          input=encoded_vector)
    ```
 
-   1. 构造解码器RNN的初始状态。由于解码器需要预测时序目标序列，但在0时刻并没有初始值，所以我们希望对其进行初始化。这里采用的是将源语言序列逆序编码后的最后一个状态进行非线性映射，作为该初始值，即$c_0=h_T$。
+   - 构造解码器RNN的初始状态。由于解码器需要预测时序目标序列，但在0时刻并没有初始值，所以我们希望对其进行初始化。这里采用的是将源语言序列逆序编码后的最后一个状态进行非线性映射，作为该初始值，即$c_0=h_T$。
 
    ```python
    backward_first = paddle.layer.first_seq(input=src_backward)
-    with paddle.layer.mixed(
-            size=decoder_size, act=paddle.activation.Tanh()) as decoder_boot:
-        decoder_boot += paddle.layer.full_matrix_projection(
+   decoder_boot = paddle.layer.fc(
+         size=decoder_size,
+         act=paddle.activation.Tanh(),
+         bias_attr=False,
          input=backward_first)
    ```
 
-   1. 定义解码阶段每一个时间步的RNN行为，即根据当前时刻的源语言上下文向量$c_i$、解码器隐层状态$z_i$和目标语言中第$i$个词$u_i$，来预测第$i+1$个词的概率$p_{i+1}$。
+   - 定义解码阶段每一个时间步的RNN行为，即根据当前时刻的源语言上下文向量$c_i$、解码器隐层状态$z_i$和目标语言中第$i$个词$u_i$，来预测第$i+1$个词的概率$p_{i+1}$。
       - decoder_mem记录了前一个时间步的隐层状态$z_i$，其初始状态是decoder_boot。
       - context通过调用`simple_attention`函数，实现公式$c_i=\sum {j=1}^{T}a_{ij}h_j$。其中，enc_vec是$h_j$，enc_proj是$h_j$的映射（见3.1），权重$a_{ij}$的计算已经封装在`simple_attention`函数中。
       - decoder_inputs融合了$c_i$和当前目标词current_word（即$u_i$）的表示。
@@ -266,7 +253,6 @@ wmt14_reader = paddle.batch(
 
    ```python
    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
-
         decoder_mem = paddle.layer.memory(
             name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
 
@@ -275,10 +261,13 @@ wmt14_reader = paddle.batch(
             encoded_proj=enc_proj,
             decoder_state=decoder_mem)
 
-        with paddle.layer.mixed(size=decoder_size * 3) as decoder_inputs:
-            decoder_inputs += paddle.layer.full_matrix_projection(input=context)
-            decoder_inputs += paddle.layer.full_matrix_projection(
-                input=current_word)
+        decoder_inputs = paddle.layer.fc(
+            act=paddle.activation.Linear(),
+            size=decoder_size * 3,
+            bias_attr=False,
+            input=[context, current_word],
+            layer_attr=paddle.attr.ExtraLayerAttribute(
+                error_clipping_threshold=100.0))
 
         gru_step = paddle.layer.gru_step(
             name='gru_decoder',
@@ -286,24 +275,24 @@ wmt14_reader = paddle.batch(
             output_mem=decoder_mem,
             size=decoder_size)
 
-        with paddle.layer.mixed(
+        out = paddle.layer.mixed(
             size=target_dict_dim,
             bias_attr=True,
-                act=paddle.activation.Softmax()) as out:
-            out += paddle.layer.full_matrix_projection(input=gru_step)
+            act=paddle.activation.Softmax(),
+            input=paddle.layer.full_matrix_projection(input=gru_step))
         return out
    ```
 
-1. 定义解码器框架名字，和`gru_decoder_with_attention`函数的前两个输入。注意：这两个输入使用`StaticInput`，具体说明可见[StaticInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
+4. 定义解码器框架名字，和`gru_decoder_with_attention`函数的前两个输入。注意：这两个输入使用`StaticInput`，具体说明可见[StaticInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
 
     ```python
     decoder_group_name = "decoder_group"
-    group_input1 = paddle.layer.StaticInputV2(input=encoded_vector, is_seq=True)
-    group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
+    group_input1 = paddle.layer.StaticInput(input=encoded_vector)
+    group_input2 = paddle.layer.StaticInput(input=encoded_proj)
     group_inputs = [group_input1, group_input2]
     ```
 
-1. 训练模式下的解码器调用：
+5. 训练模式下的解码器调用：
 
    - 首先，将目标语言序列的词向量trg_embedding，直接作为训练模式下的current_word传给`gru_decoder_with_attention`函数。
    - 其次，使用`recurrent_group`函数循环调用`gru_decoder_with_attention`函数。
@@ -311,6 +300,7 @@ wmt14_reader = paddle.batch(
    - 最后，用多类交叉熵损失函数`classification_cost`来计算损失值。
 
    ```python
+   if not is_generating:
        trg_embedding = paddle.layer.embedding(
            input=paddle.layer.data(
                name='target_language_word',
@@ -335,30 +325,70 @@ wmt14_reader = paddle.batch(
        cost = paddle.layer.classification_cost(input=decoder, label=lbl)
     ```
 
-注意：我们提供的配置在Bahdanau的论文\[[4](#参考文献)\]上做了一些简化，可参考[issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133)。
+6. 生成模式下的解码器调用：
 
-### 参数定义
+   - 首先，在序列生成任务中，由于解码阶段的RNN总是引用上一时刻生成出的词的词向量，作为当前时刻的输入，因此，使用`GeneratedInput`来自动完成这一过程。具体说明可见[GeneratedInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
+   - 其次，使用`beam_search`函数循环调用`gru_decoder_with_attention`函数，生成出序列id。
 
-首先依据模型配置的`cost`定义模型参数。
+   ```python
+   if is_generating:
+      # In generation, the decoder predicts a next target word based on
+      # the encoded source sequence and the previous generated target word.
 
-```python
-parameters = paddle.parameters.create(cost)
-```
+      # The encoded source sequence (encoder's output) must be specified by
+      # StaticInput, which is a read-only memory.
+      # Embedding of the previous generated word is automatically retrieved
+      # by GeneratedInputs initialized by a start mark <s>.
+
+       trg_embedding = paddle.layer.GeneratedInput(
+           size=target_dict_dim,
+           embedding_name='_target_language_embedding',
+           embedding_size=word_vector_dim)
+       group_inputs.append(trg_embedding)
 
-可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
+       beam_gen = paddle.layer.beam_search(
+           name=decoder_group_name,
+           step=gru_decoder_with_attention,
+           input=group_inputs,
+           bos_id=0,
+           eos_id=1,
+           beam_size=beam_size,
+           max_length=max_length)
+   ```
 
-```python
-for param in parameters.keys():
-    print param
-```
+注意：我们提供的配置在Bahdanau的论文\[[4](#参考文献)\]上做了一些简化，可参考[issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133)。
 
 ### 训练模型
 
-1. 构造trainer
+1. 参数定义
+
+    依据模型配置的`cost`定义模型参数。可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
+
+    ```python
+    if not is_generating:
+        parameters = paddle.parameters.create(cost)
+        for param in parameters.keys():
+            print param
+    ```
+
+2. 数据定义
+
+    获取wmt14的dataset reader。
+
+    ```python
+    if not is_generating:
+        wmt14_reader = paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
+            batch_size=5)
+    ```
+
+3. 构造trainer
 
     根据优化目标cost,网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的SGD方法。
 
     ```python
+    if not is_generating:
         optimizer = paddle.optimizer.Adam(
             learning_rate=5e-5,
             regularization=paddle.optimizer.L2Regularization(rate=8e-4))
@@ -367,73 +397,104 @@ for param in parameters.keys():
                                      update_equation=optimizer)
     ```
 
-1. 构造event_handler
+4. 构造event_handler
 
-    可以通过自定义回调函数来评估训练过程中的各种状态，比如错误率等。下面的代码通过event.batch_id % 10 == 0 指定没10个batch打印一次日志，包含cost等信息。
+    可以通过自定义回调函数来评估训练过程中的各种状态，比如错误率等。下面的代码通过event.batch_id % 2 == 0 指定每2个batch打印一次日志，包含cost等信息。
 
     ```python
+    if not is_generating:
         def event_handler(event):
             if isinstance(event, paddle.event.EndIteration):
-            if event.batch_id % 10 == 0:
+                if event.batch_id % 2 == 0:
                     print "\nPass %d, Batch %d, Cost %f, %s" % (
                         event.pass_id, event.batch_id, event.cost, event.metrics)
-            else:
-                sys.stdout.write('.')
-                sys.stdout.flush()
     ```
 
-1. 启动训练：
+5. 启动训练
 
     ```python
+    if not is_generating:
         trainer.train(
-        reader=wmt14_reader,
-        event_handler=event_handler,
-        num_passes=2,
-        feeding=feeding)
+                reader=wmt14_reader, event_handler=event_handler, num_passes=2)
     ```
 
-训练开始后，可以观察到event_handler输出的日志如下：
+ 训练开始后，可以观察到event_handler输出的日志如下：
+ ```text
+ Pass 0, Batch 0, Cost 148.444983, {'classification_error_evaluator': 1.0}
+ .........
+ Pass 0, Batch 10, Cost 335.896802, {'classification_error_evaluator': 0.9325153231620789}
+ .........
+ ```
 
-```text
-Pass 0, Batch 0, Cost 148.444983, {'classification_error_evaluator': 1.0}
-.........
-Pass 0, Batch 10, Cost 335.896802, {'classification_error_evaluator': 0.9325153231620789}
-.........
-```
+### 生成模型
 
-    当`classification_error_evaluator`的值低于0.35的时候，表示训练成功。
+1. 加载预训练的模型
 
-## 应用模型
+    由于NMT模型的训练非常耗时，我们在50个物理节点（每节点含有2颗6核CPU）的集群中，花了5天时间训练了一个模型供大家直接下载使用。该模型大小为205MB，[BLEU评估](#BLEU评估)值为26.92。
 
-### 下载预训练的模型
+    ```python
+    if is_generating:
+        parameters = paddle.dataset.wmt14.model()
+    ```
+2. 数据定义
 
-由于NMT模型的训练非常耗时，我们在50个物理节点（每节点含有2颗6核CPU）的集群中，花了5天时间训练了16个pass，其中每个pass耗时7个小时。因此，我们提供了一个预先训练好的模型（pass-00012）供大家直接下载使用。该模型大小为205MB，在所有16个模型中有最高的[BLEU评估](#BLEU评估)值26.92。下载并解压模型的命令如下：
+    从wmt14的生成集中读取前3个样本作为源语言句子。
 
-```bash
-cd pretrained
-./wmt14_model.sh
-```
+    ```python
+    if is_generating:
+        gen_creator = paddle.dataset.wmt14.gen(dict_size)
+        gen_data = []
+        gen_num = 3
+        for item in gen_creator():
+            gen_data.append((item[0], ))
+            if len(gen_data) == gen_num:
+                break
+    ```
+3. 构造infer
 
-### BLEU评估
+    根据网络拓扑结构和模型参数构造出infer用来生成，在预测时还需要指定输出域`field`，这里使用生成句子的概率`prob`和句子中每个词的`id`。
 
-BLEU(Bilingual Evaluation understudy)是一种广泛使用的机器翻译自动评测指标，由IBM的watson研究中心于2002年提出\[[5](#参考文献)\]，基本出发点是：机器译文越接近专业翻译人员的翻译结果，翻译系统的性能越好。其中，机器译文与人工参考译文之间的接近程度，采用句子精确度（precision）的计算方法，即比较两者的n元词组相匹配的个数，匹配的个数越多，BLEU得分越好。
+    ```python
+    if is_generating:
+        beam_result = paddle.infer(
+            output_layer=beam_gen,
+            parameters=parameters,
+            input=gen_data,
+            field=['prob', 'id'])
+    ```
 
-[Moses](http://www.statmt.org/moses/) 是一个统计学的开源机器翻译系统，我们使用其中的 [multi-bleu.perl](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/multi-bleu.perl) 来做BLEU评估。下载脚本的命令如下：
-```bash
-./moses_bleu.sh
-```
-BLEU评估可以使用`eval_bleu`脚本如下，其中FILE为需要评估的文件名，BEAMSIZE为柱宽度，默认使用`data/wmt14/gen/ntst14.trg`作为标准的翻译结果。
-```bash
-./eval_bleu.sh FILE BEAMSIZE
-```
-本教程的具体命令如下：
-```bash
-./eval_bleu.sh gen_result 3
-```
-您会在屏幕上看到：
-```text
-BLEU = 26.92
-```
+4. 打印生成结果
+
+    根据源/目标语言字典，将源语言句子和`beam_size`个生成句子打印输出。
+
+    ```python
+    if is_generating:
+        # load the dictionary
+        src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
+
+        gen_sen_idx = np.where(beam_result[1] == -1)[0]
+        assert len(gen_sen_idx) == len(gen_data) * beam_size
+
+        # -1 is the delimiter of generated sequences.
+        # the first element of each generated sequence its length.
+        start_pos, end_pos = 1, 0
+        for i, sample in enumerate(gen_data):
+            print(" ".join([src_dict[w] for w in sample[0][1:-1]]))
+            for j in xrange(beam_size):
+                end_pos = gen_sen_idx[i * beam_size + j]
+                print("%.4f\t%s" % (beam_result[0][i][j], " ".join(
+                    trg_dict[w] for w in beam_result[1][start_pos:end_pos])))
+                start_pos = end_pos + 2
+            print("\n")
+    ```
+
+  生成开始后，可以观察到输出的日志如下：
+  ```text
+  Les <unk> se <unk> au sujet de la largeur des sièges alors que de grosses commandes sont en jeu
+  -19.0196        The <unk> will be rotated about the width of the seats , while large orders are at stake . <e>
+  -19.1131        The <unk> will be rotated about the width of the seats , while large commands are at stake . <e>
+  -19.5129        The <unk> will be rotated about the width of the seats , while large commands are at play . <e>
+  ```
 
 ## 总结
 
@@ -448,4 +509,4 @@ BLEU = 26.92
 5. Papineni K, Roukos S, Ward T, et al. [BLEU: a method for automatic evaluation of machine translation](http://dl.acm.org/citation.cfm?id=1073135)[C]//Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 2002: 311-318.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

07.machine_translation/README.en.md → 08.machine_translation/README.md

 # Machine Translation
 
-The source code of this tutorial is live at [book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/07.machine_translation). Please refer to the [installation tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst) for getting started with Paddle.
+The source code of this tutorial is live at [book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation). Please refer to the [book running tutorial](https://github.com/PaddlePaddle/book#running-the-book) for getting started with Paddle.
 
 ## Background
 
 Machine translation (MT) leverages computers to translate from one language to another. The language to be translated is referred to as the source language, while the language to be translated into is referred to as the target language. Thus, Machine translation is the process of translating from the source language to the target language. It is one of the most important research topics in the field of natural language processing.
 
-Early machine translation systems are mainly rule-based i.e. they rely on a language expert to specify the translation rules between the two languages. It is quite difficult to cover all the rules used in one languge. So it is quite a challenge for language experts to specify all possible rules in two or more different languages. Hence, a major challenge in conventional machine translation has been the difficulty in obtaining a complete rule set \[[1](#References)\]。
+Early machine translation systems are mainly rule-based i.e. they rely on a language expert to specify the translation rules between the two languages. It is quite difficult to cover all the rules used in one languge. So it is quite a challenge for language experts to specify all possible rules in two or more different languages. Hence, a major challenge in conventional machine translation has been the difficulty in obtaining a complete rule set \[[1](#References)\].
 
 
 To address the aforementioned problems, statistical machine translation techniques have been developed. These techniques learn the translation rules from a large corpus, instead of being designed by a language expert. While these techniques overcome the bottleneck of knowledge acquisition, there are still quite a lot of challenges, for example:
@@ -41,9 +41,9 @@ Let's consider an example of Chinese-to-English translation. The model is given
 ```
 After training and with a beam-search size of 3, the generated translations are as follows:
 ```text
-0 -5.36816   these are signs of hope and relief . <e>
-1 -6.23177   these are the light of hope and relief . <e>
-2 -7.7914  these are the light of hope and the relief of hope . <e>
+0 -5.36816   These are signs of hope and relief . <e>
+1 -6.23177   These are the light of hope and relief . <e>
+2 -7.7914  These are the light of hope and the relief of hope . <e>
 ```
 - The first column corresponds to the id of the generated sentence; the second column corresponds to the score of the generated sentence (in descending order), where a larger value indicates better quality; the last column corresponds to the generated sentence.
 - There are two special tokens: `<e>` denotes the end of a sentence while `<unk>` denotes unknown word, i.e., a word not in the training dictionary.
@@ -94,7 +94,7 @@ Figure 4. Encoder-Decoder Framework
 
 There are three steps for encoding a sentence:
 
-1. One-hot vector representation of a word: Each word $x_i$ in the source sentence $x=\left \{ x_1,x_2,...,x_T \right \}$ is represented as a vector $w_i\epsilon R^{\left | V \right |},i=1,2,...,T$   where $w_i$ has the same dimensionality as the size of the dictionary, i.e., $\left | V \right |$, and has an element of one at the location corresponding to the location of the word in the dictionary and zero elsewhere.
+1. One-hot vector representation of a word: Each word $x_i$ in the source sentence $x=\left \{ x_1,x_2,...,x_T \right \}$ is represented as a vector $w_i\epsilon \left \{ 0,1 \right \}^{\left | V \right |},i=1,2,...,T$   where $w_i$ has the same dimensionality as the size of the dictionary, i.e., $\left | V \right |$, and has an element of one at the location corresponding to the location of the word in the dictionary and zero elsewhere.
 
 2. Word embedding as a representation in the low-dimensional semantic space: There are two problems with one-hot vector representation
 
@@ -183,6 +183,12 @@ The goal is to maximize the probability of the generated sequence when using bea
 
 Note: $z_{i+1}$ and $p_{i+1}$ are computed the same way as in [Decoder](#Decoder). In generation mode, each step is greedy in so there is no guarantee of a global optimum.
 
+## BLEU Score
+
+Bilingual Evaluation understudy (BLEU) is a metric widely used for automatic machine translation proposed by IBM Watson Research Center in 2002\[[5](#References)\]. The closer the translation produced by a machine is to the translation produced by a human expert, the better the performance of the translation system.
+
+To measure the closeness between machine translation and human translation, sentence precision is used. It compares the number of matched n-grams. More matches will lead to higher BLEU scores.
+
 ## Data Preparation
 
 This tutorial uses a dataset from [WMT-14](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/), where [bitexts (after selection)](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/bitexts.tgz) is used as the training set, and [dev+test data](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz) is used as test and generation set.
@@ -213,25 +219,8 @@ import paddle.v2 as paddle
 
 # train with a single CPU
 paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### Define DataSet
-
-We will define dictionary size, and create [**data reader**](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader#python-data-reader-design-doc) for WMT-14 dataset.
-
-```python
-# source and target dict dim.
-dict_size = 30000
-
-feeding = {
-    'source_language_word': 0,
-    'target_language_word': 1,
-    'target_language_next_word': 2
-}
-wmt14_reader = paddle.batch(
-    paddle.reader.shuffle(
-        paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
-    batch_size=5)
+# False: training, True: generating
+is_generating = False
 ```
 
 ### Model Configuration
@@ -239,15 +228,18 @@ wmt14_reader = paddle.batch(
 1. Define some global variables
 
    ```python
+   dict_size = 30000 # dict dim
    source_dict_dim = dict_size # source language dictionary size
    target_dict_dim = dict_size # destination language dictionary size
    word_vector_dim = 512 # word embedding dimension
    encoder_size = 512 # hidden layer size of GRU in encoder
    decoder_size = 512 # hidden layer size of GRU in decoder
+   beam_size = 3 # expand width in beam search
+   max_length = 250 # a stop condition of sequence generation
    ```
 
-1. Implement Encoder as follows:
-   1. Input is a sequence of words represented by an integer word index sequence. So we define data layer of data type `integer_value_sequence`. The value range of each element in the sequence is `[0, source_dict_dim)`
+2. Implement Encoder as follows:
+   - Input is a sequence of words represented by an integer word index sequence. So we define data layer of data type `integer_value_sequence`. The value range of each element in the sequence is `[0, source_dict_dim)`
 
    ```python
    src_word_id = paddle.layer.data(
@@ -255,16 +247,14 @@ wmt14_reader = paddle.batch(
        type=paddle.data_type.integer_value_sequence(source_dict_dim))
    ```
 
-   1. Map the one-hot vector (represented by word index) into a word vector $\mathbf{s}$ in a low-dimensional semantic space
+   - Map the one-hot vector (represented by word index) into a word vector $\mathbf{s}$ in a low-dimensional semantic space
 
    ```python
    src_embedding = paddle.layer.embedding(
-        input=src_word_id,
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_source_language_embedding'))
+       input=src_word_id, size=word_vector_dim)
    ```
 
-   1. Use bi-direcitonal GRU to encode the source language sequence, and concatenate the encoding outputs from the two GRUs to get $\mathbf{h}$
+   - Use bi-direcitonal GRU to encode the source language sequence, and concatenate the encoding outputs from the two GRUs to get $\mathbf{h}$
 
    ```python
    src_forward = paddle.networks.simple_gru(
@@ -274,27 +264,30 @@ wmt14_reader = paddle.batch(
    encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
    ```
 
-1. Implement Attention-based Decoder as follows:
+3. Implement Attention-based Decoder as follows:
 
-   1. Get a projection of the encoding (c.f. 2.3) of the source language sequence by passing it into a feed forward neural network
+   - Get a projection of the encoding (c.f. 2.3) of the source language sequence by passing it into a feed forward neural network
 
    ```python
-    with paddle.layer.mixed(size=decoder_size) as encoded_proj:
-        encoded_proj += paddle.layer.full_matrix_projection(
+   encoded_proj = paddle.layer.fc(
+         act=paddle.activation.Linear(),
+         size=decoder_size,
+         bias_attr=False,
          input=encoded_vector)
    ```
 
-   1. Use a non-linear transformation of the last hidden state of the backward GRU on the source language sentence as the initial state of the decoder RNN $c_0=h_T$
+   - Use a non-linear transformation of the last hidden state of the backward GRU on the source language sentence as the initial state of the decoder RNN $c_0=h_T$
 
    ```python
    backward_first = paddle.layer.first_seq(input=src_backward)
-    with paddle.layer.mixed(
-            size=decoder_size, act=paddle.activation.Tanh()) as decoder_boot:
-        decoder_boot += paddle.layer.full_matrix_projection(
+   decoder_boot = paddle.layer.fc(
+         size=decoder_size,
+         act=paddle.activation.Tanh(),
+         bias_attr=False,
          input=backward_first)
    ```
 
-   1. Define the computation in each time step for the decoder RNN, i.e., according to the current context vector $c_i$, hidden state for the decoder $z_i$ and the $i$-th word $u_i$ in the target language to predict the probability $p_{i+1}$ for the $i+1$-th word.
+   - Define the computation in each time step for the decoder RNN, i.e., according to the current context vector $c_i$, hidden state for the decoder $z_i$ and the $i$-th word $u_i$ in the target language to predict the probability $p_{i+1}$ for the $i+1$-th word.
 
       - decoder_mem records the hidden state $z_i$ from the previous time step, with an initial state as decoder_boot.
       - context is computed via `simple_attention` as $c_i=\sum {j=1}^{T}a_{ij}h_j$, where enc_vec is the projection of $h_j$ and enc_proj is the projection of $h_j$ (c.f. 3.1). $a_{ij}$ is calculated within `simple_attention`.
@@ -304,7 +297,6 @@ wmt14_reader = paddle.batch(
 
    ```python
    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
-
         decoder_mem = paddle.layer.memory(
             name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
 
@@ -313,10 +305,13 @@ wmt14_reader = paddle.batch(
             encoded_proj=enc_proj,
             decoder_state=decoder_mem)
 
-        with paddle.layer.mixed(size=decoder_size * 3) as decoder_inputs:
-            decoder_inputs += paddle.layer.full_matrix_projection(input=context)
-            decoder_inputs += paddle.layer.full_matrix_projection(
-                input=current_word)
+        decoder_inputs = paddle.layer.fc(
+            act=paddle.activation.Linear(),
+            size=decoder_size * 3,
+            bias_attr=False,
+            input=[context, current_word],
+            layer_attr=paddle.attr.ExtraLayerAttribute(
+            error_clipping_threshold=100.0))
 
         gru_step = paddle.layer.gru_step(
             name='gru_decoder',
@@ -324,24 +319,24 @@ wmt14_reader = paddle.batch(
             output_mem=decoder_mem,
             size=decoder_size)
 
-        with paddle.layer.mixed(
+        out = paddle.layer.fc(
             size=target_dict_dim,
             bias_attr=True,
-                act=paddle.activation.Softmax()) as out:
-            out += paddle.layer.full_matrix_projection(input=gru_step)
+            act=paddle.activation.Softmax(),
+            input=gru_step)
         return out
    ```
 
-1. Define the name for the decoder and the first two input for `gru_decoder_with_attention`. Note that `StaticInput` is used for the two inputs. Please refer to [StaticInput Document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入) for more details.
+4. Define the name for the decoder and the first two input for `gru_decoder_with_attention`. Note that `StaticInput` is used for the two inputs. Please refer to [StaticInput Document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入) for more details.
 
     ```python
     decoder_group_name = "decoder_group"
-    group_input1 = paddle.layer.StaticInputV2(input=encoded_vector, is_seq=True)
-    group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
+    group_input1 = paddle.layer.StaticInput(input=encoded_vector)
+    group_input2 = paddle.layer.StaticInput(input=encoded_proj)
     group_inputs = [group_input1, group_input2]
     ```
 
-1. Training mode:
+5. Training mode:
 
    - word embedding from the target language trg_embedding is passed to `gru_decoder_with_attention` as current_word.
    - `recurrent_group` calls `gru_decoder_with_attention` in a recurrent way
@@ -349,6 +344,7 @@ wmt14_reader = paddle.batch(
    - multi-class cross-entropy (`classification_cost`) is used to calculate the cost
 
    ```python
+   if not is_generating:
        trg_embedding = paddle.layer.embedding(
            input=paddle.layer.data(
                name='target_language_word',
@@ -373,30 +369,69 @@ wmt14_reader = paddle.batch(
        cost = paddle.layer.classification_cost(input=decoder, label=lbl)
    ```
 
-Note: Our configuration is based on Bahdanau et al. \[[4](#Reference)\] but with a few simplifications. Please refer to [issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133) for more details.
+6. Generating mode:
+
+   - the decoder predicts a next target word based on the the last generated target word. Embedding of the last generated word is automatically gotten by GeneratedInputs.
+   - `beam_search` calls `gru_decoder_with_attention` in a recurrent way, to predict sequence id.
 
-### Create Parameters
+   ```python
+   if is_generating:
+       # In generation, the decoder predicts a next target word based on
+       # the encoded source sequence and the previous generated target word.
 
-Create every parameter that `cost` layer needs.
+       # The encoded source sequence (encoder's output) must be specified by
+       # StaticInput, which is a read-only memory.
+       # Embedding of the previous generated word is automatically retrieved
+       # by GeneratedInputs initialized by a start mark <s>.
 
-```python
-parameters = paddle.parameters.create(cost)
-```
+       trg_embedding = paddle.layer.GeneratedInput(
+           size=target_dict_dim,
+           embedding_name='_target_language_embedding',
+           embedding_size=word_vector_dim)
+       group_inputs.append(trg_embedding)
 
-We can get parameter names. If the parameter name is not specified during model configuration, it will be generated.
+       beam_gen = paddle.layer.beam_search(
+           name=decoder_group_name,
+           step=gru_decoder_with_attention,
+           input=group_inputs,
+           bos_id=0,
+           eos_id=1,
+           beam_size=beam_size,
+           max_length=max_length)
+   ```
 
-```python
-for param in parameters.keys():
-    print param
-```
+Note: Our configuration is based on Bahdanau et al. \[[4](#Reference)\] but with a few simplifications. Please refer to [issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133) for more details.
 
 ## Model Training
 
-1. Create trainer
+1. Create Parameters
+
+    Create every parameter that `cost` layer needs. And we can get parameter names. If the parameter name is not specified during model configuration, it will be generated.
+
+    ```python
+    if not is_generating:
+        parameters = paddle.parameters.create(cost)
+        for param in parameters.keys():
+            print param
+    ```
+
+2. Define DataSet
+
+    Create [**data reader**](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader#python-data-reader-design-doc) for WMT-14 dataset.
+
+    ```python
+    if not is_generating:
+        wmt14_reader = paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
+            batch_size=5)
+    ```
+3. Create trainer
 
     We need to tell trainer what to optimize, and how to optimize. Here trainer will optimize `cost` layer using stochastic gradient descent (SDG).
 
     ```python
+    if not is_generating:
         optimizer = paddle.optimizer.Adam(
             learning_rate=5e-5,
             regularization=paddle.optimizer.L2Regularization(rate=8e-4))
@@ -405,68 +440,103 @@ for param in parameters.keys():
                                      update_equation=optimizer)
     ```
 
-1. Define event handler
+4. Define event handler
 
     The event handler is a callback function invoked by trainer when an event happens. Here we will print log in event handler.
 
     ```python
+    if not is_generating:
         def event_handler(event):
             if isinstance(event, paddle.event.EndIteration):
-            if event.batch_id % 10 == 0:
+                if event.batch_id % 2 == 0:
                     print "\nPass %d, Batch %d, Cost %f, %s" % (
                         event.pass_id, event.batch_id, event.cost, event.metrics)
     ```
 
-1. Start training
+5. Start training
 
     ```python
+    if not is_generating:
         trainer.train(
-        reader=wmt14_reader,
-        event_handler=event_handler,
-        num_passes=2,
-        feeding=feeding)
+                reader=wmt14_reader, event_handler=event_handler, num_passes=2)
     ```
 
+  The training log is as follows:
   ```text
   Pass 0, Batch 0, Cost 247.408008, {'classification_error_evaluator': 1.0}
   Pass 0, Batch 10, Cost 212.058789, {'classification_error_evaluator': 0.8737863898277283}
   ...
   ```
 
-The model training is successful when the `classification_error_evaluator` is lower than 0.35.
-
 ## Model Usage
 
-### Download Pre-trained Model
+1. Download Pre-trained Model
 
-As the training of an NMT model is very time consuming, we provide a pre-trained model (pass-00012, ~205M). The model is trained with a cluster of 50 physical nodes (each node has two 6-core CPU). We trained 16 passes (taking about 5 days) with each pass taking about 7 hours. The provided model (pass-00012) has the highest [BLEU Score](#BLEU Score) of 26.92. Run the following command to download the model:
+    As the training of an NMT model is very time consuming, we provide a pre-trained model. The model is trained with a cluster of 50 physical nodes (each node has two 6-core CPU) over 5 days. The provided model has the [BLEU Score](#BLEU Score) of 26.92, and the size of 205M.
 
-```bash
-cd pretrained
-./wmt14_model.sh
-```
+    ```python
+    if is_generating:
+        parameters = paddle.dataset.wmt14.model()
+    ```
+2. Define DataSet
 
-### BLEU Evaluation
+    Get the first 3 samples of wmt14 generating set as the source language sequences.
 
-Bilingual Evaluation understudy (BLEU) is a metric widely used for automatic machine translation proposed by IBM Watson Research Center in 2002\[[5](#References)\]. The closer the translation produced by a machine is to the translation produced by a human expert, the better the performance of the translation system.
-To measure the closeness between machine translation and human translation, sentence precision is used. It compares the number of matched n-grams. More matches will lead to higher BLEU scores.
+   ```python
+   if is_generating:
+        gen_creator = paddle.dataset.wmt14.gen(dict_size)
+        gen_data = []
+        gen_num = 3
+        for item in gen_creator():
+            gen_data.append((item[0], ))
+            if len(gen_data) == gen_num:
+                break
+   ```
 
-[Moses](http://www.statmt.org/moses/) is an open-source machine translation system, we used [multi-bleu.perl](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/multi-bleu.perl) for BLEU evaluation. Run the following command for downloading:
-```bash
-./moses_bleu.sh
-```
-BLEU evaluation can be performed using the `eval_bleu` script as follows, where FILE is the name of the file to be evaluated, BEAMSIZE is the beam size value, and `data/wmt14/gen/ntst14.trg` is used as the standard translation in default.
-```bash
-./eval_bleu.sh FILE BEAMSIZE
-```
-Specificaly, the script is run as follows:
-```bash
-./eval_bleu.sh gen_result 3
-```
-You will see the following message as output:
-```text
-BLEU = 26.92
-```
+3. Create infer
+
+    Use inference interface `paddle.infer` return the prediction probability (see field `prob`) and labels (see field `id`) of each generated sequence.
+
+   ```python
+   if is_generating:
+        beam_result = paddle.infer(
+            output_layer=beam_gen,
+            parameters=parameters,
+            input=gen_data,
+            field=['prob', 'id'])
+   ```
+4. Print generated translation
+
+    Print sequence and its `beam_size` generated translation results based on the dictionary.
+
+   ```python
+   if is_generating:
+       # load the dictionary
+       src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
+
+       gen_sen_idx = np.where(beam_result[1] == -1)[0]
+       assert len(gen_sen_idx) == len(gen_data) * beam_size
+
+       # -1 is the delimiter of generated sequences.
+       # the first element of each generated sequence its length.
+       start_pos, end_pos = 1, 0
+       for i, sample in enumerate(gen_data):
+           print(" ".join([src_dict[w] for w in sample[0][1:-1]]))
+           for j in xrange(beam_size):
+               end_pos = gen_sen_idx[i * beam_size + j]
+               print("%.4f\t%s" % (beam_result[0][i][j], " ".join(
+                     trg_dict[w] for w in beam_result[1][start_pos:end_pos])))
+               start_pos = end_pos + 2
+           print("\n")
+   ```
+
+  The generating log is as follows:
+  ```text
+  Les <unk> se <unk> au sujet de la largeur des sièges alors que de grosses commandes sont en jeu
+  -19.0196        The <unk> will be rotated about the width of the seats , while large orders are at stake . <e>
+  -19.1131        The <unk> will be rotated about the width of the seats , while large commands are at stake . <e>
+  -19.5129        The <unk> will be rotated about the width of the seats , while large commands are at play . <e>
+  ```
 
 ## Summary
 
@@ -481,4 +551,4 @@ End-to-end neural machine translation is a recently developed way to perform mac
 5. Papineni K, Roukos S, Ward T, et al. [BLEU: a method for automatic evaluation of machine translation](http://dl.acm.org/citation.cfm?id=1073135)[C]//Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 2002: 311-318.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

07.machine_translation/index.html → 08.machine_translation/index.cn.html

@@ -42,7 +42,7 @@
 <div id="markdown" style='display:none'>
 # 机器翻译
 
-本教程源代码目录在[book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/07.machine_translation)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)。
+本教程源代码目录在[book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation)， 初次使用请参考PaddlePaddle[安装教程](https://github.com/PaddlePaddle/book/blob/develop/README.cn.md#运行这本书)，更多内容请参考本教程的[视频课堂](http://bit.baidu.com/course/detail/id/179.html)。
 
 ## 背景介绍
 
@@ -68,9 +68,9 @@
 ```
 如果设定显示翻译结果的条数（即[柱搜索算法](#柱搜索算法)的宽度）为3，生成的英语句子如下：
 ```text
-0 -5.36816   these are signs of hope and relief . <e>
-1 -6.23177   these are the light of hope and relief . <e>
-2 -7.7914  these are the light of hope and the relief of hope . <e>
+0 -5.36816   These are signs of hope and relief . <e>
+1 -6.23177   These are the light of hope and relief . <e>
+2 -7.7914  These are the light of hope and the relief of hope . <e>
 ```
 - 左起第一列是生成句子的序号；左起第二列是该条句子的得分（从大到小），分值越高越好；左起第三列是生成的英语句子。
 - 另外有两个特殊标志：`<e>`表示句子的结尾，`<unk>`表示未登录词（unknown word），即未在训练字典中出现的词。
@@ -81,7 +81,7 @@
 
 ### GRU
 
-我们已经在[情感分析](https://github.com/PaddlePaddle/book/blob/develop/understand_sentiment/README.md)一章中介绍了循环神经网络（RNN）及长短时间记忆网络（LSTM）。相比于简单的RNN，LSTM增加了记忆单元（memory cell）、输入门（input gate）、遗忘门（forget gate）及输出门（output gate），这些门及记忆单元组合起来大大提升了RNN处理远距离依赖问题的能力。
+我们已经在[情感分析](https://github.com/PaddlePaddle/book/blob/develop/06.understand_sentiment/README.cn.md)一章中介绍了循环神经网络（RNN）及长短时间记忆网络（LSTM）。相比于简单的RNN，LSTM增加了记忆单元（memory cell）、输入门（input gate）、遗忘门（forget gate）及输出门（output gate），这些门及记忆单元组合起来大大提升了RNN处理远距离依赖问题的能力。
 
 GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN的一种扩展，如下图所示。GRU单元只有两个门：
 - 重置门（reset gate）：如果重置门关闭，会忽略掉历史信息，即历史不相干的信息不会影响未来的输出。
@@ -95,7 +95,7 @@ GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN
 
 ### 双向循环神经网络
 
-我们已经在[语义角色标注](https://github.com/PaddlePaddle/book/blob/develop/label_semantic_roles/README.md)一章中介绍了一种双向循环神经网络，这里介绍Bengio团队在论文\[[2](#参考文献),[4](#参考文献)\]中提出的另一种结构。该结构的目的是输入一个序列，得到其在每个时刻的特征表示，即输出的每个时刻都用定长向量表示到该时刻的上下文语义信息。
+我们已经在[语义角色标注](https://github.com/PaddlePaddle/book/blob/develop/07.label_semantic_roles/README.cn.md)一章中介绍了一种双向循环神经网络，这里介绍Bengio团队在论文\[[2](#参考文献),[4](#参考文献)\]中提出的另一种结构。该结构的目的是输入一个序列，得到其在每个时刻的特征表示，即输出的每个时刻都用定长向量表示到该时刻的上下文语义信息。
 
 具体来说，该双向循环神经网络分别在时间维以顺序和逆序——即前向（forward）和后向（backward）——依次处理输入序列，并将每个时间步RNN的输出拼接成为最终的输出层。这样每个时间步的输出节点，都包含了输入序列中当前时刻完整的过去和未来的上下文信息。下图展示的是一个按时间步展开的双向循环神经网络。该网络包含一个前向和一个后向RNN，其中有六个权重矩阵：输入到前向隐层和后向隐层的权重矩阵（$W_1, W_3$），隐层到隐层自己的权重矩阵（$W_2,W_5$），前向隐层和后向隐层到输出层的权重矩阵（$W_4, W_6$）。注意，该网络的前向隐层和后向隐层之间没有连接。
 
@@ -116,7 +116,7 @@ GRU\[[2](#参考文献)\]是Cho等人在LSTM上提出的简化版本，也是RNN
 
 编码阶段分为三步：
 
-1. one-hot vector表示：将源语言句子$x=\left \{ x_1,x_2,...,x_T \right \}$的每个词$x_i$表示成一个列向量$w_i\epsilon R^{\left | V \right |},i=1,2,...,T$。这个向量$w_i$的维度与词汇表大小$\left | V \right |$ 相同，并且只有一个维度上有值1（该位置对应该词在词汇表中的位置），其余全是0。
+1. one-hot vector表示：将源语言句子$x=\left \{ x_1,x_2,...,x_T \right \}$的每个词$x_i$表示成一个列向量$w_i\epsilon \left \{ 0,1 \right \}^{\left | V \right |},i=1,2,...,T$。这个向量$w_i$的维度与词汇表大小$\left | V \right |$ 相同，并且只有一个维度上有值1（该位置对应该词在词汇表中的位置），其余全是0。
 
 2. 映射到低维语义空间的词向量：one-hot vector表示存在两个问题，1）生成的向量维度往往很大，容易造成维数灾难；2）难以刻画词与词之间的关系（如语义相似性，也就是无法很好地表达语义）。因此，需再one-hot vector映射到低维的语义空间，由一个固定维度的稠密向量（称为词向量）表示。记映射矩阵为$C\epsilon R^{K\times \left | V \right |}$，用$s_i=Cw_i$表示第$i$个词的词向量，$K$为向量维度。
 
@@ -209,7 +209,7 @@ e_{ij}&=align(z_i,h_j)\\\\
 
 该数据集有193319条训练数据，6003条测试数据，词典长度为30000。因为数据规模限制，使用该数据集训练出来的模型效果无法保证。
 
-## 训练流程说明
+## 流程说明
 
 ### paddle初始化
 
@@ -220,56 +220,40 @@ import paddle.v2 as paddle
 
 # 配置只使用cpu，并且使用一个cpu进行训练
 paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### 数据定义
-
-首先要定义词典大小，数据生成和网络配置都需要用到。然后获取wmt14的dataset reader。
-
-```python
-# source and target dict dim.
-dict_size = 30000
-
-feeding = {
-    'source_language_word': 0,
-    'target_language_word': 1,
-    'target_language_next_word': 2
-}
-wmt14_reader = paddle.batch(
-    paddle.reader.shuffle(
-        paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
-    batch_size=5)
+# 训练模式False，生成模式True
+is_generating = False
 ```
 
 ### 模型结构
 1. 首先，定义了一些全局变量。
 
     ```python
+    dict_size = 30000 # 字典维度
     source_dict_dim = dict_size # 源语言字典维度
     target_dict_dim = dict_size # 目标语言字典维度
     word_vector_dim = 512 # 词向量维度
     encoder_size = 512 # 编码器中的GRU隐层大小
     decoder_size = 512 # 解码器中的GRU隐层大小
+    beam_size = 3 # 柱宽度
+    max_length = 250 # 生成句子的最大长度
     ```
 
-1. 其次，实现编码器框架。分为三步：
+2. 其次，实现编码器框架。分为三步：
 
-   1 输入是一个文字序列，被表示成整型的序列。序列中每个元素是文字在字典中的索引。所以，我们定义数据层的数据类型为`integer_value_sequence`（整型序列），序列中每个元素的范围是`[0, source_dict_dim)`。
+   - 输入是一个文字序列，被表示成整型的序列。序列中每个元素是文字在字典中的索引。所以，我们定义数据层的数据类型为`integer_value_sequence`（整型序列），序列中每个元素的范围是`[0, source_dict_dim)`。
 
    ```python
     src_word_id = paddle.layer.data(
         name='source_language_word',
         type=paddle.data_type.integer_value_sequence(source_dict_dim))
    ```
-   1. 将上述编码映射到低维语言空间的词向量$\mathbf{s}$。
+   - 将上述编码映射到低维语言空间的词向量$\mathbf{s}$。
 
    ```python
     src_embedding = paddle.layer.embedding(
-        input=src_word_id,
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_source_language_embedding'))
+        input=src_word_id, size=word_vector_dim)
    ```
-   1. 用双向GRU编码源语言序列，拼接两个GRU的编码结果得到$\mathbf{h}$。
+   - 用双向GRU编码源语言序列，拼接两个GRU的编码结果得到$\mathbf{h}$。
 
    ```python
     src_forward = paddle.networks.simple_gru(
@@ -279,27 +263,30 @@ wmt14_reader = paddle.batch(
     encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
    ```
 
-1. 接着，定义基于注意力机制的解码器框架。分为三步：
+3. 接着，定义基于注意力机制的解码器框架。分为三步：
 
-   1. 对源语言序列编码后的结果（见2.3），过一个前馈神经网络（Feed Forward Neural Network），得到其映射。
+   - 对源语言序列编码后的结果（见2的最后一步），过一个前馈神经网络（Feed Forward Neural Network），得到其映射。
 
    ```python
-    with paddle.layer.mixed(size=decoder_size) as encoded_proj:
-        encoded_proj += paddle.layer.full_matrix_projection(
+   encoded_proj = paddle.layer.fc(
+         act=paddle.activation.Linear(),
+         size=decoder_size,
+         bias_attr=False,
          input=encoded_vector)
    ```
 
-   1. 构造解码器RNN的初始状态。由于解码器需要预测时序目标序列，但在0时刻并没有初始值，所以我们希望对其进行初始化。这里采用的是将源语言序列逆序编码后的最后一个状态进行非线性映射，作为该初始值，即$c_0=h_T$。
+   - 构造解码器RNN的初始状态。由于解码器需要预测时序目标序列，但在0时刻并没有初始值，所以我们希望对其进行初始化。这里采用的是将源语言序列逆序编码后的最后一个状态进行非线性映射，作为该初始值，即$c_0=h_T$。
 
    ```python
    backward_first = paddle.layer.first_seq(input=src_backward)
-    with paddle.layer.mixed(
-            size=decoder_size, act=paddle.activation.Tanh()) as decoder_boot:
-        decoder_boot += paddle.layer.full_matrix_projection(
+   decoder_boot = paddle.layer.fc(
+         size=decoder_size,
+         act=paddle.activation.Tanh(),
+         bias_attr=False,
          input=backward_first)
    ```
 
-   1. 定义解码阶段每一个时间步的RNN行为，即根据当前时刻的源语言上下文向量$c_i$、解码器隐层状态$z_i$和目标语言中第$i$个词$u_i$，来预测第$i+1$个词的概率$p_{i+1}$。
+   - 定义解码阶段每一个时间步的RNN行为，即根据当前时刻的源语言上下文向量$c_i$、解码器隐层状态$z_i$和目标语言中第$i$个词$u_i$，来预测第$i+1$个词的概率$p_{i+1}$。
       - decoder_mem记录了前一个时间步的隐层状态$z_i$，其初始状态是decoder_boot。
       - context通过调用`simple_attention`函数，实现公式$c_i=\sum {j=1}^{T}a_{ij}h_j$。其中，enc_vec是$h_j$，enc_proj是$h_j$的映射（见3.1），权重$a_{ij}$的计算已经封装在`simple_attention`函数中。
       - decoder_inputs融合了$c_i$和当前目标词current_word（即$u_i$）的表示。
@@ -308,7 +295,6 @@ wmt14_reader = paddle.batch(
 
    ```python
    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
-
         decoder_mem = paddle.layer.memory(
             name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
 
@@ -317,10 +303,13 @@ wmt14_reader = paddle.batch(
             encoded_proj=enc_proj,
             decoder_state=decoder_mem)
 
-        with paddle.layer.mixed(size=decoder_size * 3) as decoder_inputs:
-            decoder_inputs += paddle.layer.full_matrix_projection(input=context)
-            decoder_inputs += paddle.layer.full_matrix_projection(
-                input=current_word)
+        decoder_inputs = paddle.layer.fc(
+            act=paddle.activation.Linear(),
+            size=decoder_size * 3,
+            bias_attr=False,
+            input=[context, current_word],
+            layer_attr=paddle.attr.ExtraLayerAttribute(
+                error_clipping_threshold=100.0))
 
         gru_step = paddle.layer.gru_step(
             name='gru_decoder',
@@ -328,24 +317,24 @@ wmt14_reader = paddle.batch(
             output_mem=decoder_mem,
             size=decoder_size)
 
-        with paddle.layer.mixed(
+        out = paddle.layer.mixed(
             size=target_dict_dim,
             bias_attr=True,
-                act=paddle.activation.Softmax()) as out:
-            out += paddle.layer.full_matrix_projection(input=gru_step)
+            act=paddle.activation.Softmax(),
+            input=paddle.layer.full_matrix_projection(input=gru_step))
         return out
    ```
 
-1. 定义解码器框架名字，和`gru_decoder_with_attention`函数的前两个输入。注意：这两个输入使用`StaticInput`，具体说明可见[StaticInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
+4. 定义解码器框架名字，和`gru_decoder_with_attention`函数的前两个输入。注意：这两个输入使用`StaticInput`，具体说明可见[StaticInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
 
     ```python
     decoder_group_name = "decoder_group"
-    group_input1 = paddle.layer.StaticInputV2(input=encoded_vector, is_seq=True)
-    group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
+    group_input1 = paddle.layer.StaticInput(input=encoded_vector)
+    group_input2 = paddle.layer.StaticInput(input=encoded_proj)
     group_inputs = [group_input1, group_input2]
     ```
 
-1. 训练模式下的解码器调用：
+5. 训练模式下的解码器调用：
 
    - 首先，将目标语言序列的词向量trg_embedding，直接作为训练模式下的current_word传给`gru_decoder_with_attention`函数。
    - 其次，使用`recurrent_group`函数循环调用`gru_decoder_with_attention`函数。
@@ -353,6 +342,7 @@ wmt14_reader = paddle.batch(
    - 最后，用多类交叉熵损失函数`classification_cost`来计算损失值。
 
    ```python
+   if not is_generating:
        trg_embedding = paddle.layer.embedding(
            input=paddle.layer.data(
                name='target_language_word',
@@ -377,30 +367,70 @@ wmt14_reader = paddle.batch(
        cost = paddle.layer.classification_cost(input=decoder, label=lbl)
     ```
 
-注意：我们提供的配置在Bahdanau的论文\[[4](#参考文献)\]上做了一些简化，可参考[issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133)。
+6. 生成模式下的解码器调用：
 
-### 参数定义
+   - 首先，在序列生成任务中，由于解码阶段的RNN总是引用上一时刻生成出的词的词向量，作为当前时刻的输入，因此，使用`GeneratedInput`来自动完成这一过程。具体说明可见[GeneratedInput文档](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入)。
+   - 其次，使用`beam_search`函数循环调用`gru_decoder_with_attention`函数，生成出序列id。
 
-首先依据模型配置的`cost`定义模型参数。
+   ```python
+   if is_generating:
+      # In generation, the decoder predicts a next target word based on
+      # the encoded source sequence and the previous generated target word.
 
-```python
-parameters = paddle.parameters.create(cost)
-```
+      # The encoded source sequence (encoder's output) must be specified by
+      # StaticInput, which is a read-only memory.
+      # Embedding of the previous generated word is automatically retrieved
+      # by GeneratedInputs initialized by a start mark <s>.
 
-可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
+       trg_embedding = paddle.layer.GeneratedInput(
+           size=target_dict_dim,
+           embedding_name='_target_language_embedding',
+           embedding_size=word_vector_dim)
+       group_inputs.append(trg_embedding)
 
-```python
-for param in parameters.keys():
-    print param
-```
+       beam_gen = paddle.layer.beam_search(
+           name=decoder_group_name,
+           step=gru_decoder_with_attention,
+           input=group_inputs,
+           bos_id=0,
+           eos_id=1,
+           beam_size=beam_size,
+           max_length=max_length)
+   ```
+
+注意：我们提供的配置在Bahdanau的论文\[[4](#参考文献)\]上做了一些简化，可参考[issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133)。
 
 ### 训练模型
 
-1. 构造trainer
+1. 参数定义
+
+    依据模型配置的`cost`定义模型参数。可以打印参数名字，如果在网络配置中没有指定名字，则默认生成。
+
+    ```python
+    if not is_generating:
+        parameters = paddle.parameters.create(cost)
+        for param in parameters.keys():
+            print param
+    ```
+
+2. 数据定义
+
+    获取wmt14的dataset reader。
+
+    ```python
+    if not is_generating:
+        wmt14_reader = paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
+            batch_size=5)
+    ```
+
+3. 构造trainer
 
     根据优化目标cost,网络拓扑结构和模型参数来构造出trainer用来训练，在构造时还需指定优化方法，这里使用最基本的SGD方法。
 
     ```python
+    if not is_generating:
         optimizer = paddle.optimizer.Adam(
             learning_rate=5e-5,
             regularization=paddle.optimizer.L2Regularization(rate=8e-4))
@@ -409,73 +439,104 @@ for param in parameters.keys():
                                      update_equation=optimizer)
     ```
 
-1. 构造event_handler
+4. 构造event_handler
 
-    可以通过自定义回调函数来评估训练过程中的各种状态，比如错误率等。下面的代码通过event.batch_id % 10 == 0 指定没10个batch打印一次日志，包含cost等信息。
+    可以通过自定义回调函数来评估训练过程中的各种状态，比如错误率等。下面的代码通过event.batch_id % 2 == 0 指定每2个batch打印一次日志，包含cost等信息。
 
     ```python
+    if not is_generating:
         def event_handler(event):
             if isinstance(event, paddle.event.EndIteration):
-            if event.batch_id % 10 == 0:
+                if event.batch_id % 2 == 0:
                     print "\nPass %d, Batch %d, Cost %f, %s" % (
                         event.pass_id, event.batch_id, event.cost, event.metrics)
-            else:
-                sys.stdout.write('.')
-                sys.stdout.flush()
     ```
 
-1. 启动训练：
+5. 启动训练
 
     ```python
+    if not is_generating:
         trainer.train(
-        reader=wmt14_reader,
-        event_handler=event_handler,
-        num_passes=2,
-        feeding=feeding)
+                reader=wmt14_reader, event_handler=event_handler, num_passes=2)
     ```
 
-训练开始后，可以观察到event_handler输出的日志如下：
+ 训练开始后，可以观察到event_handler输出的日志如下：
+ ```text
+ Pass 0, Batch 0, Cost 148.444983, {'classification_error_evaluator': 1.0}
+ .........
+ Pass 0, Batch 10, Cost 335.896802, {'classification_error_evaluator': 0.9325153231620789}
+ .........
+ ```
 
-```text
-Pass 0, Batch 0, Cost 148.444983, {'classification_error_evaluator': 1.0}
-.........
-Pass 0, Batch 10, Cost 335.896802, {'classification_error_evaluator': 0.9325153231620789}
-.........
-```
+### 生成模型
 
-    当`classification_error_evaluator`的值低于0.35的时候，表示训练成功。
+1. 加载预训练的模型
 
-## 应用模型
+    由于NMT模型的训练非常耗时，我们在50个物理节点（每节点含有2颗6核CPU）的集群中，花了5天时间训练了一个模型供大家直接下载使用。该模型大小为205MB，[BLEU评估](#BLEU评估)值为26.92。
 
-### 下载预训练的模型
+    ```python
+    if is_generating:
+        parameters = paddle.dataset.wmt14.model()
+    ```
+2. 数据定义
 
-由于NMT模型的训练非常耗时，我们在50个物理节点（每节点含有2颗6核CPU）的集群中，花了5天时间训练了16个pass，其中每个pass耗时7个小时。因此，我们提供了一个预先训练好的模型（pass-00012）供大家直接下载使用。该模型大小为205MB，在所有16个模型中有最高的[BLEU评估](#BLEU评估)值26.92。下载并解压模型的命令如下：
+    从wmt14的生成集中读取前3个样本作为源语言句子。
 
-```bash
-cd pretrained
-./wmt14_model.sh
-```
+    ```python
+    if is_generating:
+        gen_creator = paddle.dataset.wmt14.gen(dict_size)
+        gen_data = []
+        gen_num = 3
+        for item in gen_creator():
+            gen_data.append((item[0], ))
+            if len(gen_data) == gen_num:
+                break
+    ```
+3. 构造infer
 
-### BLEU评估
+    根据网络拓扑结构和模型参数构造出infer用来生成，在预测时还需要指定输出域`field`，这里使用生成句子的概率`prob`和句子中每个词的`id`。
 
-BLEU(Bilingual Evaluation understudy)是一种广泛使用的机器翻译自动评测指标，由IBM的watson研究中心于2002年提出\[[5](#参考文献)\]，基本出发点是：机器译文越接近专业翻译人员的翻译结果，翻译系统的性能越好。其中，机器译文与人工参考译文之间的接近程度，采用句子精确度（precision）的计算方法，即比较两者的n元词组相匹配的个数，匹配的个数越多，BLEU得分越好。
+    ```python
+    if is_generating:
+        beam_result = paddle.infer(
+            output_layer=beam_gen,
+            parameters=parameters,
+            input=gen_data,
+            field=['prob', 'id'])
+    ```
 
-[Moses](http://www.statmt.org/moses/) 是一个统计学的开源机器翻译系统，我们使用其中的 [multi-bleu.perl](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/multi-bleu.perl) 来做BLEU评估。下载脚本的命令如下：
-```bash
-./moses_bleu.sh
-```
-BLEU评估可以使用`eval_bleu`脚本如下，其中FILE为需要评估的文件名，BEAMSIZE为柱宽度，默认使用`data/wmt14/gen/ntst14.trg`作为标准的翻译结果。
-```bash
-./eval_bleu.sh FILE BEAMSIZE
-```
-本教程的具体命令如下：
-```bash
-./eval_bleu.sh gen_result 3
-```
-您会在屏幕上看到：
-```text
-BLEU = 26.92
-```
+4. 打印生成结果
+
+    根据源/目标语言字典，将源语言句子和`beam_size`个生成句子打印输出。
+
+    ```python
+    if is_generating:
+        # load the dictionary
+        src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
+
+        gen_sen_idx = np.where(beam_result[1] == -1)[0]
+        assert len(gen_sen_idx) == len(gen_data) * beam_size
+
+        # -1 is the delimiter of generated sequences.
+        # the first element of each generated sequence its length.
+        start_pos, end_pos = 1, 0
+        for i, sample in enumerate(gen_data):
+            print(" ".join([src_dict[w] for w in sample[0][1:-1]]))
+            for j in xrange(beam_size):
+                end_pos = gen_sen_idx[i * beam_size + j]
+                print("%.4f\t%s" % (beam_result[0][i][j], " ".join(
+                    trg_dict[w] for w in beam_result[1][start_pos:end_pos])))
+                start_pos = end_pos + 2
+            print("\n")
+    ```
+
+  生成开始后，可以观察到输出的日志如下：
+  ```text
+  Les <unk> se <unk> au sujet de la largeur des sièges alors que de grosses commandes sont en jeu
+  -19.0196        The <unk> will be rotated about the width of the seats , while large orders are at stake . <e>
+  -19.1131        The <unk> will be rotated about the width of the seats , while large commands are at stake . <e>
+  -19.5129        The <unk> will be rotated about the width of the seats , while large commands are at play . <e>
+  ```
 
 ## 总结
 
@@ -490,7 +551,7 @@ BLEU = 26.92
 5. Papineni K, Roukos S, Ward T, et al. [BLEU: a method for automatic evaluation of machine translation](http://dl.acm.org/citation.cfm?id=1073135)[C]//Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 2002: 311-318.
 
 <br/>
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。
 
 </div>
 <!-- You can change the lines below now. -->

07.machine_translation/index.en.html → 08.machine_translation/index.html

@@ -42,13 +42,13 @@
 <div id="markdown" style='display:none'>
 # Machine Translation
 
-The source code of this tutorial is live at [book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/07.machine_translation). Please refer to the [installation tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst) for getting started with Paddle.
+The source code of this tutorial is live at [book/machine_translation](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation). Please refer to the [book running tutorial](https://github.com/PaddlePaddle/book#running-the-book) for getting started with Paddle.
 
 ## Background
 
 Machine translation (MT) leverages computers to translate from one language to another. The language to be translated is referred to as the source language, while the language to be translated into is referred to as the target language. Thus, Machine translation is the process of translating from the source language to the target language. It is one of the most important research topics in the field of natural language processing.
 
-Early machine translation systems are mainly rule-based i.e. they rely on a language expert to specify the translation rules between the two languages. It is quite difficult to cover all the rules used in one languge. So it is quite a challenge for language experts to specify all possible rules in two or more different languages. Hence, a major challenge in conventional machine translation has been the difficulty in obtaining a complete rule set \[[1](#References)\]。
+Early machine translation systems are mainly rule-based i.e. they rely on a language expert to specify the translation rules between the two languages. It is quite difficult to cover all the rules used in one languge. So it is quite a challenge for language experts to specify all possible rules in two or more different languages. Hence, a major challenge in conventional machine translation has been the difficulty in obtaining a complete rule set \[[1](#References)\].
 
 
 To address the aforementioned problems, statistical machine translation techniques have been developed. These techniques learn the translation rules from a large corpus, instead of being designed by a language expert. While these techniques overcome the bottleneck of knowledge acquisition, there are still quite a lot of challenges, for example:
@@ -83,9 +83,9 @@ Let's consider an example of Chinese-to-English translation. The model is given
 ```
 After training and with a beam-search size of 3, the generated translations are as follows:
 ```text
-0 -5.36816   these are signs of hope and relief . <e>
-1 -6.23177   these are the light of hope and relief . <e>
-2 -7.7914  these are the light of hope and the relief of hope . <e>
+0 -5.36816   These are signs of hope and relief . <e>
+1 -6.23177   These are the light of hope and relief . <e>
+2 -7.7914  These are the light of hope and the relief of hope . <e>
 ```
 - The first column corresponds to the id of the generated sentence; the second column corresponds to the score of the generated sentence (in descending order), where a larger value indicates better quality; the last column corresponds to the generated sentence.
 - There are two special tokens: `<e>` denotes the end of a sentence while `<unk>` denotes unknown word, i.e., a word not in the training dictionary.
@@ -136,7 +136,7 @@ Figure 4. Encoder-Decoder Framework
 
 There are three steps for encoding a sentence:
 
-1. One-hot vector representation of a word: Each word $x_i$ in the source sentence $x=\left \{ x_1,x_2,...,x_T \right \}$ is represented as a vector $w_i\epsilon R^{\left | V \right |},i=1,2,...,T$   where $w_i$ has the same dimensionality as the size of the dictionary, i.e., $\left | V \right |$, and has an element of one at the location corresponding to the location of the word in the dictionary and zero elsewhere.
+1. One-hot vector representation of a word: Each word $x_i$ in the source sentence $x=\left \{ x_1,x_2,...,x_T \right \}$ is represented as a vector $w_i\epsilon \left \{ 0,1 \right \}^{\left | V \right |},i=1,2,...,T$   where $w_i$ has the same dimensionality as the size of the dictionary, i.e., $\left | V \right |$, and has an element of one at the location corresponding to the location of the word in the dictionary and zero elsewhere.
 
 2. Word embedding as a representation in the low-dimensional semantic space: There are two problems with one-hot vector representation
 
@@ -225,6 +225,12 @@ The goal is to maximize the probability of the generated sequence when using bea
 
 Note: $z_{i+1}$ and $p_{i+1}$ are computed the same way as in [Decoder](#Decoder). In generation mode, each step is greedy in so there is no guarantee of a global optimum.
 
+## BLEU Score
+
+Bilingual Evaluation understudy (BLEU) is a metric widely used for automatic machine translation proposed by IBM Watson Research Center in 2002\[[5](#References)\]. The closer the translation produced by a machine is to the translation produced by a human expert, the better the performance of the translation system.
+
+To measure the closeness between machine translation and human translation, sentence precision is used. It compares the number of matched n-grams. More matches will lead to higher BLEU scores.
+
 ## Data Preparation
 
 This tutorial uses a dataset from [WMT-14](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/), where [bitexts (after selection)](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/bitexts.tgz) is used as the training set, and [dev+test data](http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz) is used as test and generation set.
@@ -255,25 +261,8 @@ import paddle.v2 as paddle
 
 # train with a single CPU
 paddle.init(use_gpu=False, trainer_count=1)
-```
-
-### Define DataSet
-
-We will define dictionary size, and create [**data reader**](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader#python-data-reader-design-doc) for WMT-14 dataset.
-
-```python
-# source and target dict dim.
-dict_size = 30000
-
-feeding = {
-    'source_language_word': 0,
-    'target_language_word': 1,
-    'target_language_next_word': 2
-}
-wmt14_reader = paddle.batch(
-    paddle.reader.shuffle(
-        paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
-    batch_size=5)
+# False: training, True: generating
+is_generating = False
 ```
 
 ### Model Configuration
@@ -281,15 +270,18 @@ wmt14_reader = paddle.batch(
 1. Define some global variables
 
    ```python
+   dict_size = 30000 # dict dim
    source_dict_dim = dict_size # source language dictionary size
    target_dict_dim = dict_size # destination language dictionary size
    word_vector_dim = 512 # word embedding dimension
    encoder_size = 512 # hidden layer size of GRU in encoder
    decoder_size = 512 # hidden layer size of GRU in decoder
+   beam_size = 3 # expand width in beam search
+   max_length = 250 # a stop condition of sequence generation
    ```
 
-1. Implement Encoder as follows:
-   1. Input is a sequence of words represented by an integer word index sequence. So we define data layer of data type `integer_value_sequence`. The value range of each element in the sequence is `[0, source_dict_dim)`
+2. Implement Encoder as follows:
+   - Input is a sequence of words represented by an integer word index sequence. So we define data layer of data type `integer_value_sequence`. The value range of each element in the sequence is `[0, source_dict_dim)`
 
    ```python
    src_word_id = paddle.layer.data(
@@ -297,16 +289,14 @@ wmt14_reader = paddle.batch(
        type=paddle.data_type.integer_value_sequence(source_dict_dim))
    ```
 
-   1. Map the one-hot vector (represented by word index) into a word vector $\mathbf{s}$ in a low-dimensional semantic space
+   - Map the one-hot vector (represented by word index) into a word vector $\mathbf{s}$ in a low-dimensional semantic space
 
    ```python
    src_embedding = paddle.layer.embedding(
-        input=src_word_id,
-        size=word_vector_dim,
-        param_attr=paddle.attr.ParamAttr(name='_source_language_embedding'))
+       input=src_word_id, size=word_vector_dim)
    ```
 
-   1. Use bi-direcitonal GRU to encode the source language sequence, and concatenate the encoding outputs from the two GRUs to get $\mathbf{h}$
+   - Use bi-direcitonal GRU to encode the source language sequence, and concatenate the encoding outputs from the two GRUs to get $\mathbf{h}$
 
    ```python
    src_forward = paddle.networks.simple_gru(
@@ -316,27 +306,30 @@ wmt14_reader = paddle.batch(
    encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
    ```
 
-1. Implement Attention-based Decoder as follows:
+3. Implement Attention-based Decoder as follows:
 
-   1. Get a projection of the encoding (c.f. 2.3) of the source language sequence by passing it into a feed forward neural network
+   - Get a projection of the encoding (c.f. 2.3) of the source language sequence by passing it into a feed forward neural network
 
    ```python
-    with paddle.layer.mixed(size=decoder_size) as encoded_proj:
-        encoded_proj += paddle.layer.full_matrix_projection(
+   encoded_proj = paddle.layer.fc(
+         act=paddle.activation.Linear(),
+         size=decoder_size,
+         bias_attr=False,
          input=encoded_vector)
    ```
 
-   1. Use a non-linear transformation of the last hidden state of the backward GRU on the source language sentence as the initial state of the decoder RNN $c_0=h_T$
+   - Use a non-linear transformation of the last hidden state of the backward GRU on the source language sentence as the initial state of the decoder RNN $c_0=h_T$
 
    ```python
    backward_first = paddle.layer.first_seq(input=src_backward)
-    with paddle.layer.mixed(
-            size=decoder_size, act=paddle.activation.Tanh()) as decoder_boot:
-        decoder_boot += paddle.layer.full_matrix_projection(
+   decoder_boot = paddle.layer.fc(
+         size=decoder_size,
+         act=paddle.activation.Tanh(),
+         bias_attr=False,
          input=backward_first)
    ```
 
-   1. Define the computation in each time step for the decoder RNN, i.e., according to the current context vector $c_i$, hidden state for the decoder $z_i$ and the $i$-th word $u_i$ in the target language to predict the probability $p_{i+1}$ for the $i+1$-th word.
+   - Define the computation in each time step for the decoder RNN, i.e., according to the current context vector $c_i$, hidden state for the decoder $z_i$ and the $i$-th word $u_i$ in the target language to predict the probability $p_{i+1}$ for the $i+1$-th word.
 
       - decoder_mem records the hidden state $z_i$ from the previous time step, with an initial state as decoder_boot.
       - context is computed via `simple_attention` as $c_i=\sum {j=1}^{T}a_{ij}h_j$, where enc_vec is the projection of $h_j$ and enc_proj is the projection of $h_j$ (c.f. 3.1). $a_{ij}$ is calculated within `simple_attention`.
@@ -346,7 +339,6 @@ wmt14_reader = paddle.batch(
 
    ```python
    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
-
         decoder_mem = paddle.layer.memory(
             name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
 
@@ -355,10 +347,13 @@ wmt14_reader = paddle.batch(
             encoded_proj=enc_proj,
             decoder_state=decoder_mem)
 
-        with paddle.layer.mixed(size=decoder_size * 3) as decoder_inputs:
-            decoder_inputs += paddle.layer.full_matrix_projection(input=context)
-            decoder_inputs += paddle.layer.full_matrix_projection(
-                input=current_word)
+        decoder_inputs = paddle.layer.fc(
+            act=paddle.activation.Linear(),
+            size=decoder_size * 3,
+            bias_attr=False,
+            input=[context, current_word],
+            layer_attr=paddle.attr.ExtraLayerAttribute(
+            error_clipping_threshold=100.0))
 
         gru_step = paddle.layer.gru_step(
             name='gru_decoder',
@@ -366,24 +361,24 @@ wmt14_reader = paddle.batch(
             output_mem=decoder_mem,
             size=decoder_size)
 
-        with paddle.layer.mixed(
+        out = paddle.layer.fc(
             size=target_dict_dim,
             bias_attr=True,
-                act=paddle.activation.Softmax()) as out:
-            out += paddle.layer.full_matrix_projection(input=gru_step)
+            act=paddle.activation.Softmax(),
+            input=gru_step)
         return out
    ```
 
-1. Define the name for the decoder and the first two input for `gru_decoder_with_attention`. Note that `StaticInput` is used for the two inputs. Please refer to [StaticInput Document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入) for more details.
+4. Define the name for the decoder and the first two input for `gru_decoder_with_attention`. Note that `StaticInput` is used for the two inputs. Please refer to [StaticInput Document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/deep_model/rnn/recurrent_group_cn.md#输入) for more details.
 
     ```python
     decoder_group_name = "decoder_group"
-    group_input1 = paddle.layer.StaticInputV2(input=encoded_vector, is_seq=True)
-    group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
+    group_input1 = paddle.layer.StaticInput(input=encoded_vector)
+    group_input2 = paddle.layer.StaticInput(input=encoded_proj)
     group_inputs = [group_input1, group_input2]
     ```
 
-1. Training mode:
+5. Training mode:
 
    - word embedding from the target language trg_embedding is passed to `gru_decoder_with_attention` as current_word.
    - `recurrent_group` calls `gru_decoder_with_attention` in a recurrent way
@@ -391,6 +386,7 @@ wmt14_reader = paddle.batch(
    - multi-class cross-entropy (`classification_cost`) is used to calculate the cost
 
    ```python
+   if not is_generating:
        trg_embedding = paddle.layer.embedding(
            input=paddle.layer.data(
                name='target_language_word',
@@ -415,30 +411,69 @@ wmt14_reader = paddle.batch(
        cost = paddle.layer.classification_cost(input=decoder, label=lbl)
    ```
 
-Note: Our configuration is based on Bahdanau et al. \[[4](#Reference)\] but with a few simplifications. Please refer to [issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133) for more details.
+6. Generating mode:
 
-### Create Parameters
+   - the decoder predicts a next target word based on the the last generated target word. Embedding of the last generated word is automatically gotten by GeneratedInputs.
+   - `beam_search` calls `gru_decoder_with_attention` in a recurrent way, to predict sequence id.
 
-Create every parameter that `cost` layer needs.
+   ```python
+   if is_generating:
+       # In generation, the decoder predicts a next target word based on
+       # the encoded source sequence and the previous generated target word.
 
-```python
-parameters = paddle.parameters.create(cost)
-```
+       # The encoded source sequence (encoder's output) must be specified by
+       # StaticInput, which is a read-only memory.
+       # Embedding of the previous generated word is automatically retrieved
+       # by GeneratedInputs initialized by a start mark <s>.
 
-We can get parameter names. If the parameter name is not specified during model configuration, it will be generated.
+       trg_embedding = paddle.layer.GeneratedInput(
+           size=target_dict_dim,
+           embedding_name='_target_language_embedding',
+           embedding_size=word_vector_dim)
+       group_inputs.append(trg_embedding)
 
-```python
-for param in parameters.keys():
-    print param
-```
+       beam_gen = paddle.layer.beam_search(
+           name=decoder_group_name,
+           step=gru_decoder_with_attention,
+           input=group_inputs,
+           bos_id=0,
+           eos_id=1,
+           beam_size=beam_size,
+           max_length=max_length)
+   ```
+
+Note: Our configuration is based on Bahdanau et al. \[[4](#Reference)\] but with a few simplifications. Please refer to [issue #1133](https://github.com/PaddlePaddle/Paddle/issues/1133) for more details.
 
 ## Model Training
 
-1. Create trainer
+1. Create Parameters
+
+    Create every parameter that `cost` layer needs. And we can get parameter names. If the parameter name is not specified during model configuration, it will be generated.
+
+    ```python
+    if not is_generating:
+        parameters = paddle.parameters.create(cost)
+        for param in parameters.keys():
+            print param
+    ```
+
+2. Define DataSet
+
+    Create [**data reader**](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/design/reader#python-data-reader-design-doc) for WMT-14 dataset.
+
+    ```python
+    if not is_generating:
+        wmt14_reader = paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
+            batch_size=5)
+    ```
+3. Create trainer
 
     We need to tell trainer what to optimize, and how to optimize. Here trainer will optimize `cost` layer using stochastic gradient descent (SDG).
 
     ```python
+    if not is_generating:
         optimizer = paddle.optimizer.Adam(
             learning_rate=5e-5,
             regularization=paddle.optimizer.L2Regularization(rate=8e-4))
@@ -447,68 +482,103 @@ for param in parameters.keys():
                                      update_equation=optimizer)
     ```
 
-1. Define event handler
+4. Define event handler
 
     The event handler is a callback function invoked by trainer when an event happens. Here we will print log in event handler.
 
     ```python
+    if not is_generating:
         def event_handler(event):
             if isinstance(event, paddle.event.EndIteration):
-            if event.batch_id % 10 == 0:
+                if event.batch_id % 2 == 0:
                     print "\nPass %d, Batch %d, Cost %f, %s" % (
                         event.pass_id, event.batch_id, event.cost, event.metrics)
     ```
 
-1. Start training
+5. Start training
 
     ```python
+    if not is_generating:
         trainer.train(
-        reader=wmt14_reader,
-        event_handler=event_handler,
-        num_passes=2,
-        feeding=feeding)
+                reader=wmt14_reader, event_handler=event_handler, num_passes=2)
     ```
 
+  The training log is as follows:
   ```text
   Pass 0, Batch 0, Cost 247.408008, {'classification_error_evaluator': 1.0}
   Pass 0, Batch 10, Cost 212.058789, {'classification_error_evaluator': 0.8737863898277283}
   ...
   ```
 
-The model training is successful when the `classification_error_evaluator` is lower than 0.35.
-
 ## Model Usage
 
-### Download Pre-trained Model
+1. Download Pre-trained Model
 
-As the training of an NMT model is very time consuming, we provide a pre-trained model (pass-00012, ~205M). The model is trained with a cluster of 50 physical nodes (each node has two 6-core CPU). We trained 16 passes (taking about 5 days) with each pass taking about 7 hours. The provided model (pass-00012) has the highest [BLEU Score](#BLEU Score) of 26.92. Run the following command to download the model:
+    As the training of an NMT model is very time consuming, we provide a pre-trained model. The model is trained with a cluster of 50 physical nodes (each node has two 6-core CPU) over 5 days. The provided model has the [BLEU Score](#BLEU Score) of 26.92, and the size of 205M.
 
-```bash
-cd pretrained
-./wmt14_model.sh
-```
+    ```python
+    if is_generating:
+        parameters = paddle.dataset.wmt14.model()
+    ```
+2. Define DataSet
 
-### BLEU Evaluation
+    Get the first 3 samples of wmt14 generating set as the source language sequences.
 
-Bilingual Evaluation understudy (BLEU) is a metric widely used for automatic machine translation proposed by IBM Watson Research Center in 2002\[[5](#References)\]. The closer the translation produced by a machine is to the translation produced by a human expert, the better the performance of the translation system.
-To measure the closeness between machine translation and human translation, sentence precision is used. It compares the number of matched n-grams. More matches will lead to higher BLEU scores.
+   ```python
+   if is_generating:
+        gen_creator = paddle.dataset.wmt14.gen(dict_size)
+        gen_data = []
+        gen_num = 3
+        for item in gen_creator():
+            gen_data.append((item[0], ))
+            if len(gen_data) == gen_num:
+                break
+   ```
 
-[Moses](http://www.statmt.org/moses/) is an open-source machine translation system, we used [multi-bleu.perl](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/multi-bleu.perl) for BLEU evaluation. Run the following command for downloading:
-```bash
-./moses_bleu.sh
-```
-BLEU evaluation can be performed using the `eval_bleu` script as follows, where FILE is the name of the file to be evaluated, BEAMSIZE is the beam size value, and `data/wmt14/gen/ntst14.trg` is used as the standard translation in default.
-```bash
-./eval_bleu.sh FILE BEAMSIZE
-```
-Specificaly, the script is run as follows:
-```bash
-./eval_bleu.sh gen_result 3
-```
-You will see the following message as output:
-```text
-BLEU = 26.92
-```
+3. Create infer
+
+    Use inference interface `paddle.infer` return the prediction probability (see field `prob`) and labels (see field `id`) of each generated sequence.
+
+   ```python
+   if is_generating:
+        beam_result = paddle.infer(
+            output_layer=beam_gen,
+            parameters=parameters,
+            input=gen_data,
+            field=['prob', 'id'])
+   ```
+4. Print generated translation
+
+    Print sequence and its `beam_size` generated translation results based on the dictionary.
+
+   ```python
+   if is_generating:
+       # load the dictionary
+       src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
+
+       gen_sen_idx = np.where(beam_result[1] == -1)[0]
+       assert len(gen_sen_idx) == len(gen_data) * beam_size
+
+       # -1 is the delimiter of generated sequences.
+       # the first element of each generated sequence its length.
+       start_pos, end_pos = 1, 0
+       for i, sample in enumerate(gen_data):
+           print(" ".join([src_dict[w] for w in sample[0][1:-1]]))
+           for j in xrange(beam_size):
+               end_pos = gen_sen_idx[i * beam_size + j]
+               print("%.4f\t%s" % (beam_result[0][i][j], " ".join(
+                     trg_dict[w] for w in beam_result[1][start_pos:end_pos])))
+               start_pos = end_pos + 2
+           print("\n")
+   ```
+
+  The generating log is as follows:
+  ```text
+  Les <unk> se <unk> au sujet de la largeur des sièges alors que de grosses commandes sont en jeu
+  -19.0196        The <unk> will be rotated about the width of the seats , while large orders are at stake . <e>
+  -19.1131        The <unk> will be rotated about the width of the seats , while large commands are at stake . <e>
+  -19.5129        The <unk> will be rotated about the width of the seats , while large commands are at play . <e>
+  ```
 
 ## Summary
 
@@ -523,7 +593,7 @@ End-to-end neural machine translation is a recently developed way to perform mac
 5. Papineni K, Roukos S, Ward T, et al. [BLEU: a method for automatic evaluation of machine translation](http://dl.acm.org/citation.cfm?id=1073135)[C]//Proceedings of the 40th annual meeting on association for computational linguistics. Association for Computational Linguistics, 2002: 311-318.
 
 <br/>
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.
 
 </div>
 <!-- You can change the lines below now. -->

08.machine_translation/train.py

+import sys
+import numpy as np
+
+import paddle.v2 as paddle
+
+
+def save_model(parameters, save_path):
+    with open(save_path, 'w') as f:
+        parameters.to_tar(f)
+
+
+def seq_to_seq_net(source_dict_dim,
+                   target_dict_dim,
+                   is_generating,
+                   beam_size=3,
+                   max_length=250):
+    ### Network Architecture
+    word_vector_dim = 512  # dimension of word vector
+    decoder_size = 512  # dimension of hidden unit of GRU decoder
+    encoder_size = 512  # dimension of hidden unit of GRU encoder
+
+    #### Encoder
+    src_word_id = paddle.layer.data(
+        name='source_language_word',
+        type=paddle.data_type.integer_value_sequence(source_dict_dim))
+    src_embedding = paddle.layer.embedding(
+        input=src_word_id, size=word_vector_dim)
+    src_forward = paddle.networks.simple_gru(
+        input=src_embedding, size=encoder_size)
+    src_backward = paddle.networks.simple_gru(
+        input=src_embedding, size=encoder_size, reverse=True)
+    encoded_vector = paddle.layer.concat(input=[src_forward, src_backward])
+
+    #### Decoder
+    encoded_proj = paddle.layer.fc(
+        act=paddle.activation.Linear(),
+        size=decoder_size,
+        bias_attr=False,
+        input=encoded_vector)
+
+    backward_first = paddle.layer.first_seq(input=src_backward)
+
+    decoder_boot = paddle.layer.fc(
+        size=decoder_size,
+        act=paddle.activation.Tanh(),
+        bias_attr=False,
+        input=backward_first)
+
+    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
+
+        decoder_mem = paddle.layer.memory(
+            name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
+
+        context = paddle.networks.simple_attention(
+            encoded_sequence=enc_vec,
+            encoded_proj=enc_proj,
+            decoder_state=decoder_mem)
+
+        decoder_inputs = paddle.layer.fc(
+            act=paddle.activation.Linear(),
+            size=decoder_size * 3,
+            bias_attr=False,
+            input=[context, current_word],
+            layer_attr=paddle.attr.ExtraLayerAttribute(
+                error_clipping_threshold=100.0))
+
+        gru_step = paddle.layer.gru_step(
+            name='gru_decoder',
+            input=decoder_inputs,
+            output_mem=decoder_mem,
+            size=decoder_size)
+
+        out = paddle.layer.fc(
+            size=target_dict_dim,
+            bias_attr=True,
+            act=paddle.activation.Softmax(),
+            input=gru_step)
+        return out
+
+    decoder_group_name = 'decoder_group'
+    group_input1 = paddle.layer.StaticInput(input=encoded_vector)
+    group_input2 = paddle.layer.StaticInput(input=encoded_proj)
+    group_inputs = [group_input1, group_input2]
+
+    if not is_generating:
+        trg_embedding = paddle.layer.embedding(
+            input=paddle.layer.data(
+                name='target_language_word',
+                type=paddle.data_type.integer_value_sequence(target_dict_dim)),
+            size=word_vector_dim,
+            param_attr=paddle.attr.ParamAttr(name='_target_language_embedding'))
+        group_inputs.append(trg_embedding)
+
+        # For decoder equipped with attention mechanism, in training,
+        # target embeding (the groudtruth) is the data input,
+        # while encoded source sequence is accessed to as an unbounded memory.
+        # Here, the StaticInput defines a read-only memory
+        # for the recurrent_group.
+        decoder = paddle.layer.recurrent_group(
+            name=decoder_group_name,
+            step=gru_decoder_with_attention,
+            input=group_inputs)
+
+        lbl = paddle.layer.data(
+            name='target_language_next_word',
+            type=paddle.data_type.integer_value_sequence(target_dict_dim))
+        cost = paddle.layer.classification_cost(input=decoder, label=lbl)
+
+        return cost
+    else:
+        # In generation, the decoder predicts a next target word based on
+        # the encoded source sequence and the previous generated target word.
+
+        # The encoded source sequence (encoder's output) must be specified by
+        # StaticInput, which is a read-only memory.
+        # Embedding of the previous generated word is automatically retrieved
+        # by GeneratedInputs initialized by a start mark <s>.
+
+        trg_embedding = paddle.layer.GeneratedInput(
+            size=target_dict_dim,
+            embedding_name='_target_language_embedding',
+            embedding_size=word_vector_dim)
+        group_inputs.append(trg_embedding)
+
+        beam_gen = paddle.layer.beam_search(
+            name=decoder_group_name,
+            step=gru_decoder_with_attention,
+            input=group_inputs,
+            bos_id=0,
+            eos_id=1,
+            beam_size=beam_size,
+            max_length=max_length)
+
+        return beam_gen
+
+
+def main():
+    paddle.init(use_gpu=False, trainer_count=1)
+    is_generating = False
+
+    # source and target dict dim.
+    dict_size = 30000
+    source_dict_dim = target_dict_dim = dict_size
+
+    # train the network
+    if not is_generating:
+        # define optimize method and trainer
+        optimizer = paddle.optimizer.Adam(
+            learning_rate=5e-5,
+            regularization=paddle.optimizer.L2Regularization(rate=8e-4))
+
+        cost = seq_to_seq_net(source_dict_dim, target_dict_dim, is_generating)
+        parameters = paddle.parameters.create(cost)
+
+        trainer = paddle.trainer.SGD(
+            cost=cost, parameters=parameters, update_equation=optimizer)
+        # define data reader
+        wmt14_reader = paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.wmt14.train(dict_size), buf_size=8192),
+            batch_size=4)
+
+        # define event_handler callback
+        def event_handler(event):
+            if isinstance(event, paddle.event.EndIteration):
+                if event.batch_id % 10 == 0:
+                    print("\nPass %d, Batch %d, Cost %f, %s" %
+                          (event.pass_id, event.batch_id, event.cost,
+                           event.metrics))
+                else:
+                    sys.stdout.write('.')
+                    sys.stdout.flush()
+
+                if not event.batch_id % 10:
+                    save_path = 'params_pass_%05d_batch_%05d.tar' % (
+                        event.pass_id, event.batch_id)
+                    save_model(parameters, save_path)
+
+            if isinstance(event, paddle.event.EndPass):
+                # save parameters
+                save_path = 'params_pass_%05d.tar' % (event.pass_id)
+                save_model(parameters, save_path)
+
+        # start to train
+        trainer.train(
+            reader=wmt14_reader, event_handler=event_handler, num_passes=2)
+
+    # generate a english sequence to french
+    else:
+        # use the first 3 samples for generation
+        gen_data = []
+        gen_num = 3
+        for item in paddle.dataset.wmt14.gen(dict_size)():
+            gen_data.append([item[0]])
+            if len(gen_data) == gen_num:
+                break
+
+        beam_size = 3
+        beam_gen = seq_to_seq_net(source_dict_dim, target_dict_dim,
+                                  is_generating, beam_size)
+
+        # get the trained model, whose bleu = 26.92
+        parameters = paddle.dataset.wmt14.model()
+
+        # prob is the prediction probabilities, and id is the prediction word.
+        beam_result = paddle.infer(
+            output_layer=beam_gen,
+            parameters=parameters,
+            input=gen_data,
+            field=['prob', 'id'])
+
+        # load the dictionary
+        src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
+
+        gen_sen_idx = np.where(beam_result[1] == -1)[0]
+        assert len(gen_sen_idx) == len(gen_data) * beam_size
+
+        # -1 is the delimiter of generated sequences.
+        # the first element of each generated sequence its length.
+        start_pos, end_pos = 1, 0
+        for i, sample in enumerate(gen_data):
+            print(
+                " ".join([src_dict[w] for w in sample[0][1:-1]])
+            )  # skip the start and ending mark when printing the source sentence
+            for j in xrange(beam_size):
+                end_pos = gen_sen_idx[i * beam_size + j]
+                print("%.4f\t%s" % (beam_result[0][i][j], " ".join(
+                    trg_dict[w] for w in beam_result[1][start_pos:end_pos])))
+                start_pos = end_pos + 2
+            print("\n")
+
+
+if __name__ == '__main__':
+    main()

README.cn.md

+# 深度学习入门
+
+[![Build Status](https://travis-ci.org/PaddlePaddle/book.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/book)
+[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://book.paddlepaddle.org/)
+[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://book.paddlepaddle.org/index.cn.html)
+
+1. [新手入门](http://book.paddlepaddle.org/01.fit_a_line/index.cn.html)
+1. [识别数字](http://book.paddlepaddle.org/02.recognize_digits/index.cn.html)
+1. [图像分类](http://book.paddlepaddle.org/03.image_classification/index.cn.html)
+1. [词向量](http://book.paddlepaddle.org/04.word2vec/index.cn.html)
+1. [个性化推荐](http://book.paddlepaddle.org/05.recommender_system/index.cn.html)
+1. [情感分析](http://book.paddlepaddle.org/06.understand_sentiment/index.cn.html)
+1. [语义角色标注](http://book.paddlepaddle.org/07.label_semantic_roles/index.cn.html)
+1. [机器翻译](http://book.paddlepaddle.org/08.machine_translation/index.cn.html)
+
+更多学习内容请访问PaddlePaddle[视频课堂](http://bit.baidu.com/Course/datalist/column/117.html)。
+
+## 运行这本书
+
+您现在在看的这本书是一本“交互式”电子书 —— 每一章都可以运行在一个Jupyter Notebook里。
+
+我们把Jupyter、PaddlePaddle、以及各种被依赖的软件都打包进一个Docker image了。所以您不需要自己来安装各种软件，只需要安装Docker即可。对于各种Linux发行版，请参考 https://www.docker.com 。如果您使用[Windows](https://www.docker.com/docker-windows)或者[Mac](https://www.docker.com/docker-mac)，可以考虑[给Docker更多内存和CPU资源](http://stackoverflow.com/a/39720010/724872)。
+
+只需要在命令行窗口里运行：
+
+```bash
+docker run -d -p 8888:8888 paddlepaddle/book
+```
+
+会从DockerHub.com下载和运行本书的Docker image。阅读和在线编辑本书请在浏览器里访问 http://localhost:8888 。
+
+如果您访问DockerHub.com很慢，可以试试我们的另一个镜像docker.paddlepaddle.org：
+
+```bash
+docker run -d -p 8888:8888 docker.paddlepaddle.org/book
+```
+
+### 使用GPU训练
+
+本书默认使用CPU训练，若是要使用GPU训练，使用步骤会稍有变化。为了保证GPU驱动能够在镜像里面正常运行，我们推荐使用[nvidia-docker](https://github.com/NVIDIA/nvidia-docker)来运行镜像。请先安装nvidia-docker，之后请运行：
+
+```bash
+nvidia-docker run -d -p 8888:8888 paddlepaddle/book:latest-gpu
+```
+
+或者使用国内的镜像请运行：
+
+```bash
+nvidia-docker run -d -p 8888:8888 docker.paddlepaddle.org/book:latest-gpu
+```
+
+还需要将以下代码
+```python
+paddle.init(use_gpu=False, trainer_count=1)
+```
+
+改成：
+```python
+paddle.init(use_gpu=True, trainer_count=1)
+```
+
+
+## 贡献内容
+
+您要是能贡献新的章节那就太好了！请发Pull Requests把您写的章节加入到`/pending`下面的一个子目录里。当这一章稳定下来，我们一起把您的目录挪到根目录。
+
+为了写作、运行、调试，您需要安装Python 2.x和Go >1.5, 并可以用[脚本程序](https://github.com/PaddlePaddle/book/blob/develop/.tools/convert-markdown-into-ipynb-and-test.sh)来生成新的Docker image。
+
+**Note:** We also provide [English Readme](https://github.com/PaddlePaddle/book/blob/develop/README.md) for PaddlePaddle book.
+
+
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">知识共享 署名-相同方式共享 4.0 国际 许可协议</a>进行许可。

README.en.md

-# Deep Learning with PaddlePaddle
-
-1. [Fit a Line](http://book.paddlepaddle.org/01.fit_a_line/index.en.html)
-1. [Recognize Digits](http://book.paddlepaddle.org/02.recognize_digits/index.en.html)
-1. [Image Classification](http://book.paddlepaddle.org/03.image_classification/index.en.html)
-1. [Word to Vector](http://book.paddlepaddle.org/04.word2vec/index.en.html)
-1. [Understand Sentiment](http://book.paddlepaddle.org/05.understand_sentiment/index.en.html)
-1. [Label Semantic Roles](http://book.paddlepaddle.org/06.label_semantic_roles/index.en.html)
-1. [Machine Translation](http://book.paddlepaddle.org/07.machine_translation/index.en.html)
-1. [Recommender System](http://book.paddlepaddle.org/08.recommender_system/index.en.html)
-
-## Running the Book
-
-This book you are reading is interactive -- each chapter can run as a Jupyter Notebook.
-
-We packed this book, Jupyter, PaddlePaddle, and all dependencies into a Docker image. So you don't need to install anything except Docker. If you are using Windows, please follow [this installation guide](https://www.docker.com/docker-windows).  If you are running Mac, please follow [this](https://www.docker.com/docker-mac). For various Linux distros, please refer to https://www.docker.com.  If you are using Windows or Mac, you might want to give Docker [more memory and CPUs/cores](http://stackoverflow.com/a/39720010/724872).
-
-Just type
-
-```bash
-docker run -d -p 8888:8888 paddlepaddle/book
-```
-
-This command will download the pre-built Docker image from DockerHub.com and run it in a container.  Please direct your Web browser to http://localhost:8888 to read the book.
-
-If you are living in somewhere slow to access DockerHub.com, you might try our mirror server docker.paddlepaddle.org:
-
-```bash
-docker run -d -p 8888:8888 docker.paddlepaddle.org/book
-```
-
-## Contribute
-
-Your contribution is welcome!  Please feel free to file Pull Requests to add your chapter as a directory under `/pending`. Once it is going stable, the community would like to move it to `/`.
-
-To write, run, and debug your chapters, you will need Python 2.x, Go >1.5. You can build the Docker image using [this script](https://github.com/PaddlePaddle/book/blob/develop/.tools/convert-markdown-into-ipynb-and-test.sh).
-This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.

README.md

-# 深度学习入门
+# Deep Learning with PaddlePaddle
 
 [![Build Status](https://travis-ci.org/PaddlePaddle/book.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/book)
-[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://book.paddlepaddle.org/index.en.html)
-[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://book.paddlepaddle.org)
-[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)
+[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://book.paddlepaddle.org/)
+[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://book.paddlepaddle.org/index.cn.html)
 
-1. [新手入门](http://book.paddlepaddle.org/01.fit_a_line)
-1. [识别数字](http://book.paddlepaddle.org/02.recognize_digits)
-1. [图像分类](http://book.paddlepaddle.org/03.image_classification)
-1. [词向量](http://book.paddlepaddle.org/04.word2vec)
-1. [情感分析](http://book.paddlepaddle.org/05.understand_sentiment)
-1. [语义角色标注](http://book.paddlepaddle.org/06.label_semantic_roles)
-1. [机器翻译](http://book.paddlepaddle.org/07.machine_translation)
-1. [个性化推荐](http://book.paddlepaddle.org/08.recommender_system)
+1. [Fit a Line](http://book.paddlepaddle.org/01.fit_a_line/)
+1. [Recognize Digits](http://book.paddlepaddle.org/02.recognize_digits/)
+1. [Image Classification](http://book.paddlepaddle.org/03.image_classification/)
+1. [Word to Vector](http://book.paddlepaddle.org/04.word2vec/)
+1. [Recommender System](http://book.paddlepaddle.org/05.recommender_system/)
+1. [Understand Sentiment](http://book.paddlepaddle.org/06.understand_sentiment/)
+1. [Label Semantic Roles](http://book.paddlepaddle.org/07.label_semantic_roles/)
+1. [Machine Translation](http://book.paddlepaddle.org/08.machine_translation/)
 
-## 运行这本书
+## Running the Book
 
-您现在在看的这本书是一本“交互式”电子书 —— 每一章都可以运行在一个Jupyter Notebook里。
+This book you are reading is interactive -- each chapter can run as a Jupyter Notebook.
 
-我们把Jupyter、PaddlePaddle、以及各种被依赖的软件都打包进一个Docker image了。所以您不需要自己来安装各种软件，只需要安装Docker即可。对于各种Linux发行版，请参考 https://www.docker.com 。如果您使用[Windows](https://www.docker.com/docker-windows)或者[Mac](https://www.docker.com/docker-mac)，可以考虑[给Docker更多内存和CPU资源](http://stackoverflow.com/a/39720010/724872)。
+We packed this book, Jupyter, PaddlePaddle, and all dependencies into a Docker image. So you don't need to install anything except Docker. If you are using Windows, please follow [this installation guide](https://www.docker.com/docker-windows).  If you are running Mac, please follow [this](https://www.docker.com/docker-mac). For various Linux distros, please refer to https://www.docker.com.  If you are using Windows or Mac, you might want to give Docker [more memory and CPUs/cores](http://stackoverflow.com/a/39720010/724872).
 
-只需要在命令行窗口里运行：
+Just type
 
 ```bash
 docker run -d -p 8888:8888 paddlepaddle/book
+
 ```
 
-会从DockerHub.com下载和运行本书的Docker image。阅读和在线编辑本书请在浏览器里访问 http://localhost:8888 。
+This command will download the pre-built Docker image from DockerHub.com and run it in a container.  Please direct your Web browser to http://localhost:8888 to read the book.
 
-如果您访问DockerHub.com很慢，可以试试我们的另一个镜像docker.paddlepaddle.org：
+If you are living in somewhere slow to access DockerHub.com, you might try our mirror server docker.paddlepaddle.org:
 
 ```bash
 docker run -d -p 8888:8888 docker.paddlepaddle.org/book
+
+```
+
+### Training with GPU
+
+By default we are using CPU for training, if you want to train with GPU, the steps are a little different.
+
+To make sure GPU can be successfully used from inside container, please install [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Then run:
+
+```bash
+nvidia-docker run -d -p 8888:8888 paddlepaddle/book:latest-gpu
+
+```
+
+Or you can use the image registry mirror in China:
+
+```bash
+nvidia-docker run -d -p 8888:8888 docker.paddlepaddle.org/book:latest-gpu
+
 ```
 
-## 贡献内容
+Change the code in the chapter that you are reading from
+```python
+paddle.init(use_gpu=False, trainer_count=1)
+```
 
-您要是能贡献新的章节那就太好了！请发Pull Requests把您写的章节加入到`/pending`下面的一个子目录里。当这一章稳定下来，我们一起把您的目录挪到根目录。
+to:
+```python
+paddle.init(use_gpu=True, trainer_count=1)
+```
 
-为了写作、运行、调试，您需要安装Python 2.x和Go >1.5, 并可以用[脚本程序](https://github.com/PaddlePaddle/book/blob/develop/.tools/convert-markdown-into-ipynb-and-test.sh)来生成新的Docker image。
 
-**Note:** We also provide [English Readme](https://github.com/PaddlePaddle/book/blob/develop/README.en.md) for PaddlePaddle book.
+## Contribute
 
+Your contribution is welcome!  Please feel free to file Pull Requests to add your chapter as a directory under `/pending`. Once it is going stable, the community would like to move it to `/`.
 
-<a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/"><img alt="知识共享许可协议" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" /></a><br /><span xmlns:dct="http://purl.org/dc/terms/" href="http://purl.org/dc/dcmitype/Text" property="dct:title" rel="dct:type">本教程</span> 由 <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a> 创作，采用 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">知识共享 署名-非商业性使用-相同方式共享 4.0 国际 许可协议</a>进行许可。
+To write, run, and debug your chapters, you will need Python 2.x, Go >1.5. You can build the Docker image using [this script](https://github.com/PaddlePaddle/book/blob/develop/.tools/convert-markdown-into-ipynb-and-test.sh).
+This tutorial is contributed by <a xmlns:cc="http://creativecommons.org/ns#" href="http://book.paddlepaddle.org" property="cc:attributionName" rel="cc:attributionURL">PaddlePaddle</a>, and licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative Commons Attribution-ShareAlike 4.0 International License</a>.

index.en.html → index.cn.html

 <!DOCTYPE html>
-<html lang="en">
+<html lang="zh-Hans">
 <head>
     <meta charset="UTF-8">
-    <title>Deep Learning 101</title>
+    <title>深度学习入门</title>
     <link href="http://cdn.bootcss.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" rel="stylesheet">
     <script src="http://cdn.bootcss.com/tether/1.4.0/js/tether.js"></script>
     <script src="http://cdn.bootcss.com/jquery/3.1.0/jquery.slim.js"></script>
@@ -26,6 +26,10 @@
 }
 
 
+.left-panel .card-header a {
+    font-size: 14px;
+}
+
 
 .left-panel .card-block a:not(.click_active) {
     background: #e4e6e9;
@@ -70,17 +74,17 @@
         <ul class="nav navbar-nav">
           <li class="nav-item">
             <a class="nav-link" href="http://paddlepaddle.org">
-                Home
-            </a>
-          </li>
-          <li class="nav-item ">
-            <a class="nav-link" href="./index.html">
-                中文
+                首页
             </a>
           </li>
           <li class="nav-item active">
             <a class="nav-link" href="#">
-                Deep Learning 101
+                深度学习入门
+            </a>
+          </li>
+          <li class="nav-item ">
+            <a class="nav-link" href="./index.html">
+                English
             </a>
           </li>
           <li class="nav-item">
@@ -96,7 +100,7 @@
                     <div class="card-header" role="tab" id="headingOne">
                       <h5 class="mb-0">
                         <a aria-expanded="true" aria-controls="collapseOne">
-                            Deep Learning 101<span class="sr-only">(current)</span>
+                            深度学习入门
                         </a>
                       </h5>
                     </div>
@@ -104,36 +108,36 @@
                       <div class="card-block pl-0 pr-0 pt-0 pb-0">
                         <div class="list-group ">
                           
-                            <a href="./01.fit_a_line/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Linear Regression
+                            <a href="./01.fit_a_line/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                新手入门
                             </a>
                           
-                            <a href="./02.recognize_digits/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Recognize Digits
+                            <a href="./02.recognize_digits/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                识别数字
                             </a>
                           
-                            <a href="./03.image_classification/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Image Classification
+                            <a href="./03.image_classification/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                图像分类
                             </a>
                           
-                            <a href="./04.word2vec/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Word2Vec
+                            <a href="./04.word2vec/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                词向量
                             </a>
                           
-                            <a href="./05.understand_sentiment/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Sentiment Analysis
+                            <a href="./05.recommender_system/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                个性化推荐
                             </a>
                           
-                            <a href="./06.label_semantic_roles/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Semantic Role Labeling
+                            <a href="./06.understand_sentiment/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                情感分析
                             </a>
                           
-                            <a href="./07.machine_translation/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Machine Translation
+                            <a href="./07.label_semantic_roles/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                语义角色标注
                             </a>
                           
-                            <a href="./08.recommender_system/index.en.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                Personalized Recommendation
+                            <a href="./08.machine_translation/index.cn.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                机器翻译
                             </a>
                           
                         </div>
@@ -143,7 +147,7 @@
                 </div>
             </div>
             <div class="col">
-                <iframe src="./01.fit_a_line/index.en.html" style="border: none; overflow-y : hidden" width="100%" height="100%" name="content_iframe" id="content_iframe">
+                <iframe src="./01.fit_a_line/index.cn.html" style="border: none; overflow-y : hidden" width="100%" height="100%" name="content_iframe" id="content_iframe">
                 </iframe>
             </div>
         </div>

index.html

 <!DOCTYPE html>
-<html lang="zh-Hans">
+<html lang="en">
 <head>
     <meta charset="UTF-8">
-    <title>深度学习入门</title>
+    <title>Deep Learning 101</title>
     <link href="http://cdn.bootcss.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css" rel="stylesheet">
     <script src="http://cdn.bootcss.com/tether/1.4.0/js/tether.js"></script>
     <script src="http://cdn.bootcss.com/jquery/3.1.0/jquery.slim.js"></script>
@@ -26,10 +26,6 @@
 }
 
 
-.left-panel .card-header a {
-    font-size: 14px;
-}
-
 
 .left-panel .card-block a:not(.click_active) {
     background: #e4e6e9;
@@ -74,17 +70,17 @@
         <ul class="nav navbar-nav">
           <li class="nav-item">
             <a class="nav-link" href="http://paddlepaddle.org">
-                首页
+                Home
             </a>
           </li>
-          <li class="nav-item active">
-            <a class="nav-link" href="#">
-                深度学习入门
+          <li class="nav-item ">
+            <a class="nav-link" href="./index.cn.html">
+                中文
             </a>
           </li>
-          <li class="nav-item ">
-            <a class="nav-link" href="./index.en.html">
-                English
+          <li class="nav-item active">
+            <a class="nav-link" href="#">
+                Deep Learning 101
             </a>
           </li>
           <li class="nav-item">
@@ -100,7 +96,7 @@
                     <div class="card-header" role="tab" id="headingOne">
                       <h5 class="mb-0">
                         <a aria-expanded="true" aria-controls="collapseOne">
-                            深度学习入门
+                            Deep Learning 101<span class="sr-only">(current)</span>
                         </a>
                       </h5>
                     </div>
@@ -109,35 +105,35 @@
                         <div class="list-group ">
                           
                             <a href="./01.fit_a_line/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                新手入门
+                                Linear Regression
                             </a>
                           
                             <a href="./02.recognize_digits/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                识别数字
+                                Recognize Digits
                             </a>
                           
                             <a href="./03.image_classification/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                图像分类
+                                Image Classification
                             </a>
                           
                             <a href="./04.word2vec/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                词向量
+                                Word2Vec
                             </a>
                           
-                            <a href="./05.understand_sentiment/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                情感分析
+                            <a href="./05.recommender_system/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                Personalized Recommendation
                             </a>
                           
-                            <a href="./06.label_semantic_roles/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                语义角色标注
+                            <a href="./06.understand_sentiment/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                Sentiment Analysis
                             </a>
                           
-                            <a href="./07.machine_translation/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                机器翻译
+                            <a href="./07.label_semantic_roles/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                Semantic Role Labeling
                             </a>
                           
-                            <a href="./08.recommender_system/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
-                                个性化推荐
+                            <a href="./08.machine_translation/index.html" target="content_iframe" class="list-group-item list-group-item-action" style="border: 0px; border-bottom: 2px solid #dddfe3;">
+                                Machine Translation
                             </a>
                           
                         </div>

mnist-client/.eslintrc

+{
+    "rules": {
+        "indent": [
+            2,
+            4
+        ],
+        "quotes": [
+            2,
+            "single"
+        ],
+        "linebreak-style": [
+            2,
+            "unix"
+        ],
+        "semi": [
+            2,
+            "always"
+        ]
+    },
+    "env": {
+        "es6": true,
+        "node": true,
+        "browser": true
+    },
+    "extends": "eslint:recommended"
+}
\ No newline at end of file

mnist-client/.gitignore

+venv
+*.pyc
+node_modules
+static/js/main.js
+index.html

mnist-client/Procfile

+web: gunicorn main:app --log-file=-

mnist-client/README.md

+# MNIST classification by PaddlePaddle
+
+Forked from https://github.com/sugyan/tensorflow-mnist
+
+![screencast](https://cloud.githubusercontent.com/assets/80381/11339453/f04f885e-923c-11e5-8845-33c16978c54d.gif)
+
+## Build
+
+    $ docker build -t paddle-mnist .
+
+## Usage
+
+
+1. Download `inference_topology.pkl` and `param.tar` to current directory
+1. Run following commands:
+```bash
+docker run -v `pwd`:/data -d -p 8000:80 -e WITH_GPU=0 paddlepaddle/book:serve
+docker run -it -p 5000:5000 paddlepaddle/book:mnist
+```
+1. Visit http://localhost:5000

mnist-client/app.json

+{
+    "name": "paddlepaddle-mnist",
+    "buildpacks": [
+        { "url": "https://github.com/heroku/heroku-buildpack-nodejs" },
+        { "url": "https://github.com/heroku/heroku-buildpack-python" }
+    ]
+}

mnist-client/gulpfile.js

+var gulp = require('gulp');
+var babel = require('gulp-babel');
+var sourcemaps = require('gulp-sourcemaps');
+var uglify = require('gulp-uglify');
+
+gulp.task('build', function() {
+    return gulp.src('src/js/*.js')
+        .pipe(babel({ presets: ['es2015'] }))
+        .pipe(sourcemaps.init({ loadMaps: true }))
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('static/js'));
+});
+
+gulp.task('watch', function() {
+    gulp.watch('src/js/*.js', ['build']);
+});
+
+gulp.task('default', ['build']);

mnist-client/main.py

+from flask import Flask, jsonify, render_template, request
+
+# webapp
+app = Flask(__name__)
+
+
+@app.route('/')
+def main():
+    return render_template('index.html')
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000, threaded=True)

mnist-client/package.json

+{
+  "name": "paddlepaddle-mnist",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "postinstall": "gulp"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sugyan/tensorflow-mnist.git"
+  },
+  "engines": {
+    "node": "6.x"
+  },
+  "dependencies": {
+    "babel-preset-es2015": "^6.1.18",
+    "bootstrap": "^3.3.5",
+    "gulp": "^3.9.0",
+    "gulp-babel": "^6.1.0",
+    "gulp-sourcemaps": "^1.6.0",
+    "gulp-uglify": "^1.5.1",
+    "jquery": "^2.1.4"
+  }
+}

mnist-client/requirements.txt

+Flask==0.12

mnist-client/runtime.txt

+python-3.6.0
\ No newline at end of file

mnist-client/src/js/main.js

+/* global $ */
+class Main {
+    constructor() {
+        this.canvas = document.getElementById('main');
+        this.input = document.getElementById('input');
+        this.canvas.width  = 449; // 16 * 28 + 1
+        this.canvas.height = 449; // 16 * 28 + 1
+        this.ctx = this.canvas.getContext('2d');
+        this.canvas.addEventListener('mousedown', this.onMouseDown.bind(this));
+        this.canvas.addEventListener('mouseup',   this.onMouseUp.bind(this));
+        this.canvas.addEventListener('mousemove', this.onMouseMove.bind(this));
+        this.initialize();
+    }
+    initialize() {
+        this.ctx.fillStyle = '#FFFFFF';
+        this.ctx.fillRect(0, 0, 449, 449);
+        this.ctx.lineWidth = 1;
+        this.ctx.strokeRect(0, 0, 449, 449);
+        this.ctx.lineWidth = 0.05;
+        for (var i = 0; i < 27; i++) {
+            this.ctx.beginPath();
+            this.ctx.moveTo((i + 1) * 16,   0);
+            this.ctx.lineTo((i + 1) * 16, 449);
+            this.ctx.closePath();
+            this.ctx.stroke();
+
+            this.ctx.beginPath();
+            this.ctx.moveTo(  0, (i + 1) * 16);
+            this.ctx.lineTo(449, (i + 1) * 16);
+            this.ctx.closePath();
+            this.ctx.stroke();
+        }
+        this.drawInput();
+        $('#output td').text('').removeClass('success');
+    }
+    onMouseDown(e) {
+        this.canvas.style.cursor = 'default';
+        this.drawing = true;
+        this.prev = this.getPosition(e.clientX, e.clientY);
+    }
+    onMouseUp() {
+        this.drawing = false;
+        this.drawInput();
+    }
+    onMouseMove(e) {
+        if (this.drawing) {
+            var curr = this.getPosition(e.clientX, e.clientY);
+            this.ctx.lineWidth = 16;
+            this.ctx.lineCap = 'round';
+            this.ctx.beginPath();
+            this.ctx.moveTo(this.prev.x, this.prev.y);
+            this.ctx.lineTo(curr.x, curr.y);
+            this.ctx.stroke();
+            this.ctx.closePath();
+            this.prev = curr;
+        }
+    }
+    getPosition(clientX, clientY) {
+        var rect = this.canvas.getBoundingClientRect();
+        return {
+            x: clientX - rect.left,
+            y: clientY - rect.top
+        };
+    }
+    drawInput() {
+        var ctx = this.input.getContext('2d');
+        var img = new Image();
+        img.onload = () => {
+            var inputs = [];
+            var small = document.createElement('canvas').getContext('2d');
+            small.drawImage(img, 0, 0, img.width, img.height, 0, 0, 28, 28);
+            var data = small.getImageData(0, 0, 28, 28).data;
+            for (var i = 0; i < 28; i++) {
+                for (var j = 0; j < 28; j++) {
+                    var n = 4 * (i * 28 + j);
+                    inputs[i * 28 + j] = (data[n + 0] + data[n + 1] + data[n + 2]) / 3;
+                    ctx.fillStyle = 'rgb(' + [data[n + 0], data[n + 1], data[n + 2]].join(',') + ')';
+                    ctx.fillRect(j * 5, i * 5, 5, 5);
+                }
+            }
+            if (Math.min(...inputs) === 255) {
+                return;
+            }
+	    for (var i = 0; i < 784; i++) {
+		if (inputs[i] == 255) {
+		    // background
+		    inputs[i] = -1.0
+		} else {
+		    inputs[i] = 1.0
+		}
+	    }
+            $.ajax({
+                url: 'http://localhost:8000/',
+                method: 'POST',
+                contentType: 'application/json',
+                data: JSON.stringify({"img":inputs}),
+                success: (data) => {
+		    data = data["data"][0]
+                    var max = 0;
+                    var max_index = 0;
+                    for (let j = 0; j < 10; j++) {
+                        var value = Math.round(data[j] * 1000);
+                        if (value > max) {
+                            max = value;
+                            max_index = j;
+                        }
+                        var digits = String(value).length;
+                        for (var k = 0; k < 3 - digits; k++) {
+                            value = '0' + value;
+                        }
+                        var text = '0.' + value;
+                        if (value > 999) {
+                            text = '1.000';
+                        }
+                        $('#output tr').eq(j + 1).find('td').text(text);
+                    }
+                    for (let j = 0; j < 10; j++) {
+                        if (j === max_index) {
+                            $('#output tr').eq(j + 1).find('td').addClass('success');
+                        } else {
+                            $('#output tr').eq(j + 1).find('td').removeClass('success');
+                        }
+                    }
+                }
+            });
+        };
+        img.src = this.canvas.toDataURL();
+    }
+}
+
+$(() => {
+    var main = new Main();
+    $('#clear').click(() => {
+        main.initialize();
+    });
+});

mnist-client/static/css/bootstrap.min.css

+../../node_modules/bootstrap/dist/css/bootstrap.min.css
\ No newline at end of file

mnist-client/static/js/jquery.min.js

+../../node_modules/jquery/dist/jquery.min.js
\ No newline at end of file

mnist-client/templates/index.html

+<!DOCTYPE html>
+<html>
+  <head>
+    <title>MNIST</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/bootstrap.min.css') }}">
+    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery.min.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/main.js') }}"></script>
+  </head>
+  <body>
+    <a href="https://github.com/sugyan/tensorflow-mnist"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://camo.githubusercontent.com/a6677b08c955af8400f44c6298f40e7d19cc5b2d/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f677261795f3664366436642e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_gray_6d6d6d.png"></a>
+    <div class="container">
+      <h1>MNIST</h1>
+      <div class="row">
+        <div class="col-md-6">
+          <p>draw a digit here!</p>
+          <canvas id="main"></canvas>
+          <p>
+            <button id="clear" class="btn btn-default">clear</button>
+          </p>
+        </div>
+        <div class="col-md-6">
+          <p>input:</p>
+          <canvas id="input" style="border:1px solid" width="140" height="140"></canvas>
+          <hr>
+          <p>output:</p>
+          <table id="output" class="table">
+            <tr>
+              <th class="col-md-1">class</th>
+              <th class="col-md-2">confidence</th>
+            </tr>
+            <tr>
+              <th>0</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>1</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>2</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>3</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>4</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>5</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>6</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>7</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>8</th>
+              <td></td>
+            </tr>
+            <tr>
+              <th>9</th>
+              <td></td>
+            </tr>
+          </table>
+        </div>
+      </div>
+    </div>
+  </body>
+</html>

serve/.gitignore

+*~
+.idea
+index.html

serve/README.md

+# PaddlePaddle Serving Example
+
+
+## Build
+
+    $ docker build -t serve .
+
+## Run
+
+    $ docker run -v `pwd`:/data -it -p 8000:80 -e WITH_GPU=0 paddlepaddle/book:serve
+    $ curl -H "Content-Type: application/json" -X POST -d '{"img":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]}' http://localhost:8000/

serve/main.py

+import os
+import traceback
+
+import paddle.v2 as paddle
+from flask import Flask, jsonify, request
+from flask_cors import CORS
+
+tarfn = os.getenv('PARAMETER_TAR_PATH', None)
+
+if tarfn is None:
+    raise ValueError(
+        "please specify parameter tar file path with environment variable PARAMETER_TAR_PATH"
+    )
+
+topology_filepath = os.getenv('TOPOLOGY_FILE_PATH', None)
+
+if topology_filepath is None:
+    raise ValueError(
+        "please specify topology file path with environment variable TOPOLOGY_FILE_PATH"
+    )
+
+with_gpu = os.getenv('WITH_GPU', '0') != '0'
+
+port = int(os.getenv('PORT', '80'))
+
+app = Flask(__name__)
+CORS(app)
+
+
+def errorResp(msg):
+    return jsonify(code=-1, message=msg)
+
+
+def successResp(data):
+    return jsonify(code=0, message="success", data=data)
+
+
+@app.route('/', methods=['POST'])
+def infer():
+    global inferer
+    try:
+        feeding = {}
+        d = []
+        for i, key in enumerate(request.json):
+            d.append(request.json[key])
+            feeding[key] = i
+        r = inferer.infer([d], feeding=feeding)
+    except:
+        trace = traceback.format_exc()
+        return errorResp(trace)
+    return successResp(r.tolist())
+
+
+if __name__ == '__main__':
+    paddle.init(use_gpu=with_gpu)
+    with open(tarfn) as param_f, open(topology_filepath) as topo_f:
+        params = paddle.parameters.Parameters.from_tar(param_f)
+        inferer = paddle.inference.Inference(parameters=params, fileobj=topo_f)
+    print 'serving on port', port
+    app.run(host='0.0.0.0', port=port, threaded=True)

serve/requirements.txt

+Flask==0.12.2
+Flask-CORS==3.0.3