merge develop

WORKSPACE

-# External dependency to Google protobuf.
-http_archive(
-    name="protobuf",
-    url="http://github.com/google/protobuf/archive/v3.1.0.tar.gz",
-    sha256="0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7",
-    strip_prefix="protobuf-3.1.0")
-
-# External dependency to gtest 1.7.0.  This method comes from
-# https://www.bazel.io/versions/master/docs/tutorial/cpp.html.
-new_http_archive(
-    name="gtest",
-    url="https://github.com/google/googletest/archive/release-1.7.0.zip",
-    sha256="b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0",
-    build_file="third_party/gtest.BUILD",
-    strip_prefix="googletest-release-1.7.0")
-
-# External dependency to gflags.  This method comes from
-# https://github.com/gflags/example/blob/master/WORKSPACE.
-new_git_repository(
-    name="gflags",
-    tag="v2.2.0",
-    remote="https://github.com/gflags/gflags.git",
-    build_file="third_party/gflags.BUILD")
-
-# External dependency to glog.  This method comes from
-# https://github.com/reyoung/bazel_playground/blob/master/WORKSPACE
-new_git_repository(
-    name="glog",
-    remote="https://github.com/google/glog.git",
-    commit="b6a5e0524c28178985f0d228e9eaa43808dbec3c",
-    build_file="third_party/glog.BUILD")

demo/quick_start/api_predict.sh

@@ -17,7 +17,7 @@ set -e
 #Note the default model is pass-00002, you shold make sure the model path
 #exists or change the mode path.
 #only test on trainer_config.lr.py
-model=output/pass-00001/
+model=output/model/pass-00001/
 config=trainer_config.lr.py
 label=data/labels.list
 dict=data/dict.txt

demo/quick_start/cluster/cluster_train.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+# Should run pserver.sh before run this script.
+bin_dir=$(cd `dirname $0`; pwd)
+home_dir=$(cd "${bin_dir}/.."; pwd)
+source "$bin_dir/env.sh"
+
+model_dir="$bin_dir/output"
+log_file="$bin_dir/train.log"
+
+pushd "$home_dir"
+cfg=trainer_config.lr.py
+paddle train \
+  --config=$cfg \
+  --save_dir=${model_dir} \
+  --trainer_count=4 \
+  --local=0 \
+  --log_period=100 \
+  --num_passes=15 \
+  --use_gpu=false \
+  --show_parameter_stats_period=100 \
+  --test_all_data_in_one_period=1 \
+  --num_gradient_servers=1 \
+  --nics=`get_nics` \
+  --port=7164 \
+  --ports_num=1 \
+  --pservers="127.0.0.1" \
+  --comment="paddle_trainer" \
+  2>&1 | tee "$log_file"
+popd

demo/quick_start/cluster/env.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+
+function get_nics() {
+  machine=`uname -s`
+  local nics=""
+  if [ "$machine" == "Linux" ]; then
+    nics="lo"
+  elif [ "$machine" == "Darwin" ]; then
+    nics="lo0"
+  else
+    nics="unsupport"
+  fi
+  echo $nics
+}

demo/quick_start/cluster/pserver.sh

+#!/bin/bash
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+bin_dir=$(cd `dirname $0`; pwd)
+source "$bin_dir/env.sh"
+
+paddle pserver \
+  --nics=`get_nics` \
+  --port=7164 \
+  --ports_num=1 \
+  --ports_num_for_sparse=1 \
+  --num_gradient_servers=1 \
+  --comment="paddle_pserver" \
+  2>&1 | tee 'pserver.log'

doc/getstarted/build_and_install/docker_install_en.rst

@@ -65,16 +65,13 @@ The general development workflow with Docker and Bazel is as follows:
        --name paddle   \
        -p 2022:22      \
        -v $PWD:/paddle \
-       -v $HOME/.cache/bazel:/root/.cache/bazel \
        paddle:dev
 
    where :code:`-d` makes the container running in background,
    :code:`--name paddle` allows us to run a nginx container to serve
    documents in this container, :code:`-p 2022:22` allows us to SSH
    into this container, :code:`-v $PWD:/paddle` shares the source code
-   on the host with the container, :code:`-v
-   $HOME/.cache/bazel:/root/.cache/bazel` shares Bazel cache on the
-   host with the container.
+   on the host with the container.
 
 4. SSH into the container:
 
@@ -94,13 +91,6 @@ The general development workflow with Docker and Bazel is as follows:
       make -j `nproc`
       CTEST_OUTPUT_ON_FAILURE=1 ctest
 
-   or Bazel in the container:
-
-   .. code-block:: bash
-
-      cd /paddle
-      bazel test ...
-
 
 CPU-only and GPU Images
 -----------------------

doc/tutorials/gan/index_en.md

+# Generative Adversarial Networks (GAN) 
+
+This demo implements GAN training described in the original [GAN paper](https://arxiv.org/abs/1406.2661) and deep convolutional generative adversarial networks [DCGAN paper](https://arxiv.org/abs/1511.06434).
+
+The high-level structure of GAN is shown in Figure. 1 below. It is composed of two major parts: a generator and a discriminator, both of which are based on neural networks. The generator takes in some kind of noise with a known distribution and transforms it into an image. The discriminator takes in an image and determines whether it is artificially generated by the generator or a real image. So the generator and the discriminator are in a competitive game in which generator is trying to generate image to look as real as possible to fool the discriminator, while the discriminator is trying to distinguish between real and fake images. 
+
+<p align="center">
+    <img src="./gan.png" width="500" height="300"> 
+</p>
+<p align="center">
+    Figure 1. GAN-Model-Structure
+    <a href="https://ishmaelbelghazi.github.io/ALI/">figure credit</a>
+</p>
+
+The generator and discriminator take turn to be trained using SGD. The objective function of the generator is for its generated images being classified as real by the discriminator, and the objective function of the discriminator is to correctly classify real and fake images. When the GAN model is trained to converge to the equilibrium state, the generator will transform the given noise distribution to the distribution of real images, and the discriminator will not be able to distinguish between real and fake images at all. 
+
+## Implementation of GAN Model Structure
+Since GAN model involves multiple neural networks, it requires to use paddle python API. So the code walk-through below can also partially serve as an introduction to the usage of Paddle Python API.
+
+There are three networks defined in gan_conf.py, namely **generator_training**, **discriminator_training** and **generator**. The relationship to the model structure we defined above is that **discriminator_training** is the discriminator, **generator** is the generator, and the **generator_training** combined the generator and discriminator since training generator would require the discriminator to provide loss function. This relationship is described in the following code:
+```python
+if is_generator_training:
+    noise = data_layer(name="noise", size=noise_dim)
+    sample = generator(noise)
+
+if is_discriminator_training:
+    sample = data_layer(name="sample", size=sample_dim)
+
+if is_generator_training or is_discriminator_training:
+    label = data_layer(name="label", size=1)
+    prob = discriminator(sample)
+    cost = cross_entropy(input=prob, label=label)
+    classification_error_evaluator(
+        input=prob, label=label, name=mode + '_error')
+    outputs(cost)
+
+if is_generator:
+    noise = data_layer(name="noise", size=noise_dim)
+    outputs(generator(noise))
+```
+
+In order to train the networks defined in gan_conf.py, one first needs to initialize a Paddle environment, parse the config, create GradientMachine from the config and create trainer from GradientMachine as done in the code chunk below:
+```python
+import py_paddle.swig_paddle as api
+# init paddle environment
+api.initPaddle('--use_gpu=' + use_gpu, '--dot_period=10',
+               '--log_period=100', '--gpu_id=' + args.gpu_id,
+               '--save_dir=' + "./%s_params/" % data_source)
+
+# Parse config
+gen_conf = parse_config(conf, "mode=generator_training,data=" + data_source)
+dis_conf = parse_config(conf, "mode=discriminator_training,data=" + data_source)
+generator_conf = parse_config(conf, "mode=generator,data=" + data_source)
+
+# Create GradientMachine
+dis_training_machine = api.GradientMachine.createFromConfigProto(
+dis_conf.model_config)
+gen_training_machine = api.GradientMachine.createFromConfigProto(
+gen_conf.model_config)
+generator_machine = api.GradientMachine.createFromConfigProto(
+generator_conf.model_config)
+
+# Create trainer
+dis_trainer = api.Trainer.create(dis_conf, dis_training_machine)
+gen_trainer = api.Trainer.create(gen_conf, gen_training_machine)
+```
+
+In order to balance the strength between generator and discriminator, we schedule to train whichever one is performing worse by comparing their loss function value. The loss function value can be calculated by a forward pass through the GradientMachine.
+```python
+def get_training_loss(training_machine, inputs):
+    outputs = api.Arguments.createArguments(0)
+    training_machine.forward(inputs, outputs, api.PASS_TEST)
+    loss = outputs.getSlotValue(0).copyToNumpyMat()
+    return numpy.mean(loss)
+```
+
+After training one network, one needs to sync the new parameters to the other networks. The code below demonstrates one example of such use case:
+```python
+# Train the gen_training
+gen_trainer.trainOneDataBatch(batch_size, data_batch_gen)
+
+# Copy the parameters from gen_training to dis_training and generator
+copy_shared_parameters(gen_training_machine,
+dis_training_machine)
+copy_shared_parameters(gen_training_machine, generator_machine)
+```
+
+
+## A Toy Example 
+With the infrastructure explained above, we can now walk you through a toy example of generating two dimensional uniform distribution using 10 dimensional Gaussian noise. 
+
+The Gaussian noises are generated using the code below:
+```python
+def get_noise(batch_size, noise_dim):
+    return numpy.random.normal(size=(batch_size, noise_dim)).astype('float32')
+```
+
+The real samples (2-D uniform) are generated using the code below:
+```python
+# synthesize 2-D uniform data in gan_trainer.py:114
+def load_uniform_data():
+    data = numpy.random.rand(1000000, 2).astype('float32')
+    return data
+```
+
+The generator and discriminator network are built using fully-connected layer and batch_norm layer, and are defined in gan_conf.py. 
+
+To train the GAN model, one can use the command below. The flag -d specifies the training data (cifar, mnist or uniform) and flag --useGpu specifies whether to use gpu for training (0 is cpu, 1 is gpu).  
+```bash
+$python gan_trainer.py -d uniform --useGpu 1
+```
+The generated samples can be found in ./uniform_samples/ and one example is shown below as Figure 2. One can see that it roughly recovers the 2D uniform distribution. 
+
+<p align="center">
+    <img src="./uniform_sample.png" width="300" height="300"> 
+</p>
+<p align="center">
+    Figure 2. Uniform Sample
+</p>
+
+## MNIST Example
+### Data preparation
+To download the MNIST data, one can use the following commands:
+```bash
+$cd data/
+$./get_mnist_data.sh
+```
+
+### Model description
+Following the DC-Gan paper (https://arxiv.org/abs/1511.06434), we use convolution/convolution-transpose layer in the discriminator/generator network to better deal with images. The details of the network structures are defined in gan_conf_image.py. 
+
+### Training the model
+To train the GAN model on mnist data, one can use the following command:
+```bash
+$python gan_trainer.py -d mnist --useGpu 1
+```
+The generated sample images can be found at ./mnist_samples/ and one example is shown below as Figure 3. 
+<p align="center">
+    <img src="./mnist_sample.png" width="300" height="300"> 
+</p>
+<p align="center">
+    Figure 3. MNIST Sample
+</p>

doc/tutorials/image_classification/index_cn.md

+图像分类教程
+==========
+
+在本教程中，我们将使用CIFAR-10数据集训练一个卷积神经网络，并使用这个神经网络来对图片进行分类。如下图所示，卷积神经网络可以辨识图片中的主体，并给出分类结果。
+<center>![Image Classification](./image_classification.png)</center>
+
+## 数据准备
+首先下载CIFAR-10数据集。下面是CIFAR-10数据集的官方网址：
+
+<https://www.cs.toronto.edu/~kriz/cifar.html>
+
+我们准备了一个脚本，可以用于从官方网站上下载CIFAR-10数据集，转为jpeg文件并存入特定的目录。使用这个脚本前请确认已经安装了pillow及相关依赖模块。可以参照下面的命令进行安装：
+
+1. 安装pillow
+
+```bash
+sudo apt-get install libjpeg-dev
+pip install pillow
+```
+
+2. 下载数据集
+
+```bash
+cd demo/image_classification/data/
+sh download_cifar.sh
+```
+
+CIFAR-10数据集包含60000张32x32的彩色图片。图片分为10类，每个类包含6000张。其中50000张图片作为训练集，10000张作为测试集。
+
+下图展示了所有的图片类别，每个类别中随机抽取了10张图片。
+<center>![Image Classification](./cifar.png)</center>
+
+脚本运行完成后，我们应当会得到一个名为cifar-out的文件夹，其下子文件夹的结构如下
+
+
+```
+train
+---airplane
+---automobile
+---bird
+---cat
+---deer
+---dog
+---frog
+---horse
+---ship
+---truck
+test
+---airplane
+---automobile
+---bird
+---cat
+---deer
+---dog
+---frog
+---horse
+---ship
+---truck
+```
+
+cifar-out下包含`train`和`test`两个文件夹，其中分别包含了CIFAR-10中的训练集和测试集。这两个文件夹下各自有10个子文件夹，每个子文件夹下存储相应分类的图片。将图片按照上述结构存储好之后，我们就可以着手对分类模型进行训练了。
+
+## 预处理
+数据下载之后，还需要进行预处理，将数据转换为Paddle的格式。我们可以通过如下命令进行预处理工作：
+
+```
+cd demo/image_classification/
+sh preprocess.sh
+```
+
+其中`preprocess.sh` 调用 `./demo/image_classification/preprocess.py` 对图片进行预处理
+```sh
+export PYTHONPATH=$PYTHONPATH:../../
+data_dir=./data/cifar-out
+python preprocess.py -i $data_dir -s 32 -c 1
+```
+
+`./demo/image_classification/preprocess.py` 使用如下参数：
+
+- `-i` 或 `--input` 给出输入数据所在路径；
+- `-s` 或 `--size` 给出图片尺寸；
+- `-c` 或 `--color` 标示图片是彩色图或灰度图
+
+## 模型训练
+在开始训练之前，我们需要先创建一个模型配置文件。下面我们给出了一个配置示例。**注意**，这里的列出的和`vgg_16_cifar.py`文件稍有差别，因为该文件可适用于预测。
+
+```python
+from paddle.trainer_config_helpers import *
+data_dir='data/cifar-out/batches/'
+meta_path=data_dir+'batches.meta'
+args = {'meta':meta_path, 'mean_img_size': 32,
+        'img_size': 32, 'num_classes': 10,
+        'use_jpeg': 1, 'color': "color"}
+define_py_data_sources2(train_list=data_dir+"train.list",
+                        test_list=data_dir+'test.list',
+                        module='image_provider',
+                        obj='processData',
+                        args=args)
+settings(
+    batch_size = 128,
+    learning_rate = 0.1 / 128.0,
+    learning_method = MomentumOptimizer(0.9),
+    regularization = L2Regularization(0.0005 * 128))
+
+img = data_layer(name='image', size=3*32*32)
+lbl = data_layer(name="label", size=10)
+# small_vgg is predined in trainer_config_helpers.network
+predict = small_vgg(input_image=img, num_channels=3)
+outputs(classification_cost(input=predict, label=lbl))
+```
+
+在第一行中我们载入用于定义网络的函数。
+```python
+from paddle.trainer_config_helpers import *
+```
+
+之后定义的`define_py_data_sources2`使用Python数据提供器，其中 `args`将在`image_provider.py`进行使用，该文件负责产生图片数据并传递给Paddle系统
+ - `meta`: 训练集平均值。
+ - `mean_img_size`: 平均特征图的高度及宽度。
+ - `img_size`：输入图片的高度及宽度。
+ - `num_classes`：类别个数。
+ - `use_jpeg`：处理过程中数据存储格式。
+ - `color`：标示是否为彩色图片。
+ 
+ `settings`用于设置训练算法。在下面的例子中，learning rate被设置为0.1除以batch size，而weight decay则为0.0005乘以batch size。
+ 
+ ```python
+settings(
+    batch_size = 128,
+    learning_rate = 0.1 / 128.0,
+    learning_method = MomentumOptimizer(0.9),
+    regularization = L2Regularization(0.0005 * 128)
+)
+```
+
+`small_vgg`定义了网络结构。这里我们使用的是一个小的VGG网络。关于VGG卷积神经网络的描述可以参考：[http://www.robots.ox.ac.uk/~vgg/research/very_deep/](http://www.robots.ox.ac.uk/~vgg/research/very_deep/)。
+```python
+# small_vgg is predined in trainer_config_helpers.network
+predict = small_vgg(input_image=img, num_channels=3)
+```
+配置创建完毕后，可以运行脚本train.sh来训练模型。
+
+```bash
+config=vgg_16_cifar.py
+output=./cifar_vgg_model
+log=train.log
+
+paddle train \
+--config=$config \
+--dot_period=10 \
+--log_period=100 \
+--test_all_data_in_one_period=1 \
+--use_gpu=1 \
+--save_dir=$output \
+2>&1 | tee $log
+
+python -m paddle.utils.plotcurve -i $log > plot.png
+```
+- 这里我们使用的是GPU模式进行训练。如果你没有GPU环境，可以设置`use_gpu=0`。
+- `./demo/image_classification/vgg_16_cifar.py`是网络和数据配置文件。各项参数的详细说明可以在命令行参数相关文档中找到。
+- 脚本`plotcurve.py`依赖于python的`matplotlib`模块。因此如果这个脚本运行失败，也许是因为需要安装`matplotlib`。
+在训练完成后，训练及测试误差曲线图会被`plotcurve.py`脚本保存在 `plot.png`中。下面是一个误差曲线图的示例：
+
+<center>![Training and testing curves.](./plot.png)</center>
+
+## 预测
+在训练完成后，模型及参数会被保存在路径`./cifar_vgg_model/pass-%05d`下。例如第300个pass的模型会被保存在`./cifar_vgg_model/pass-00299`。
+
+要对一个图片的进行分类预测，我们可以使用`predict.sh`，该脚本将输出预测分类的标签：
+
+```
+sh predict.sh
+```
+
+predict.sh:
+```
+model=cifar_vgg_model/pass-00299/
+image=data/cifar-out/test/airplane/seaplane_s_000978.png
+use_gpu=1
+python prediction.py $model $image $use_gpu
+```
+
+## 练习
+在CUB-200数据集上使用VGG模型训练一个鸟类图片分类模型。相关的鸟类数据集可以从如下地址下载，其中包含了200种鸟类的照片（主要来自北美洲）。
+
+<http://www.vision.caltech.edu/visipedia/CUB-200.html>
+
+
+
+
+## 细节探究
+### 卷积神经网络
+卷积神经网络是一种使用卷积层的前向神经网络，很适合构建用于理解图片内容的模型。一个典型的神经网络如下图所示：
+
+![Convolutional Neural Network](./lenet.png)
+
+一个卷积神经网络包含如下层：
+
+- 卷积层：通过卷积操作从图片或特征图中提取特征
+- 池化层：使用max-pooling对特征图下采样
+- 全连接层：使输入层到隐藏层的神经元是全部连接的。
+
+卷积神经网络在图片分类上有着惊人的性能，这是因为它发掘出了图片的两类重要信息：局部关联性质和空间不变性质。通过交替使用卷积和池化处理， 卷积神经网络能够很好的表示这两类信息。
+
+关于如何定义网络中的层，以及如何在层之间进行连接，请参考Layer文档。

doc/tutorials/image_classification/index_en.md

@@ -147,7 +147,7 @@ for classification. A description of VGG network can be found here [http://www.r
 # small_vgg is predined in trainer_config_helpers.network
 predict = small_vgg(input_image=img, num_channels=3)
 ```
-After writing the config, we can train the model by running the script train.sh. Notice that the following script assumes the you run the script in the `./demo/image_classification` folder. If you run the script in a different folder, you need to change the paths of the scripts and the configuration files accordingly.
+After writing the config, we can train the model by running the script train.sh.
 
 ```bash
 config=vgg_16_cifar.py

paddle/api/GradientMachine.cpp

@@ -68,6 +68,14 @@ void GradientMachine::start() { m->machine->start(); }
 
 void GradientMachine::finish() { m->machine->finish(); }
 
+void GradientMachine::onPassEnd() { m->machine->onPassEnd(); }
+
+void GradientMachine::prefetch(const Arguments& inArgs) {
+  auto& in =
+      m->cast<std::vector<paddle::Argument>>(inArgs.getInternalArgumentsPtr());
+  m->machine->prefetch(in);
+}
+
 void GradientMachine::forward(const Arguments& inArgs,
                               Arguments* outArgs,
                               PassType passType) {

paddle/api/PaddleAPI.h

@@ -20,15 +20,11 @@ limitations under the License. */
 #include <string>
 #include <vector>
 #include "paddle/utils/GlobalConstants.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 /// Import PaddlePaddle's enumeration into global namespace.
 using namespace paddle::enumeration_wrapper;  // NOLINT
 
-#define DISABLE_COPY_AND_ASSIGN(classname) \
-  classname(const classname& other);       \
-  classname& operator=(const classname& other)
-
 /**
  * @brief Initialize paddle.
  *
@@ -102,7 +98,7 @@ const size_t NO_SPARSE_ID = -1UL;
 struct MatrixPrivate;
 class Matrix {
   Matrix();  // User Cannot Create Matrix.
-  DISABLE_COPY_AND_ASSIGN(Matrix);
+  DISABLE_COPY(Matrix);
   static Matrix* createByPaddleMatrixPtr(void* sharedPtr);
 
 public:
@@ -242,7 +238,7 @@ private:
 
 struct VectorPrivate;
 class Vector {
-  DISABLE_COPY_AND_ASSIGN(Vector);
+  DISABLE_COPY(Vector);
   Vector();
   static Vector* createByPaddleVectorPtr(void* ptr);
 
@@ -322,7 +318,7 @@ private:
 struct IVectorPrivate;
 class IVector {
   IVector();
-  DISABLE_COPY_AND_ASSIGN(IVector);
+  DISABLE_COPY(IVector);
   static IVector* createByPaddleVectorPtr(void* ptr);
 
 public:
@@ -402,7 +398,7 @@ struct ArgumentsPrivate;
 class Arguments {
 private:
   Arguments();  // Internal Create.
-  DISABLE_COPY_AND_ASSIGN(Arguments);
+  DISABLE_COPY(Arguments);
 
 public:
   /**
@@ -472,7 +468,7 @@ enum GradientMatchineCreateMode {
 
 struct ParameterConfigPrivate;
 class ParameterConfig {
-  DISABLE_COPY_AND_ASSIGN(ParameterConfig);
+  DISABLE_COPY(ParameterConfig);
   ParameterConfig();
 
   /**
@@ -502,7 +498,7 @@ private:
 
 struct OptimizationConfigPrivate;
 class OptimizationConfig {
-  DISABLE_COPY_AND_ASSIGN(OptimizationConfig);
+  DISABLE_COPY(OptimizationConfig);
   OptimizationConfig();
 
 public:
@@ -527,7 +523,7 @@ struct ParameterPrivate;
 class Parameter {
 private:
   Parameter();
-  DISABLE_COPY_AND_ASSIGN(Parameter);
+  DISABLE_COPY(Parameter);
 
 public:
   virtual ~Parameter();
@@ -572,7 +568,7 @@ struct ModelConfigPrivate;
 class ModelConfig {
 private:
   ModelConfig();
-  DISABLE_COPY_AND_ASSIGN(ModelConfig);
+  DISABLE_COPY(ModelConfig);
 
 public:
   virtual ~ModelConfig();
@@ -593,7 +589,7 @@ struct TrainerConfigPrivate;
 class TrainerConfig {
 private:
   TrainerConfig();
-  DISABLE_COPY_AND_ASSIGN(TrainerConfig);
+  DISABLE_COPY(TrainerConfig);
 
 public:
   virtual ~TrainerConfig();
@@ -633,7 +629,7 @@ public:
 
 struct ParameterTraverseCallbackPrivate;
 class ParameterTraverseCallback {
-  DISABLE_COPY_AND_ASSIGN(ParameterTraverseCallback);
+  DISABLE_COPY(ParameterTraverseCallback);
   ParameterTraverseCallback();
 
 public:
@@ -655,7 +651,7 @@ private:
  */
 struct ParameterOptimizerPrivate;
 class ParameterOptimizer {
-  DISABLE_COPY_AND_ASSIGN(ParameterOptimizer);
+  DISABLE_COPY(ParameterOptimizer);
   ParameterOptimizer();
 
 public:
@@ -692,7 +688,7 @@ struct GradientMachinePrivate;
 class GradientMachine {
 private:
   GradientMachine();
-  DISABLE_COPY_AND_ASSIGN(GradientMachine);
+  DISABLE_COPY(GradientMachine);
 
 public:
   virtual ~GradientMachine();
@@ -725,6 +721,16 @@ public:
 
   void start();
 
+  /**
+   * Prefetch row ids of sparse parameter.
+   */
+  void prefetch(const Arguments& inArgs);
+
+  /**
+   * Do some thing when train pass ended.
+   */
+  void onPassEnd();
+
   /**
    * The forward stage of GradientMachine.
    *
@@ -866,7 +872,7 @@ struct EvaluatorPrivate;
 class Evaluator {
 private:
   Evaluator();
-  DISABLE_COPY_AND_ASSIGN(Evaluator);
+  DISABLE_COPY(Evaluator);
 
 public:
   ~Evaluator();
@@ -900,7 +906,7 @@ private:
   TrainerPrivate* m;
   Trainer();
   Trainer(TrainerConfig* optConfig, GradientMachine* gm);
-  DISABLE_COPY_AND_ASSIGN(Trainer);
+  DISABLE_COPY(Trainer);
 
 public:
   virtual ~Trainer();
@@ -966,7 +972,7 @@ public:
 
 struct SequenceGeneratorPrivate;
 class SequenceGenerator {
-  DISABLE_COPY_AND_ASSIGN(SequenceGenerator);
+  DISABLE_COPY(SequenceGenerator);
   SequenceGenerator();
 
 public:

paddle/api/paddle_ld_flags.py

@@ -141,9 +141,12 @@ try:
 
         def c_flag(self):
             if self.with_coverage:
-                return ["-fprofile-arcs", "-ftest-coverage", "-O0", "-g"]
+                return [
+                    "-fprofile-arcs", "-ftest-coverage", "-O0", "-g",
+                    "-std=c++11"
+                ]
             else:
-                return None
+                return ["-std=c++11"]
 except ImportError:
 
     class PaddleLDFlag(object):

paddle/cuda/include/hl_base.h

@@ -16,7 +16,31 @@ limitations under the License. */
 #define HL_BASE_H_
 
 #include <cstddef>
-#include "paddle/utils/TypeDefs.h"
+
+#ifdef PADDLE_TYPE_DOUBLE
+#define HL_FLOAT_MAX 3.40282347e+38F
+#define HL_FLOAT_MIN 1.17549435e-38F
+using real = double;
+#else
+#define HL_FLOAT_MAX 1.7976931348623157e+308
+#define HL_FLOAT_MIN 2.2250738585072014e-308
+using real = float;
+#endif
+
+/**
+ * The maximum input value for exp, used to avoid overflow problem.
+ * currently only used for tanh function.
+ */
+#define EXP_MAX_INPUT 40.0
+
+/**
+ * @brief DIVUP(x, y) is similar to ceil(x / y).
+ * @note  For CUDA, DIVUP will be used to specify
+ *        the size of blockDim.
+ */
+#ifndef DIVUP
+#define DIVUP(x, y) (((x) + (y)-1) / (y))
+#endif
 
 /**
  * HPPL is an internal high performance parallel computing library
@@ -181,46 +205,6 @@ typedef struct {
   size_t nnz;
 } _hl_sparse_matrix_s, *hl_sparse_matrix_s;
 
-#ifndef PADDLE_TYPE_DOUBLE
-/**
- * HPPL data type: real (float or double)
- *
- * if real == float
- *
- * HL_FLOAT_MAX: 3.40282347e+38F
- *
- * HL_FLOAT_MIN: 1.17549435e-38F
- */
-#define HL_FLOAT_MAX 3.40282347e+38F
-/**
- * if real == double
- *
- * HL_FLOAT_MAX: 1.7976931348623157e+308
- *
- * HL_FLOAT_MIN: 2.2250738585072014e-308
- */
-#define HL_FLOAT_MIN 1.17549435e-38F
-#else
-#define HL_FLOAT_MAX 1.7976931348623157e+308
-#define HL_FLOAT_MIN 2.2250738585072014e-308
-#endif
-
-/**
- * The maximum input value for exp, used to avoid overflow problem.
- *
- * Currently only used for tanh function.
- */
-#define EXP_MAX_INPUT 40.0
-
-/**
- * @brief DIVUP(x, y) is similar to ceil(x / y).
- * @note  For CUDA, DIVUP will be used to specify
- *        the size of blockDim.
- */
-#ifndef DIVUP
-#define DIVUP(x, y) (((x) + (y)-1) / (y))
-#endif
-
 #ifdef __NVCC__
 
 #include "cuda_runtime.h"

paddle/cuda/src/hl_cuda_cudnn.cc

@@ -14,11 +14,11 @@ limitations under the License. */
 
 #include "hl_cuda_cudnn.h"
 #include <cudnn.h>
+#include <gflags/gflags.h>
 #include <mutex>
 #include "hl_cuda_cudnn.ph"
 #include "hl_dso_loader.h"
 #include "hl_thread.ph"
-#include "paddle/utils/CommandLineParser.h"
 #include "paddle/utils/Logging.h"
 
 DEFINE_int32(cudnn_conv_workspace_limit_in_mb,

paddle/cuda/src/hl_dso_loader.cc

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "hl_dso_loader.h"
-#include "paddle/utils/CommandLineParser.h"
+#include <gflags/gflags.h>
 #include "paddle/utils/Logging.h"
 
 DEFINE_string(cudnn_dir,

paddle/gserver/dataproviders/DataProvider.h

@@ -34,8 +34,8 @@ limitations under the License. */
 #include "paddle/utils/Logging.h"
 #include "paddle/utils/Queue.h"
 #include "paddle/utils/ThreadLocal.h"
-#include "paddle/utils/TypeDefs.h"
 #include "paddle/utils/Util.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 /**

paddle/gserver/layers/BatchNormalizationLayer.h

@@ -58,6 +58,8 @@ protected:
   /// to batch, channels* imagePixels.
   void shrinkMat(const MatrixPtr& in, MatrixPtr& out);
 
+  void onPassEnd() { firstTest_ = true; }
+
   MatrixPtr tmpMat_, tmpGrad_;
   MatrixPtr expandedIn_, expandedOut_;
   MatrixPtr expandedInGrad_, expandedOutGrad_, inGrad_;

paddle/gserver/layers/GruCompute.h

@@ -16,7 +16,7 @@ limitations under the License. */
 
 #include "ModelConfig.pb.h"
 #include "hl_gpu.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/gserver/layers/LstmCompute.h

@@ -16,7 +16,7 @@ limitations under the License. */
 
 #include "ModelConfig.pb.h"
 #include "hl_gpu.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/gserver/layers/MultinomialSampler.h

@@ -16,7 +16,7 @@ limitations under the License. */
 
 #include <memory>
 #include <random>
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/gserver/layers/RecurrentLayer.cpp

@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
+#include <gflags/gflags.h>
 #include "Layer.h"
 #include "SequenceToBatch.h"
-#include "paddle/utils/CommandLineParser.h"
 #include "paddle/utils/Stat.h"
 
 DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");

paddle/gserver/tests/TestUtil.cpp

@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "TestUtil.h"
-
+#include <gflags/gflags.h>
 #include "paddle/math/SparseMatrix.h"
-#include "paddle/utils/CommandLineParser.h"
 
 DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
 

paddle/math/BaseMatrix.h

@@ -16,7 +16,7 @@ limitations under the License. */
 #include <stdint.h>
 #include <cstddef>
 #include "TensorExpression.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/math/Matrix.h

@@ -27,7 +27,7 @@ limitations under the License. */
 #include "MemoryHandle.h"
 #include "Vector.h"
 #include "paddle/utils/ThreadLocal.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/math/SparseRowMatrix.h

@@ -14,10 +14,10 @@ limitations under the License. */
 
 #pragma once
 
+#include <gflags/gflags.h>
 #include <string.h>
 #include <algorithm>
 #include "Matrix.h"
-#include "paddle/utils/CommandLineParser.h"
 #include "paddle/utils/Util.h"
 
 DECLARE_bool(allow_inefficient_sparse_update);

paddle/math/TensorExpression.h

@@ -17,7 +17,7 @@ limitations under the License. */
 #include <cstddef>
 #include "hl_tensor_ops.h"
 #include "paddle/utils/Logging.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/math/Vector.h

@@ -22,7 +22,7 @@ limitations under the License. */
 #include "BaseMatrix.h"
 #include "MemoryHandle.h"
 #include "paddle/utils/Thread.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/parameter/ParallelParameter.h

@@ -28,7 +28,7 @@ limitations under the License. */
 #include "paddle/parameter/ParameterUpdateFunctions.h"
 #include "paddle/utils/Flags.h"
 #include "paddle/utils/Locks.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 #include "ParameterConfig.pb.h"
 

paddle/parameter/Parameter.cpp

@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "Parameter.h"
+#include <gflags/gflags.h>
 #include <fstream>
 #include "AverageOptimizer.h"
 #include "FirstOrderOptimizer.h"
@@ -23,7 +24,6 @@ limitations under the License. */
 #include "paddle/math/CpuSparseMatrix.h"
 #include "paddle/math/MathUtils.h"
 #include "paddle/math/SparseRowMatrix.h"
-#include "paddle/utils/CommandLineParser.h"
 #include "paddle/utils/Logging.h"
 
 DEFINE_int32(enable_grad_share,

paddle/parameter/Parameter.h

@@ -29,8 +29,8 @@ limitations under the License. */
 #include "paddle/utils/GlobalConstants.h"
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/ThreadLocal.h"
-#include "paddle/utils/TypeDefs.h"
 #include "paddle/utils/Util.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/parameter/ParameterUpdateFunctions.h

@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 
 #include "paddle/math/Vector.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/pserver/BaseClient.cpp

@@ -13,9 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "BaseClient.h"
+#include <gflags/gflags.h>
 #include <string.h>
 #include <vector>
-#include "paddle/utils/CommandLineParser.h"
 #include "paddle/utils/Stat.h"
 
 DECLARE_string(pservers);

paddle/pserver/BaseClient.h

@@ -18,7 +18,7 @@ limitations under the License. */
 #include "paddle/math/Matrix.h"
 #include "paddle/pserver/ProtoServer.h"
 #include "paddle/utils/Queue.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 namespace paddle {
 

paddle/pserver/LightNetwork.cpp

@@ -18,6 +18,7 @@ limitations under the License. */
 #include <netinet/tcp.h>
 #include <sys/socket.h>
 #include <sys/types.h>
+#include <chrono>
 
 #include <arpa/inet.h>
 #include <net/if.h>
@@ -382,8 +383,20 @@ void SocketClient::TcpClient(const std::string &serverAddr, int serverPort) {
   setOption(sockfd);
 
   /// Now connect to the server
-  PCHECK(connect(sockfd, (sockaddr *)&serv_addr, sizeof(serv_addr)) >= 0)
-      << "ERROR connecting to " << serverAddr;
+  int retry_second = 0;
+  int error = 0;
+  do {
+    error = connect(sockfd, (sockaddr *)&serv_addr, sizeof(serv_addr));
+    if (error == ECONNREFUSED) {
+      LOG(WARNING) << "connection refused by pserver, try again!";
+      if (retry_second++ >= 7) {
+        LOG(FATAL) << "connection refused by pserver, maybe pserver failed!";
+      }
+      std::this_thread::sleep_for(std::chrono::seconds(1));
+    } else {
+      PCHECK(error >= 0) << "ERROR connecting to " << serverAddr;
+    }
+  } while (error == ECONNREFUSED);
 
   channel_.reset(new SocketChannel(sockfd, serverAddr));
   tcpRdma_ = F_TCP;

paddle/pserver/ParameterClient2.h

@@ -26,8 +26,8 @@ limitations under the License. */
 #include "paddle/utils/Flags.h"
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/Queue.h"
-#include "paddle/utils/TypeDefs.h"
 #include "paddle/utils/Util.h"
+#include "paddle/utils/common.h"
 
 #include "ParameterService.pb.h"
 

paddle/pserver/ParameterServer2.h

@@ -32,7 +32,7 @@ limitations under the License. */
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/Stat.h"
 #include "paddle/utils/ThreadLocal.h"
-#include "paddle/utils/TypeDefs.h"
+#include "paddle/utils/common.h"
 
 #include "ParameterService.pb.h"
 

paddle/scripts/docker/Dockerfile

@@ -17,18 +17,6 @@ RUN cd /usr/src/gtest && cmake . && make && cp *.a /usr/lib
 RUN pip install -U BeautifulSoup docopt PyYAML pillow \
     sphinx sphinx_rtd_theme recommonmark
 
-# cmake tends to hide and blur the dependencies between code modules, as
-# noted here https://github.com/PaddlePaddle/Paddle/issues/763. We are
-# thinking about using Bazel to fix this problem, e.g.,
-# https://github.com/PaddlePaddle/Paddle/issues/681#issuecomment-263996102. To
-# start the trail of fixing, we add Bazel to our Dockerfiles.
-RUN apt-get update && apt-get install -y curl software-properties-common \
-    && add-apt-repository ppa:webupd8team/java \
-    && echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections \
-    && echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
-    && curl https://bazel.build/bazel-release.pub.gpg | apt-key add - \
-    && apt-get update && apt-get install -y oracle-java8-installer bazel
-
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_SWIG_PY

paddle/scripts/docker/Dockerfile.gpu

@@ -17,18 +17,6 @@ RUN cd /usr/src/gtest && cmake . && make && cp *.a /usr/lib
 RUN pip install -U BeautifulSoup docopt PyYAML pillow \
     sphinx sphinx_rtd_theme recommonmark
 
-# cmake tends to hide and blur the dependencies between code modules, as
-# noted here https://github.com/PaddlePaddle/Paddle/issues/763. We are
-# thinking about using Bazel to fix this problem, e.g.,
-# https://github.com/PaddlePaddle/Paddle/issues/681#issuecomment-263996102. To
-# start the trail of fixing, we add Bazel to our Dockerfiles.
-RUN apt-get update && apt-get install -y curl software-properties-common \
-    && add-apt-repository ppa:webupd8team/java \
-    && echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections \
-    && echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
-    && curl https://bazel.build/bazel-release.pub.gpg | apt-key add - \
-    && apt-get update && apt-get install -y oracle-java8-installer bazel
-
 ARG WITH_AVX
 ARG WITH_DOC
 ARG WITH_SWIG_PY

paddle/setup.py.in

@@ -30,8 +30,10 @@ is_lin = (system == 'linux')
 # The extra links will passed from COMAKE
 #   because generate paddle LDFLAGS is too complicated to do in setup.py
 #   it just read COMAKE generated LDFLAGS.
+extra_comps = []
 extra_links = []
 obj = api.paddle_ld_flags.PaddleLDFlag()
+extra_comps = obj.c_flag()
 ldflags = obj.ldflag_str()
 if ldflags is not None:
   extra_links.extend(ldflags.split(" "))
@@ -51,20 +53,15 @@ elif is_osx == True:
 
 include_dirs = [np.get_include(), "../"]    # include numpy and paddle.
 
-extra_c = obj.c_flag()
-
-attr=dict()
-if extra_c is not None:
-  attr["extra_compile_args"] = extra_c
-
 setup(name="py_paddle",
   version="@PADDLE_VERSION@",
   ext_modules=[
     Extension('py_paddle._swig_paddle',      # Build SWIG Extension.
        ['Paddle_wrap.cxx'],
+       language = "c++",
        include_dirs = include_dirs,
        extra_link_args = extra_links,
-       **attr
+       extra_compile_args = extra_comps
     )
   ],
   packages=['py_paddle'],

paddle/trainer/ThreadParameterUpdater.h

@@ -33,8 +33,8 @@ namespace paddle {
    because at the current moment, the merging on CPU is happening on the
    main thread, and the its parameter size can be much larger than the one GPU.
    Thus, for GPU, the parameter updates happens in updateImpl() function, which
-   is called by gradient machines as a callback function as a callback function
-   supplied to backward() and forwardBackward().
+   is called by gradient machines as a callback function supplied to backward()
+   and forwardBackward().
    For CPU, the parameter updates happens in separate threads maintained by this
    class.
  */

paddle/utils/CommandLineParser.cpp

-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#include "CommandLineParser.h"
-
-namespace paddle {
-#ifndef GFLAGS_NS
-#define GFLAGS_NS google
-#endif
-
-namespace gflags_ns = GFLAGS_NS;
-
-void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
-  if (withHelp) {
-    gflags_ns::ParseCommandLineFlags(argc, &argv, true);
-  } else {
-    gflags_ns::ParseCommandLineNonHelpFlags(argc, &argv, true);
-  }
-}
-
-}  // namespace paddle

paddle/utils/CommandLineParser.h

-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#include <gflags/gflags.h>
-
-namespace paddle {
-void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
-
-}  // namespace paddle

paddle/utils/CpuId.h

@@ -11,7 +11,7 @@ limitations under the License. */
 
 #pragma once
 
-#include "DisableCopy.h"
+#include "common.h"
 
 namespace paddle {
 

paddle/utils/CustomStackTrace.cpp

@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "CustomStackTrace.h"
+#include <gflags/gflags.h>
 #include <iostream>
-#include "CommandLineParser.h"
 
 DEFINE_bool(
     layer_stack_error_only_current_thread,

paddle/utils/DisableCopy.h

-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-/**
- * Disable copy macro.
- */
-#define DISABLE_COPY(CLASS_NAME)                \
-  CLASS_NAME(CLASS_NAME &&) = delete;           \
-  CLASS_NAME(const CLASS_NAME &other) = delete; \
-  CLASS_NAME &operator=(const CLASS_NAME &other) = delete

paddle/utils/Flags.h

@@ -14,7 +14,7 @@ limitations under the License. */
 
 #pragma once
 
-#include "CommandLineParser.h"
+#include <gflags/gflags.h>
 
 DECLARE_bool(parallel_nn);
 DECLARE_int32(async_count);

paddle/utils/Locks.h

@@ -19,7 +19,7 @@ limitations under the License. */
 #include <condition_variable>
 #include <mutex>
 
-#include "DisableCopy.h"
+#include "common.h"
 
 namespace paddle {
 

paddle/utils/ThreadLocal.cpp

@@ -13,7 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "ThreadLocal.h"
-#include "CommandLineParser.h"
+
+#include <gflags/gflags.h>
+
 #include "Util.h"
 
 DEFINE_bool(thread_local_rand_use_global_seed,

paddle/utils/Util.cpp

@@ -24,10 +24,10 @@ limitations under the License. */
 #include <fstream>
 #include <mutex>
 
-#include "paddle/utils/Logging.h"
+#include <gflags/gflags.h>
 
-#include "CommandLineParser.h"
 #include "CustomStackTrace.h"
+#include "Logging.h"
 #include "StringUtil.h"
 #include "Thread.h"
 #include "ThreadLocal.h"
@@ -152,7 +152,12 @@ void initMain(int argc, char** argv) {
     line += ' ';
   }
   LOG(INFO) << "commandline: " << line;
-  ParseCommandLineFlags(&argc, argv, true);
+
+#ifndef GFLAGS_GFLAGS_H_
+  namespace gflags = google;
+#endif
+
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
   CHECK_EQ(argc, 1) << "Unknown commandline argument: " << argv[1];
 
   installProfilerSwitch();

paddle/utils/Util.h

@@ -26,13 +26,11 @@ limitations under the License. */
 #include <unordered_map>
 #include <vector>
 
-#include "CommandLineParser.h"
-#include "DisableCopy.h"
 #include "Logging.h"
 #include "TrainerConfig.pb.h"
+#include "common.h"
 
 #include "Flags.h"
-#include "TypeDefs.h"
 #include "hl_gpu.h"
 
 /**

paddle/utils/Version.h

@@ -15,7 +15,7 @@ limitations under the License. */
 #pragma once
 #include <stddef.h>
 #include <iostream>
-#include "TypeDefs.h"
+#include "common.h"
 
 namespace paddle {
 

paddle/utils/TypeDefs.h → paddle/utils/common.h

@@ -14,13 +14,20 @@ limitations under the License. */
 
 #pragma once
 
+/**
+ * Disable copy macro.
+ */
+#define DISABLE_COPY(class_name)                \
+  class_name(class_name &&) = delete;           \
+  class_name(const class_name &other) = delete; \
+  class_name &operator=(const class_name &other) = delete
+
 namespace paddle {
+
 #ifdef PADDLE_TYPE_DOUBLE
-typedef double real;
+using real = double;
 #else
-typedef float real;
+using real = float;
 #endif
 
 }  // namespace paddle
-
-using paddle::real;

paddle/utils/tests/test_CustomStackTrace.cpp

@@ -12,10 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include <gtest/gtest.h>
 #include <chrono>
 
-#include "paddle/utils/CommandLineParser.h"
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+
 #include "paddle/utils/CustomStackTrace.h"
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/Util.h"

paddle/utils/tests/test_SpinLock.cpp

@@ -12,9 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include <gtest/gtest.h>
 #include <vector>
-#include "paddle/utils/CommandLineParser.h"
+
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/Logging.h"
 #include "paddle/utils/Util.h"

paddle/utils/tests/test_ThreadBarrier.cpp

@@ -12,10 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include <gtest/gtest.h>
 #include <set>
 #include <vector>
-#include "paddle/utils/CommandLineParser.h"
+
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+
 #include "paddle/utils/Locks.h"
 #include "paddle/utils/Logging.h"
 #include "paddle/utils/Util.h"

python/paddle/trainer_config_helpers/__init__.py

@@ -21,6 +21,5 @@ from networks import *
 from optimizers import *
 from attrs import *
 from config_parser_utils import *
-
 # This will enable operator overload for LayerOutput
-import math as layer_math
+import layer_math

python/paddle/trainer_config_helpers/attrs.py

@@ -19,34 +19,34 @@ __all__ = [
 
 
 def convert_and_compare(x, Type):
-    """                                                                                                                                                                                                
-    Convert x to be the same type as Type and then convert back to                                                                                                                                      
-    check whether there is a loss of information                                                                                                                                                        
-    :param x: object to be checked                                                                                                                                                                      
-    :param Type: target type to check x over                                                                                                                                                           
-    
+    """
+    Convert x to be the same type as Type and then convert back to
+    check whether there is a loss of information
+    :param x: object to be checked
+    :param Type: target type to check x over
+
     """
     return type(x)(Type(x)) == x
 
 
 def is_compatible_with(x, Type):
-    """                                                                                                                                                                                                
-    Check if x has a type compatible with Type                                                                                                                                                         
-    :param x: object to be checked                                                                                                                                                                     
-    :param Type: target type to check x over                                                                                                                                                           
-    
+    """
+    Check if x has a type compatible with Type
+    :param x: object to be checked
+    :param Type: target type to check x over
+
     """
     if type(x) == Type:
         return True
     try:
         if float == Type or int == Type:
-            # avoid those types that can be converted to float/int but not very                                                                                                                            
-            # meaningful and  could potentially lead to error                                                                                                                                              
-            # i.e., str and bool typed value should not be used for initializing float/int variable                                                                                                        
+            # avoid those types that can be converted to float/int but not very
+            # meaningful and  could potentially lead to error
+            # i.e., str and bool typed value should not be used for initializing float/int variable
             if not isinstance(x, str) and not isinstance(x, bool):
                 return convert_and_compare(x, Type)
         elif bool == Type:
-            # should not use string type to initialize bool variable                                                                                                                                   
+            # should not use string type to initialize bool variable
             if not isinstance(x, str):
                 return convert_and_compare(x, Type)
         else:
@@ -88,6 +88,10 @@ class ParameterAttribute(object):
     :type learning_rate: float or None
     :param momentum: The parameter momentum. None means use global value.
     :type momentum: float or None
+    :param gradient_clipping_threshold: gradient clipping threshold. If gradient
+                                        value larger than some value, will be
+                                        clipped.
+    :type gradient_clipping_threshold: float
     :param sparse_update: Enable sparse update for this parameter. It will
                           enable both local and remote sparse update.
     :type sparse_update: bool
@@ -104,6 +108,7 @@ class ParameterAttribute(object):
                  l2_rate=None,
                  learning_rate=None,
                  momentum=None,
+                 gradient_clipping_threshold=None,
                  sparse_update=False):
         # initialize strategy.
         if is_static:
@@ -152,6 +157,11 @@ class ParameterAttribute(object):
             self.attr['sparse_update'] = True
             self.attr['sparse_remote_update'] = True
 
+        if gradient_clipping_threshold is not None and \
+                is_compatible_with(gradient_clipping_threshold, float):
+            self.attr['gradient_clipping_threshold'] = \
+                gradient_clipping_threshold
+
     def set_default_parameter_name(self, name):
         """
         Set default parameter name. If parameter not set, then will use default

third_party/gflags.BUILD

-# Bazel (http://bazel.io/) BUILD file for gflags.
-#
-# See INSTALL.md for instructions for adding gflags to a Bazel workspace.
-
-licenses(["notice"])
-
-exports_files(["src/gflags_complections.sh", "COPYING.txt"])
-
-load(":bazel/gflags.bzl", "gflags_sources", "gflags_library")
-(hdrs, srcs) = gflags_sources(namespace=["google", "gflags"])
-gflags_library(hdrs=hdrs, srcs=srcs, threads=0)
-gflags_library(hdrs=hdrs, srcs=srcs, threads=1)

third_party/gflags_test/BUILD

-licenses(["notice"])  # Apache 2.0
-
-cc_test(
-    name="gflags_test",
-    srcs=["gflags_test.cc"],
-    copts=["-Iexternal/gtest/include"],
-    deps=[
-        "@gtest//:gtest",
-        "@gflags//:gflags",
-    ], )

third_party/gflags_test/gflags_test.cc

-#include <iostream>
-#include <string>
-
-#include "gflags/gflags.h"
-#include "gtest/gtest.h"
-
-DEFINE_bool(verbose, false, "Display program name before message");
-DEFINE_string(message, "Hello world!", "Message to print");
-
-static bool IsNonEmptyMessage(const char *flagname, const std::string &value) {
-  return value[0] != '\0';
-}
-DEFINE_validator(message, &IsNonEmptyMessage);
-
-namespace third_party {
-namespace gflags_test {
-
-TEST(GflagsTest, ParseAndPrint) {
-  gflags::SetUsageMessage("some usage message");
-  gflags::SetVersionString("1.0.0");
-  int argc = 1;
-  char program_name[] = "gflags_test";
-  char **argv = new char *[2];
-  argv[0] = program_name;
-  argv[1] = NULL;
-  gflags::ParseCommandLineFlags(&argc, reinterpret_cast<char ***>(&argv), true);
-  EXPECT_EQ("gflags_test", std::string(gflags::ProgramInvocationShortName()));
-  EXPECT_EQ("Hello world!", FLAGS_message);
-  gflags::ShutDownCommandLineFlags();
-}
-
-}  // namespace gflags_test
-}  // namespace third_party

third_party/glog.BUILD

-licenses(["notice"])
-
-cc_library(
-    visibility=["//visibility:public"],
-    name="glog",
-    includes=[
-        ".",
-        "src",
-    ],
-    copts=[
-        "-D_START_GOOGLE_NAMESPACE_='namespace google {'",
-        "-D_END_GOOGLE_NAMESPACE_='}'",
-        "-DGOOGLE_NAMESPACE='google'",
-        "-DGOOGLE_GLOG_DLL_DECL=''",
-        "-DHAVE_DLADDR",
-        "-DHAVE_SNPRINTF",
-        "-DHAVE_DLFCN_H",
-        "-DHAVE_FCNTL",
-        "-DHAVE_GLOB_H",
-        "-DHAVE_INTTYPES_H",
-        "-DHAVE_LIBPTHREAD",
-        "-DHAVE_SYS_SYSCALL_H",
-        "-DHAVE_MEMORY_H",
-        "-DHAVE_NAMESPACES",
-        "-DHAVE_PREAD",
-        "-DHAVE_PTHREAD",
-        "-DHAVE_PWD_H",
-        "-DHAVE_PWRITE",
-        "-DHAVE_RWLOCK",
-        "-DHAVE_SIGACTION",
-        "-DHAVE_SIGALTSTACK",
-        "-DHAVE_STDINT_H",
-        "-DHAVE_STRING_H",
-        "-DHAVE_SYS_TIME_H",
-        "-DHAVE_SYS_TYPES_H",
-        "-DHAVE_SYS_UCONTEXT_H",
-        "-DHAVE_SYS_UTSNAME_H",
-        "-DHAVE_UNISTD_H",
-        "-DHAVE_USING_OPERATOR",
-        "-DHAVE_HAVE___ATTRIBUTE___",
-        "-DHAVE_HAVE___BUILTIN_EXPECT",
-        #"-DNO_FRAME_POINTER",
-        "-D_GNU_SOURCE",
-        #"-fno-sanitize=thread",
-        #"-fno-sanitize=address",
-        "-Iexternal/glog/src",
-    ],
-    srcs=[
-        "src/demangle.cc",
-        "src/logging.cc",
-        "src/raw_logging.cc",
-        "src/signalhandler.cc",
-        "src/symbolize.cc",
-        "src/utilities.cc",
-        "src/vlog_is_on.cc",
-        ":config_h",
-        ":logging_h",
-        ":raw_logging_h",
-        ":stl_logging_h",
-        ":vlog_is_on_h",
-    ],
-    hdrs=[
-        "src/demangle.h",
-        "src/mock-log.h",
-        "src/stacktrace.h",
-        "src/symbolize.h",
-        "src/utilities.h",
-        "src/base/commandlineflags.h",
-        "src/base/googleinit.h",
-        "src/base/mutex.h",
-        "src/glog/log_severity.h",
-    ])
-
-genrule(
-    name="config_h",
-    srcs=["src/config.h.cmake.in"],
-    outs=["config.h"],
-    cmd="awk '{ gsub(/^#cmakedefine/, \"//cmakedefine\"); print; }' $(<) > $(@)",
-)
-
-genrule(
-    name="logging_h",
-    srcs=["src/glog/logging.h.in"],
-    outs=["glog/logging.h"],
-    cmd="$(location :gen_sh) < $(<) > $(@)",
-    tools=[":gen_sh"])
-
-genrule(
-    name="raw_logging_h",
-    srcs=["src/glog/raw_logging.h.in"],
-    outs=["glog/raw_logging.h"],
-    cmd="$(location :gen_sh) < $(<) > $(@)",
-    tools=[":gen_sh"])
-
-genrule(
-    name="stl_logging_h",
-    srcs=["src/glog/stl_logging.h.in"],
-    outs=["glog/stl_logging.h"],
-    cmd="$(location :gen_sh) < $(<) > $(@)",
-    tools=[":gen_sh"])
-
-genrule(
-    name="vlog_is_on_h",
-    srcs=["src/glog/vlog_is_on.h.in"],
-    outs=["glog/vlog_is_on.h"],
-    cmd="$(location :gen_sh) < $(<) > $(@)",
-    tools=[":gen_sh"])
-
-genrule(
-    name="gen_sh",
-    outs=["gen.sh"],
-    cmd="""
-cat > $@ <<"EOF"
-#! /bin/sh
-sed -e 's/@ac_cv_have_unistd_h@/1/g' \
-    -e 's/@ac_cv_have_stdint_h@/1/g' \
-    -e 's/@ac_cv_have_systypes_h@/1/g' \
-    -e 's/@ac_cv_have_libgflags_h@/1/g' \
-    -e 's/@ac_cv_have_uint16_t@/1/g' \
-    -e 's/@ac_cv_have___builtin_expect@/1/g' \
-    -e 's/@ac_cv_have_.*@/0/g' \
-    -e 's/@ac_google_start_namespace@/namespace google {/g' \
-    -e 's/@ac_google_end_namespace@/}/g' \
-    -e 's/@ac_google_namespace@/google/g' \
-    -e 's/@ac_cv___attribute___noinline@/__attribute__((noinline))/g' \
-    -e 's/@ac_cv___attribute___noreturn@/__attribute__((noreturn))/g' \
-    -e 's/@ac_cv___attribute___printf_4_5@/__attribute__((__format__ (__printf__, 4, 5)))/g'
-EOF""")

third_party/glog_test/BUILD

-licenses(["notice"])  # Apache 2.0
-
-cc_test(
-    name="glog_test",
-    srcs=["glog_test.cc"],
-    copts=["-Iexternal/gtest/include"],
-    deps=[
-        "@gtest//:gtest",
-        "@glog//:glog",
-    ], )

third_party/glog_test/glog_test.cc

-#include <iostream>
-#include <string>
-
-#include "glog/logging.h"
-#include "gtest/gtest.h"
-
-TEST(GlogTest, Logging) { LOG(INFO) << "Hello world"; }

third_party/gtest.BUILD

-cc_library(
-    name="gtest",
-    srcs=glob(
-        ["src/*.cc"], exclude=["src/gtest-all.cc"]),
-    hdrs=glob(["include/**/*.h", "src/*.h"]),
-    copts=["-Iexternal/gtest/include"],
-    linkopts=["-pthread"],
-    visibility=["//visibility:public"], )

third_party/protobuf_test/BUILD

-licenses(["notice"])  # Apache 2.0
-
-load("@protobuf//:protobuf.bzl", "cc_proto_library")
-
-cc_proto_library(
-    name="example_proto",
-    srcs=["example.proto"],
-    protoc="@protobuf//:protoc",
-    default_runtime="@protobuf//:protobuf", )
-
-cc_library(
-    name="example_lib",
-    srcs=["example_lib.cc"],
-    hdrs=["example_lib.h"],
-    deps=[":example_proto"], )
-
-cc_test(
-    name="example_lib_test",
-    srcs=["example_lib_test.cc"],
-    copts=["-Iexternal/gtest/include"],
-    deps=[
-        "@gtest//:gtest",
-        ":example_lib",
-    ], )

third_party/protobuf_test/README.md

-This package tests that Bazel can build protobuf related rules.

third_party/protobuf_test/example.proto

-syntax = "proto3";
-
-package third_party.protobuf_test;
-
-message Greeting {
-  string name = 1;
-}

third_party/protobuf_test/example_lib.cc

-#include "third_party/protobuf_test/example_lib.h"
-
-namespace third_party {
-namespace protobuf_test {
-
-std::string get_greet(const Greeting& who) { return "Hello " + who.name(); }
-
-}  // namespace protobuf_test
-}  // namespace thrid_party

third_party/protobuf_test/example_lib.h

-#pragma once
-
-#include "third_party/protobuf_test/example.pb.h"
-
-#include <string>
-
-namespace third_party {
-namespace protobuf_test {
-
-std::string get_greet(const Greeting &who);
-
-}  // namespace protobuf_test
-}  // namespace third_party

third_party/protobuf_test/example_lib_test.cc

-#include "third_party/protobuf_test/example_lib.h"
-
-#include "gtest/gtest.h"
-
-namespace third_party {
-namespace protobuf_test {
-
-TEST(ProtobufTest, GetGreet) {
-  Greeting g;
-  g.set_name("Paddle");
-  EXPECT_EQ("Hello Paddle", get_greet(g));
-}
-
-}  // namespace protobuf_test
-}  // namespace third_party