Skip to content

P
Paddle

Crayon鑫 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
提交 b0e8b477 编写于 作者: Z zhouyingfeng
浏览文件
操作

Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into cn_doc_gpu_profiling

上级 bcf6f828 8591e70b
展开全部 隐藏空白更改
内联 并排
Showing with 983 addition and 1483 deletion
CMakeLists.txt
浏览文件 @ b0e8b477
......@@ -25,8 +25,8 @@ find_package(ZLIB REQUIRED)
find_package(NumPy REQUIRED)
find_package(Threads REQUIRED)
find_package(AVX QUIET)
find_package(Glog)
find_package(Gflags QUIET)
find_package(Glog REQUIRED)
find_package(Gflags REQUIRED)
find_package(GTest)
find_package(Sphinx)
find_package(Doxygen)
......@@ -40,8 +40,6 @@ option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND})
option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
option(WITH_GLOG "Compile PaddlePaddle use glog, otherwise use a log implement internally" ${LIBGLOG_FOUND})
option(WITH_GFLAGS "Compile PaddlePaddle use gflags, otherwise use a flag implement internally" ${GFLAGS_FOUND})
option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND})
......@@ -136,16 +134,12 @@ else(WITH_RDMA)
add_definitions(-DPADDLE_DISABLE_RDMA)
endif(WITH_RDMA)
if(WITH_GLOG)
add_definitions(-DPADDLE_USE_GLOG)
include_directories(${LIBGLOG_INCLUDE_DIR})
endif()
# glog
include_directories(${LIBGLOG_INCLUDE_DIR})
if(WITH_GFLAGS)
add_definitions(-DPADDLE_USE_GFLAGS)
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
endif()
#gflags
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
if(WITH_TESTING)
enable_testing()
......
CONTRIBUTING.md 0 → 120000
浏览文件 @ b0e8b477
./doc/howto/contribute_to_paddle_en.md
\ No newline at end of file
WORKSPACE
浏览文件 @ b0e8b477
......@@ -3,7 +3,7 @@ http_archive(
name="protobuf",
url="http://github.com/google/protobuf/archive/v3.1.0.tar.gz",
sha256="0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7",
strip_prefix="protobuf-3.1.0", )
strip_prefix="protobuf-3.1.0")
# External dependency to gtest 1.7.0. This method comes from
# https://www.bazel.io/versions/master/docs/tutorial/cpp.html.
......@@ -12,4 +12,20 @@ new_http_archive(
url="https://github.com/google/googletest/archive/release-1.7.0.zip",
sha256="b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0",
build_file="third_party/gtest.BUILD",
strip_prefix="googletest-release-1.7.0", )
strip_prefix="googletest-release-1.7.0")
# External dependency to gflags. This method comes from
# https://github.com/gflags/example/blob/master/WORKSPACE.
new_git_repository(
name="gflags",
tag="v2.2.0",
remote="https://github.com/gflags/gflags.git",
build_file="third_party/gflags.BUILD")
# External dependency to glog. This method comes from
# https://github.com/reyoung/bazel_playground/blob/master/WORKSPACE
new_git_repository(
name="glog",
remote="https://github.com/google/glog.git",
commit="b6a5e0524c28178985f0d228e9eaa43808dbec3c",
build_file="third_party/glog.BUILD")
cmake/FindSphinx.cmake
浏览文件 @ b0e8b477
......@@ -72,6 +72,7 @@ function( Sphinx_add_target target_name builder conf cache source destination )
${source}
${destination}
COMMENT "Generating sphinx documentation: ${builder}"
COMMAND ln -s ${destination}/index_*.html ${destination}/index.html
)
set_property(
......@@ -143,4 +144,4 @@ function( Sphinx_add_targets target_base_name conf source base_destination )
add_dependencies( ${target_base_name}_linkcheck ${_dependencies} )
endif()
endfunction()
\ No newline at end of file
endfunction()
cmake/check_packages.cmake
浏览文件 @ b0e8b477
......@@ -14,13 +14,9 @@ if(WITH_STYLE_CHECK)
find_package(PythonInterp REQUIRED)
endif()
if(WITH_GLOG)
find_package(Glog REQUIRED)
endif()
find_package(Glog REQUIRED)
if(WITH_GFLAGS)
find_package(Gflags REQUIRED)
endif()
find_package(Gflags REQUIRED)
if(WITH_TESTING)
find_package(GTest REQUIRED)
......
cmake/util.cmake
浏览文件 @ b0e8b477
......@@ -65,7 +65,7 @@ endmacro()
# link_paddle_exe
# add paddle library for a paddle executable, such as trainer, pserver.
#
# It will handle WITH_PYTHON/WITH_GLOG etc.
# It will handle WITH_PYTHON etc.
function(link_paddle_exe TARGET_NAME)
if(WITH_RDMA)
generate_rdma_links()
......@@ -108,6 +108,8 @@ function(link_paddle_exe TARGET_NAME)
paddle_cuda
${METRIC_LIBS}
${PROTOBUF_LIBRARY}
${LIBGLOG_LIBRARY}
${GFLAGS_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${CBLAS_LIBS}
${ZLIB_LIBRARIES}
......@@ -125,16 +127,6 @@ function(link_paddle_exe TARGET_NAME)
${PYTHON_LIBRARIES})
endif()
if(WITH_GLOG)
target_link_libraries(${TARGET_NAME}
${LIBGLOG_LIBRARY})
endif()
if(WITH_GFLAGS)
target_link_libraries(${TARGET_NAME}
${GFLAGS_LIBRARIES})
endif()
if(WITH_GPU)
if(NOT WITH_DSO OR WITH_METRIC)
target_link_libraries(${TARGET_NAME}
......
demo/semantic_role_labeling/data/extract_dict_feature.py
浏览文件 @ b0e8b477
......@@ -43,13 +43,13 @@ def extract_dict_features(pair_file, feature_file):
mark[verb_index] = 1
ctx_0 = sentence_list[verb_index]
if verb_index < len(labels_list) - 2:
if verb_index < len(labels_list) - 1:
mark[verb_index + 1] = 1
ctx_p1 = sentence_list[verb_index + 1]
else:
ctx_p1 = 'eos'
if verb_index < len(labels_list) - 3:
if verb_index < len(labels_list) - 2:
mark[verb_index + 2] = 1
ctx_p2 = sentence_list[verb_index + 2]
else:
......
doc/api/data_provider/pydataprovider2_en.rst
浏览文件 @ b0e8b477
.. _api_pydataprovider2_en:
.. _api_pydataprovider2:
PyDataProvider2
===============
......@@ -104,7 +104,7 @@ And PaddlePadle will do all of the rest things\:
Is this cool?
.. _api_pydataprovider2_en_sequential_model:
.. _api_pydataprovider2_sequential_model:
DataProvider for the sequential model
-------------------------------------
......
doc/api/predict/swig_py_paddle_en.rst
浏览文件 @ b0e8b477
......@@ -23,7 +23,7 @@ python's :code:`help()` function. Let's walk through the above python script:
* At the beginning, use :code:`swig_paddle.initPaddle()` to initialize
PaddlePaddle with command line arguments, for more about command line arguments
see :ref:`cmd_detail_introduction_en` .
see :ref:`cmd_detail_introduction` .
* Parse the configuration file that is used in training with :code:`parse_config()`.
Because data to predict with always have no label, and output of prediction work
normally is the output layer rather than the cost layer, so you should modify
......@@ -36,7 +36,7 @@ python's :code:`help()` function. Let's walk through the above python script:
- Note: As swig_paddle can only accept C++ matrices, we offer a utility
class DataProviderConverter that can accept the same input data with
PyDataProvider2, for more information please refer to document
of :ref:`api_pydataprovider2_en` .
of :ref:`api_pydataprovider2` .
* Do the prediction with :code:`forwardTest()`, which takes the converted
input data and outputs the activations of the output layer.
......
doc/conf.py.cn.in
浏览文件 @ b0e8b477
......@@ -79,7 +79,7 @@ language = 'zh_CN'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ['_build', '**/*_en*', '*_en*']
# The reST default role (used for this markup: `text`) to use for all
# documents.
......
doc/conf.py.en.in
浏览文件 @ b0e8b477
......@@ -80,7 +80,7 @@ language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ['_build', '**/*_cn*', '*_cn*']
# The reST default role (used for this markup: `text`) to use for all
# documents.
......
doc/getstarted/build_and_install/build_from_source_en.md
浏览文件 @ b0e8b477
......@@ -49,10 +49,8 @@ PaddlePaddle supports some build options. To enable it, first you need to instal
<tbody>
<tr><td class="left">WITH_GPU</td><td class="left">Compile with GPU mode.</td></tr>
<tr><td class="left">WITH_DOUBLE</td><td class="left">Compile with double precision floating-point, default: single precision.</td></tr>
<tr><td class="left">WITH_GLOG</td><td class="left">Compile with glog. If not found, default: an internal log implementation.</td></tr>
<tr><td class="left">WITH_GFLAGS</td><td class="left">Compile with gflags. If not found, default: an internal flag implementation.</td></tr>
<tr><td class="left">WITH_TESTING</td><td class="left">Compile with gtest for PaddlePaddle's unit testing.</td></tr>
<tr><td class="left">WITH_DOC</td><td class="left"> Compile to generate PaddlePaddle's docs, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_DOC</td><td class="left"> Compile to generate PaddlePaddle's docs, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_SWIG_PY</td><td class="left">Compile with python predict API, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_STYLE_CHECK</td><td class="left">Compile with code style check, default: enabled (ON).</td></tr>
</tbody>
......
doc/getstarted/build_and_install/cmake/compile_options.csv
浏览文件 @ b0e8b477
......@@ -6,8 +6,6 @@ WITH_AVX,是否编译含有AVX指令集的PaddlePaddle二进制文件,是
WITH_PYTHON,是否内嵌PYTHON解释器。方便今后的嵌入式移植工作。,是
WITH_STYLE_CHECK,是否编译时进行代码风格检查,是
WITH_RDMA,是否开启RDMA,否
WITH_GLOG,是否开启GLOG。如果不开启，则会使用一个简化版的日志，同时方便今后的嵌入式移植工作。,取决于是否寻找到GLOG
WITH_GFLAGS,是否使用GFLAGS。如果不开启，则会使用一个简化版的命令行参数解析器，同时方便今后的嵌入式移植工作。,取决于是否寻找到GFLAGS
WITH_TIMER,是否开启计时功能。如果开启会导致运行略慢，打印的日志变多，但是方便调试和测Benchmark,否
WITH_TESTING,是否开启单元测试,取决于是否寻找到GTEST
WITH_DOC,是否编译中英文文档,否
......
doc/getstarted/build_and_install/ubuntu_install_cn.rst
浏览文件 @ b0e8b477
......@@ -46,8 +46,6 @@ PaddlePaddle提供了ubuntu 14.04 deb安装包。
with_double: OFF
with_python: ON
with_rdma: OFF
with_glog: ON
with_gflags: ON
with_metric_learning:
with_timer: OFF
with_predict_sdk:
......
doc/howto/cmd_parameter/detail_introduction_en.md
浏览文件 @ b0e8b477
```eval_rst
.. _cmd_detail_introduction_en:
.. _cmd_detail_introduction:
```
# Detail Description
......
doc/howto/cmd_parameter/index_en.md
浏览文件 @ b0e8b477
```eval_rst
.. _cmd_line_index_en:
.. _cmd_line_index:
```
# How to Set Command-line Parameters
......
doc/howto/contribute_to_paddle_en.md
浏览文件 @ b0e8b477
# How to Contribute Code
We sincerely appreciate your contributions. You can use fork and pull request
workflow to merge your code.
workflow to merge your code.
## Code Requirements
- Your code must be fully documented by
[doxygen](http://www.stack.nl/~dimitri/doxygen/) style.
......@@ -12,11 +12,11 @@ workflow to merge your code.
- Pass all unit tests.
The following tutorial guides you into submitting your contibution.
## [Creating a Fork](https://help.github.com/articles/fork-a-repo/)
Just head over to the GitHub page and click the "Fork" button.
It's just that simple.
It's just that simple.
## Clone
......@@ -25,7 +25,7 @@ The **develop** is the main branch, and other user's branches are feature branch
Once you've created a fork, you can use your favorite git client to clone your
repo or just head straight to the command line:
```shell
# Clone your fork to your local machine
git clone --branch develop https://github.com/USERNAME/Paddle.git
......@@ -47,6 +47,22 @@ Then you can start to develop by making a local developement branch
git checkout -b MY_COOL_STUFF_BRANCH
```
## Using `pre-commit` hook
Paddle developers use [pre-commit](http://pre-commit.com/) tool to manage git
pre-commit hooks. It can help us format source codes (cpp, python), check some
basic thing before commit (only one EOL for each file, do not add a huge file
in git). `pre-commit` tests is a part of unit tests in Travis-CI now, every
PR doesn't fit hook can not be merged into Paddle.
To use [pre-commit](http://pre-commit.com/), you should install it by
`pip install pre-commit`, and currently, Paddle uses `clang-format` to format
c/cpp sources. Please make sure clang-format 3.8+ installed.
Then just run `pre-commit install` in your Paddle clone directory. When you
commit your code, the pre-commit hook will check the local code if there is
anything not suitable to commit, and so on.
## Commit
Commit your changes by following command lines:
......@@ -83,7 +99,7 @@ git pull --rebase upstream develop
If there are no unique commits locally, git will simply perform a fast-forward.
However, if you have been making changes (in the vast majority of cases you
probably shouldn't be), you may have to deal with conflicts.
probably shouldn't be), you may have to deal with conflicts.
Now, your local master branch is up-to-date with everything modified upstream.
......
doc/howto/deep_model/rnn/rnn_en.rst
浏览文件 @ b0e8b477
......@@ -30,7 +30,7 @@ Then at the :code:`process` function, each :code:`yield` function will return th
yield src_ids, trg_ids, trg_ids_next
For more details description of how to write a data provider, please refer to :ref:`api_pydataprovider2_en` . The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`.
For more details description of how to write a data provider, please refer to :ref:`api_pydataprovider2` . The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`.
===============================================
Configure Recurrent Neural Network Architecture
......@@ -246,6 +246,6 @@ The code is listed below:
outputs(beam_gen)
Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to :ref:`semantic_role_labeling_en` for more details.
Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to :ref:`semantic_role_labeling` for more details.
The full configuration file is located at :code:`demo/seqToseq/seqToseq_net.py`.
doc/tutorials/imagenet_model/resnet_model_cn.md 0 → 100644
浏览文件 @ b0e8b477
# Model Zoo - ImageNet #
[ImageNet](http://www.image-net.org/) 是通用物体分类领域一个众所周知的数据库。本教程提供了一个用于ImageNet上的卷积分类网络模型。
## ResNet 介绍
论文 [Deep Residual Learning for Image Recognition](http://arxiv.org/abs/1512.03385) 中提出的ResNet网络结构在2015年ImageNet大规模视觉识别竞赛(ILSVRC 2015)的分类任务中赢得了第一名。他们提出残差学习的框架来简化网络的训练，所构建网络结构的的深度比之前使用的网络有大幅度的提高。下图展示的是基于残差的连接方式。左图构造网络模块的方式被用于34层的网络中，而右图的瓶颈连接模块用于50层，101层和152层的网络结构中。
<center>![resnet_block](./resnet_block.jpg)</center>
<center>图 1. ResNet 网络模块</center>
本教程中我们给出了三个ResNet模型，这些模型都是由原作者提供的模型<https://github.com/KaimingHe/deep-residual-networks>转换过来的。我们使用PaddlePaddle在ILSVRC的验证集共50,000幅图像上测试了模型的分类错误率，其中输入图像的颜色通道顺序为**BGR**，保持宽高比缩放到短边为256，只截取中心方形的图像区域。分类错误率和模型大小由下表给出。
<center>
<table border="2" cellspacing="0" cellpadding="6" rules="all" frame="border">
<colgroup>
<col class="left" />
<col class="left" />
<col class="left" />
</colgroup>
<thead>
<tr>
<th scope="col" class="left">ResNet</th>
<th scope="col" class="left">Top-1</th>
<th scope="col" class="left">Model Size</th>
</tr>
</thead>
<tbody>
<tr>
<td class="left">ResNet-50</td>
<td class="left">24.9%</td>
<td class="left">99M</td>
</tr>
<tr>
<td class="left">ResNet-101</td>
<td class="left">23.7%</td>
<td class="left">173M</td>
</tr>
<tr>
<td class="left">ResNet-152</td>
<td class="left">23.2%</td>
<td class="left">234M</td>
</tr>
</tbody>
</table></center>
<br>
## ResNet 模型
50层，101层和152层的网络配置文件可参照```demo/model_zoo/resnet/resnet.py```。你也可以通过在命令行参数中增加一个参数如```--config_args=layer_num=50```来指定网络层的数目。
### 网络可视化
你可以通过执行下面的命令来得到ResNet网络的结构可视化图。该脚本会生成一个dot文件，然后可以转换为图片。需要安装graphviz来转换dot文件为图片。
```
cd demo/model_zoo/resnet
./net_diagram.sh
```
### 模型下载
```
cd demo/model_zoo/resnet
./get_model.sh
```
你可以执行上述命令来下载所有的模型和均值文件，如果下载成功，这些文件将会被保存在```demo/model_zoo/resnet/model```路径下。
```
mean_meta_224 resnet_101 resnet_152 resnet_50
```
* resnet_50: 50层网络模型。
* resnet_101: 101层网络模型。
* resnet_152: 152层网络模型。
* mean\_meta\_224: 均值图像文件，图像大小为3 x 224 x 224，颜色通道顺序为**BGR**。你也可以使用这三个值: 103.939, 116.779, 123.68。
### 参数信息
* **卷积层权重**
由于每个卷积层后面连接的是batch normalization层，因此该层中没有偏置(bias)参数，并且只有一个权重。
形状: `(Co, ky, kx, Ci)`
* Co: 输出特征图的通道数目
* ky: 滤波器核在垂直方向上的尺寸
* kx: 滤波器核在水平方向上的尺寸
* Ci: 输入特征图的通道数目
二维矩阵: (Co * ky * kx, Ci), 行优先次序存储。
* **全连接层权重**
二维矩阵: (输入层尺寸, 本层尺寸), 行优先次序存储。
* **[Batch Normalization](<http://arxiv.org/abs/1502.03167>) 层权重**
本层有四个参数，实际上只有.w0和.wbias是需要学习的参数，另外两个分别是滑动均值和方差。在测试阶段它们将会被加载到模型中。下表展示了batch normalization层的参数。
<center>
<table border="2" cellspacing="0" cellpadding="6" rules="all" frame="border">
<colgroup>
<col class="left" />
<col class="left" />
<col class="left" />
</colgroup>
<thead>
<tr>
<th scope="col" class="left">参数名</th>
<th scope="col" class="left">尺寸</th>
<th scope="col" class="left">含义</th>
</tr>
</thead>
<tbody>
<tr>
<td class="left">_res2_1_branch1_bn.w0</td>
<td class="left">256</td>
<td class="left">gamma, 缩放参数</td>
</tr>
<tr>
<td class="left">_res2_1_branch1_bn.w1</td>
<td class="left">256</td>
<td class="left">特征图均值</td>
</tr>
<tr>
<td class="left">_res2_1_branch1_bn.w2</td>
<td class="left">256</td>
<td class="left">特征图方差</td>
</tr>
<tr>
<td class="left">_res2_1_branch1_bn.wbias</td>
<td class="left">256</td>
<td class="left">beta, 偏置参数</td>
</tr>
</tbody>
</table></center>
<br>
### 参数读取
使用者可以使用下面的Python脚本来读取参数值:
```
import sys
import numpy as np
def load(file_name):
with open(file_name, 'rb') as f:
f.read(16) # skip header for float type.
return np.fromfile(f, dtype=np.float32)
if __name__=='__main__':
weight = load(sys.argv[1])
```
或者直接使用下面的shell命令:
```
od -j 16 -f _res2_1_branch1_bn.w0
```
## 特征提取
我们提供了C++和Python接口来提取特征。下面的例子使用了`demo/model_zoo/resnet/example`中的数据，详细地展示了整个特征提取的过程。
### C++接口
首先，在配置文件中的`define_py_data_sources2`里指定图像数据列表，具体请参照示例`demo/model_zoo/resnet/resnet.py`。
```
train_list = 'train.list' if not is_test else None
# mean.meta is mean file of ImageNet dataset.
# mean.meta size : 3 x 224 x 224.
# If you use three mean value, set like:
# "mean_value:103.939,116.779,123.68;"
args={
'mean_meta': "model/mean_meta_224/mean.meta",
'image_size': 224, 'crop_size': 224,
'color': True,'swap_channel:': [2, 1, 0]}
define_py_data_sources2(train_list,
'example/test.list',
module="example.image_list_provider",
obj="processData",
args=args)
```
第二步，在`resnet.py`文件中指定要提取特征的网络层的名字。例如，
```
Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn")
```
第三步，在`extract_fea_c++.sh`文件中指定模型路径和输出的目录，然后执行下面的命令。
```
cd demo/model_zoo/resnet
./extract_fea_c++.sh
```
如果执行成功，特征将会存到`fea_output/rank-00000`文件中，如下所示。同时你可以使用`load_feature.py`文件中的`load_feature_c`接口来加载该文件。
```
-0.115318 -0.108358 ... -0.087884;-1.27664 ... -1.11516 -2.59123;
-0.126383 -0.116248 ... -0.00534909;-1.42593 ... -1.04501 -1.40769;
```
* 每行存储的是一个样本的特征。其中，第一行存的是图像`example/dog.jpg`的特征，第二行存的是图像`example/cat.jpg`的特征。
* 不同层的特征由分号`;`隔开，并且它们的顺序与`Outputs()`中指定的层顺序一致。这里，左边是`res5_3_branch2c_conv`层的特征，右边是`res5_3_branch2c_bn`层特征。
### Python接口
示例`demo/model_zoo/resnet/classify.py`中展示了如何使用Python来提取特征。下面的例子同样使用了`./example/test.list`中的数据。执行的命令如下：
```
cd demo/model_zoo/resnet
./extract_fea_py.sh
```
extract_fea_py.sh:
```
python classify.py \
--job=extract \
--conf=resnet.py\
--use_gpu=1 \
--mean=model/mean_meta_224/mean.meta \
--model=model/resnet_50 \
--data=./example/test.list \
--output_layer="res5_3_branch2c_conv,res5_3_branch2c_bn" \
--output_dir=features
```
* \--job=extract: 指定工作模式来提取特征。
* \--conf=resnet.py: 网络配置文件。
* \--use_gpu=1: 指定是否使用GPU。
* \--model=model/resnet_50: 模型路径。
* \--data=./example/test.list: 数据列表。
* \--output_layer="xxx,xxx": 指定提取特征的层。
* \--output_dir=features: 输出目录。
如果运行成功，你将会看到特征存储在`features/batch_0`文件中，该文件是由cPickle产生的。你可以使用`load_feature.py`中的`load_feature_py`接口来打开该文件，它将返回如下的字典：
```
{
'cat.jpg': {'res5_3_branch2c_conv': array([[-0.12638293, -0.116248 , -0.11883899, ..., -0.00895038, 0.01994277, -0.00534909]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.42593431, -1.28918779, -1.32414699, ..., -1.45933616, -1.04501402, -1.40769434]], dtype=float32)},
'dog.jpg': {'res5_3_branch2c_conv': array([[-0.11531784, -0.10835785, -0.08809858, ...,0.0055237, 0.01505112, -0.08788397]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.27663755, -1.18272924, -0.90937918, ..., -1.25178063, -1.11515927, -2.59122872]], dtype=float32)}
}
```
仔细观察，这些特征值与上述使用C++接口提取的结果是一致的。
## 预测
`classify.py`文件也可以用于对样本进行预测。我们提供了一个示例脚本`predict.sh`，它使用50层的ResNet模型来对`example/test.list`中的数据进行预测。
```
cd demo/model_zoo/resnet
./predict.sh
```
predict.sh调用了`classify.py`:
```
python classify.py \
--job=predict \
--conf=resnet.py\
--multi_crop \
--model=model/resnet_50 \
--use_gpu=1 \
--data=./example/test.list
```
* \--job=extract: 指定工作模型进行预测。
* \--conf=resnet.py: 网络配置文件。network configure.
* \--multi_crop: 使用10个裁剪图像块，预测概率取平均。
* \--use_gpu=1: 指定是否使用GPU。
* \--model=model/resnet_50: 模型路径。
* \--data=./example/test.list: 数据列表。
如果运行成功，你将会看到如下结果，其中156和285是这些图像的分类标签。
```
Label of example/dog.jpg is: 156
Label of example/cat.jpg is: 282
```
doc/tutorials/imagenet_model/resnet_model_en.md
浏览文件 @ b0e8b477
......@@ -52,7 +52,7 @@ See ```demo/model_zoo/resnet/resnet.py```. This config contains network of 50, 1
### Network Visualization
You can get a diagram of ResNet network by running the following commands. The script generates dot file and then converts dot file to PNG file, which uses installed draw_dot tool in our server. If you can not access the server, just install graphviz to convert dot file.
You can get a diagram of ResNet network by running the following commands. The script generates dot file and then converts dot file to PNG file, which needs to install graphviz to convert.
```
cd demo/model_zoo/resnet
......@@ -138,7 +138,7 @@ There are four parameters in this layer. In fact, only .w0 and .wbias are the le
### Parameter Observation
Users who want to observe the parameters can use python to read:
Users who want to observe the parameters can use Python to read:
```
import sys
......@@ -209,7 +209,7 @@ If successful, features are saved in `fea_output/rank-00000` as follows. And you
### Python Interface
`demo/model_zoo/resnet/classify.py` is an example to show how to use python to extract features. Following example still uses data of `./example/test.list`. Command is as follows:
`demo/model_zoo/resnet/classify.py` is an example to show how to use Python to extract features. Following example still uses data of `./example/test.list`. Command is as follows:
```
cd demo/model_zoo/resnet
......@@ -238,8 +238,6 @@ python classify.py \
* \--output_layer="xxx,xxx": specify layers to extract features.
* \--output_dir=features: output diretcoty.
Note, since the convolution layer in these ResNet models is suitable for the cudnn implementation which only support GPU. It not support CPU mode because of compatibility issue and we will fix later.
If run successfully, you will see features saved in `features/batch_0`, this file is produced with cPickle. You can use `load_feature_py` interface in `load_feature.py` to open the file, and it returns a dictionary as follows:
```
......
doc/tutorials/rec/ml_dataset_en.md
浏览文件 @ b0e8b477
```eval_rst
.. _demo_ml_dataset_en:
.. _demo_ml_dataset:
```
# MovieLens Dataset
......
doc/tutorials/rec/ml_regression_en.rst
浏览文件 @ b0e8b477
......@@ -16,7 +16,7 @@ Data Preparation
````````````````
Download and extract dataset
''''''''''''''''''''''''''''
We use :ref:`demo_ml_dataset_en` here.
We use :ref:`demo_ml_dataset` here.
To download and unzip the dataset, simply run the following commands.
.. code-block:: bash
......@@ -264,7 +264,7 @@ In this :code:`dataprovider.py`, we should set\:
* use_seq\: Whether this :code:`dataprovider.py` in sequence mode or not.
* process\: Return each sample of data to :code:`paddle`.
The data provider details document see :ref:`api_pydataprovider2_en`.
The data provider details document see :ref:`api_pydataprovider2`.
Train
`````
......@@ -280,7 +280,7 @@ The run.sh is shown as follow:
It just start a paddle training process, write the log to `log.txt`,
then print it on screen.
Each command line argument in :code:`run.sh`, please refer to the :ref:`cmd_line_index_en` page. The short description of these arguments is shown as follow.
Each command line argument in :code:`run.sh`, please refer to the :ref:`cmd_line_index` page. The short description of these arguments is shown as follow.
* config\: Tell paddle which file is neural network configuration.
* save_dir\: Tell paddle save model into './output'
......
doc/tutorials/semantic_role_labeling/index_en.md
浏览文件 @ b0e8b477
```eval_rst
.. _semantic_role_labeling_en:
.. _semantic_role_labeling:
```
# Semantic Role labeling Tutorial #
......
paddle/api/CMakeLists.txt
浏览文件 @ b0e8b477
......@@ -17,22 +17,18 @@ add_library(paddle_api STATIC
${API_SOURCES})
add_dependencies(paddle_api gen_proto_cpp)
list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH)
if(WITH_GFLAGS)
list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH)
if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}")
# Because gflags compiled by cmake, so it is imported by cmake target,
# not a real library path. Get the real library path here.
message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}")
get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION)
message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}")
else()
set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES})
endif()
if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}")
# Because gflags compiled by cmake, so it is imported by cmake target,
# not a real library path. Get the real library path here.
message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}")
get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION)
message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}")
else()
set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES})
endif()
configure_file(
paddle_api_config.py.in
${PROJ_ROOT}/paddle/api/paddle_api_config.py
......@@ -57,7 +53,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp
paddle_trainer
paddle_api
paddle_cuda
${PY_PADDLE_PYTHON_FILES}
${PY_PADDLE_PYTHON_FILES}
)
install(DIRECTORY ${PROJ_ROOT}/paddle/dist/
......
paddle/api/Trainer.cpp
浏览文件 @ b0e8b477
......@@ -27,9 +27,9 @@ limitations under the License. */
using paddle::real;
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
struct TrainerPrivate : public paddle::Trainer {
bool _trainOneBatch(size_t batchSize);
......
paddle/api/paddle_api_config.py.in
浏览文件 @ b0e8b477
......@@ -8,9 +8,7 @@ CMAKE_DL_LIBS="@CMAKE_DL_LIBS@"
WITH_PYTHON="@WITH_PYTHON@"
PYTHON_LIBRARIES="@PYTHON_LIBRARIES@"
WITH_GLOG="@WITH_GLOG@"
LIBGLOG_LIBRARY="@LIBGLOG_LIBRARY@"
WITH_GFLAGS="@WITH_GFLAGS@"
GFLAGS_LIBRARIES="@GFLAGS_LIBRARIES@"
GFLAGS_LOCATION="@GFLAGS_LOCATION@"
CBLAS_LIBRARIES="@CBLAS_LIBS@"
......
paddle/api/paddle_ld_flags.py
浏览文件 @ b0e8b477
......@@ -47,10 +47,8 @@ try:
self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON)
self.python_libs = PYTHON_LIBRARIES
self.with_glog = PaddleLDFlag.cmake_bool(WITH_GLOG)
self.glog_libs = LIBGLOG_LIBRARY
self.with_gflags = PaddleLDFlag.cmake_bool(WITH_GFLAGS)
self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS)
self.gflags_libs = GFLAGS_LIBRARIES
self.gflags_location = GFLAGS_LOCATION
......@@ -88,6 +86,8 @@ try:
"-lpaddle_cuda",
"-lpaddle_api",
self.normalize_flag(self.protolib),
self.normalize_flag(self.glog_libs),
self.normalize_flag(self.gflags_libs),
self.normalize_flag(self.zlib),
self.normalize_flag(self.thread),
self.normalize_flag(self.dl_libs),
......@@ -96,10 +96,6 @@ try:
if self.with_python:
libs.append(self.normalize_flag(self.python_libs))
if self.with_glog:
libs.append(self.normalize_flag(self.glog_libs))
if self.with_gflags:
libs.append(self.normalize_flag(self.gflags_libs))
if self.with_gpu:
libs.append(self.normalize_flag(self.curt))
if self.with_coverage:
......
paddle/cuda/src/hl_cuda_cudnn.cc
浏览文件 @ b0e8b477
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
namespace dynload {
......
paddle/cuda/src/hl_cuda_device.cc
浏览文件 @ b0e8b477
......@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
// clang-format off
// Because clang-format 4.X and clang-format 3.8+ format
// Because clang-format 4.X and clang-format 3.8+ format
// following lines in different. So disable clang-format.
#include "hl_cuda.h"
#include <cuda_profiler_api.h>
......@@ -22,6 +22,7 @@ limitations under the License. */
#include <sys/time.h>
#include <unistd.h>
#include <mutex>
#include "hl_cuda.h"
#include "hl_cuda.ph"
#include "hl_dso_loader.h"
#include "hl_thread.ph"
......
paddle/cuda/src/hl_dso_loader.cc
浏览文件 @ b0e8b477
......@@ -16,21 +16,21 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
P_DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
P_DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
static inline std::string join(const std::string& part1,
const std::string& part2) {
......
paddle/gserver/dataproviders/ProtoDataProvider.cpp
浏览文件 @ b0e8b477
......@@ -22,9 +22,9 @@ limitations under the License. */
#include "DataProviderGroup.h"
#include "paddle/utils/Logging.h"
P_DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
namespace paddle {
......
paddle/gserver/evaluators/Evaluator.cpp
浏览文件 @ b0e8b477
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
浏览文件 @ b0e8b477
......@@ -21,11 +21,11 @@ limitations under the License. */
#include "NeuralNetwork.h"
#include "ParallelNeuralNetwork.h"
P_DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
#ifdef PADDLE_METRIC_LEARNING
P_DECLARE_bool(external);
DECLARE_bool(external);
#endif
namespace paddle {
......
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
浏览文件 @ b0e8b477
......@@ -24,7 +24,7 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
static const char* DIY_CALC_PROB_SYMBOL_NAME = "calc_prob";
static const char* DIY_START_CALC_PROB_SYMBOL_NAME = "start_calc_prob";
......
paddle/gserver/layers/DataLayer.cpp
浏览文件 @ b0e8b477
......@@ -54,7 +54,7 @@ void DataLayer::copyDataToOutput(Argument& output) {
output.setFrameWidth(config_.width());
} else {
output.setFrameHeight(data_.getFrameHeight());
output.setFrameHeight(data_.getFrameHeight());
output.setFrameWidth(data_.getFrameWidth());
}
output.cpuSequenceDims = data_.cpuSequenceDims;
output.sequenceStartPositions = data_.sequenceStartPositions;
......
paddle/gserver/layers/Layer.cpp
浏览文件 @ b0e8b477
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "TransLayer.h"
#include "ValidationLayer.h"
P_DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
namespace paddle {
......
paddle/gserver/layers/LstmLayer.cpp
浏览文件 @ b0e8b477
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/Matrix.h"
#include "paddle/utils/Stat.h"
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(prev_batch_state);
namespace paddle {
......
paddle/gserver/layers/RecurrentLayer.cpp
浏览文件 @ b0e8b477
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
namespace paddle {
......
paddle/gserver/layers/ValidationLayer.h
浏览文件 @ b0e8b477
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "Layer.h"
#include "paddle/gserver/evaluators/Evaluator.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/tests/LayerGradUtil.cpp
浏览文件 @ b0e8b477
......@@ -14,7 +14,7 @@ limitations under the License. */
#include "LayerGradUtil.h"
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(thread_local_rand_use_global_seed);
namespace paddle {
real getCostSum(LayerPtr& testLayer, MatrixPtr weights) {
......
paddle/gserver/tests/TestUtil.cpp
浏览文件 @ b0e8b477
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/CommandLineParser.h"
P_DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
namespace paddle {
......
paddle/gserver/tests/test_ActivationGrad.cpp
浏览文件 @ b0e8b477
......@@ -25,8 +25,8 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_bool(thread_local_rand_use_global_seed);
void testActivation(const string& act) {
LOG(INFO) << "test activation: " << act;
......
paddle/gserver/tests/test_BatchNorm.cpp
浏览文件 @ b0e8b477
......@@ -27,11 +27,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the batchNormLayer can be followed by a ConvLayer
TEST(Layer, batchNorm) {
......
paddle/gserver/tests/test_ConvTrans.cpp
浏览文件 @ b0e8b477
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the convTrans forward is the same as conv backward
TEST(Layer, convTransLayerFwd) {
......
paddle/gserver/tests/test_ConvUnify.cpp
浏览文件 @ b0e8b477
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Do one forward pass of convTrans layer and check to see if its output
// matches the given result
......
paddle/gserver/tests/test_Evaluator.cpp
浏览文件 @ b0e8b477
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_bool(thread_local_rand_use_global_seed);
enum InputType {
INPUT_DATA, // dense vector
......
paddle/gserver/tests/test_LayerGrad.cpp
浏览文件 @ b0e8b477
......@@ -26,11 +26,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
TEST(Operator, dot_mul) {
TestConfig config;
......
paddle/gserver/tests/test_NetworkCompare.cpp
浏览文件 @ b0e8b477
......@@ -25,10 +25,10 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DEFINE_bool(use_label, true, "input label or sequence label");
P_DEFINE_bool(static_para, false, "static parameter");
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DEFINE_bool(use_label, true, "input label or sequence label");
DEFINE_bool(static_para, false, "static parameter");
struct DataIn {
std::vector<Argument> inArgs;
......@@ -267,8 +267,8 @@ TEST(Compare, img_conv2) {
}
#endif
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
TEST(Compare, network) {
if (FLAGS_config_file_a != "" && FLAGS_config_file_b != "") {
compareNetwork(FLAGS_config_file_a, FLAGS_config_file_b);
......
paddle/gserver/tests/test_PyDataProvider2.cpp
浏览文件 @ b0e8b477
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/PythonUtil.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(train_list, "unittest.list", "file list for unittest");
DEFINE_string(train_list, "unittest.list", "file list for unittest");
namespace paddle {
namespace unittest {
......
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
浏览文件 @ b0e8b477
......@@ -20,7 +20,7 @@ limitations under the License. */
#include <paddle/utils/Util.h>
#include <paddle/utils/Version.h>
P_DECLARE_int32(seed);
DECLARE_int32(seed);
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/gserver/tests/test_RecurrentLayer.cpp
浏览文件 @ b0e8b477
......@@ -23,9 +23,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(rnn_use_batch);
P_DECLARE_int32(fixed_seq_length);
DECLARE_bool(use_gpu);
DECLARE_bool(rnn_use_batch);
DECLARE_int32(fixed_seq_length);
void checkError(const Matrix& matrix1, const Matrix& matrix2) {
CHECK(matrix1.getHeight() == matrix2.getHeight());
......
paddle/gserver/tests/test_SelectiveFCLayer.cpp
浏览文件 @ b0e8b477
......@@ -29,11 +29,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(num_passes);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(config_args);
DECLARE_bool(use_gpu);
DECLARE_int32(num_passes);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(config_args);
size_t fcLayerWidth = 1024;
......
paddle/gserver/tests/test_WarpCTCLayer.cpp
浏览文件 @ b0e8b477
......@@ -25,7 +25,7 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
DECLARE_bool(use_gpu);
const real* getData(const Matrix& matrix) {
if (matrix.useGpu()) {
......
paddle/math/SparseRowMatrix.cpp
浏览文件 @ b0e8b477
......@@ -24,9 +24,9 @@ limitations under the License. */
#include "paddle/utils/Thread.h"
#include "paddle/utils/Util.h"
P_DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
namespace paddle {
......
paddle/math/SparseRowMatrix.h
浏览文件 @ b0e8b477
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Util.h"
P_DECLARE_bool(allow_inefficient_sparse_update);
DECLARE_bool(allow_inefficient_sparse_update);
namespace paddle {
......
paddle/math/Storage.cpp
浏览文件 @ b0e8b477
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "Allocator.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
namespace paddle {
......
paddle/math/tests/test_TrainingAlgorithm.cpp
浏览文件 @ b0e8b477
......@@ -22,9 +22,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
#ifndef PADDLE_TYPE_DOUBLE
P_DEFINE_double(max_diff, 1e-5, "max diff allowed");
DEFINE_double(max_diff, 1e-5, "max diff allowed");
#else
P_DEFINE_double(max_diff, 1e-13, "max diff allowed");
DEFINE_double(max_diff, 1e-13, "max diff allowed");
#endif
class SetMaxDiff {
......
paddle/parameter/Argument.cpp
浏览文件 @ b0e8b477
......@@ -245,6 +245,8 @@ int32_t Argument::resizeAndCopyFrom(const Argument& src,
bool useGpu,
hl_stream_t stream) {
dataId = src.dataId;
frameWidth = src.frameWidth;
frameHeight = src.frameHeight;
if (!src.sequenceStartPositions) {
// non-sequence input, copy samples directly
......
paddle/parameter/FirstOrderOptimizer.cpp
浏览文件 @ b0e8b477
......@@ -19,7 +19,7 @@ limitations under the License. */
#include <cmath>
P_DEFINE_bool(log_clipping, false, "enable log clipping or not");
DEFINE_bool(log_clipping, false, "enable log clipping or not");
namespace paddle {
......
paddle/parameter/Parameter.cpp
浏览文件 @ b0e8b477
......@@ -26,11 +26,11 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
P_DEFINE_int32(
DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
DEFINE_int32(
grad_share_block_num,
64,
"block number of gradient parameter share for batch multi-cpu training");
......
paddle/pserver/BaseClient.cpp
浏览文件 @ b0e8b477
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DECLARE_string(pservers);
DECLARE_string(pservers);
namespace paddle {
......
paddle/pserver/LightNetwork.cpp
浏览文件 @ b0e8b477
......@@ -31,23 +31,23 @@ limitations under the License. */
#include "paddle/utils/Util.h"
/// quick ack can reduce the latency of small message
P_DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
/// reasonable size can hold bursted packets and reduce packets loss
P_DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
namespace paddle {
......
paddle/pserver/ParameterClient2.cpp
浏览文件 @ b0e8b477
......@@ -20,8 +20,8 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/StringUtil.h"
P_DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
P_DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
namespace paddle {
......
paddle/pserver/ParameterClient2.h
浏览文件 @ b0e8b477
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "ProtoServer.h"
#include "SparseParameterDistribution.h"
P_DECLARE_int32(parallel_thread_num);
DECLARE_int32(parallel_thread_num);
namespace paddle {
......
paddle/pserver/ParameterServer2.cpp
浏览文件 @ b0e8b477
......@@ -30,11 +30,11 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
P_DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
P_DEFINE_double(
DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
DEFINE_double(
async_lagged_ratio_default,
1.5,
"if async_lagged_grad_discard_ratio is not set in trainer_config.conf"
......
paddle/pserver/ParameterServer2.h
浏览文件 @ b0e8b477
......@@ -38,7 +38,7 @@ limitations under the License. */
#include "ProtoServer.h"
P_DECLARE_int32(port);
DECLARE_int32(port);
namespace paddle {
......
paddle/pserver/SparseParameterDistribution.cpp
浏览文件 @ b0e8b477
......@@ -20,26 +20,26 @@ limitations under the License. */
#include "SparseParameterDistribution.h"
P_DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
P_DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
P_DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
P_DEFINE_double(
DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
DEFINE_double(
check_sparse_distribution_ratio,
0.6,
"if parameters dispatched to different pservers exhibit unbalanced "
" distribution for check_sparse_distribution_ratio * "
" check_sparse_distribution_batches times, crash program");
P_DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
namespace paddle {
......
paddle/pserver/test/SocketTest.cpp
浏览文件 @ b0e8b477
......@@ -195,9 +195,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) {
channel_.reset(new SocketChannel(sockfd));
}
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 10000000, "Data size");
P_DEFINE_int32(loop_time, 100000, "test loop time");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 10000000, "Data size");
DEFINE_int32(loop_time, 100000, "test loop time");
using namespace paddle; // NOLINT
......
paddle/pserver/test/test_ParameterServer2.cpp
浏览文件 @ b0e8b477
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(num_gradient_servers);
P_DEFINE_string(server_addr, "127.0.0.1", "assign server address");
P_DEFINE_int32(server_cpu, 0, "assign server cpu");
DECLARE_int32(num_gradient_servers);
DEFINE_string(server_addr, "127.0.0.1", "assign server address");
DEFINE_int32(server_cpu, 0, "assign server cpu");
class ParameterServer2Tester : public ParameterServer2 {
public:
......
paddle/pserver/test/test_ProtoServer.cpp
浏览文件 @ b0e8b477
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/pserver/ProtoServer.h"
#include "paddle/utils/Stat.h"
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 50000000, "Data size");
P_DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
P_DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 50000000, "Data size");
DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
using namespace paddle; // NOLINT
......
paddle/scripts/submit_local.sh.in
浏览文件 @ b0e8b477
......@@ -21,8 +21,6 @@ function version(){
echo " with_double: @WITH_DOUBLE@"
echo " with_python: @WITH_PYTHON@"
echo " with_rdma: @WITH_RDMA@"
echo " with_glog: @WITH_GLOG@"
echo " with_gflags: @WITH_GFLAGS@"
echo " with_metric_learning: @WITH_METRIC@"
echo " with_timer: @WITH_TIMER@"
echo " with_predict_sdk: @WITH_PREDICT_SDK@"
......
paddle/trainer/MergeModel.cpp
浏览文件 @ b0e8b477
......@@ -19,8 +19,8 @@ limitations under the License. */
#include "paddle/pserver/ParameterServer2.h"
#include "paddle/utils/PythonUtil.h"
P_DEFINE_string(model_dir, "", "Directory for separated model files");
P_DEFINE_string(model_file, "", "File for merged model file");
DEFINE_string(model_dir, "", "Directory for separated model files");
DEFINE_string(model_file, "", "File for merged model file");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/RemoteParameterUpdater.cpp
浏览文件 @ b0e8b477
......@@ -17,8 +17,8 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DECLARE_int32(trainer_id);
P_DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_string(save_dir);
namespace paddle {
......
paddle/trainer/ThreadParameterUpdater.cpp
浏览文件 @ b0e8b477
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/math/SparseRowMatrix.h"
#include "paddle/utils/Thread.h"
P_DECLARE_int32(trainer_count);
DECLARE_int32(trainer_count);
namespace paddle {
......
paddle/trainer/Trainer.cpp
浏览文件 @ b0e8b477
......@@ -38,60 +38,56 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
#include "paddle/gserver/layers/ValidationLayer.h"
P_DEFINE_string(config, "", "Trainer config file");
P_DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
P_DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
P_DEFINE_bool(local, true, "Train in local mode or not");
P_DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
P_DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
P_DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
P_DEFINE_string(save_dir, "", "Directory for saving model parameter");
P_DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
P_DEFINE_int32(test_pass,
-1,
"Will load parameter start from this pass to test");
P_DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
P_DEFINE_bool(with_cost, true, "enable cost layer or not");
P_DEFINE_bool(distribute_test, false, "test in distribute mode");
P_DEFINE_int32(num_passes, 100, "train for so many passes");
P_DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
P_DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
P_DEFINE_string(feat_file, "", "File name of extracted feature.");
P_DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
P_DEFINE_string(model_list,
"",
"File that saves the model list when evaluation");
DEFINE_string(config, "", "Trainer config file");
DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
DEFINE_bool(local, true, "Train in local mode or not");
DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
DEFINE_string(save_dir, "", "Directory for saving model parameter");
DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
DEFINE_int32(test_pass, -1, "Will load parameter start from this pass to test");
DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
DEFINE_bool(with_cost, true, "enable cost layer or not");
DEFINE_bool(distribute_test, false, "test in distribute mode");
DEFINE_int32(num_passes, 100, "train for so many passes");
DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
DEFINE_string(feat_file, "", "File name of extracted feature.");
DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
DEFINE_string(model_list, "", "File that saves the model list when evaluation");
namespace paddle {
......
paddle/trainer/Trainer.h
浏览文件 @ b0e8b477
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "paddle/internals/metric_learning/MetricTrainer.h"
#endif
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
namespace paddle {
......
paddle/trainer/TrainerBenchmark.cpp
浏览文件 @ b0e8b477
......@@ -18,9 +18,9 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DECLARE_int32(test_period);
DECLARE_int32(test_period);
P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
namespace paddle {
......
paddle/trainer/TrainerConfigHelper.cpp
浏览文件 @ b0e8b477
......@@ -18,16 +18,16 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/PythonUtil.h"
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
P_DECLARE_string(save_dir);
P_DECLARE_int32(trainer_id);
P_DECLARE_bool(local);
P_DECLARE_bool(with_cost);
P_DECLARE_bool(with_gpu);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_bool(local);
DECLARE_bool(with_cost);
DECLARE_bool(with_gpu);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
const char *kConfigParserModuleName = "paddle.trainer.config_parser";
const char *kConfigParserFuncName = "parse_config_and_serialize";
......
paddle/trainer/TrainerInternalConfig.cpp
浏览文件 @ b0e8b477
......@@ -14,17 +14,17 @@ limitations under the License. */
#include "TrainerInternalConfig.h"
P_DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
P_DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
P_DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
P_DECLARE_bool(local);
DECLARE_bool(local);
namespace paddle {
......
paddle/trainer/TrainerMain.cpp
浏览文件 @ b0e8b477
......@@ -22,21 +22,20 @@ limitations under the License. */
#include "Trainer.h"
#include "paddle/pserver/RDMANetwork.h"
P_DEFINE_bool(start_pserver, false, "Whether to start pserver");
P_DECLARE_int32(gpu_id);
P_DEFINE_string(job, "train", "one of (train, test, checkgrad)");
P_DECLARE_int32(start_pass);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(rdma_tcp);
DEFINE_bool(start_pserver, false, "Whether to start pserver");
DECLARE_int32(gpu_id);
DEFINE_string(job, "train", "one of (train, test, checkgrad)");
DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(rdma_tcp);
using namespace paddle; // NOLINT
int main(int argc, char** argv) {
// write logs instantly (never buffer log messages)
#ifdef PADDLE_USE_GLOG
// write logs instantly (never buffer log messages)
FLAGS_logbuflevel = -1;
#endif
initMain(argc, argv);
initPython(argc, argv);
......
paddle/trainer/tests/test_Compare.cpp
浏览文件 @ b0e8b477
......@@ -24,10 +24,10 @@ using namespace std; // NOLINT
static const string& configFile = "trainer/tests/sample_trainer_config.conf";
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(config_args);
DECLARE_int32(gpu_id);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_string(config_args);
struct comData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareSparse.cpp
浏览文件 @ b0e8b477
......@@ -25,22 +25,22 @@ using namespace std; // NOLINT
static const string& configFile1 =
"trainer/tests/sample_trainer_config_qb_rnn.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
P_DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
int gNumDevices = 0;
......
paddle/trainer/tests/test_CompareTwoNets.cpp
浏览文件 @ b0e8b477
......@@ -22,25 +22,25 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_int32(seed);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_int32(seed);
struct ComData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareTwoOpts.cpp
浏览文件 @ b0e8b477
......@@ -22,20 +22,20 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
......
paddle/trainer/tests/test_Prediction.cpp
浏览文件 @ b0e8b477
......@@ -18,11 +18,11 @@ limitations under the License. */
#include <gtest/gtest.h>
P_DECLARE_string(config);
P_DECLARE_string(config_args);
P_DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
DECLARE_string(config);
DECLARE_string(config_args);
DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/tests/test_Trainer.cpp
浏览文件 @ b0e8b477
......@@ -28,10 +28,10 @@ static const string& configFile3 = "trainer/tests/chunking.conf";
static const string& configFile4 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(allow_only_one_model_on_one_gpu);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_bool(allow_only_one_model_on_one_gpu);
void checkGradientTest(const string& configFile,
bool useGpu,
......
paddle/trainer/tests/test_TrainerOnePass.cpp
浏览文件 @ b0e8b477
......@@ -27,12 +27,12 @@ static const string& configFile1 = "trainer/tests/sample_trainer_config.conf";
static const string& configFile2 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
class TrainerForTest : public paddle::Trainer {
public:
......@@ -122,10 +122,10 @@ TEST(average_window_cpu, gpu4) {
#endif
// 3. test trainer + pserver.
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
double checkRemoteParameterUpdater(TrainerForTest& trainer) {
auto gradientMachine = trainer.getGradientMachine();
......
paddle/trainer/tests/test_recurrent_machine_generation.cpp
浏览文件 @ b0e8b477
......@@ -30,7 +30,7 @@ static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT
static string expectFile = // NOLINT
"trainer/tests/rnn_gen_test_model_dir/r1.test"; // NOLINT
P_DECLARE_string(config_args);
DECLARE_string(config_args);
vector<float> readRetFile(const string& fname) {
ifstream inFile(fname);
......
paddle/utils/BarrierStat.cpp
浏览文件 @ b0e8b477
......@@ -20,15 +20,15 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
P_DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
P_DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
namespace paddle {
......
paddle/utils/CommandLineParser.cpp
浏览文件 @ b0e8b477
......@@ -13,220 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "CommandLineParser.h"
#ifndef PADDLE_USE_GFLAGS
#include <stdlib.h>
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "paddle/utils/StringUtil.h"
namespace paddle {
static constexpr int kStatusOK = 0;
static constexpr int kStatusInvalid = 1;
static constexpr int kStatusNotFound = 2;
/**
* \brief: Convert a string to any type value.
*
* \note: It will specialize by type T that is supported.
*/
template <typename T>
bool StringToValue(const std::string& content, T* value) {
bool ok;
*value = str::toWithStatus<T>(content, &ok);
return ok;
}
template <>
bool StringToValue<bool>(const std::string& content, bool* value) {
std::string tmp = content;
std::transform(tmp.begin(), tmp.end(), tmp.begin(), [](char in) -> char {
if (in <= 'Z' && in >= 'A') {
return in - ('Z' - 'z');
} else {
return in;
}
}); // tolower.
if (tmp == "true" || tmp == "1") {
*value = true;
return true;
} else if (tmp == "false" || tmp == "0") {
*value = false;
return true;
} else {
return false;
}
}
template <>
bool StringToValue<std::string>(const std::string& content,
std::string* value) {
*value = content;
return true;
}
/**
* \brief Parse argument "--blah=blah".
*
* \param argument: The command line argument string, such as "--blah=blah"
* \param [out] extraInfo: The details error message for parse argument.
* \return: kStatusOK, kStatusInvalid, kStatusNotFound
*/
template <typename T>
int ParseArgument(const std::string& argument, std::string* extraInfo) {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
T* value = command.value;
std::string prefix = "--";
prefix += name;
prefix += "=";
std::string content;
if (str::startsWith(argument, prefix)) {
content = argument.substr(prefix.size(), argument.size() - prefix.size());
} else {
prefix = "-";
prefix += name;
prefix += "=";
if (str::startsWith(argument, prefix)) {
content =
argument.substr(prefix.size(), argument.size() - prefix.size());
}
}
if (!content.empty()) {
if (StringToValue(content, value)) {
return kStatusOK;
} else {
*extraInfo = name;
return kStatusInvalid;
}
}
}
return kStatusNotFound;
}
/**
* @brief ParseBoolArgumentExtra
* parse '--flag_name', '-flag_name' as true; '--noflag_name', '-noflag_name' as
* false
*/
static int ParseBoolArgumentExtra(const std::string& argument,
std::string* extraInfo) {
(void)(extraInfo); // unused extraInfo, just make api same.
//! @warning: The order and content of prefixes is DESIGNED for parsing
//! command line. The length of prefixes are 1, 2, 3, 4. The parse logic takes
//! use of this fact. DO NOT CHANGE IT without reading how to parse command
//! below.
static const std::vector<std::pair<const char*, bool>> prefixes = {
{"-", true}, {"--", true}, {"-no", false}, {"--no", false}};
for (flags_internal::CommandLineFlagRegistry<bool>::Command& command :
flags_internal::CommandLineFlagRegistry<bool>::Instance()->commands) {
if (argument.size() > command.name.size()) {
//! Use the length of prefix is 1, 2, 3, 4.
size_t diff = argument.size() - command.name.size() - 1UL;
if (diff < prefixes.size()) {
const std::string& prefix = std::get<0>(prefixes[diff]);
if (argument == prefix + command.name) {
*command.value = std::get<1>(prefixes[diff]);
return kStatusOK;
}
}
}
}
return kStatusNotFound;
}
/**
* \brief: Print command line arguments' usage with type T.
*/
template <typename T>
static void PrintTypeUsage() {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
name = "--" + name; // Program will exit, so modify name is safe.
std::string& desc = command.text;
T& defaultValue = command.defaultValue;
std::cerr << std::setw(20) << name << ": " << desc
<< "[default:" << defaultValue << "]." << std::endl;
}
}
template <typename... TS>
static void PrintTypeUsages() {
int unused[] = {0, (PrintTypeUsage<TS>(), 0)...};
(void)(unused);
}
/**
* \brief: Print all usage, and exit(1)
*/
static void PrintUsageAndExit(const char* argv0) {
std::cerr << "Program " << argv0 << " Flags: " << std::endl;
PrintTypeUsages<bool, int32_t, std::string, double, int64_t, uint64_t>();
exit(1);
}
/**
* \brief: Print the error flags, usage, and exit.
*/
static void PrintParseError(const std::string& name,
const char* actualInput,
const char* arg0) {
std::cerr << "Parse command flag " << name << " error! User input is "
<< actualInput << std::endl;
PrintUsageAndExit(arg0);
}
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
int unused_argc = 1;
std::string extra;
for (int i = 1; i < *argc; ++i) {
std::string arg = argv[i];
int s = kStatusInvalid;
#define ParseArgumentWithType(type) \
s = ParseArgument<type>(arg, &extra); \
if (s == kStatusOK) { \
continue; \
} else if (s == kStatusInvalid) { \
PrintParseError(extra, argv[i], argv[0]); \
}
ParseArgumentWithType(bool); // NOLINT
ParseArgumentWithType(int32_t);
ParseArgumentWithType(double); // NOLINT
ParseArgumentWithType(int64_t);
ParseArgumentWithType(uint64_t);
ParseArgumentWithType(std::string);
#undef ParseArgumentWithType
s = ParseBoolArgumentExtra(arg, &extra);
if (s == kStatusOK) {
continue;
}
if (withHelp && (arg == "--help" || arg == "-h")) {
PrintUsageAndExit(argv[0]);
}
// NOT Found for all flags.
std::swap(argv[unused_argc++], argv[i]);
}
*argc = unused_argc;
}
} // namespace paddle
#else
namespace paddle {
#ifndef GFLAGS_NS
#define GFLAGS_NS google
......@@ -243,4 +30,3 @@ void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
}
} // namespace paddle
#endif
paddle/utils/CommandLineParser.h
浏览文件 @ b0e8b477
......@@ -13,167 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#ifndef PADDLE_USE_GFLAGS
#include <stdint.h>
#include <string>
#include <vector>
#include "DisableCopy.h"
namespace paddle {
namespace flags_internal {
/**
* Command line flag registry for special type T. It will store all command
* arguments settings. such as name, default value.
*/
template <typename T>
struct CommandLineFlagRegistry {
/**
* The factory method of CommandLineFlagRegistry
*
* \return: The singleton instance of CommandLineFlagRegistry.
*/
static CommandLineFlagRegistry* Instance() {
static CommandLineFlagRegistry instance_;
return &instance_;
}
struct Command {
/// name of argument.
std::string name;
/// address of actual variable. such as FLAGS_xxx.
T* value;
/// usage text.
std::string text;
/// default value of this command.
T defaultValue;
};
/// the command line arguments of type T.
std::vector<Command> commands;
DISABLE_COPY(CommandLineFlagRegistry);
private:
inline CommandLineFlagRegistry() {}
};
/**
*Helper class to register command line flag.
*/
template <typename T>
struct CommandLineFlagRegister {
/**
* \brief: Register a command line argument
*
* \param [in] name: The command line name.
* \param [inout] val: The command line argument instance, FLAGS_xxx.
* \param [in] desc: The command line helper message.
*/
CommandLineFlagRegister(const std::string& name,
T* val,
const std::string desc) {
CommandLineFlagRegistry<T>::Instance()->commands.push_back(
{name, val, desc, *val});
}
};
/**
* \brief: Define a command line arguments.
*
* \param type: The variable type, such as int, double, etc.
* \param name: The variable name. The command line argument is '--name', the
*variable
*is 'FLAGS_name'
* \param default_value: The default value of command line argument.
* \param text: The description in command line argument.
*/
#define PADDLE_DEFINE_variable(type, name, default_value, text) \
type FLAGS_##name = default_value; \
namespace paddle_flags_internal { \
paddle::flags_internal::CommandLineFlagRegister<type> \
flags_internal_var_##name(#name, &FLAGS_##name, text); \
} // namespace paddle_flags_internal
/**
* Declare a variable to use.
*/
#define PADDLE_DECLARE_variable(type, name) extern type FLAGS_##name;
// DEFINE macro for each types.
#define P_DEFINE_int32(name, default_value, text) \
PADDLE_DEFINE_variable(int32_t, name, default_value, text)
#define P_DEFINE_bool(name, default_value, text) \
PADDLE_DEFINE_variable(bool, name, default_value, text)
#define P_DEFINE_string(name, default_value, text) \
PADDLE_DEFINE_variable(std::string, name, default_value, text)
#define P_DEFINE_double(name, default_value, text) \
PADDLE_DEFINE_variable(double, name, default_value, text)
#define P_DEFINE_int64(name, default_value, text) \
PADDLE_DEFINE_variable(int64_t, name, default_value, text)
#define P_DEFINE_uint64(name, default_value, text) \
PADDLE_DEFINE_variable(uint64_t, name, default_value, text)
// Declare macro for each types.
#define P_DECLARE_int32(name) PADDLE_DECLARE_variable(int32_t, name)
#define P_DECLARE_bool(name) PADDLE_DECLARE_variable(bool, name)
#define P_DECLARE_string(name) PADDLE_DECLARE_variable(std::string, name)
#define P_DECLARE_double(name) PADDLE_DECLARE_variable(double, name)
#define P_DECLARE_int64(name) PADDLE_DECLARE_variable(int64_t, name)
#define P_DECLARE_uint64(name) PADDLE_DECLARE_variable(uint64_t, name)
} // namespace flags_internal
/**
* \brief Parse command line flags. If parse error, just failed and exit 1.
*
* \param [inout] argc: The command argument count. This method will modify
*argc, and left unused arguments.
* \param [inout] argv: The command argument values. This method will modify
*argv, and left unused arguments.
* \param [in] withHelp: True will parse '-h' and '--help' to print usage.
*
* \note: The Command line flags format basically as follow:
*
* * If the type of flag is not bool, then the follow format of command line
* will be parsed:
* * --flag_name=value
* * -flag_name=value
*
* * If the flag is bool, then:
* * --flag_name=value, -flag_name=value will be parsed.
* * if value.tolower() == "true"| "1" will be treated as true.
* * else if value.tolower() == "false" | "0" will be treated as false.
* * --flag_name will be parsed as true.
* * --noflag_name will be parsed as false.
*/
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#else // if use gflags.
#include <gflags/gflags.h>
#define P_DEFINE_int32 DEFINE_int32
#define P_DEFINE_bool DEFINE_bool
#define P_DEFINE_string DEFINE_string
#define P_DEFINE_double DEFINE_double
#define P_DEFINE_int64 DEFINE_int64
#define P_DEFINE_uint64 DEFINE_uint64
#define P_DECLARE_int32 DECLARE_int32
#define P_DECLARE_bool DECLARE_bool
#define P_DECLARE_string DECLARE_string
#define P_DECLARE_double DECLARE_double
#define P_DECLARE_int64 DECLARE_int64
#define P_DECLARE_uint64 DECLARE_uint64
namespace paddle {
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#endif
paddle/utils/CustomStackTrace.cpp
浏览文件 @ b0e8b477
......@@ -16,7 +16,7 @@ limitations under the License. */
#include <iostream>
#include "CommandLineParser.h"
P_DEFINE_bool(
DEFINE_bool(
layer_stack_error_only_current_thread,
true,
"Dump current thread or whole process layer stack when signal error "
......
paddle/utils/Flags.cpp
浏览文件 @ b0e8b477
......@@ -15,65 +15,61 @@ limitations under the License. */
#include "Flags.h"
#ifdef PADDLE_ONLY_CPU
P_DEFINE_bool(use_gpu, false, "Only support CPU training");
DEFINE_bool(use_gpu, false, "Only support CPU training");
#else
P_DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
#endif
P_DEFINE_bool(
parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
P_DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
P_DEFINE_int32(gpu_id, 0, "Which gpu core to use");
P_DEFINE_int32(port, 20134, "Listening port for pserver");
P_DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
P_DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
P_DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
P_DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
P_DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
P_DEFINE_int32(
trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
P_DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
P_DEFINE_string(comment, "", "A string for commenting this training task");
P_DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
P_DEFINE_int32(log_period, 100, "Log progress every so many batches");
P_DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
P_DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
P_DEFINE_int32(enable_parallel_vector,
0,
"threshold for enable parallel vector");
P_DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
P_DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
DEFINE_bool(parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
DEFINE_int32(gpu_id, 0, "Which gpu core to use");
DEFINE_int32(port, 20134, "Listening port for pserver");
DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
DEFINE_int32(trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
DEFINE_string(comment, "", "A string for commenting this training task");
DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
DEFINE_int32(log_period, 100, "Log progress every so many batches");
DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
DEFINE_int32(enable_parallel_vector, 0, "threshold for enable parallel vector");
DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
P_DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
P_DEFINE_string(predict_file, "", "File name for saving predict result");
P_DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
P_DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
DEFINE_string(predict_file, "", "File name for saving predict result");
DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
paddle/utils/Flags.h
浏览文件 @ b0e8b477
......@@ -16,28 +16,28 @@ limitations under the License. */
#include "CommandLineParser.h"
P_DECLARE_bool(parallel_nn);
P_DECLARE_int32(async_count);
P_DECLARE_int32(port);
P_DECLARE_int32(data_server_port);
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(trainer_count);
P_DECLARE_int32(ports_num);
P_DECLARE_int32(ports_num_for_sparse);
P_DECLARE_string(nics);
P_DECLARE_string(rdma_tcp);
P_DECLARE_int32(trainer_id);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_string(comment);
P_DECLARE_string(load_missing_parameter_strategy);
P_DECLARE_int32(log_period);
P_DECLARE_int32(log_period_server);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_int32(enable_parallel_vector);
P_DECLARE_bool(loadsave_parameters_in_pserver);
P_DECLARE_int32(beam_size);
P_DECLARE_bool(show_layer_stat);
P_DECLARE_string(predict_file);
P_DECLARE_bool(prev_batch_state);
P_DECLARE_string(init_model_path);
DECLARE_bool(parallel_nn);
DECLARE_int32(async_count);
DECLARE_int32(port);
DECLARE_int32(data_server_port);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_int32(trainer_count);
DECLARE_int32(ports_num);
DECLARE_int32(ports_num_for_sparse);
DECLARE_string(nics);
DECLARE_string(rdma_tcp);
DECLARE_int32(trainer_id);
DECLARE_int32(num_gradient_servers);
DECLARE_string(comment);
DECLARE_string(load_missing_parameter_strategy);
DECLARE_int32(log_period);
DECLARE_int32(log_period_server);
DECLARE_double(checkgrad_eps);
DECLARE_int32(enable_parallel_vector);
DECLARE_bool(loadsave_parameters_in_pserver);
DECLARE_int32(beam_size);
DECLARE_bool(show_layer_stat);
DECLARE_string(predict_file);
DECLARE_bool(prev_batch_state);
DECLARE_string(init_model_path);
paddle/utils/Logging.cpp
浏览文件 @ b0e8b477
......@@ -18,175 +18,9 @@ limitations under the License. */
*/
#include "Logging.h"
#ifndef PADDLE_USE_GLOG
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mutex>
#include <thread>
#include <vector>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
namespace paddle {
namespace internal {
std::string join(const std::string& part1, const std::string& part2) {
const char sep = '/';
if (!part2.empty() && part2.front() == sep) {
return part2;
}
std::string ret;
ret.reserve(part1.size() + part2.size() + 1);
ret = part1;
if (!ret.empty() && ret.back() != sep) {
ret += sep;
}
ret += part2;
return ret;
}
static inline bool env2bool(const char* envName, bool defaultValue = false) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
return memchr("tTyY1\0", envValue[0], 6) != nullptr;
}
}
static inline int env2int(const char* envName, int defaultValue = 0) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
int retValue = defaultValue;
try {
retValue = std::stoi(envValue);
} catch (...) {
// pass
}
return retValue;
}
}
static inline int env2index(const char* envName,
const std::vector<std::string>& options,
int defaultValue) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
for (size_t i = 0; i < options.size(); ++i) {
if (options[i] == envValue) {
return static_cast<int>(i);
}
}
return defaultValue;
}
}
static bool gLogToStderr = env2bool("PLOG_LOGTOSTDERR", true);
static const std::vector<std::string> gLevelName = {
"INFO", "WARNING", "ERROR", "FATAL"};
static int gMinLogLevel =
env2int("PLOG_MINLOGLEVEL", env2index("PLOG_MINLOGLEVEL", gLevelName, 0));
static std::vector<std::vector<int>> gLogFds;
static std::vector<int> gLogFileFds;
static bool gLogInited = false;
static void freeLogFileFds() {
for (auto fd : gLogFileFds) {
close(fd);
}
}
static void initializeLogFds(char* argv0) {
gLogFds.resize(NUM_SEVERITIES);
for (int i = gMinLogLevel; i < NUM_SEVERITIES && gLogToStderr;
++i) { // Add stderr
std::vector<int>& fds = gLogFds[i];
fds.push_back(STDERR_FILENO);
}
char* logDir = getenv("PLOG_LOGDIR");
for (int i = gMinLogLevel; i < NUM_SEVERITIES && logDir != nullptr; ++i) {
std::string filename =
join(logDir, std::string(argv0) + "." + gLevelName[i]);
int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644);
if (fd == -1) {
fprintf(stderr, "Open log file error!");
exit(1);
}
gLogFileFds.push_back(fd);
std::vector<int>& curFds = gLogFds[i];
curFds.insert(curFds.end(), gLogFileFds.begin(), gLogFileFds.end());
}
atexit(freeLogFileFds);
gLogInited = true;
}
static void (*gFailureFunctionPtr)() ATTR_NORETURN = abort;
LogMessage::LogMessage(const char* fname, int line, int severity)
: fname_(fname), line_(line), severity_(severity) {}
LogMessage::~LogMessage() { this->generateLogMessage(); }
void LogMessage::generateLogMessage() {
if (!gLogInited) {
fprintf(stderr,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
} else {
for (auto& fd : gLogFds[this->severity_]) {
dprintf(fd,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
}
}
}
LogMessageFatal::LogMessageFatal(const char* file, int line)
: LogMessage(file, line, FATAL) {}
LogMessageFatal::~LogMessageFatal() {
generateLogMessage();
gFailureFunctionPtr();
}
} // namespace internal
void initializeLogging(int argc, char** argv) {
internal::initializeLogFds(argv[0]);
}
namespace logging {
void setMinLogLevel(int level) { paddle::internal::gMinLogLevel = level; }
void installFailureFunction(void (*callback)() ATTR_NORETURN) {
paddle::internal::gFailureFunctionPtr = callback;
}
} // namespace logging
} // namespace paddle
#else
namespace paddle {
void initializeLogging(int argc, char** argv) {
(void)(argc);
if (!getenv("GLOG_logtostderr")) {
......@@ -197,13 +31,16 @@ void initializeLogging(int argc, char** argv) {
}
namespace logging {
void setMinLogLevel(int level) { FLAGS_minloglevel = level; }
void installFailureFunction(void (*callback)()) {
google::InstallFailureFunction(callback);
}
void installFailureWriter(void (*callback)(const char*, int)) {
google::InstallFailureWriter(callback);
}
} // namespace logging
} // namespace paddle
#endif
paddle/utils/Logging.h
浏览文件 @ b0e8b477
......@@ -22,175 +22,21 @@ limitations under the License. */
#include <sstream>
#include <string>
#ifndef PADDLE_USE_GLOG
#include "CompilerMacros.h"
//! TODO(yuyang18): Move this utility macro into some global header.
#define PP_CAT(a, b) PP_CAT_I(a, b)
#define PP_CAT_I(a, b) PP_CAT_II(~, a##b)
#define PP_CAT_II(p, res) res
/**
* Generate Unique Variable Name, Usefully in macro.
* @SEE
* http://stackoverflow.com/questions/1082192/how-to-generate-random-variable-names-in-c-using-macros
*/
#define UNIQUE_NAME(base) PP_CAT(base, __LINE__)
#include <glog/logging.h>
namespace paddle {
//! Log levels.
const int INFO = 0;
const int WARNING = 1;
const int ERROR = 2;
const int FATAL = 3;
const int NUM_SEVERITIES = 4;
namespace internal {
class LogMessage : public std::basic_ostringstream<char> {
public:
LogMessage(const char* fname, int line, int severity);
~LogMessage();
protected:
/**
* @brief Print log message to stderr, files, etc.
*/
void generateLogMessage();
private:
const char* fname_;
int line_;
int severity_;
};
// LogMessageFatal ensures the process will exit in failure after
// logging this message.
class LogMessageFatal : public LogMessage {
public:
LogMessageFatal(const char* file, int line) __attribute__((cold));
~LogMessageFatal() __attribute__((noreturn));
};
#define _P_LOG_INFO \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::INFO)
#define _P_LOG_WARNING \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::WARNING)
#define _P_LOG_ERROR \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::ERROR)
#define _P_LOG_FATAL ::paddle::internal::LogMessageFatal(__FILE__, __LINE__)
#define P_LOG(severity) _P_LOG_##severity
#define P_LOG_FIRST_N(severity, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) ++UNIQUE_NAME(LOG_OCCURRENCES); \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) P_LOG(severity)
#define P_LOG_IF_EVERY_N(severity, condition, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (condition && ((UNIQUE_NAME(LOG_OCCURRENCES) = \
(UNIQUE_NAME(LOG_OCCURRENCES) + 1) % n) == (1 % n))) \
P_LOG(severity)
#define P_LOG_EVERY_N(severity, n) P_LOG_IF_EVERY_N(severity, true, n)
// TODO(jeff): Define a proper implementation of VLOG_IS_ON
#define P_VLOG_IS_ON(lvl) ((lvl) <= 0)
#define P_LOG_IF(severity, condition) \
if (condition) P_LOG(severity)
#define P_VLOG(lvl) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl))
#define P_VLOG_IF(lvl, cond) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl) && cond)
#define P_VLOG_EVERY_N(lvl, n) P_LOG_IF_EVERY_N(INFO, P_VLOG_IS_ON(lvl), n)
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
// CHECK dies with a fatal error if condition is not true. It is *not*
// controlled by NDEBUG, so the check will be executed regardless of
// compilation mode. Therefore, it is safe to do things like:
// CHECK(fp->Write(x) == 4)
#define P_CHECK(condition) \
if (PREDICT_FALSE(!(condition))) \
P_LOG(FATAL) << "Check failed: " #condition " "
#define P_CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define P_CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define P_CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define P_CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define P_CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define P_CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define P_CHECK_NOTNULL(val) P_CHECK((val) != NULL)
//! GLOG compatible APIs
//! NOTE: only implement Paddle actually used APIs.
#define LOG(x) P_LOG(x)
#define VLOG(x) P_VLOG(x)
#define DLOG(x) P_VLOG(5)
#define CHECK(x) P_CHECK(x)
#define PCHECK(x) P_CHECK(x)
#define CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define CHECK_NOTNULL(val) P_CHECK((val) != NULL)
#define VLOG_IS_ON(x) P_VLOG_IS_ON(x)
#define LOG_FIRST_N(severity, n) P_LOG_FIRST_N(severity, n)
#define LOG_IF(severity, condition) P_LOG_IF(severity, condition)
#define VLOG_EVERY_N(lvl, n) P_VLOG_EVERY_N(lvl, n)
#define VLOG_IF(lvl, cond) P_VLOG_IF(lvl, cond)
#define LOG_EVERY_N(severity, n) P_LOG_EVERY_N(severity, n)
} // namespace internal
/**
* @brief initialize logging
* @note: Current implement of logging is lack of:
* PrintCallStack when fatal.
* VLOG_IS_ON
* But it is portable to multi-platform, and simple enough to modify.
*/
void initializeLogging(int argc, char** argv);
namespace logging {
/**
* @brief Set Min Log Level. if Log.level < minLogLevel, then will not print log
* to stream
* @param level. Any integer is OK, but only 0 <= x <= NUM_SEVERITIES is useful.
*/
void setMinLogLevel(int level);
/**
* @brief Install Log(Fatal) failure function. Default is abort();
* @param callback: The failure function.
*/
void installFailureFunction(void (*callback)() ATTR_NORETURN);
/**
* @brief installFailureWriter
* @note: not implemented currently.
*/
inline void installFailureWriter(void (*callback)(const char*, int)) {
(void)(callback); // unused callback.
}
} // namespace logging
} // namespace paddle
#else
#include <glog/logging.h>
namespace paddle {
void initializeLogging(int argc, char** argv);
namespace logging {
void setMinLogLevel(int level);
void installFailureFunction(void (*callback)());
void installFailureWriter(void (*callback)(const char*, int));
} // namespace logging
}
#endif // PADDLE_USE_GLOG
} // namespace logging
} // namespace paddle
#ifndef NDEBUG
#define DEBUG_LEVEL 5
......
paddle/utils/PythonUtil.cpp
浏览文件 @ b0e8b477
......@@ -20,8 +20,8 @@ namespace paddle {
#ifdef PADDLE_NO_PYTHON
P_DEFINE_string(python_path, "", "python path");
P_DEFINE_string(python_bin, "python2.7", "python bin");
DEFINE_string(python_path, "", "python path");
DEFINE_string(python_bin, "python2.7", "python bin");
constexpr int kExecuteCMDBufLength = 204800;
......
paddle/utils/ThreadLocal.cpp
浏览文件 @ b0e8b477
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "CommandLineParser.h"
#include "Util.h"
P_DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
namespace paddle {
......
paddle/utils/Util.cpp
浏览文件 @ b0e8b477
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "ThreadLocal.h"
#include "Version.h"
P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#ifdef WITH_GOOGLE_PERFTOOLS
/*
......@@ -52,10 +52,8 @@ P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#include <gperftools/profiler.h>
P_DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
P_DEFINE_string(profile_data_file,
"gperf.prof",
"file for storing profile data");
DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
DEFINE_string(profile_data_file, "gperf.prof", "file for storing profile data");
static void profilerSwitch(int signalNumber) {
bool static started = false;
......
paddle/utils/Version.cpp
浏览文件 @ b0e8b477
......@@ -18,13 +18,8 @@ limitations under the License. */
#include <numeric>
#include "Flags.h"
#include "Util.h"
//! TODO(yuyang18) in gflags, version has another define. Use another flag
//! instead.
#ifndef PADDLE_USE_GFLAGS
P_DEFINE_bool(version, false, "print version");
#else
P_DECLARE_bool(version);
#endif
DECLARE_bool(version);
namespace paddle {
namespace version {
......
paddle/utils/tests/CMakeLists.txt
浏览文件 @ b0e8b477
此差异已折叠。
paddle/utils/tests/test_CommandLineParser.cpp 已删除 100644 → 0
浏览文件 @ bcf6f828
此差异已折叠。
paddle/utils/tests/test_Logging.cpp 已删除 100644 → 0
浏览文件 @ bcf6f828
此差异已折叠。
proto/ModelConfig.proto
浏览文件 @ b0e8b477
此差异已折叠。
third_party/gflags.BUILD 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/gflags_test/BUILD 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/gflags_test/gflags_test.cc 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/glog.BUILD 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/glog_test/BUILD 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/glog_test/glog_test.cc 0 → 100644
浏览文件 @ b0e8b477
此差异已折叠。
third_party/gtest.BUILD
浏览文件 @ b0e8b477
此差异已折叠。
third_party/protobuf_test/BUILD
浏览文件 @ b0e8b477
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或