diff --git a/CMakeLists.txt b/CMakeLists.txt index d82d8f633c4794ffce29e02a86702f4bd8b34774..65fbbb481c432f7b905f4dec7ea39c51ec853ae8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,8 @@ find_package(ZLIB REQUIRED) find_package(NumPy REQUIRED) find_package(Threads REQUIRED) find_package(AVX QUIET) -find_package(Glog) -find_package(Gflags QUIET) +find_package(Glog REQUIRED) +find_package(Gflags REQUIRED) find_package(GTest) find_package(Sphinx) find_package(Doxygen) @@ -40,8 +40,6 @@ option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND}) option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON) option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND}) option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF) -option(WITH_GLOG "Compile PaddlePaddle use glog, otherwise use a log implement internally" ${LIBGLOG_FOUND}) -option(WITH_GFLAGS "Compile PaddlePaddle use gflags, otherwise use a flag implement internally" ${GFLAGS_FOUND}) option(WITH_TIMER "Compile PaddlePaddle use timer" OFF) option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF) option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND}) @@ -136,16 +134,12 @@ else(WITH_RDMA) add_definitions(-DPADDLE_DISABLE_RDMA) endif(WITH_RDMA) -if(WITH_GLOG) - add_definitions(-DPADDLE_USE_GLOG) - include_directories(${LIBGLOG_INCLUDE_DIR}) -endif() +# glog +include_directories(${LIBGLOG_INCLUDE_DIR}) -if(WITH_GFLAGS) - add_definitions(-DPADDLE_USE_GFLAGS) - add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE}) - include_directories(${GFLAGS_INCLUDE_DIRS}) -endif() +#gflags +add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE}) +include_directories(${GFLAGS_INCLUDE_DIRS}) if(WITH_TESTING) enable_testing() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 120000 index 0000000000000000000000000000000000000000..f3eb8b4edba28a689ae662232c0b47de47bd0699 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +./doc/howto/contribute_to_paddle_en.md \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 0b8299905abb844bfbd8b27f47b8fafded31ef7a..f097c41da85affd1ff0b24200dbdbc63bf9c3ab6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,7 +3,7 @@ http_archive( name="protobuf", url="http://github.com/google/protobuf/archive/v3.1.0.tar.gz", sha256="0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7", - strip_prefix="protobuf-3.1.0", ) + strip_prefix="protobuf-3.1.0") # External dependency to gtest 1.7.0. This method comes from # https://www.bazel.io/versions/master/docs/tutorial/cpp.html. @@ -12,4 +12,20 @@ new_http_archive( url="https://github.com/google/googletest/archive/release-1.7.0.zip", sha256="b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", build_file="third_party/gtest.BUILD", - strip_prefix="googletest-release-1.7.0", ) + strip_prefix="googletest-release-1.7.0") + +# External dependency to gflags. This method comes from +# https://github.com/gflags/example/blob/master/WORKSPACE. +new_git_repository( + name="gflags", + tag="v2.2.0", + remote="https://github.com/gflags/gflags.git", + build_file="third_party/gflags.BUILD") + +# External dependency to glog. This method comes from +# https://github.com/reyoung/bazel_playground/blob/master/WORKSPACE +new_git_repository( + name="glog", + remote="https://github.com/google/glog.git", + commit="b6a5e0524c28178985f0d228e9eaa43808dbec3c", + build_file="third_party/glog.BUILD") diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index 6702f45a168bf0dfc6cfca3ff8e68fbc79c92b11..05aa100eaefcf0d9119763b1440c96e341c80387 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -72,6 +72,7 @@ function( Sphinx_add_target target_name builder conf cache source destination ) ${source} ${destination} COMMENT "Generating sphinx documentation: ${builder}" + COMMAND ln -s ${destination}/index_*.html ${destination}/index.html ) set_property( @@ -143,4 +144,4 @@ function( Sphinx_add_targets target_base_name conf source base_destination ) add_dependencies( ${target_base_name}_linkcheck ${_dependencies} ) endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cmake/check_packages.cmake b/cmake/check_packages.cmake index 06887455418797f7162a5970669a0483e42a2db8..4b7cadfc85cd5c5bd9f1ff7e4dc833cf4f438005 100644 --- a/cmake/check_packages.cmake +++ b/cmake/check_packages.cmake @@ -14,13 +14,9 @@ if(WITH_STYLE_CHECK) find_package(PythonInterp REQUIRED) endif() -if(WITH_GLOG) - find_package(Glog REQUIRED) -endif() +find_package(Glog REQUIRED) -if(WITH_GFLAGS) - find_package(Gflags REQUIRED) -endif() +find_package(Gflags REQUIRED) if(WITH_TESTING) find_package(GTest REQUIRED) diff --git a/cmake/util.cmake b/cmake/util.cmake index eb7db7ce2ef20bf48b134378cfe7672f65a970a3..38366373c6dbcc1d05c359484ae73ace1bbc59be 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -65,7 +65,7 @@ endmacro() # link_paddle_exe # add paddle library for a paddle executable, such as trainer, pserver. # -# It will handle WITH_PYTHON/WITH_GLOG etc. +# It will handle WITH_PYTHON etc. function(link_paddle_exe TARGET_NAME) if(WITH_RDMA) generate_rdma_links() @@ -108,6 +108,8 @@ function(link_paddle_exe TARGET_NAME) paddle_cuda ${METRIC_LIBS} ${PROTOBUF_LIBRARY} + ${LIBGLOG_LIBRARY} + ${GFLAGS_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${CBLAS_LIBS} ${ZLIB_LIBRARIES} @@ -125,16 +127,6 @@ function(link_paddle_exe TARGET_NAME) ${PYTHON_LIBRARIES}) endif() - if(WITH_GLOG) - target_link_libraries(${TARGET_NAME} - ${LIBGLOG_LIBRARY}) - endif() - - if(WITH_GFLAGS) - target_link_libraries(${TARGET_NAME} - ${GFLAGS_LIBRARIES}) - endif() - if(WITH_GPU) if(NOT WITH_DSO OR WITH_METRIC) target_link_libraries(${TARGET_NAME} diff --git a/demo/semantic_role_labeling/data/extract_dict_feature.py b/demo/semantic_role_labeling/data/extract_dict_feature.py index a02a49a86ed31f44058c192525a2acd979c5de0b..da44111976a0dec68345fc139d0aa459ca9211c2 100644 --- a/demo/semantic_role_labeling/data/extract_dict_feature.py +++ b/demo/semantic_role_labeling/data/extract_dict_feature.py @@ -43,13 +43,13 @@ def extract_dict_features(pair_file, feature_file): mark[verb_index] = 1 ctx_0 = sentence_list[verb_index] - if verb_index < len(labels_list) - 2: + if verb_index < len(labels_list) - 1: mark[verb_index + 1] = 1 ctx_p1 = sentence_list[verb_index + 1] else: ctx_p1 = 'eos' - if verb_index < len(labels_list) - 3: + if verb_index < len(labels_list) - 2: mark[verb_index + 2] = 1 ctx_p2 = sentence_list[verb_index + 2] else: diff --git a/doc/api/data_provider/pydataprovider2_en.rst b/doc/api/data_provider/pydataprovider2_en.rst index 2e3cbd89babe1127b37eca4826718828de662bce..30357be32538db4423ad0eaf899138256c84edc7 100644 --- a/doc/api/data_provider/pydataprovider2_en.rst +++ b/doc/api/data_provider/pydataprovider2_en.rst @@ -1,4 +1,4 @@ -.. _api_pydataprovider2_en: +.. _api_pydataprovider2: PyDataProvider2 =============== @@ -104,7 +104,7 @@ And PaddlePadle will do all of the rest things\: Is this cool? -.. _api_pydataprovider2_en_sequential_model: +.. _api_pydataprovider2_sequential_model: DataProvider for the sequential model ------------------------------------- diff --git a/doc/api/predict/swig_py_paddle_en.rst b/doc/api/predict/swig_py_paddle_en.rst index 8dfea1d3345711255255ac423d2e82095312c98e..1c628e6971fa5643e6a9ca629488049957686193 100644 --- a/doc/api/predict/swig_py_paddle_en.rst +++ b/doc/api/predict/swig_py_paddle_en.rst @@ -23,7 +23,7 @@ python's :code:`help()` function. Let's walk through the above python script: * At the beginning, use :code:`swig_paddle.initPaddle()` to initialize PaddlePaddle with command line arguments, for more about command line arguments - see :ref:`cmd_detail_introduction_en` . + see :ref:`cmd_detail_introduction` . * Parse the configuration file that is used in training with :code:`parse_config()`. Because data to predict with always have no label, and output of prediction work normally is the output layer rather than the cost layer, so you should modify @@ -36,7 +36,7 @@ python's :code:`help()` function. Let's walk through the above python script: - Note: As swig_paddle can only accept C++ matrices, we offer a utility class DataProviderConverter that can accept the same input data with PyDataProvider2, for more information please refer to document - of :ref:`api_pydataprovider2_en` . + of :ref:`api_pydataprovider2` . * Do the prediction with :code:`forwardTest()`, which takes the converted input data and outputs the activations of the output layer. diff --git a/doc/conf.py.cn.in b/doc/conf.py.cn.in index 92d72f797ea2018f9a7e9bef076c738cc9a1d6d2..418d718fbd9c61bff3acb9c2dab0638c0b650bab 100644 --- a/doc/conf.py.cn.in +++ b/doc/conf.py.cn.in @@ -79,7 +79,7 @@ language = 'zh_CN' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ['_build', '**/*_en*', '*_en*'] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/doc/conf.py.en.in b/doc/conf.py.en.in index f942f166fc7599c3bac7eb197e3918bc736ba3f6..e96c25cb75bee20d2e2949423d80ddab1d3450a1 100644 --- a/doc/conf.py.en.in +++ b/doc/conf.py.en.in @@ -80,7 +80,7 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ['_build', '**/*_cn*', '*_cn*'] # The reST default role (used for this markup: `text`) to use for all # documents. diff --git a/doc/getstarted/build_and_install/build_from_source_en.md b/doc/getstarted/build_and_install/build_from_source_en.md index 5db871d59ae83666263d03a6ea3b504d323293ee..aaa07d49d3148266db27670a98c2b27db4dc0a8f 100644 --- a/doc/getstarted/build_and_install/build_from_source_en.md +++ b/doc/getstarted/build_and_install/build_from_source_en.md @@ -49,10 +49,8 @@ PaddlePaddle supports some build options. To enable it, first you need to instal WITH_GPUCompile with GPU mode. WITH_DOUBLECompile with double precision floating-point, default: single precision. -WITH_GLOGCompile with glog. If not found, default: an internal log implementation. -WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation. WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing. -WITH_DOC Compile to generate PaddlePaddle's docs, default: disabled (OFF). +WITH_DOC Compile to generate PaddlePaddle's docs, default: disabled (OFF). WITH_SWIG_PYCompile with python predict API, default: disabled (OFF). WITH_STYLE_CHECKCompile with code style check, default: enabled (ON). diff --git a/doc/getstarted/build_and_install/cmake/compile_options.csv b/doc/getstarted/build_and_install/cmake/compile_options.csv index 171d8fba71d14394c5bcda0bfde2231c3b173086..463b825470579d0c3736a408b1e82dd33e6f8d42 100644 --- a/doc/getstarted/build_and_install/cmake/compile_options.csv +++ b/doc/getstarted/build_and_install/cmake/compile_options.csv @@ -6,8 +6,6 @@ WITH_AVX,是否编译含有AVX指令集的PaddlePaddle二进制文件,是 WITH_PYTHON,是否内嵌PYTHON解释器。方便今后的嵌入式移植工作。,是 WITH_STYLE_CHECK,是否编译时进行代码风格检查,是 WITH_RDMA,是否开启RDMA,否 -WITH_GLOG,是否开启GLOG。如果不开启，则会使用一个简化版的日志，同时方便今后的嵌入式移植工作。,取决于是否寻找到GLOG -WITH_GFLAGS,是否使用GFLAGS。如果不开启，则会使用一个简化版的命令行参数解析器，同时方便今后的嵌入式移植工作。,取决于是否寻找到GFLAGS WITH_TIMER,是否开启计时功能。如果开启会导致运行略慢，打印的日志变多，但是方便调试和测Benchmark,否 WITH_TESTING,是否开启单元测试,取决于是否寻找到GTEST WITH_DOC,是否编译中英文文档,否 diff --git a/doc/getstarted/build_and_install/ubuntu_install_cn.rst b/doc/getstarted/build_and_install/ubuntu_install_cn.rst index f923a1917cf8eb837d776c1221f39591efd533d6..d02d9c63bbfb50954d7b75f2c685ce167a3b7146 100644 --- a/doc/getstarted/build_and_install/ubuntu_install_cn.rst +++ b/doc/getstarted/build_and_install/ubuntu_install_cn.rst @@ -46,8 +46,6 @@ PaddlePaddle提供了ubuntu 14.04 deb安装包。 with_double: OFF with_python: ON with_rdma: OFF - with_glog: ON - with_gflags: ON with_metric_learning: with_timer: OFF with_predict_sdk: diff --git a/doc/howto/cmd_parameter/detail_introduction_en.md b/doc/howto/cmd_parameter/detail_introduction_en.md index 82136b7d4f65ffcdff60243feb25b31a4a468637..27b2faf1d8a9367ff9498a76d363791ab7fbe61c 100644 --- a/doc/howto/cmd_parameter/detail_introduction_en.md +++ b/doc/howto/cmd_parameter/detail_introduction_en.md @@ -1,5 +1,5 @@ ```eval_rst -.. _cmd_detail_introduction_en: +.. _cmd_detail_introduction: ``` # Detail Description diff --git a/doc/howto/cmd_parameter/index_en.md b/doc/howto/cmd_parameter/index_en.md index fb658f2aa5bc0edef7b5dcb24a582d2c4182caa7..a6c236db61f834372152dba0f5049fe37b731086 100644 --- a/doc/howto/cmd_parameter/index_en.md +++ b/doc/howto/cmd_parameter/index_en.md @@ -1,5 +1,5 @@ ```eval_rst -.. _cmd_line_index_en: +.. _cmd_line_index: ``` # How to Set Command-line Parameters diff --git a/doc/howto/contribute_to_paddle_en.md b/doc/howto/contribute_to_paddle_en.md index 1decc91d62cc25c5b3157bdc6e0835421be23252..f4b67d64e7db821539e1cc31177de5caf7b14766 100644 --- a/doc/howto/contribute_to_paddle_en.md +++ b/doc/howto/contribute_to_paddle_en.md @@ -1,8 +1,8 @@ # How to Contribute Code We sincerely appreciate your contributions. You can use fork and pull request -workflow to merge your code. - +workflow to merge your code. + ## Code Requirements - Your code must be fully documented by [doxygen](http://www.stack.nl/~dimitri/doxygen/) style. @@ -12,11 +12,11 @@ workflow to merge your code. - Pass all unit tests. The following tutorial guides you into submitting your contibution. - + ## [Creating a Fork](https://help.github.com/articles/fork-a-repo/) - + Just head over to the GitHub page and click the "Fork" button. -It's just that simple. +It's just that simple. ## Clone @@ -25,7 +25,7 @@ The **develop** is the main branch, and other user's branches are feature branch Once you've created a fork, you can use your favorite git client to clone your repo or just head straight to the command line: - + ```shell # Clone your fork to your local machine git clone --branch develop https://github.com/USERNAME/Paddle.git @@ -47,6 +47,22 @@ Then you can start to develop by making a local developement branch git checkout -b MY_COOL_STUFF_BRANCH ``` +## Using `pre-commit` hook + +Paddle developers use [pre-commit](http://pre-commit.com/) tool to manage git +pre-commit hooks. It can help us format source codes (cpp, python), check some +basic thing before commit (only one EOL for each file, do not add a huge file +in git). `pre-commit` tests is a part of unit tests in Travis-CI now, every +PR doesn't fit hook can not be merged into Paddle. + +To use [pre-commit](http://pre-commit.com/), you should install it by +`pip install pre-commit`, and currently, Paddle uses `clang-format` to format +c/cpp sources. Please make sure clang-format 3.8+ installed. + +Then just run `pre-commit install` in your Paddle clone directory. When you +commit your code, the pre-commit hook will check the local code if there is +anything not suitable to commit, and so on. + ## Commit Commit your changes by following command lines: @@ -83,7 +99,7 @@ git pull --rebase upstream develop If there are no unique commits locally, git will simply perform a fast-forward. However, if you have been making changes (in the vast majority of cases you -probably shouldn't be), you may have to deal with conflicts. +probably shouldn't be), you may have to deal with conflicts. Now, your local master branch is up-to-date with everything modified upstream. diff --git a/doc/howto/deep_model/rnn/rnn_en.rst b/doc/howto/deep_model/rnn/rnn_en.rst index a0740bae61be1bcd63fa78891cb526ad17a45ae9..73f5d5371fcd3ce95253cad47b0d8e738284441c 100644 --- a/doc/howto/deep_model/rnn/rnn_en.rst +++ b/doc/howto/deep_model/rnn/rnn_en.rst @@ -30,7 +30,7 @@ Then at the :code:`process` function, each :code:`yield` function will return th yield src_ids, trg_ids, trg_ids_next -For more details description of how to write a data provider, please refer to :ref:`api_pydataprovider2_en` . The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`. +For more details description of how to write a data provider, please refer to :ref:`api_pydataprovider2` . The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`. =============================================== Configure Recurrent Neural Network Architecture @@ -246,6 +246,6 @@ The code is listed below: outputs(beam_gen) -Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to :ref:`semantic_role_labeling_en` for more details. +Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to :ref:`semantic_role_labeling` for more details. The full configuration file is located at :code:`demo/seqToseq/seqToseq_net.py`. diff --git a/doc/tutorials/imagenet_model/resnet_model_cn.md b/doc/tutorials/imagenet_model/resnet_model_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..82ec9d70b345c11aba3aa86f8206eedc8072bb88 --- /dev/null +++ b/doc/tutorials/imagenet_model/resnet_model_cn.md @@ -0,0 +1,284 @@ +# Model Zoo - ImageNet # + +[ImageNet](http://www.image-net.org/) 是通用物体分类领域一个众所周知的数据库。本教程提供了一个用于ImageNet上的卷积分类网络模型。 + +## ResNet 介绍 + +论文 [Deep Residual Learning for Image Recognition](http://arxiv.org/abs/1512.03385) 中提出的ResNet网络结构在2015年ImageNet大规模视觉识别竞赛(ILSVRC 2015)的分类任务中赢得了第一名。他们提出残差学习的框架来简化网络的训练，所构建网络结构的的深度比之前使用的网络有大幅度的提高。下图展示的是基于残差的连接方式。左图构造网络模块的方式被用于34层的网络中，而右图的瓶颈连接模块用于50层，101层和152层的网络结构中。 + +
![resnet_block](./resnet_block.jpg)
+
图 1. ResNet 网络模块
+ +本教程中我们给出了三个ResNet模型，这些模型都是由原作者提供的模型转换过来的。我们使用PaddlePaddle在ILSVRC的验证集共50,000幅图像上测试了模型的分类错误率，其中输入图像的颜色通道顺序为**BGR**，保持宽高比缩放到短边为256，只截取中心方形的图像区域。分类错误率和模型大小由下表给出。 +
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + +
ResNetTop-1Model Size
ResNet-5024.9%99M
ResNet-10123.7%173M
ResNet-15223.2%234M
+
+ +## ResNet 模型 + +50层，101层和152层的网络配置文件可参照```demo/model_zoo/resnet/resnet.py```。你也可以通过在命令行参数中增加一个参数如```--config_args=layer_num=50```来指定网络层的数目。 + +### 网络可视化 + +你可以通过执行下面的命令来得到ResNet网络的结构可视化图。该脚本会生成一个dot文件，然后可以转换为图片。需要安装graphviz来转换dot文件为图片。 + +``` +cd demo/model_zoo/resnet +./net_diagram.sh +``` + +### 模型下载 + +``` +cd demo/model_zoo/resnet +./get_model.sh +``` +你可以执行上述命令来下载所有的模型和均值文件，如果下载成功，这些文件将会被保存在```demo/model_zoo/resnet/model```路径下。 + +``` +mean_meta_224 resnet_101 resnet_152 resnet_50 +``` + * resnet_50: 50层网络模型。 + * resnet_101: 101层网络模型。 + * resnet_152: 152层网络模型。 + * mean\_meta\_224: 均值图像文件，图像大小为3 x 224 x 224，颜色通道顺序为**BGR**。你也可以使用这三个值: 103.939, 116.779, 123.68。 + +### 参数信息 + +* **卷积层权重** + + 由于每个卷积层后面连接的是batch normalization层，因此该层中没有偏置(bias)参数，并且只有一个权重。 + 形状: `(Co, ky, kx, Ci)` + * Co: 输出特征图的通道数目 + * ky: 滤波器核在垂直方向上的尺寸 + * kx: 滤波器核在水平方向上的尺寸 + * Ci: 输入特征图的通道数目 + + 二维矩阵: (Co * ky * kx, Ci), 行优先次序存储。 + +* **全连接层权重** + + 二维矩阵: (输入层尺寸, 本层尺寸), 行优先次序存储。 + +* **[Batch Normalization]() 层权重** + +本层有四个参数，实际上只有.w0和.wbias是需要学习的参数，另外两个分别是滑动均值和方差。在测试阶段它们将会被加载到模型中。下表展示了batch normalization层的参数。 +
+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
参数名尺寸含义
_res2_1_branch1_bn.w0256gamma, 缩放参数
_res2_1_branch1_bn.w1256特征图均值
_res2_1_branch1_bn.w2256特征图方差
_res2_1_branch1_bn.wbias256beta, 偏置参数
+
+ +### 参数读取 + +使用者可以使用下面的Python脚本来读取参数值: + +``` +import sys +import numpy as np + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +if __name__=='__main__': + weight = load(sys.argv[1]) +``` + +或者直接使用下面的shell命令: + +``` +od -j 16 -f _res2_1_branch1_bn.w0 +``` + +## 特征提取 + +我们提供了C++和Python接口来提取特征。下面的例子使用了`demo/model_zoo/resnet/example`中的数据，详细地展示了整个特征提取的过程。 + +### C++接口 + +首先，在配置文件中的`define_py_data_sources2`里指定图像数据列表，具体请参照示例`demo/model_zoo/resnet/resnet.py`。 + +``` + train_list = 'train.list' if not is_test else None + # mean.meta is mean file of ImageNet dataset. + # mean.meta size : 3 x 224 x 224. + # If you use three mean value, set like: + # "mean_value:103.939,116.779,123.68;" + args={ + 'mean_meta': "model/mean_meta_224/mean.meta", + 'image_size': 224, 'crop_size': 224, + 'color': True,'swap_channel:': [2, 1, 0]} + define_py_data_sources2(train_list, + 'example/test.list', + module="example.image_list_provider", + obj="processData", + args=args) +``` + +第二步，在`resnet.py`文件中指定要提取特征的网络层的名字。例如， + +``` +Outputs("res5_3_branch2c_conv", "res5_3_branch2c_bn") +``` + +第三步，在`extract_fea_c++.sh`文件中指定模型路径和输出的目录，然后执行下面的命令。 + +``` +cd demo/model_zoo/resnet +./extract_fea_c++.sh +``` + +如果执行成功，特征将会存到`fea_output/rank-00000`文件中，如下所示。同时你可以使用`load_feature.py`文件中的`load_feature_c`接口来加载该文件。 + +``` +-0.115318 -0.108358 ... -0.087884;-1.27664 ... -1.11516 -2.59123; +-0.126383 -0.116248 ... -0.00534909;-1.42593 ... -1.04501 -1.40769; +``` + +* 每行存储的是一个样本的特征。其中，第一行存的是图像`example/dog.jpg`的特征，第二行存的是图像`example/cat.jpg`的特征。 +* 不同层的特征由分号`;`隔开，并且它们的顺序与`Outputs()`中指定的层顺序一致。这里，左边是`res5_3_branch2c_conv`层的特征，右边是`res5_3_branch2c_bn`层特征。 + +### Python接口 + +示例`demo/model_zoo/resnet/classify.py`中展示了如何使用Python来提取特征。下面的例子同样使用了`./example/test.list`中的数据。执行的命令如下： + +``` +cd demo/model_zoo/resnet +./extract_fea_py.sh +``` + +extract_fea_py.sh: + +``` +python classify.py \ + --job=extract \ + --conf=resnet.py\ + --use_gpu=1 \ + --mean=model/mean_meta_224/mean.meta \ + --model=model/resnet_50 \ + --data=./example/test.list \ + --output_layer="res5_3_branch2c_conv,res5_3_branch2c_bn" \ + --output_dir=features + +``` +* \--job=extract: 指定工作模式来提取特征。 +* \--conf=resnet.py: 网络配置文件。 +* \--use_gpu=1: 指定是否使用GPU。 +* \--model=model/resnet_50: 模型路径。 +* \--data=./example/test.list: 数据列表。 +* \--output_layer="xxx,xxx": 指定提取特征的层。 +* \--output_dir=features: 输出目录。 + +如果运行成功，你将会看到特征存储在`features/batch_0`文件中，该文件是由cPickle产生的。你可以使用`load_feature.py`中的`load_feature_py`接口来打开该文件，它将返回如下的字典： + +``` +{ +'cat.jpg': {'res5_3_branch2c_conv': array([[-0.12638293, -0.116248 , -0.11883899, ..., -0.00895038, 0.01994277, -0.00534909]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.42593431, -1.28918779, -1.32414699, ..., -1.45933616, -1.04501402, -1.40769434]], dtype=float32)}, +'dog.jpg': {'res5_3_branch2c_conv': array([[-0.11531784, -0.10835785, -0.08809858, ...,0.0055237, 0.01505112, -0.08788397]], dtype=float32), 'res5_3_branch2c_bn': array([[-1.27663755, -1.18272924, -0.90937918, ..., -1.25178063, -1.11515927, -2.59122872]], dtype=float32)} +} +``` + +仔细观察，这些特征值与上述使用C++接口提取的结果是一致的。 + +## 预测 + +`classify.py`文件也可以用于对样本进行预测。我们提供了一个示例脚本`predict.sh`，它使用50层的ResNet模型来对`example/test.list`中的数据进行预测。 + +``` +cd demo/model_zoo/resnet +./predict.sh +``` + +predict.sh调用了`classify.py`: + +``` +python classify.py \ + --job=predict \ + --conf=resnet.py\ + --multi_crop \ + --model=model/resnet_50 \ + --use_gpu=1 \ + --data=./example/test.list +``` +* \--job=extract: 指定工作模型进行预测。 +* \--conf=resnet.py: 网络配置文件。network configure. +* \--multi_crop: 使用10个裁剪图像块，预测概率取平均。 +* \--use_gpu=1: 指定是否使用GPU。 +* \--model=model/resnet_50: 模型路径。 +* \--data=./example/test.list: 数据列表。 + +如果运行成功，你将会看到如下结果，其中156和285是这些图像的分类标签。 + +``` +Label of example/dog.jpg is: 156 +Label of example/cat.jpg is: 282 +``` diff --git a/doc/tutorials/imagenet_model/resnet_model_en.md b/doc/tutorials/imagenet_model/resnet_model_en.md index 5403ab9f17d2399fee878d0f3c512cb166aba06f..478ad06193b14ba7fe02238df621db1f7b0804d4 100644 --- a/doc/tutorials/imagenet_model/resnet_model_en.md +++ b/doc/tutorials/imagenet_model/resnet_model_en.md @@ -52,7 +52,7 @@ See ```demo/model_zoo/resnet/resnet.py```. This config contains network of 50, 1 ### Network Visualization -You can get a diagram of ResNet network by running the following commands. The script generates dot file and then converts dot file to PNG file, which uses installed draw_dot tool in our server. If you can not access the server, just install graphviz to convert dot file. +You can get a diagram of ResNet network by running the following commands. The script generates dot file and then converts dot file to PNG file, which needs to install graphviz to convert. ``` cd demo/model_zoo/resnet @@ -138,7 +138,7 @@ There are four parameters in this layer. In fact, only .w0 and .wbias are the le ### Parameter Observation -Users who want to observe the parameters can use python to read: +Users who want to observe the parameters can use Python to read: ``` import sys @@ -209,7 +209,7 @@ If successful, features are saved in `fea_output/rank-00000` as follows. And you ### Python Interface -`demo/model_zoo/resnet/classify.py` is an example to show how to use python to extract features. Following example still uses data of `./example/test.list`. Command is as follows: +`demo/model_zoo/resnet/classify.py` is an example to show how to use Python to extract features. Following example still uses data of `./example/test.list`. Command is as follows: ``` cd demo/model_zoo/resnet @@ -238,8 +238,6 @@ python classify.py \ * \--output_layer="xxx,xxx": specify layers to extract features. * \--output_dir=features: output diretcoty. -Note, since the convolution layer in these ResNet models is suitable for the cudnn implementation which only support GPU. It not support CPU mode because of compatibility issue and we will fix later. - If run successfully, you will see features saved in `features/batch_0`, this file is produced with cPickle. You can use `load_feature_py` interface in `load_feature.py` to open the file, and it returns a dictionary as follows: ``` diff --git a/doc/tutorials/rec/ml_dataset_en.md b/doc/tutorials/rec/ml_dataset_en.md index dc11a5e06031b62d9f86e4dd83a14b2f1a72afc3..25dea5c4afbf1ce1c1ac6195cbd245b116459e2e 100644 --- a/doc/tutorials/rec/ml_dataset_en.md +++ b/doc/tutorials/rec/ml_dataset_en.md @@ -1,6 +1,5 @@ ```eval_rst -.. _demo_ml_dataset_en: - +.. _demo_ml_dataset: ``` # MovieLens Dataset diff --git a/doc/tutorials/rec/ml_regression_en.rst b/doc/tutorials/rec/ml_regression_en.rst index 6346090a84fad71ab9dff21de0dcc536b5760b83..4bb2586e342e7321375e99f760bfd464d0480872 100644 --- a/doc/tutorials/rec/ml_regression_en.rst +++ b/doc/tutorials/rec/ml_regression_en.rst @@ -16,7 +16,7 @@ Data Preparation ```````````````` Download and extract dataset '''''''''''''''''''''''''''' -We use :ref:`demo_ml_dataset_en` here. +We use :ref:`demo_ml_dataset` here. To download and unzip the dataset, simply run the following commands. .. code-block:: bash @@ -264,7 +264,7 @@ In this :code:`dataprovider.py`, we should set\: * use_seq\: Whether this :code:`dataprovider.py` in sequence mode or not. * process\: Return each sample of data to :code:`paddle`. -The data provider details document see :ref:`api_pydataprovider2_en`. +The data provider details document see :ref:`api_pydataprovider2`. Train ````` @@ -280,7 +280,7 @@ The run.sh is shown as follow: It just start a paddle training process, write the log to `log.txt`, then print it on screen. -Each command line argument in :code:`run.sh`, please refer to the :ref:`cmd_line_index_en` page. The short description of these arguments is shown as follow. +Each command line argument in :code:`run.sh`, please refer to the :ref:`cmd_line_index` page. The short description of these arguments is shown as follow. * config\: Tell paddle which file is neural network configuration. * save_dir\: Tell paddle save model into './output' diff --git a/doc/tutorials/semantic_role_labeling/index_en.md b/doc/tutorials/semantic_role_labeling/index_en.md index 9a9f0d757bac845b858811e09dd64056f6bc08e5..92d7c634832119c718711a57c16f69492d405f28 100644 --- a/doc/tutorials/semantic_role_labeling/index_en.md +++ b/doc/tutorials/semantic_role_labeling/index_en.md @@ -1,5 +1,5 @@ ```eval_rst -.. _semantic_role_labeling_en: +.. _semantic_role_labeling: ``` # Semantic Role labeling Tutorial # diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 9b2d122a09adabd766014a9d21a167eec5b2de32..6ad1d79e59b11b2c1f7aacf22d13347b3fd8e0e2 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -17,22 +17,18 @@ add_library(paddle_api STATIC ${API_SOURCES}) add_dependencies(paddle_api gen_proto_cpp) +list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH) -if(WITH_GFLAGS) - list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH) - - if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}") - # Because gflags compiled by cmake, so it is imported by cmake target, - # not a real library path. Get the real library path here. - message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}") - get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION) - message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}") - else() - set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES}) - endif() +if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}") +# Because gflags compiled by cmake, so it is imported by cmake target, +# not a real library path. Get the real library path here. +message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}") +get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION) +message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}") +else() +set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES}) endif() - configure_file( paddle_api_config.py.in ${PROJ_ROOT}/paddle/api/paddle_api_config.py @@ -57,7 +53,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp paddle_trainer paddle_api paddle_cuda - ${PY_PADDLE_PYTHON_FILES} + ${PY_PADDLE_PYTHON_FILES} ) install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ diff --git a/paddle/api/Trainer.cpp b/paddle/api/Trainer.cpp index 59b47d4b1c7b6d586e89624c155d7ba6f3885eb6..d83dc380beeec3747451a483f4811eb833e8c226 100644 --- a/paddle/api/Trainer.cpp +++ b/paddle/api/Trainer.cpp @@ -27,9 +27,9 @@ limitations under the License. */ using paddle::real; -P_DECLARE_string(config); -P_DECLARE_string(init_model_path); -P_DECLARE_int32(start_pass); +DECLARE_string(config); +DECLARE_string(init_model_path); +DECLARE_int32(start_pass); struct TrainerPrivate : public paddle::Trainer { bool _trainOneBatch(size_t batchSize); diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index a2352250c31efa7ee3c4c8338d95dce5a5b9a511..23542b952b7699d66cf64b47d0354e9078ae06d9 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -8,9 +8,7 @@ CMAKE_DL_LIBS="@CMAKE_DL_LIBS@" WITH_PYTHON="@WITH_PYTHON@" PYTHON_LIBRARIES="@PYTHON_LIBRARIES@" -WITH_GLOG="@WITH_GLOG@" LIBGLOG_LIBRARY="@LIBGLOG_LIBRARY@" -WITH_GFLAGS="@WITH_GFLAGS@" GFLAGS_LIBRARIES="@GFLAGS_LIBRARIES@" GFLAGS_LOCATION="@GFLAGS_LOCATION@" CBLAS_LIBRARIES="@CBLAS_LIBS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index 85cc54700faceb5a514cebe665a2da5ed2f7aa3c..51d7dfee58b786512201577872559ae510051ba9 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -47,10 +47,8 @@ try: self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON) self.python_libs = PYTHON_LIBRARIES - self.with_glog = PaddleLDFlag.cmake_bool(WITH_GLOG) self.glog_libs = LIBGLOG_LIBRARY - self.with_gflags = PaddleLDFlag.cmake_bool(WITH_GFLAGS) self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS) self.gflags_libs = GFLAGS_LIBRARIES self.gflags_location = GFLAGS_LOCATION @@ -88,6 +86,8 @@ try: "-lpaddle_cuda", "-lpaddle_api", self.normalize_flag(self.protolib), + self.normalize_flag(self.glog_libs), + self.normalize_flag(self.gflags_libs), self.normalize_flag(self.zlib), self.normalize_flag(self.thread), self.normalize_flag(self.dl_libs), @@ -96,10 +96,6 @@ try: if self.with_python: libs.append(self.normalize_flag(self.python_libs)) - if self.with_glog: - libs.append(self.normalize_flag(self.glog_libs)) - if self.with_gflags: - libs.append(self.normalize_flag(self.gflags_libs)) if self.with_gpu: libs.append(self.normalize_flag(self.curt)) if self.with_coverage: diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index 7111224d599f0d67395254a95d7f63110a6a87c4..8cddf10d40c6277c6bb29a4fe11e5845a2770213 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -21,10 +21,10 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Logging.h" -P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb, - 4096, - "Specify cuDNN max workspace limit, in units MB, " - "4096MB=4GB by default."); +DEFINE_int32(cudnn_conv_workspace_limit_in_mb, + 4096, + "Specify cuDNN max workspace limit, in units MB, " + "4096MB=4GB by default."); namespace dynload { diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index 41787f6c0a067fa677f1be5e4b2461365e8b28c4..a71eecba2736234dafaf6b67e5efac5358a30871 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ // clang-format off -// Because clang-format 4.X and clang-format 3.8+ format +// Because clang-format 4.X and clang-format 3.8+ format // following lines in different. So disable clang-format. #include "hl_cuda.h" #include @@ -22,6 +22,7 @@ limitations under the License. */ #include #include #include +#include "hl_cuda.h" #include "hl_cuda.ph" #include "hl_dso_loader.h" #include "hl_thread.ph" diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index f509b8924319dcaa48adaf1de32fec03e45d61c5..54c7620fc081f681d9d33bcd711008fa5029df05 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -16,21 +16,21 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Logging.h" -P_DEFINE_string(cudnn_dir, - "", - "Specify path for loading libcudnn.so. For instance, " - "/usr/local/cudnn/lib. If empty [default], dlopen " - "will search cudnn from LD_LIBRARY_PATH"); - -P_DEFINE_string(cuda_dir, - "", - "Specify path for loading cuda library, such as libcublas, " - "libcurand. For instance, /usr/local/cuda/lib64. (Note: " - "libcudart can not be specified by cuda_dir, since some " - "build-in function in cudart already ran before main entry). " - "If default, dlopen will search cuda from LD_LIBRARY_PATH"); - -P_DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so."); +DEFINE_string(cudnn_dir, + "", + "Specify path for loading libcudnn.so. For instance, " + "/usr/local/cudnn/lib. If empty [default], dlopen " + "will search cudnn from LD_LIBRARY_PATH"); + +DEFINE_string(cuda_dir, + "", + "Specify path for loading cuda library, such as libcublas, " + "libcurand. For instance, /usr/local/cuda/lib64. (Note: " + "libcudart can not be specified by cuda_dir, since some " + "build-in function in cudart already ran before main entry). " + "If default, dlopen will search cuda from LD_LIBRARY_PATH"); + +DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so."); static inline std::string join(const std::string& part1, const std::string& part2) { diff --git a/paddle/gserver/dataproviders/ProtoDataProvider.cpp b/paddle/gserver/dataproviders/ProtoDataProvider.cpp index d16ecca2d977478e7e7f8819f3b5a5ea48e69b07..c6f5cab1915b7f41d505c37a7fef762a392bad7f 100644 --- a/paddle/gserver/dataproviders/ProtoDataProvider.cpp +++ b/paddle/gserver/dataproviders/ProtoDataProvider.cpp @@ -22,9 +22,9 @@ limitations under the License. */ #include "DataProviderGroup.h" #include "paddle/utils/Logging.h" -P_DEFINE_double(memory_threshold_on_load_data, - 1.0, - "stop loading data when memory is not sufficient"); +DEFINE_double(memory_threshold_on_load_data, + 1.0, + "stop loading data when memory is not sufficient"); namespace paddle { diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 7556d21e01e0314d3ee17fa37642081174ec41f3..2f9928191170aa6cf25417362cb360b5e2865b69 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/gserver/gradientmachines/NeuralNetwork.h" -P_DECLARE_int32(trainer_id); +DECLARE_int32(trainer_id); namespace paddle { diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index a7324f55451e696176a040b470c2d3bdf9eaa392..88c098b3559d8d2918309aa48329af067f79bdd5 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -21,11 +21,11 @@ limitations under the License. */ #include "NeuralNetwork.h" #include "ParallelNeuralNetwork.h" -P_DEFINE_bool(allow_only_one_model_on_one_gpu, - true, - "If true, do not allow multiple models on one GPU device"); +DEFINE_bool(allow_only_one_model_on_one_gpu, + true, + "If true, do not allow multiple models on one GPU device"); #ifdef PADDLE_METRIC_LEARNING -P_DECLARE_bool(external); +DECLARE_bool(external); #endif namespace paddle { diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index ee1c92bdf531d9e5cc4bbd63c4f6d91b32b8cee9..8f68b3d66bd263b8df34801878efee3e2de2622d 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -24,7 +24,7 @@ limitations under the License. */ #include "paddle/utils/Stat.h" #include "paddle/utils/Util.h" -P_DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so"); +DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so"); static const char* DIY_CALC_PROB_SYMBOL_NAME = "calc_prob"; static const char* DIY_START_CALC_PROB_SYMBOL_NAME = "start_calc_prob"; diff --git a/paddle/gserver/layers/DataLayer.cpp b/paddle/gserver/layers/DataLayer.cpp index 66f0606a38cf597c2697ef1d9e9419ea0e94ef5a..3551df4e172f0237685127b0b3869554d9c5f97d 100644 --- a/paddle/gserver/layers/DataLayer.cpp +++ b/paddle/gserver/layers/DataLayer.cpp @@ -54,7 +54,7 @@ void DataLayer::copyDataToOutput(Argument& output) { output.setFrameWidth(config_.width()); } else { output.setFrameHeight(data_.getFrameHeight()); - output.setFrameHeight(data_.getFrameHeight()); + output.setFrameWidth(data_.getFrameWidth()); } output.cpuSequenceDims = data_.cpuSequenceDims; output.sequenceStartPositions = data_.sequenceStartPositions; diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index c9e121047b5fa1297cfca7c268205594f079e1e3..c47943f81c01589eada4b825d54be5c69314b6fa 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -33,7 +33,7 @@ limitations under the License. */ #include "TransLayer.h" #include "ValidationLayer.h" -P_DEFINE_bool(log_error_clipping, false, "enable log error clipping or not"); +DEFINE_bool(log_error_clipping, false, "enable log error clipping or not"); namespace paddle { diff --git a/paddle/gserver/layers/LstmLayer.cpp b/paddle/gserver/layers/LstmLayer.cpp index 452091eff42083537f37d89b8f8464851f2e36db..2543d1b49a801943819e05bc52e53eaeafae1edf 100644 --- a/paddle/gserver/layers/LstmLayer.cpp +++ b/paddle/gserver/layers/LstmLayer.cpp @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/math/Matrix.h" #include "paddle/utils/Stat.h" -P_DECLARE_bool(prev_batch_state); +DECLARE_bool(prev_batch_state); namespace paddle { diff --git a/paddle/gserver/layers/RecurrentLayer.cpp b/paddle/gserver/layers/RecurrentLayer.cpp index 9f3bf76a2dcf42b0ede0f21a241e83de39c5944b..85812c9d660e07e915012a7337e621c10a6597ca 100644 --- a/paddle/gserver/layers/RecurrentLayer.cpp +++ b/paddle/gserver/layers/RecurrentLayer.cpp @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Stat.h" -P_DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation."); +DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation."); namespace paddle { diff --git a/paddle/gserver/layers/ValidationLayer.h b/paddle/gserver/layers/ValidationLayer.h index 471055429d34bee591cf7e66cd28221a8ebd83ed..4c1de7b3b7d6975c2693eb065f7d3e19cc51a95c 100644 --- a/paddle/gserver/layers/ValidationLayer.h +++ b/paddle/gserver/layers/ValidationLayer.h @@ -18,7 +18,7 @@ limitations under the License. */ #include "Layer.h" #include "paddle/gserver/evaluators/Evaluator.h" -P_DECLARE_int32(trainer_id); +DECLARE_int32(trainer_id); namespace paddle { diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index dffc24936faa2d855ae2eb762efbcc1c0f545943..1d5e7de1ba624d98c953efe1cdd2318548c4e914 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -14,7 +14,7 @@ limitations under the License. */ #include "LayerGradUtil.h" -P_DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(thread_local_rand_use_global_seed); namespace paddle { real getCostSum(LayerPtr& testLayer, MatrixPtr weights) { diff --git a/paddle/gserver/tests/TestUtil.cpp b/paddle/gserver/tests/TestUtil.cpp index e656da5b8f7c0f9ebbc094c0e1548423ea060f50..e07c60861a4a6567fd1e28559b9806cb623a3bdf 100644 --- a/paddle/gserver/tests/TestUtil.cpp +++ b/paddle/gserver/tests/TestUtil.cpp @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/math/SparseMatrix.h" #include "paddle/utils/CommandLineParser.h" -P_DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length"); +DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length"); namespace paddle { diff --git a/paddle/gserver/tests/test_ActivationGrad.cpp b/paddle/gserver/tests/test_ActivationGrad.cpp index 20a6126d0b69f71eecc439854c8f97f94ec53de5..7d7e68da5c5a9dbcba024002a988f26f7613b724 100644 --- a/paddle/gserver/tests/test_ActivationGrad.cpp +++ b/paddle/gserver/tests/test_ActivationGrad.cpp @@ -25,8 +25,8 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(use_gpu); +DECLARE_bool(thread_local_rand_use_global_seed); void testActivation(const string& act) { LOG(INFO) << "test activation: " << act; diff --git a/paddle/gserver/tests/test_BatchNorm.cpp b/paddle/gserver/tests/test_BatchNorm.cpp index 3bd4e321b7d073055ea8e9d97020379276de8cdf..7f5fcb670b70aed9f0a04180d344556a0390122f 100644 --- a/paddle/gserver/tests/test_BatchNorm.cpp +++ b/paddle/gserver/tests/test_BatchNorm.cpp @@ -27,11 +27,11 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_bool(prev_batch_state); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_double(checkgrad_eps); +DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(prev_batch_state); // Test that the batchNormLayer can be followed by a ConvLayer TEST(Layer, batchNorm) { diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp index 83100e3bec7e61c7d7751754ace760729e9adb27..99202c2d5702a9569c3a9a92897a8a0e38b8e2a6 100644 --- a/paddle/gserver/tests/test_ConvTrans.cpp +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -28,11 +28,11 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_bool(prev_batch_state); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_double(checkgrad_eps); +DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(prev_batch_state); // Test that the convTrans forward is the same as conv backward TEST(Layer, convTransLayerFwd) { diff --git a/paddle/gserver/tests/test_ConvUnify.cpp b/paddle/gserver/tests/test_ConvUnify.cpp index 02763406a34da16da52fa0247c6c469c2418a914..2ab18f886848d198b9063c7559790497ce131efe 100644 --- a/paddle/gserver/tests/test_ConvUnify.cpp +++ b/paddle/gserver/tests/test_ConvUnify.cpp @@ -28,11 +28,11 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_bool(prev_batch_state); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_double(checkgrad_eps); +DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(prev_batch_state); // Do one forward pass of convTrans layer and check to see if its output // matches the given result diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 7a930aebcf4ae7ab163c497d4d9545fdcf4f8eb5..e07066dad84aa6326c2447fc5ee80fa496735fbf 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -21,9 +21,9 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_bool(thread_local_rand_use_global_seed); enum InputType { INPUT_DATA, // dense vector diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 9f8b197df554a67ebcdd3cff2c5d7d91bfc1932d..8a8d094ed357a6565dd9827c4bb10b76db6a146a 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -26,11 +26,11 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_bool(prev_batch_state); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_double(checkgrad_eps); +DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_bool(prev_batch_state); TEST(Operator, dot_mul) { TestConfig config; diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index baa55aa0252cb63db7c4aa92cf9b6933199273fa..fc60228f816e0cea30ef764c59a8c7875ed4a0e8 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -25,10 +25,10 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_int32(gpu_id); -P_DECLARE_double(checkgrad_eps); -P_DEFINE_bool(use_label, true, "input label or sequence label"); -P_DEFINE_bool(static_para, false, "static parameter"); +DECLARE_int32(gpu_id); +DECLARE_double(checkgrad_eps); +DEFINE_bool(use_label, true, "input label or sequence label"); +DEFINE_bool(static_para, false, "static parameter"); struct DataIn { std::vector inArgs; @@ -267,8 +267,8 @@ TEST(Compare, img_conv2) { } #endif -P_DEFINE_string(config_file_a, "", "config of one network to compare"); -P_DEFINE_string(config_file_b, "", "config of another network to compare"); +DEFINE_string(config_file_a, "", "config of one network to compare"); +DEFINE_string(config_file_b, "", "config of another network to compare"); TEST(Compare, network) { if (FLAGS_config_file_a != "" && FLAGS_config_file_b != "") { compareNetwork(FLAGS_config_file_a, FLAGS_config_file_b); diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index 436318d35634f4ba46781a125ace110551029439..5f8bc5ecd0f77efc6dcda0330f124ca6cab7f277 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/utils/PythonUtil.h" #include "paddle/utils/Util.h" -P_DEFINE_string(train_list, "unittest.list", "file list for unittest"); +DEFINE_string(train_list, "unittest.list", "file list for unittest"); namespace paddle { namespace unittest { diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index a351667d8b18b734022820a777c551bb11a243bf..874aabf37ccf6029bdb9b844cf144c0165154eb9 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -20,7 +20,7 @@ limitations under the License. */ #include #include -P_DECLARE_int32(seed); +DECLARE_int32(seed); using namespace paddle; // NOLINT using namespace std; // NOLINT diff --git a/paddle/gserver/tests/test_RecurrentLayer.cpp b/paddle/gserver/tests/test_RecurrentLayer.cpp index cd96ca7c848afd7a2aa38df3343bee102aa5e83a..f91c788863b6963df92b735dbfef2bacee1fff45 100644 --- a/paddle/gserver/tests/test_RecurrentLayer.cpp +++ b/paddle/gserver/tests/test_RecurrentLayer.cpp @@ -23,9 +23,9 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_bool(rnn_use_batch); -P_DECLARE_int32(fixed_seq_length); +DECLARE_bool(use_gpu); +DECLARE_bool(rnn_use_batch); +DECLARE_int32(fixed_seq_length); void checkError(const Matrix& matrix1, const Matrix& matrix2) { CHECK(matrix1.getHeight() == matrix2.getHeight()); diff --git a/paddle/gserver/tests/test_SelectiveFCLayer.cpp b/paddle/gserver/tests/test_SelectiveFCLayer.cpp index 4f3a95a535b0a47df88ac5ba7a367d25e57e5f74..ab23d00a2cb6077147f5b89664a8e2437b4cd63b 100644 --- a/paddle/gserver/tests/test_SelectiveFCLayer.cpp +++ b/paddle/gserver/tests/test_SelectiveFCLayer.cpp @@ -29,11 +29,11 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(num_passes); -P_DECLARE_string(config); -P_DECLARE_string(init_model_path); -P_DECLARE_string(config_args); +DECLARE_bool(use_gpu); +DECLARE_int32(num_passes); +DECLARE_string(config); +DECLARE_string(init_model_path); +DECLARE_string(config_args); size_t fcLayerWidth = 1024; diff --git a/paddle/gserver/tests/test_WarpCTCLayer.cpp b/paddle/gserver/tests/test_WarpCTCLayer.cpp index 700425412cebe8416f900fa702503db98722c0ee..0a4a814d5247410248f7418e1ef2c79a2da42507 100644 --- a/paddle/gserver/tests/test_WarpCTCLayer.cpp +++ b/paddle/gserver/tests/test_WarpCTCLayer.cpp @@ -25,7 +25,7 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_bool(use_gpu); +DECLARE_bool(use_gpu); const real* getData(const Matrix& matrix) { if (matrix.useGpu()) { diff --git a/paddle/math/SparseRowMatrix.cpp b/paddle/math/SparseRowMatrix.cpp index 3091743123af03561f91dfb8b03e65087310ce64..b61c6b2d49ccead5e9cfdf595a8bebae0e5b87b5 100644 --- a/paddle/math/SparseRowMatrix.cpp +++ b/paddle/math/SparseRowMatrix.cpp @@ -24,9 +24,9 @@ limitations under the License. */ #include "paddle/utils/Thread.h" #include "paddle/utils/Util.h" -P_DEFINE_bool(allow_inefficient_sparse_update, - false, - "Whether to allow inefficient sparse update"); +DEFINE_bool(allow_inefficient_sparse_update, + false, + "Whether to allow inefficient sparse update"); namespace paddle { diff --git a/paddle/math/SparseRowMatrix.h b/paddle/math/SparseRowMatrix.h index badb4b9c1cce4d93c24aac47c8ed742c4d7d38fa..9364feb4a1462a5a9d16ca0f69213ba32ad97d21 100644 --- a/paddle/math/SparseRowMatrix.h +++ b/paddle/math/SparseRowMatrix.h @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Util.h" -P_DECLARE_bool(allow_inefficient_sparse_update); +DECLARE_bool(allow_inefficient_sparse_update); namespace paddle { diff --git a/paddle/math/Storage.cpp b/paddle/math/Storage.cpp index f9a2c12cd539ab4785847a58d3fedfc384e05232..56e5442394b04230c22d668aa734dc0fa44004c2 100644 --- a/paddle/math/Storage.cpp +++ b/paddle/math/Storage.cpp @@ -16,9 +16,9 @@ limitations under the License. */ #include "Allocator.h" #include "paddle/utils/Util.h" -P_DEFINE_int32(pool_limit_size, - 536870912, - "maximum memory size managed by a memory pool, default is 512M"); +DEFINE_int32(pool_limit_size, + 536870912, + "maximum memory size managed by a memory pool, default is 512M"); namespace paddle { diff --git a/paddle/math/tests/test_TrainingAlgorithm.cpp b/paddle/math/tests/test_TrainingAlgorithm.cpp index 1bf6a0cc43ea16c955b1b1cd7ef61d2e7100726a..2c458cba9ca11e9af8a98b88a6392978c2a9be77 100644 --- a/paddle/math/tests/test_TrainingAlgorithm.cpp +++ b/paddle/math/tests/test_TrainingAlgorithm.cpp @@ -22,9 +22,9 @@ limitations under the License. */ using namespace paddle; // NOLINT #ifndef PADDLE_TYPE_DOUBLE -P_DEFINE_double(max_diff, 1e-5, "max diff allowed"); +DEFINE_double(max_diff, 1e-5, "max diff allowed"); #else -P_DEFINE_double(max_diff, 1e-13, "max diff allowed"); +DEFINE_double(max_diff, 1e-13, "max diff allowed"); #endif class SetMaxDiff { diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index e91daa371768e6f7f3846f95027cc6320926052c..65d01a15718ae2bebd4869eff0e5407524bc0e7c 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -245,6 +245,8 @@ int32_t Argument::resizeAndCopyFrom(const Argument& src, bool useGpu, hl_stream_t stream) { dataId = src.dataId; + frameWidth = src.frameWidth; + frameHeight = src.frameHeight; if (!src.sequenceStartPositions) { // non-sequence input, copy samples directly diff --git a/paddle/parameter/FirstOrderOptimizer.cpp b/paddle/parameter/FirstOrderOptimizer.cpp index 630f15c8cfbe6cca16094d0eb6677702339aa730..dbb738e98b5874f5bb33026ad585a6c3ef327d1d 100644 --- a/paddle/parameter/FirstOrderOptimizer.cpp +++ b/paddle/parameter/FirstOrderOptimizer.cpp @@ -19,7 +19,7 @@ limitations under the License. */ #include -P_DEFINE_bool(log_clipping, false, "enable log clipping or not"); +DEFINE_bool(log_clipping, false, "enable log clipping or not"); namespace paddle { diff --git a/paddle/parameter/Parameter.cpp b/paddle/parameter/Parameter.cpp index 986ae1539b6ef7745e94be6101e94b40c287be94..1673fc6e533e416dfe4db557a1a8968667d1bfff 100644 --- a/paddle/parameter/Parameter.cpp +++ b/paddle/parameter/Parameter.cpp @@ -26,11 +26,11 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Logging.h" -P_DEFINE_int32(enable_grad_share, - (100 * 1024 * 1024), - "threshold for enable gradient parameter share for batch " - "multi-cpu training"); -P_DEFINE_int32( +DEFINE_int32(enable_grad_share, + (100 * 1024 * 1024), + "threshold for enable gradient parameter share for batch " + "multi-cpu training"); +DEFINE_int32( grad_share_block_num, 64, "block number of gradient parameter share for batch multi-cpu training"); diff --git a/paddle/pserver/BaseClient.cpp b/paddle/pserver/BaseClient.cpp index a43def98c528c2cfbc65a40aa54b5d4a49961a34..b4ac7a2506921b2409baaff077cc3541f3dc8d73 100644 --- a/paddle/pserver/BaseClient.cpp +++ b/paddle/pserver/BaseClient.cpp @@ -18,7 +18,7 @@ limitations under the License. */ #include "paddle/utils/CommandLineParser.h" #include "paddle/utils/Stat.h" -P_DECLARE_string(pservers); +DECLARE_string(pservers); namespace paddle { diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index 329dfb0fb38e8ad377539d8af6be228595baa719..cbc105e651faa0f283b3becb10449f4e1bc78b38 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -31,23 +31,23 @@ limitations under the License. */ #include "paddle/utils/Util.h" /// quick ack can reduce the latency of small message -P_DEFINE_bool(small_messages, - false, - "if message size is small, recommend set it True to enable quick " - "ack and no delay"); +DEFINE_bool(small_messages, + false, + "if message size is small, recommend set it True to enable quick " + "ack and no delay"); /// reasonable sock_send_buf_size can control the traffic injected into switch /// network. Injecting too many data into traffic could cause packets loss which /// cause long latency and degrade the efficiency of communication. -P_DEFINE_int32(sock_send_buf_size, - 1024 * 1024 * 40, - "restrict sock send buff size, can reduce network congestion if " - "set carefully"); +DEFINE_int32(sock_send_buf_size, + 1024 * 1024 * 40, + "restrict sock send buff size, can reduce network congestion if " + "set carefully"); /// reasonable size can hold bursted packets and reduce packets loss -P_DEFINE_int32(sock_recv_buf_size, - 1024 * 1024 * 40, - "restrict sock recv buff size"); +DEFINE_int32(sock_recv_buf_size, + 1024 * 1024 * 40, + "restrict sock recv buff size"); namespace paddle { diff --git a/paddle/pserver/ParameterClient2.cpp b/paddle/pserver/ParameterClient2.cpp index 86fd1c5276c97f2de86b8c8eb8627721611d099c..a97859f83fe6495b298e920346c964ef2a9b146c 100644 --- a/paddle/pserver/ParameterClient2.cpp +++ b/paddle/pserver/ParameterClient2.cpp @@ -20,8 +20,8 @@ limitations under the License. */ #include "paddle/utils/Stat.h" #include "paddle/utils/StringUtil.h" -P_DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers"); -P_DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send"); +DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers"); +DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send"); namespace paddle { diff --git a/paddle/pserver/ParameterClient2.h b/paddle/pserver/ParameterClient2.h index 52553949498e1938c6dbbf8004946692cd1bfe0b..eed71ccb43b0fec76a74a7f00662c32c97c26ff4 100644 --- a/paddle/pserver/ParameterClient2.h +++ b/paddle/pserver/ParameterClient2.h @@ -34,7 +34,7 @@ limitations under the License. */ #include "ProtoServer.h" #include "SparseParameterDistribution.h" -P_DECLARE_int32(parallel_thread_num); +DECLARE_int32(parallel_thread_num); namespace paddle { diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index 2cb4c93535ffe2012abb45628cc0cfc329846fd9..856fa0ad1ab30e3fc554ac96dd3bed71b1548579 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -30,11 +30,11 @@ limitations under the License. */ #include "paddle/utils/GlobalConstants.h" #include "paddle/utils/Stat.h" -P_DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec"); -P_DEFINE_double(async_lagged_ratio_min, - 1.0, - "control config_.async_lagged_grad_discard_ratio() min value"); -P_DEFINE_double( +DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec"); +DEFINE_double(async_lagged_ratio_min, + 1.0, + "control config_.async_lagged_grad_discard_ratio() min value"); +DEFINE_double( async_lagged_ratio_default, 1.5, "if async_lagged_grad_discard_ratio is not set in trainer_config.conf" diff --git a/paddle/pserver/ParameterServer2.h b/paddle/pserver/ParameterServer2.h index 61c139981e479cf6528dfb44ab6ae2f82b33bcc5..b0cf22e1fb158e76fcee1ce6ef1f375995803ce6 100644 --- a/paddle/pserver/ParameterServer2.h +++ b/paddle/pserver/ParameterServer2.h @@ -38,7 +38,7 @@ limitations under the License. */ #include "ProtoServer.h" -P_DECLARE_int32(port); +DECLARE_int32(port); namespace paddle { diff --git a/paddle/pserver/SparseParameterDistribution.cpp b/paddle/pserver/SparseParameterDistribution.cpp index 0068f85b52be75fd2f958ad7a1e2ae76949b1fee..6dd725db30cd6d50539d1b2b30ab9e42a081c7b3 100644 --- a/paddle/pserver/SparseParameterDistribution.cpp +++ b/paddle/pserver/SparseParameterDistribution.cpp @@ -20,26 +20,26 @@ limitations under the License. */ #include "SparseParameterDistribution.h" -P_DEFINE_bool(check_sparse_distribution_in_pserver, - false, - "check whether sparse parameter exhibts balanced distribution at " - "all pservers"); -P_DEFINE_bool(show_check_sparse_distribution_log, - false, - "show logs details for sparse parameter distribution in pserver"); -P_DEFINE_int32(check_sparse_distribution_batches, - 100, - "run sparse parameter distribution check for N batches"); -P_DEFINE_double( +DEFINE_bool(check_sparse_distribution_in_pserver, + false, + "check whether sparse parameter exhibts balanced distribution at " + "all pservers"); +DEFINE_bool(show_check_sparse_distribution_log, + false, + "show logs details for sparse parameter distribution in pserver"); +DEFINE_int32(check_sparse_distribution_batches, + 100, + "run sparse parameter distribution check for N batches"); +DEFINE_double( check_sparse_distribution_ratio, 0.6, "if parameters dispatched to different pservers exhibit unbalanced " " distribution for check_sparse_distribution_ratio * " " check_sparse_distribution_batches times, crash program"); -P_DEFINE_double(check_sparse_distribution_unbalance_degree, - 2.0, - "the ratio of maximum data size and minimun data size for " - "different pserver"); +DEFINE_double(check_sparse_distribution_unbalance_degree, + 2.0, + "the ratio of maximum data size and minimun data size for " + "different pserver"); namespace paddle { diff --git a/paddle/pserver/test/SocketTest.cpp b/paddle/pserver/test/SocketTest.cpp index 6e63c4f67848bea118adde2e3ef0b0c5b64086c9..066a6c02939695e7050a7693365d7c449f70e723 100644 --- a/paddle/pserver/test/SocketTest.cpp +++ b/paddle/pserver/test/SocketTest.cpp @@ -195,9 +195,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) { channel_.reset(new SocketChannel(sockfd)); } -P_DEFINE_string(server_addr, "127.0.0.1", "Server address"); -P_DEFINE_int64(dim, 10000000, "Data size"); -P_DEFINE_int32(loop_time, 100000, "test loop time"); +DEFINE_string(server_addr, "127.0.0.1", "Server address"); +DEFINE_int64(dim, 10000000, "Data size"); +DEFINE_int32(loop_time, 100000, "test loop time"); using namespace paddle; // NOLINT diff --git a/paddle/pserver/test/test_ParameterServer2.cpp b/paddle/pserver/test/test_ParameterServer2.cpp index 4257a2308d727fc60cb5b2e7e507ec86c90cd96b..8e7231a9e1aee7b61f8dfa42f1367b79fee81a2b 100644 --- a/paddle/pserver/test/test_ParameterServer2.cpp +++ b/paddle/pserver/test/test_ParameterServer2.cpp @@ -21,9 +21,9 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_int32(num_gradient_servers); -P_DEFINE_string(server_addr, "127.0.0.1", "assign server address"); -P_DEFINE_int32(server_cpu, 0, "assign server cpu"); +DECLARE_int32(num_gradient_servers); +DEFINE_string(server_addr, "127.0.0.1", "assign server address"); +DEFINE_int32(server_cpu, 0, "assign server cpu"); class ParameterServer2Tester : public ParameterServer2 { public: diff --git a/paddle/pserver/test/test_ProtoServer.cpp b/paddle/pserver/test/test_ProtoServer.cpp index 3880dde5e3fb8a986a8fa870cb92a58d138d43de..9f86ee80f4e5cc99ea3597b3ed37a387578f032a 100644 --- a/paddle/pserver/test/test_ProtoServer.cpp +++ b/paddle/pserver/test/test_ProtoServer.cpp @@ -21,10 +21,10 @@ limitations under the License. */ #include "paddle/pserver/ProtoServer.h" #include "paddle/utils/Stat.h" -P_DEFINE_string(server_addr, "127.0.0.1", "Server address"); -P_DEFINE_int64(dim, 50000000, "Data size"); -P_DEFINE_bool(test_proto_server, true, "whether to test ProtoServer"); -P_DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests"); +DEFINE_string(server_addr, "127.0.0.1", "Server address"); +DEFINE_int64(dim, 50000000, "Data size"); +DEFINE_bool(test_proto_server, true, "whether to test ProtoServer"); +DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests"); using namespace paddle; // NOLINT diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index ace2c0dee972e338001a0e5a4045c32e64ff157e..283fd34a6d8a2268f3800ec69920e128ac75e7dc 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -21,8 +21,6 @@ function version(){ echo " with_double: @WITH_DOUBLE@" echo " with_python: @WITH_PYTHON@" echo " with_rdma: @WITH_RDMA@" - echo " with_glog: @WITH_GLOG@" - echo " with_gflags: @WITH_GFLAGS@" echo " with_metric_learning: @WITH_METRIC@" echo " with_timer: @WITH_TIMER@" echo " with_predict_sdk: @WITH_PREDICT_SDK@" diff --git a/paddle/trainer/MergeModel.cpp b/paddle/trainer/MergeModel.cpp index 1cf29a39b92cc26fa6706f857edc1eb2dff29a21..91d89b61a32259b8bbe70fda2579f87ec6b9af00 100644 --- a/paddle/trainer/MergeModel.cpp +++ b/paddle/trainer/MergeModel.cpp @@ -19,8 +19,8 @@ limitations under the License. */ #include "paddle/pserver/ParameterServer2.h" #include "paddle/utils/PythonUtil.h" -P_DEFINE_string(model_dir, "", "Directory for separated model files"); -P_DEFINE_string(model_file, "", "File for merged model file"); +DEFINE_string(model_dir, "", "Directory for separated model files"); +DEFINE_string(model_file, "", "File for merged model file"); using namespace paddle; // NOLINT using namespace std; // NOLINT diff --git a/paddle/trainer/RemoteParameterUpdater.cpp b/paddle/trainer/RemoteParameterUpdater.cpp index b7f7b93b8df091f7ccbe19a639295cba0554399a..974e78fa17d6564414962475f81497491bbb0482 100644 --- a/paddle/trainer/RemoteParameterUpdater.cpp +++ b/paddle/trainer/RemoteParameterUpdater.cpp @@ -17,8 +17,8 @@ limitations under the License. */ #include "paddle/utils/GlobalConstants.h" #include "paddle/utils/Stat.h" -P_DECLARE_int32(trainer_id); -P_DECLARE_string(save_dir); +DECLARE_int32(trainer_id); +DECLARE_string(save_dir); namespace paddle { diff --git a/paddle/trainer/ThreadParameterUpdater.cpp b/paddle/trainer/ThreadParameterUpdater.cpp index bee7f061fed3a01e8292137272c3288334ef70c2..9caa92a4d7557c0c8633d881820862bbbd5df87e 100644 --- a/paddle/trainer/ThreadParameterUpdater.cpp +++ b/paddle/trainer/ThreadParameterUpdater.cpp @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/math/SparseRowMatrix.h" #include "paddle/utils/Thread.h" -P_DECLARE_int32(trainer_count); +DECLARE_int32(trainer_count); namespace paddle { diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 85610ec04e3f64dc83575426922ac936a604b3a7..1eec2c432d235ef484b688db08aae8a39f878a85 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -38,60 +38,56 @@ limitations under the License. */ #include "paddle/gserver/gradientmachines/NeuralNetwork.h" #include "paddle/gserver/layers/ValidationLayer.h" -P_DEFINE_string(config, "", "Trainer config file"); - -P_DEFINE_int32(test_period, - 0, - "if equal 0, do test on all test data at the end of " - "each pass. While if equal non-zero, do test on all test " - "data every test_period batches"); -P_DEFINE_bool(test_all_data_in_one_period, - false, - "This option was deprecated, since we will always do " - "test on all test set "); - -P_DEFINE_bool(local, true, "Train in local mode or not"); - -P_DEFINE_int32(average_test_period, - 0, - "Do test on average parameter every so" - " many batches. MUST be devided by FLAGS_log_period." - " Default 0 means do not test average parameter"); - -P_DEFINE_int32(saving_period, 1, "Save parameteres every so many passes"); -P_DEFINE_int64(saving_period_by_batches, - 0, - "Save parameters every so many batches in one pass"); -P_DEFINE_string(save_dir, "", "Directory for saving model parameter"); -P_DEFINE_int32(start_pass, - 0, - "Start training from this pass. " - "Will load parameter from the previous pass"); -P_DEFINE_int32(test_pass, - -1, - "Will load parameter start from this pass to test"); -P_DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist"); -P_DEFINE_bool(with_cost, true, "enable cost layer or not"); -P_DEFINE_bool(distribute_test, false, "test in distribute mode"); - -P_DEFINE_int32(num_passes, 100, "train for so many passes"); - -P_DEFINE_string(config_args, - "", - "arguments passed to config file." - "Format: key1=value1,key2=value2"); - -P_DEFINE_bool(save_only_one, - false, - "Save only parameters in last pass, remove previous."); - -P_DEFINE_string(feat_file, "", "File name of extracted feature."); -P_DEFINE_string(predict_output_dir, - "", - "Directory that saves the predicted results of output layers"); -P_DEFINE_string(model_list, - "", - "File that saves the model list when evaluation"); +DEFINE_string(config, "", "Trainer config file"); + +DEFINE_int32(test_period, + 0, + "if equal 0, do test on all test data at the end of " + "each pass. While if equal non-zero, do test on all test " + "data every test_period batches"); +DEFINE_bool(test_all_data_in_one_period, + false, + "This option was deprecated, since we will always do " + "test on all test set "); + +DEFINE_bool(local, true, "Train in local mode or not"); + +DEFINE_int32(average_test_period, + 0, + "Do test on average parameter every so" + " many batches. MUST be devided by FLAGS_log_period." + " Default 0 means do not test average parameter"); + +DEFINE_int32(saving_period, 1, "Save parameteres every so many passes"); +DEFINE_int64(saving_period_by_batches, + 0, + "Save parameters every so many batches in one pass"); +DEFINE_string(save_dir, "", "Directory for saving model parameter"); +DEFINE_int32(start_pass, + 0, + "Start training from this pass. " + "Will load parameter from the previous pass"); +DEFINE_int32(test_pass, -1, "Will load parameter start from this pass to test"); +DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist"); +DEFINE_bool(with_cost, true, "enable cost layer or not"); +DEFINE_bool(distribute_test, false, "test in distribute mode"); + +DEFINE_int32(num_passes, 100, "train for so many passes"); + +DEFINE_string(config_args, + "", + "arguments passed to config file." + "Format: key1=value1,key2=value2"); + +DEFINE_bool(save_only_one, + false, + "Save only parameters in last pass, remove previous."); + +DEFINE_string(feat_file, "", "File name of extracted feature."); +DEFINE_string(predict_output_dir, + "", + "Directory that saves the predicted results of output layers"); +DEFINE_string(model_list, "", "File that saves the model list when evaluation"); namespace paddle { diff --git a/paddle/trainer/Trainer.h b/paddle/trainer/Trainer.h index cabbb4acd1135f74aca0a4a36116db412c2e89f9..7cbf18ace7a5fed053653c73e62d36c388b15123 100644 --- a/paddle/trainer/Trainer.h +++ b/paddle/trainer/Trainer.h @@ -34,7 +34,7 @@ limitations under the License. */ #include "paddle/internals/metric_learning/MetricTrainer.h" #endif -P_DECLARE_int32(num_passes); +DECLARE_int32(num_passes); namespace paddle { diff --git a/paddle/trainer/TrainerBenchmark.cpp b/paddle/trainer/TrainerBenchmark.cpp index 5c3177c8083e11ebf6b6854f86dbee8299d7e3b1..173653c81688fe4606731c68ea1854268b3f4590 100644 --- a/paddle/trainer/TrainerBenchmark.cpp +++ b/paddle/trainer/TrainerBenchmark.cpp @@ -18,9 +18,9 @@ limitations under the License. */ #include "paddle/utils/Stat.h" #include "paddle/utils/Util.h" -P_DECLARE_int32(test_period); +DECLARE_int32(test_period); -P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider."); +DEFINE_bool(feed_data, false, "Wether to read data from DataProvider."); namespace paddle { diff --git a/paddle/trainer/TrainerConfigHelper.cpp b/paddle/trainer/TrainerConfigHelper.cpp index 2017a08d20d494cbce8c3beba564ed07c1d7cc73..60ac8459a12db801321da4a9d9c1d48ac8bd6d16 100644 --- a/paddle/trainer/TrainerConfigHelper.cpp +++ b/paddle/trainer/TrainerConfigHelper.cpp @@ -18,16 +18,16 @@ limitations under the License. */ #include "paddle/utils/Flags.h" #include "paddle/utils/PythonUtil.h" -P_DECLARE_string(config); -P_DECLARE_string(init_model_path); -P_DECLARE_int32(start_pass); -P_DECLARE_string(save_dir); -P_DECLARE_int32(trainer_id); -P_DECLARE_bool(local); -P_DECLARE_bool(with_cost); -P_DECLARE_bool(with_gpu); -P_DECLARE_bool(parallel_nn); -P_DECLARE_string(config_args); +DECLARE_string(config); +DECLARE_string(init_model_path); +DECLARE_int32(start_pass); +DECLARE_string(save_dir); +DECLARE_int32(trainer_id); +DECLARE_bool(local); +DECLARE_bool(with_cost); +DECLARE_bool(with_gpu); +DECLARE_bool(parallel_nn); +DECLARE_string(config_args); const char *kConfigParserModuleName = "paddle.trainer.config_parser"; const char *kConfigParserFuncName = "parse_config_and_serialize"; diff --git a/paddle/trainer/TrainerInternalConfig.cpp b/paddle/trainer/TrainerInternalConfig.cpp index a017cdec9d06a51ddf0925280f3b60cc2dc1c17a..039fcdb524527d5e8bfa829fc403b6f2fa789991 100644 --- a/paddle/trainer/TrainerInternalConfig.cpp +++ b/paddle/trainer/TrainerInternalConfig.cpp @@ -14,17 +14,17 @@ limitations under the License. */ #include "TrainerInternalConfig.h" -P_DEFINE_int32(show_parameter_stats_period, - 0, - "Whether to show parameter stats during training"); +DEFINE_int32(show_parameter_stats_period, + 0, + "Whether to show parameter stats during training"); -P_DEFINE_int32(dot_period, 1, "Print '.' every so many batches"); +DEFINE_int32(dot_period, 1, "Print '.' every so many batches"); -P_DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater"); +DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater"); -P_DECLARE_int32(num_passes); +DECLARE_int32(num_passes); -P_DECLARE_bool(local); +DECLARE_bool(local); namespace paddle { diff --git a/paddle/trainer/TrainerMain.cpp b/paddle/trainer/TrainerMain.cpp index 0a4d56b892a2a6753f83105170ad3162da59264c..947f9cadcc983d58ce31ef462e51dc42e41eaf1b 100644 --- a/paddle/trainer/TrainerMain.cpp +++ b/paddle/trainer/TrainerMain.cpp @@ -22,21 +22,20 @@ limitations under the License. */ #include "Trainer.h" #include "paddle/pserver/RDMANetwork.h" -P_DEFINE_bool(start_pserver, false, "Whether to start pserver"); -P_DECLARE_int32(gpu_id); -P_DEFINE_string(job, "train", "one of (train, test, checkgrad)"); -P_DECLARE_int32(start_pass); -P_DECLARE_string(config); -P_DECLARE_string(init_model_path); -P_DECLARE_string(rdma_tcp); +DEFINE_bool(start_pserver, false, "Whether to start pserver"); +DECLARE_int32(gpu_id); +DEFINE_string(job, "train", "one of (train, test, checkgrad)"); +DECLARE_int32(start_pass); +DECLARE_string(config); +DECLARE_string(init_model_path); +DECLARE_string(rdma_tcp); using namespace paddle; // NOLINT int main(int argc, char** argv) { -// write logs instantly (never buffer log messages) -#ifdef PADDLE_USE_GLOG + // write logs instantly (never buffer log messages) FLAGS_logbuflevel = -1; -#endif + initMain(argc, argv); initPython(argc, argv); diff --git a/paddle/trainer/tests/test_Compare.cpp b/paddle/trainer/tests/test_Compare.cpp index 63fa48540cca81f6b463b50c4af9ea3259a7301d..72fc76bea35e433eeb08ba625b4bf6afdda491fb 100644 --- a/paddle/trainer/tests/test_Compare.cpp +++ b/paddle/trainer/tests/test_Compare.cpp @@ -24,10 +24,10 @@ using namespace std; // NOLINT static const string& configFile = "trainer/tests/sample_trainer_config.conf"; -P_DECLARE_int32(gpu_id); -P_DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_string(config_args); +DECLARE_int32(gpu_id); +DECLARE_bool(use_gpu); +DECLARE_string(config); +DECLARE_string(config_args); struct comData { vector outArgs; diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index 3fea3a3c24303b84f78f4029b0ed8e42e419c442..a7000eb77e1bbeab4f6e38c0322f82bde7164080 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -25,22 +25,22 @@ using namespace std; // NOLINT static const string& configFile1 = "trainer/tests/sample_trainer_config_qb_rnn.conf"; -P_DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_int32(gpu_id); -P_DECLARE_int32(seed); -P_DECLARE_int32(num_passes); -P_DECLARE_int32(saving_period); - -P_DECLARE_int32(num_gradient_servers); -P_DECLARE_int32(port); -P_DECLARE_bool(local); -P_DECLARE_bool(use_old_updater); -P_DECLARE_bool(parallel_nn); -P_DECLARE_string(config_args); -P_DEFINE_double(max_diff_ratio, - 0.0f, - "max diff ratio allowed for parameters value"); +DECLARE_bool(use_gpu); +DECLARE_string(config); +DECLARE_int32(gpu_id); +DECLARE_int32(seed); +DECLARE_int32(num_passes); +DECLARE_int32(saving_period); + +DECLARE_int32(num_gradient_servers); +DECLARE_int32(port); +DECLARE_bool(local); +DECLARE_bool(use_old_updater); +DECLARE_bool(parallel_nn); +DECLARE_string(config_args); +DEFINE_double(max_diff_ratio, + 0.0f, + "max diff ratio allowed for parameters value"); int gNumDevices = 0; diff --git a/paddle/trainer/tests/test_CompareTwoNets.cpp b/paddle/trainer/tests/test_CompareTwoNets.cpp index 8a4556721dda3f73fb11b36e5fdf798df3993ce8..80c61e259e71dd31d7637072248b22a2910c532e 100644 --- a/paddle/trainer/tests/test_CompareTwoNets.cpp +++ b/paddle/trainer/tests/test_CompareTwoNets.cpp @@ -22,25 +22,25 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_int32(gpu_id); +DECLARE_int32(gpu_id); -P_DECLARE_bool(local); -P_DECLARE_bool(use_gpu); +DECLARE_bool(local); +DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_string(nics); +DECLARE_string(config); +DECLARE_string(nics); -P_DEFINE_string(config_file_a, "", "config of one network to compare"); -P_DEFINE_string(config_file_b, "", "config of another network to compare"); -P_DEFINE_bool(need_high_accuracy, - false, - "whether need to run in double accuracy"); -P_DEFINE_double( +DEFINE_string(config_file_a, "", "config of one network to compare"); +DEFINE_string(config_file_b, "", "config of another network to compare"); +DEFINE_bool(need_high_accuracy, + false, + "whether need to run in double accuracy"); +DEFINE_double( max_diff_ratio, 0.0f, "max diff ratio allowed for outputs and parameters (value/gradient)"); -P_DECLARE_bool(thread_local_rand_use_global_seed); -P_DECLARE_int32(seed); +DECLARE_bool(thread_local_rand_use_global_seed); +DECLARE_int32(seed); struct ComData { vector outArgs; diff --git a/paddle/trainer/tests/test_CompareTwoOpts.cpp b/paddle/trainer/tests/test_CompareTwoOpts.cpp index 673ef289d8f5bfc0f1d6db58eb7d4e7ecba31ae3..383505f8131264844069d6f0fa13f4e0ac1f97af 100644 --- a/paddle/trainer/tests/test_CompareTwoOpts.cpp +++ b/paddle/trainer/tests/test_CompareTwoOpts.cpp @@ -22,20 +22,20 @@ limitations under the License. */ using namespace paddle; // NOLINT using namespace std; // NOLINT -P_DECLARE_int32(gpu_id); +DECLARE_int32(gpu_id); -P_DECLARE_bool(local); -P_DECLARE_bool(use_gpu); +DECLARE_bool(local); +DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_string(nics); +DECLARE_string(config); +DECLARE_string(nics); -P_DEFINE_string(config_file_a, "", "config of one network to compare"); -P_DEFINE_string(config_file_b, "", "config of another network to compare"); -P_DEFINE_bool(need_high_accuracy, - true, - "whether need to run in double accuracy (recommended)"); -P_DEFINE_double( +DEFINE_string(config_file_a, "", "config of one network to compare"); +DEFINE_string(config_file_b, "", "config of another network to compare"); +DEFINE_bool(need_high_accuracy, + true, + "whether need to run in double accuracy (recommended)"); +DEFINE_double( max_diff_ratio, 0.0f, "max diff ratio allowed for outputs and parameters (value/gradient)"); diff --git a/paddle/trainer/tests/test_Prediction.cpp b/paddle/trainer/tests/test_Prediction.cpp index 322121a579440fcf164c042b3265e5d2878e3732..0c79404eee1c0902c5c8e8eefd139da3da584636 100644 --- a/paddle/trainer/tests/test_Prediction.cpp +++ b/paddle/trainer/tests/test_Prediction.cpp @@ -18,11 +18,11 @@ limitations under the License. */ #include -P_DECLARE_string(config); -P_DECLARE_string(config_args); -P_DEFINE_string(merger, - "./paddle_merge_model", - "path to paddle_merge_model binary"); +DECLARE_string(config); +DECLARE_string(config_args); +DEFINE_string(merger, + "./paddle_merge_model", + "path to paddle_merge_model binary"); using namespace paddle; // NOLINT using namespace std; // NOLINT diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index 0fede59f8d8d62edcdb2d030952d0e738452160a..371282dd6bb9a995bc6ae8b2a5bd708f831d7e33 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -28,10 +28,10 @@ static const string& configFile3 = "trainer/tests/chunking.conf"; static const string& configFile4 = "trainer/tests/sample_trainer_config_parallel.conf"; -P_DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_int32(gpu_id); -P_DECLARE_bool(allow_only_one_model_on_one_gpu); +DECLARE_bool(use_gpu); +DECLARE_string(config); +DECLARE_int32(gpu_id); +DECLARE_bool(allow_only_one_model_on_one_gpu); void checkGradientTest(const string& configFile, bool useGpu, diff --git a/paddle/trainer/tests/test_TrainerOnePass.cpp b/paddle/trainer/tests/test_TrainerOnePass.cpp index 0b587ecce176d5b6e8e6e6c4a54cb21fc4e25a67..ee21008aec56da289dab88f72f57a1703e392fad 100644 --- a/paddle/trainer/tests/test_TrainerOnePass.cpp +++ b/paddle/trainer/tests/test_TrainerOnePass.cpp @@ -27,12 +27,12 @@ static const string& configFile1 = "trainer/tests/sample_trainer_config.conf"; static const string& configFile2 = "trainer/tests/sample_trainer_config_parallel.conf"; -P_DECLARE_bool(use_gpu); -P_DECLARE_string(config); -P_DECLARE_int32(gpu_id); -P_DECLARE_int32(seed); -P_DECLARE_int32(num_passes); -P_DECLARE_int32(saving_period); +DECLARE_bool(use_gpu); +DECLARE_string(config); +DECLARE_int32(gpu_id); +DECLARE_int32(seed); +DECLARE_int32(num_passes); +DECLARE_int32(saving_period); class TrainerForTest : public paddle::Trainer { public: @@ -122,10 +122,10 @@ TEST(average_window_cpu, gpu4) { #endif // 3. test trainer + pserver. -P_DECLARE_int32(num_gradient_servers); -P_DECLARE_int32(port); -P_DECLARE_bool(local); -P_DECLARE_bool(use_old_updater); +DECLARE_int32(num_gradient_servers); +DECLARE_int32(port); +DECLARE_bool(local); +DECLARE_bool(use_old_updater); double checkRemoteParameterUpdater(TrainerForTest& trainer) { auto gradientMachine = trainer.getGradientMachine(); diff --git a/paddle/trainer/tests/test_recurrent_machine_generation.cpp b/paddle/trainer/tests/test_recurrent_machine_generation.cpp index 7d8dfd788fd6a336379a96ea5324b8e6f60705ea..03446b3b2f6d5ff42fbf0d735a24d88bd0429747 100644 --- a/paddle/trainer/tests/test_recurrent_machine_generation.cpp +++ b/paddle/trainer/tests/test_recurrent_machine_generation.cpp @@ -30,7 +30,7 @@ static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT static string expectFile = // NOLINT "trainer/tests/rnn_gen_test_model_dir/r1.test"; // NOLINT -P_DECLARE_string(config_args); +DECLARE_string(config_args); vector readRetFile(const string& fname) { ifstream inFile(fname); diff --git a/paddle/utils/BarrierStat.cpp b/paddle/utils/BarrierStat.cpp index 9dde155aca0ec67cca7a0fb8ba9bce4732ffbfa7..a6dbdcae3f32c894d35e8114488d4a3264c6c5f2 100644 --- a/paddle/utils/BarrierStat.cpp +++ b/paddle/utils/BarrierStat.cpp @@ -20,15 +20,15 @@ limitations under the License. */ #include "paddle/utils/Flags.h" #include "paddle/utils/Stat.h" -P_DEFINE_bool(log_barrier_abstract, - true, - "if true, show abstract of barrier performance"); -P_DEFINE_int32(log_barrier_lowest_nodes, - 5, - "how many lowest node will be logged"); -P_DEFINE_bool(log_barrier_show_log, - false, // for performance tuning insight - "if true, always show barrier abstract even with little gap"); +DEFINE_bool(log_barrier_abstract, + true, + "if true, show abstract of barrier performance"); +DEFINE_int32(log_barrier_lowest_nodes, + 5, + "how many lowest node will be logged"); +DEFINE_bool(log_barrier_show_log, + false, // for performance tuning insight + "if true, always show barrier abstract even with little gap"); namespace paddle { diff --git a/paddle/utils/CommandLineParser.cpp b/paddle/utils/CommandLineParser.cpp index 51558b45a143c87be1524cabc0b8a98e8f8bc997..63f16bc54c575a0d5ae02141be3c467ee784b095 100644 --- a/paddle/utils/CommandLineParser.cpp +++ b/paddle/utils/CommandLineParser.cpp @@ -13,220 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "CommandLineParser.h" -#ifndef PADDLE_USE_GFLAGS -#include -#include -#include -#include -#include -#include -#include -#include -#include "paddle/utils/StringUtil.h" -namespace paddle { - -static constexpr int kStatusOK = 0; -static constexpr int kStatusInvalid = 1; -static constexpr int kStatusNotFound = 2; - -/** - * \brief: Convert a string to any type value. - * - * \note: It will specialize by type T that is supported. - */ -template -bool StringToValue(const std::string& content, T* value) { - bool ok; - *value = str::toWithStatus(content, &ok); - return ok; -} - -template <> -bool StringToValue(const std::string& content, bool* value) { - std::string tmp = content; - - std::transform(tmp.begin(), tmp.end(), tmp.begin(), [](char in) -> char { - if (in <= 'Z' && in >= 'A') { - return in - ('Z' - 'z'); - } else { - return in; - } - }); // tolower. - - if (tmp == "true" || tmp == "1") { - *value = true; - return true; - } else if (tmp == "false" || tmp == "0") { - *value = false; - return true; - } else { - return false; - } -} - -template <> -bool StringToValue(const std::string& content, - std::string* value) { - *value = content; - return true; -} - -/** - * \brief Parse argument "--blah=blah". - * - * \param argument: The command line argument string, such as "--blah=blah" - * \param [out] extraInfo: The details error message for parse argument. - * \return: kStatusOK, kStatusInvalid, kStatusNotFound - */ -template -int ParseArgument(const std::string& argument, std::string* extraInfo) { - for (auto& command : - flags_internal::CommandLineFlagRegistry::Instance()->commands) { - std::string& name = command.name; - T* value = command.value; - - std::string prefix = "--"; - prefix += name; - prefix += "="; - std::string content; - if (str::startsWith(argument, prefix)) { - content = argument.substr(prefix.size(), argument.size() - prefix.size()); - } else { - prefix = "-"; - prefix += name; - prefix += "="; - if (str::startsWith(argument, prefix)) { - content = - argument.substr(prefix.size(), argument.size() - prefix.size()); - } - } - - if (!content.empty()) { - if (StringToValue(content, value)) { - return kStatusOK; - } else { - *extraInfo = name; - return kStatusInvalid; - } - } - } - return kStatusNotFound; -} - -/** - * @brief ParseBoolArgumentExtra - * parse '--flag_name', '-flag_name' as true; '--noflag_name', '-noflag_name' as - * false - */ -static int ParseBoolArgumentExtra(const std::string& argument, - std::string* extraInfo) { - (void)(extraInfo); // unused extraInfo, just make api same. - - //! @warning: The order and content of prefixes is DESIGNED for parsing - //! command line. The length of prefixes are 1, 2, 3, 4. The parse logic takes - //! use of this fact. DO NOT CHANGE IT without reading how to parse command - //! below. - static const std::vector> prefixes = { - {"-", true}, {"--", true}, {"-no", false}, {"--no", false}}; - - for (flags_internal::CommandLineFlagRegistry::Command& command : - flags_internal::CommandLineFlagRegistry::Instance()->commands) { - if (argument.size() > command.name.size()) { - //! Use the length of prefix is 1, 2, 3, 4. - size_t diff = argument.size() - command.name.size() - 1UL; - if (diff < prefixes.size()) { - const std::string& prefix = std::get<0>(prefixes[diff]); - if (argument == prefix + command.name) { - *command.value = std::get<1>(prefixes[diff]); - return kStatusOK; - } - } - } - } - return kStatusNotFound; -} - -/** - * \brief: Print command line arguments' usage with type T. - */ -template -static void PrintTypeUsage() { - for (auto& command : - flags_internal::CommandLineFlagRegistry::Instance()->commands) { - std::string& name = command.name; - name = "--" + name; // Program will exit, so modify name is safe. - std::string& desc = command.text; - T& defaultValue = command.defaultValue; - std::cerr << std::setw(20) << name << ": " << desc - << "[default:" << defaultValue << "]." << std::endl; - } -} - -template -static void PrintTypeUsages() { - int unused[] = {0, (PrintTypeUsage(), 0)...}; - (void)(unused); -} -/** - * \brief: Print all usage, and exit(1) - */ -static void PrintUsageAndExit(const char* argv0) { - std::cerr << "Program " << argv0 << " Flags: " << std::endl; - PrintTypeUsages(); - exit(1); -} - -/** - * \brief: Print the error flags, usage, and exit. - */ -static void PrintParseError(const std::string& name, - const char* actualInput, - const char* arg0) { - std::cerr << "Parse command flag " << name << " error! User input is " - << actualInput << std::endl; - PrintUsageAndExit(arg0); -} - -void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) { - int unused_argc = 1; - std::string extra; - for (int i = 1; i < *argc; ++i) { - std::string arg = argv[i]; - int s = kStatusInvalid; -#define ParseArgumentWithType(type) \ - s = ParseArgument(arg, &extra); \ - if (s == kStatusOK) { \ - continue; \ - } else if (s == kStatusInvalid) { \ - PrintParseError(extra, argv[i], argv[0]); \ - } - - ParseArgumentWithType(bool); // NOLINT - ParseArgumentWithType(int32_t); - ParseArgumentWithType(double); // NOLINT - ParseArgumentWithType(int64_t); - ParseArgumentWithType(uint64_t); - ParseArgumentWithType(std::string); - -#undef ParseArgumentWithType - s = ParseBoolArgumentExtra(arg, &extra); - if (s == kStatusOK) { - continue; - } - - if (withHelp && (arg == "--help" || arg == "-h")) { - PrintUsageAndExit(argv[0]); - } - - // NOT Found for all flags. - std::swap(argv[unused_argc++], argv[i]); - } - *argc = unused_argc; -} - -} // namespace paddle -#else namespace paddle { #ifndef GFLAGS_NS #define GFLAGS_NS google @@ -243,4 +30,3 @@ void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) { } } // namespace paddle -#endif diff --git a/paddle/utils/CommandLineParser.h b/paddle/utils/CommandLineParser.h index b4449c6f095f101847c029e02fb0cb087f12f754..4e89f90bb910cee1adc7fb8dace81ff58435351f 100644 --- a/paddle/utils/CommandLineParser.h +++ b/paddle/utils/CommandLineParser.h @@ -13,167 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#ifndef PADDLE_USE_GFLAGS -#include -#include -#include -#include "DisableCopy.h" -namespace paddle { - -namespace flags_internal { - -/** - * Command line flag registry for special type T. It will store all command - * arguments settings. such as name, default value. - */ -template -struct CommandLineFlagRegistry { - /** - * The factory method of CommandLineFlagRegistry - * - * \return: The singleton instance of CommandLineFlagRegistry. - */ - static CommandLineFlagRegistry* Instance() { - static CommandLineFlagRegistry instance_; - return &instance_; - } - - struct Command { - /// name of argument. - std::string name; - /// address of actual variable. such as FLAGS_xxx. - T* value; - /// usage text. - std::string text; - /// default value of this command. - T defaultValue; - }; - - /// the command line arguments of type T. - std::vector commands; - - DISABLE_COPY(CommandLineFlagRegistry); - -private: - inline CommandLineFlagRegistry() {} -}; - -/** - *Helper class to register command line flag. - */ -template -struct CommandLineFlagRegister { - /** - * \brief: Register a command line argument - * - * \param [in] name: The command line name. - * \param [inout] val: The command line argument instance, FLAGS_xxx. - * \param [in] desc: The command line helper message. - */ - CommandLineFlagRegister(const std::string& name, - T* val, - const std::string desc) { - CommandLineFlagRegistry::Instance()->commands.push_back( - {name, val, desc, *val}); - } -}; - -/** - * \brief: Define a command line arguments. - * - * \param type: The variable type, such as int, double, etc. - * \param name: The variable name. The command line argument is '--name', the - *variable - *is 'FLAGS_name' - * \param default_value: The default value of command line argument. - * \param text: The description in command line argument. - */ -#define PADDLE_DEFINE_variable(type, name, default_value, text) \ - type FLAGS_##name = default_value; \ - namespace paddle_flags_internal { \ - paddle::flags_internal::CommandLineFlagRegister \ - flags_internal_var_##name(#name, &FLAGS_##name, text); \ - } // namespace paddle_flags_internal - -/** - * Declare a variable to use. - */ -#define PADDLE_DECLARE_variable(type, name) extern type FLAGS_##name; - -// DEFINE macro for each types. -#define P_DEFINE_int32(name, default_value, text) \ - PADDLE_DEFINE_variable(int32_t, name, default_value, text) - -#define P_DEFINE_bool(name, default_value, text) \ - PADDLE_DEFINE_variable(bool, name, default_value, text) - -#define P_DEFINE_string(name, default_value, text) \ - PADDLE_DEFINE_variable(std::string, name, default_value, text) - -#define P_DEFINE_double(name, default_value, text) \ - PADDLE_DEFINE_variable(double, name, default_value, text) - -#define P_DEFINE_int64(name, default_value, text) \ - PADDLE_DEFINE_variable(int64_t, name, default_value, text) - -#define P_DEFINE_uint64(name, default_value, text) \ - PADDLE_DEFINE_variable(uint64_t, name, default_value, text) - -// Declare macro for each types. -#define P_DECLARE_int32(name) PADDLE_DECLARE_variable(int32_t, name) -#define P_DECLARE_bool(name) PADDLE_DECLARE_variable(bool, name) -#define P_DECLARE_string(name) PADDLE_DECLARE_variable(std::string, name) -#define P_DECLARE_double(name) PADDLE_DECLARE_variable(double, name) -#define P_DECLARE_int64(name) PADDLE_DECLARE_variable(int64_t, name) -#define P_DECLARE_uint64(name) PADDLE_DECLARE_variable(uint64_t, name) -} // namespace flags_internal - -/** - * \brief Parse command line flags. If parse error, just failed and exit 1. - * - * \param [inout] argc: The command argument count. This method will modify - *argc, and left unused arguments. - * \param [inout] argv: The command argument values. This method will modify - *argv, and left unused arguments. - * \param [in] withHelp: True will parse '-h' and '--help' to print usage. - * - * \note: The Command line flags format basically as follow: - * - * * If the type of flag is not bool, then the follow format of command line - * will be parsed: - * * --flag_name=value - * * -flag_name=value - * - * * If the flag is bool, then: - * * --flag_name=value, -flag_name=value will be parsed. - * * if value.tolower() == "true"| "1" will be treated as true. - * * else if value.tolower() == "false" | "0" will be treated as false. - * * --flag_name will be parsed as true. - * * --noflag_name will be parsed as false. - */ -void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true); - -} // namespace paddle - -#else // if use gflags. #include -#define P_DEFINE_int32 DEFINE_int32 -#define P_DEFINE_bool DEFINE_bool -#define P_DEFINE_string DEFINE_string -#define P_DEFINE_double DEFINE_double -#define P_DEFINE_int64 DEFINE_int64 -#define P_DEFINE_uint64 DEFINE_uint64 -#define P_DECLARE_int32 DECLARE_int32 -#define P_DECLARE_bool DECLARE_bool -#define P_DECLARE_string DECLARE_string -#define P_DECLARE_double DECLARE_double -#define P_DECLARE_int64 DECLARE_int64 -#define P_DECLARE_uint64 DECLARE_uint64 namespace paddle { void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true); } // namespace paddle - -#endif diff --git a/paddle/utils/CustomStackTrace.cpp b/paddle/utils/CustomStackTrace.cpp index 083f5c509a26cd06d6fc6cea2a7587c7ef57d4e6..66b38218a7c7ec146f366ded516ebe22d012e47f 100644 --- a/paddle/utils/CustomStackTrace.cpp +++ b/paddle/utils/CustomStackTrace.cpp @@ -16,7 +16,7 @@ limitations under the License. */ #include #include "CommandLineParser.h" -P_DEFINE_bool( +DEFINE_bool( layer_stack_error_only_current_thread, true, "Dump current thread or whole process layer stack when signal error " diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index 1c9e602f45a818824a34aca23ef8f52a5e14cd17..59d6cbdc513660b87cb013d8aa92c5c8f9289ecb 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -15,65 +15,61 @@ limitations under the License. */ #include "Flags.h" #ifdef PADDLE_ONLY_CPU -P_DEFINE_bool(use_gpu, false, "Only support CPU training"); +DEFINE_bool(use_gpu, false, "Only support CPU training"); #else -P_DEFINE_bool(use_gpu, true, "Whether to use GPU for training"); +DEFINE_bool(use_gpu, true, "Whether to use GPU for training"); #endif -P_DEFINE_bool( - parallel_nn, - false, - "Whether to use multi-threads to calculate one neural network." - "If it was set false, use gpu_id specify which gpu core to use" - "(the device property in the trainer config file will be ingored)." - "If it was set true, the gpu core is specified by the trainer" - " config file(gpu_id will be ignored)."); -P_DEFINE_int32(trainer_count, 1, "Defined how many trainers to train"); -P_DEFINE_int32(gpu_id, 0, "Which gpu core to use"); -P_DEFINE_int32(port, 20134, "Listening port for pserver"); -P_DEFINE_int32(data_server_port, 21134, "Listening port for dserver"); -P_DEFINE_int32(ports_num, - 1, - "The ports number for parameter send," - " increment based on default port number"); -P_DEFINE_int32(ports_num_for_sparse, - 0, - "The ports number for parameter send," - " increment based on default (port + ports_num)"); -P_DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers"); -P_DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol"); -P_DEFINE_int32( - trainer_id, - 0, - "For distributed training, each trainer must be given an unique id" - " ranging from 0 to num_trainers-1. Trainer 0 is the master" - " trainer"); -P_DEFINE_int32(num_gradient_servers, 1, "number of gradient servers"); -P_DEFINE_string(comment, "", "A string for commenting this training task"); -P_DEFINE_string(load_missing_parameter_strategy, - "fail", - "which operation to take on load model fails. support " - "fail/rand/zero only."); -P_DEFINE_int32(log_period, 100, "Log progress every so many batches"); -P_DEFINE_int32(log_period_server, - 500, - "Log progress every so many batches at pserver end"); -P_DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad"); -P_DEFINE_int32(enable_parallel_vector, - 0, - "threshold for enable parallel vector"); -P_DEFINE_bool(loadsave_parameters_in_pserver, - false, - "load and save parameters in pserver. " - "only work while parameter set sparse_remote_update."); -P_DEFINE_int32(beam_size, - 1, - "Beam size used in generating most probable output sequences."); +DEFINE_bool(parallel_nn, + false, + "Whether to use multi-threads to calculate one neural network." + "If it was set false, use gpu_id specify which gpu core to use" + "(the device property in the trainer config file will be ingored)." + "If it was set true, the gpu core is specified by the trainer" + " config file(gpu_id will be ignored)."); +DEFINE_int32(trainer_count, 1, "Defined how many trainers to train"); +DEFINE_int32(gpu_id, 0, "Which gpu core to use"); +DEFINE_int32(port, 20134, "Listening port for pserver"); +DEFINE_int32(data_server_port, 21134, "Listening port for dserver"); +DEFINE_int32(ports_num, + 1, + "The ports number for parameter send," + " increment based on default port number"); +DEFINE_int32(ports_num_for_sparse, + 0, + "The ports number for parameter send," + " increment based on default (port + ports_num)"); +DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers"); +DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol"); +DEFINE_int32(trainer_id, + 0, + "For distributed training, each trainer must be given an unique id" + " ranging from 0 to num_trainers-1. Trainer 0 is the master" + " trainer"); +DEFINE_int32(num_gradient_servers, 1, "number of gradient servers"); +DEFINE_string(comment, "", "A string for commenting this training task"); +DEFINE_string(load_missing_parameter_strategy, + "fail", + "which operation to take on load model fails. support " + "fail/rand/zero only."); +DEFINE_int32(log_period, 100, "Log progress every so many batches"); +DEFINE_int32(log_period_server, + 500, + "Log progress every so many batches at pserver end"); +DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad"); +DEFINE_int32(enable_parallel_vector, 0, "threshold for enable parallel vector"); +DEFINE_bool(loadsave_parameters_in_pserver, + false, + "load and save parameters in pserver. " + "only work while parameter set sparse_remote_update."); +DEFINE_int32(beam_size, + 1, + "Beam size used in generating most probable output sequences."); -P_DEFINE_bool(show_layer_stat, false, "show the statistics of each layer"); -P_DEFINE_string(predict_file, "", "File name for saving predict result"); -P_DEFINE_bool(prev_batch_state, false, "batch is continue with next batch"); -P_DEFINE_string(init_model_path, - "", - "Path of the initial model parameters." - "If it was set, start_pass will be ignored."); +DEFINE_bool(show_layer_stat, false, "show the statistics of each layer"); +DEFINE_string(predict_file, "", "File name for saving predict result"); +DEFINE_bool(prev_batch_state, false, "batch is continue with next batch"); +DEFINE_string(init_model_path, + "", + "Path of the initial model parameters." + "If it was set, start_pass will be ignored."); diff --git a/paddle/utils/Flags.h b/paddle/utils/Flags.h index 922533d63e7f0c28a1dcec6b4d9f453f1794abb5..2ebbcb24eb061531d0807756528d7bf16e6aa124 100644 --- a/paddle/utils/Flags.h +++ b/paddle/utils/Flags.h @@ -16,28 +16,28 @@ limitations under the License. */ #include "CommandLineParser.h" -P_DECLARE_bool(parallel_nn); -P_DECLARE_int32(async_count); -P_DECLARE_int32(port); -P_DECLARE_int32(data_server_port); -P_DECLARE_bool(use_gpu); -P_DECLARE_int32(gpu_id); -P_DECLARE_int32(trainer_count); -P_DECLARE_int32(ports_num); -P_DECLARE_int32(ports_num_for_sparse); -P_DECLARE_string(nics); -P_DECLARE_string(rdma_tcp); -P_DECLARE_int32(trainer_id); -P_DECLARE_int32(num_gradient_servers); -P_DECLARE_string(comment); -P_DECLARE_string(load_missing_parameter_strategy); -P_DECLARE_int32(log_period); -P_DECLARE_int32(log_period_server); -P_DECLARE_double(checkgrad_eps); -P_DECLARE_int32(enable_parallel_vector); -P_DECLARE_bool(loadsave_parameters_in_pserver); -P_DECLARE_int32(beam_size); -P_DECLARE_bool(show_layer_stat); -P_DECLARE_string(predict_file); -P_DECLARE_bool(prev_batch_state); -P_DECLARE_string(init_model_path); +DECLARE_bool(parallel_nn); +DECLARE_int32(async_count); +DECLARE_int32(port); +DECLARE_int32(data_server_port); +DECLARE_bool(use_gpu); +DECLARE_int32(gpu_id); +DECLARE_int32(trainer_count); +DECLARE_int32(ports_num); +DECLARE_int32(ports_num_for_sparse); +DECLARE_string(nics); +DECLARE_string(rdma_tcp); +DECLARE_int32(trainer_id); +DECLARE_int32(num_gradient_servers); +DECLARE_string(comment); +DECLARE_string(load_missing_parameter_strategy); +DECLARE_int32(log_period); +DECLARE_int32(log_period_server); +DECLARE_double(checkgrad_eps); +DECLARE_int32(enable_parallel_vector); +DECLARE_bool(loadsave_parameters_in_pserver); +DECLARE_int32(beam_size); +DECLARE_bool(show_layer_stat); +DECLARE_string(predict_file); +DECLARE_bool(prev_batch_state); +DECLARE_string(init_model_path); diff --git a/paddle/utils/Logging.cpp b/paddle/utils/Logging.cpp index 20f32466a56ac4c67c16dfbea229bc78f9f4f6d9..5a1c6ecb2219f7983609c27f3215c7fc1e9e9ef2 100644 --- a/paddle/utils/Logging.cpp +++ b/paddle/utils/Logging.cpp @@ -18,175 +18,9 @@ limitations under the License. */ */ #include "Logging.h" -#ifndef PADDLE_USE_GLOG -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include namespace paddle { -namespace internal { - -std::string join(const std::string& part1, const std::string& part2) { - const char sep = '/'; - if (!part2.empty() && part2.front() == sep) { - return part2; - } - std::string ret; - ret.reserve(part1.size() + part2.size() + 1); - ret = part1; - if (!ret.empty() && ret.back() != sep) { - ret += sep; - } - ret += part2; - return ret; -} - -static inline bool env2bool(const char* envName, bool defaultValue = false) { - char* envValue = getenv(envName); - if (envValue == nullptr) { - return defaultValue; - } else { - return memchr("tTyY1\0", envValue[0], 6) != nullptr; - } -} - -static inline int env2int(const char* envName, int defaultValue = 0) { - char* envValue = getenv(envName); - if (envValue == nullptr) { - return defaultValue; - } else { - int retValue = defaultValue; - try { - retValue = std::stoi(envValue); - } catch (...) { - // pass - } - return retValue; - } -} - -static inline int env2index(const char* envName, - const std::vector& options, - int defaultValue) { - char* envValue = getenv(envName); - if (envValue == nullptr) { - return defaultValue; - } else { - for (size_t i = 0; i < options.size(); ++i) { - if (options[i] == envValue) { - return static_cast(i); - } - } - return defaultValue; - } -} - -static bool gLogToStderr = env2bool("PLOG_LOGTOSTDERR", true); -static const std::vector gLevelName = { - "INFO", "WARNING", "ERROR", "FATAL"}; -static int gMinLogLevel = - env2int("PLOG_MINLOGLEVEL", env2index("PLOG_MINLOGLEVEL", gLevelName, 0)); - -static std::vector> gLogFds; -static std::vector gLogFileFds; -static bool gLogInited = false; -static void freeLogFileFds() { - for (auto fd : gLogFileFds) { - close(fd); - } -} - -static void initializeLogFds(char* argv0) { - gLogFds.resize(NUM_SEVERITIES); - - for (int i = gMinLogLevel; i < NUM_SEVERITIES && gLogToStderr; - ++i) { // Add stderr - std::vector& fds = gLogFds[i]; - fds.push_back(STDERR_FILENO); - } - - char* logDir = getenv("PLOG_LOGDIR"); - - for (int i = gMinLogLevel; i < NUM_SEVERITIES && logDir != nullptr; ++i) { - std::string filename = - join(logDir, std::string(argv0) + "." + gLevelName[i]); - int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644); - if (fd == -1) { - fprintf(stderr, "Open log file error!"); - exit(1); - } - gLogFileFds.push_back(fd); - - std::vector& curFds = gLogFds[i]; - curFds.insert(curFds.end(), gLogFileFds.begin(), gLogFileFds.end()); - } - - atexit(freeLogFileFds); - gLogInited = true; -} - -static void (*gFailureFunctionPtr)() ATTR_NORETURN = abort; - -LogMessage::LogMessage(const char* fname, int line, int severity) - : fname_(fname), line_(line), severity_(severity) {} - -LogMessage::~LogMessage() { this->generateLogMessage(); } - -void LogMessage::generateLogMessage() { - if (!gLogInited) { - fprintf(stderr, - "%c %s:%d] %s\n", - "IWEF"[severity_], - fname_, - line_, - str().c_str()); - } else { - for (auto& fd : gLogFds[this->severity_]) { - dprintf(fd, - "%c %s:%d] %s\n", - "IWEF"[severity_], - fname_, - line_, - str().c_str()); - } - } -} - -LogMessageFatal::LogMessageFatal(const char* file, int line) - : LogMessage(file, line, FATAL) {} - -LogMessageFatal::~LogMessageFatal() { - generateLogMessage(); - gFailureFunctionPtr(); -} -} // namespace internal - -void initializeLogging(int argc, char** argv) { - internal::initializeLogFds(argv[0]); -} - -namespace logging { -void setMinLogLevel(int level) { paddle::internal::gMinLogLevel = level; } - -void installFailureFunction(void (*callback)() ATTR_NORETURN) { - paddle::internal::gFailureFunctionPtr = callback; -} - -} // namespace logging - -} // namespace paddle - -#else -namespace paddle { void initializeLogging(int argc, char** argv) { (void)(argc); if (!getenv("GLOG_logtostderr")) { @@ -197,13 +31,16 @@ void initializeLogging(int argc, char** argv) { } namespace logging { + void setMinLogLevel(int level) { FLAGS_minloglevel = level; } + void installFailureFunction(void (*callback)()) { google::InstallFailureFunction(callback); } + void installFailureWriter(void (*callback)(const char*, int)) { google::InstallFailureWriter(callback); } + } // namespace logging } // namespace paddle -#endif diff --git a/paddle/utils/Logging.h b/paddle/utils/Logging.h index 4379289f6d1b4b5a1006bd723c2177a686ed89f6..d9e551f0891fa0808b8699aea94a0d2ab4f81cb3 100644 --- a/paddle/utils/Logging.h +++ b/paddle/utils/Logging.h @@ -22,175 +22,21 @@ limitations under the License. */ #include #include -#ifndef PADDLE_USE_GLOG -#include "CompilerMacros.h" - -//! TODO(yuyang18): Move this utility macro into some global header. -#define PP_CAT(a, b) PP_CAT_I(a, b) -#define PP_CAT_I(a, b) PP_CAT_II(~, a##b) -#define PP_CAT_II(p, res) res - -/** - * Generate Unique Variable Name, Usefully in macro. - * @SEE - * http://stackoverflow.com/questions/1082192/how-to-generate-random-variable-names-in-c-using-macros - */ -#define UNIQUE_NAME(base) PP_CAT(base, __LINE__) - +#include namespace paddle { -//! Log levels. -const int INFO = 0; -const int WARNING = 1; -const int ERROR = 2; -const int FATAL = 3; -const int NUM_SEVERITIES = 4; - -namespace internal { - -class LogMessage : public std::basic_ostringstream { -public: - LogMessage(const char* fname, int line, int severity); - ~LogMessage(); - -protected: - /** - * @brief Print log message to stderr, files, etc. - */ - void generateLogMessage(); - -private: - const char* fname_; - int line_; - int severity_; -}; - -// LogMessageFatal ensures the process will exit in failure after -// logging this message. -class LogMessageFatal : public LogMessage { -public: - LogMessageFatal(const char* file, int line) __attribute__((cold)); - ~LogMessageFatal() __attribute__((noreturn)); -}; - -#define _P_LOG_INFO \ - ::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::INFO) -#define _P_LOG_WARNING \ - ::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::WARNING) -#define _P_LOG_ERROR \ - ::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::ERROR) -#define _P_LOG_FATAL ::paddle::internal::LogMessageFatal(__FILE__, __LINE__) - -#define P_LOG(severity) _P_LOG_##severity - -#define P_LOG_FIRST_N(severity, n) \ - static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \ - if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) ++UNIQUE_NAME(LOG_OCCURRENCES); \ - if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) P_LOG(severity) - -#define P_LOG_IF_EVERY_N(severity, condition, n) \ - static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \ - if (condition && ((UNIQUE_NAME(LOG_OCCURRENCES) = \ - (UNIQUE_NAME(LOG_OCCURRENCES) + 1) % n) == (1 % n))) \ - P_LOG(severity) - -#define P_LOG_EVERY_N(severity, n) P_LOG_IF_EVERY_N(severity, true, n) - -// TODO(jeff): Define a proper implementation of VLOG_IS_ON -#define P_VLOG_IS_ON(lvl) ((lvl) <= 0) - -#define P_LOG_IF(severity, condition) \ - if (condition) P_LOG(severity) - -#define P_VLOG(lvl) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl)) - -#define P_VLOG_IF(lvl, cond) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl) && cond) - -#define P_VLOG_EVERY_N(lvl, n) P_LOG_IF_EVERY_N(INFO, P_VLOG_IS_ON(lvl), n) - -#define PREDICT_FALSE(x) (__builtin_expect(x, 0)) -#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) - -// CHECK dies with a fatal error if condition is not true. It is *not* -// controlled by NDEBUG, so the check will be executed regardless of -// compilation mode. Therefore, it is safe to do things like: -// CHECK(fp->Write(x) == 4) -#define P_CHECK(condition) \ - if (PREDICT_FALSE(!(condition))) \ - P_LOG(FATAL) << "Check failed: " #condition " " - -#define P_CHECK_EQ(val1, val2) P_CHECK((val1) == (val2)) -#define P_CHECK_NE(val1, val2) P_CHECK((val1) != (val2)) -#define P_CHECK_LE(val1, val2) P_CHECK((val1) <= (val2)) -#define P_CHECK_LT(val1, val2) P_CHECK((val1) < (val2)) -#define P_CHECK_GE(val1, val2) P_CHECK((val1) >= (val2)) -#define P_CHECK_GT(val1, val2) P_CHECK((val1) > (val2)) -#define P_CHECK_NOTNULL(val) P_CHECK((val) != NULL) - -//! GLOG compatible APIs -//! NOTE: only implement Paddle actually used APIs. -#define LOG(x) P_LOG(x) -#define VLOG(x) P_VLOG(x) -#define DLOG(x) P_VLOG(5) -#define CHECK(x) P_CHECK(x) -#define PCHECK(x) P_CHECK(x) -#define CHECK_EQ(val1, val2) P_CHECK((val1) == (val2)) -#define CHECK_NE(val1, val2) P_CHECK((val1) != (val2)) -#define CHECK_LE(val1, val2) P_CHECK((val1) <= (val2)) -#define CHECK_LT(val1, val2) P_CHECK((val1) < (val2)) -#define CHECK_GE(val1, val2) P_CHECK((val1) >= (val2)) -#define CHECK_GT(val1, val2) P_CHECK((val1) > (val2)) -#define CHECK_NOTNULL(val) P_CHECK((val) != NULL) -#define VLOG_IS_ON(x) P_VLOG_IS_ON(x) -#define LOG_FIRST_N(severity, n) P_LOG_FIRST_N(severity, n) -#define LOG_IF(severity, condition) P_LOG_IF(severity, condition) -#define VLOG_EVERY_N(lvl, n) P_VLOG_EVERY_N(lvl, n) -#define VLOG_IF(lvl, cond) P_VLOG_IF(lvl, cond) -#define LOG_EVERY_N(severity, n) P_LOG_EVERY_N(severity, n) -} // namespace internal - -/** - * @brief initialize logging - * @note: Current implement of logging is lack of: - * PrintCallStack when fatal. - * VLOG_IS_ON - * But it is portable to multi-platform, and simple enough to modify. - */ void initializeLogging(int argc, char** argv); -namespace logging { -/** - * @brief Set Min Log Level. if Log.level < minLogLevel, then will not print log - * to stream - * @param level. Any integer is OK, but only 0 <= x <= NUM_SEVERITIES is useful. - */ -void setMinLogLevel(int level); - -/** - * @brief Install Log(Fatal) failure function. Default is abort(); - * @param callback: The failure function. - */ -void installFailureFunction(void (*callback)() ATTR_NORETURN); -/** - * @brief installFailureWriter - * @note: not implemented currently. - */ -inline void installFailureWriter(void (*callback)(const char*, int)) { - (void)(callback); // unused callback. -} -} // namespace logging -} // namespace paddle -#else -#include -namespace paddle { -void initializeLogging(int argc, char** argv); namespace logging { + void setMinLogLevel(int level); + void installFailureFunction(void (*callback)()); + void installFailureWriter(void (*callback)(const char*, int)); -} // namespace logging -} -#endif // PADDLE_USE_GLOG + +} // namespace logging +} // namespace paddle #ifndef NDEBUG #define DEBUG_LEVEL 5 diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp index 2ee4e4fb7ed0e2519d5e7182d6f3adc3431b493a..7faeff55c28b9065179ad27b3b604a9f411249e5 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp @@ -20,8 +20,8 @@ namespace paddle { #ifdef PADDLE_NO_PYTHON -P_DEFINE_string(python_path, "", "python path"); -P_DEFINE_string(python_bin, "python2.7", "python bin"); +DEFINE_string(python_path, "", "python path"); +DEFINE_string(python_bin, "python2.7", "python bin"); constexpr int kExecuteCMDBufLength = 204800; diff --git a/paddle/utils/ThreadLocal.cpp b/paddle/utils/ThreadLocal.cpp index 8a2878fc4bafb803a8c4b99ceccfc77405a0a3a0..75ccbd28cf21b7fafb43a072503dff14a29fec8a 100644 --- a/paddle/utils/ThreadLocal.cpp +++ b/paddle/utils/ThreadLocal.cpp @@ -16,9 +16,9 @@ limitations under the License. */ #include "CommandLineParser.h" #include "Util.h" -P_DEFINE_bool(thread_local_rand_use_global_seed, - false, - "Whether to use global seed in thread local rand."); +DEFINE_bool(thread_local_rand_use_global_seed, + false, + "Whether to use global seed in thread local rand."); namespace paddle { diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 26ff385c84564d1188120464f94f418d88c20f19..7c0d66c488f5064641c53ea7995a75c330a3e49d 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -33,7 +33,7 @@ limitations under the License. */ #include "ThreadLocal.h" #include "Version.h" -P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)"); +DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)"); #ifdef WITH_GOOGLE_PERFTOOLS /* @@ -52,10 +52,8 @@ P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)"); #include -P_DEFINE_int32(profile_signal, 12, "signal for switch google profiler"); -P_DEFINE_string(profile_data_file, - "gperf.prof", - "file for storing profile data"); +DEFINE_int32(profile_signal, 12, "signal for switch google profiler"); +DEFINE_string(profile_data_file, "gperf.prof", "file for storing profile data"); static void profilerSwitch(int signalNumber) { bool static started = false; diff --git a/paddle/utils/Version.cpp b/paddle/utils/Version.cpp index a9e351b69fcbac661142d0a3322d7bf3a4293cd9..731c30842118bce59ce45297d9c8f47fa0a69d69 100644 --- a/paddle/utils/Version.cpp +++ b/paddle/utils/Version.cpp @@ -18,13 +18,8 @@ limitations under the License. */ #include #include "Flags.h" #include "Util.h" -//! TODO(yuyang18) in gflags, version has another define. Use another flag -//! instead. -#ifndef PADDLE_USE_GFLAGS -P_DEFINE_bool(version, false, "print version"); -#else -P_DECLARE_bool(version); -#endif + +DECLARE_bool(version); namespace paddle { namespace version { diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 298ede5cd6451c9b03219dff72f6e81c374f8ef1..26fafbd1ab3f2967b765b8bcb973fb745c0e6422 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -1,5 +1,3 @@ -add_simple_unittest(test_CommandLineParser) -add_simple_unittest(test_Logging) add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) add_simple_unittest(test_CustomStackTrace) diff --git a/paddle/utils/tests/test_CommandLineParser.cpp b/paddle/utils/tests/test_CommandLineParser.cpp deleted file mode 100644 index ed2b3068d5dda710de728cfad14a98aeaf847954..0000000000000000000000000000000000000000 --- a/paddle/utils/tests/test_CommandLineParser.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef PADDLE_USE_GFLAGS -//! Test Command Line Parser for paddle internal implement. - -#include -#include - -P_DEFINE_int32(i1, 1, "test int flag 1"); -P_DEFINE_int32(i2, 2, "test int flag 2"); - -P_DEFINE_string(str1, "1", "test str flag 1"); -P_DEFINE_string(str2, "2", "test str flag 2"); - -P_DEFINE_bool(b1, true, "test bool flag 1"); -P_DEFINE_bool(b2, false, "test bool flag 2"); - -P_DEFINE_double(d1, 0.1, "test double flag 1"); -P_DEFINE_double(d2, -42.3, "test double flag 2"); - -P_DEFINE_int64(l1, 1, "test int64 flag 1"); -P_DEFINE_int64(l2, 2, "test int64 flag 2"); - -P_DEFINE_uint64(ul1, 32, "test uint64 flag 1"); -P_DEFINE_uint64(ul2, 33, "test uint64 flag 2"); - -constexpr double EPSILON = 1e-5; - -#define cc(x) const_cast((x)) - -TEST(CommandLineParser, defaultValue) { - char* argv[] = {cc("test_program"), cc("--unused_flag=134")}; - int argc = sizeof(argv) / sizeof(char*); - - paddle::ParseCommandLineFlags(&argc, argv); - - // Check Default Value - ASSERT_EQ(argc, 2); - ASSERT_EQ(FLAGS_i1, 1); - ASSERT_EQ(FLAGS_i2, 2); - ASSERT_EQ(FLAGS_str1, "1"); - ASSERT_EQ(FLAGS_str2, "2"); - ASSERT_EQ(FLAGS_b1, true); - ASSERT_EQ(FLAGS_b2, false); - ASSERT_NEAR(FLAGS_d1, 0.1, EPSILON); - ASSERT_NEAR(FLAGS_d2, -42.3, EPSILON); - ASSERT_EQ(FLAGS_i1, 1); - ASSERT_EQ(FLAGS_i2, 2); - ASSERT_EQ(FLAGS_ul1, 32UL); - ASSERT_EQ(FLAGS_ul2, 33UL); -} - -TEST(CommandLineParser, normal) { - char* argv[] = {cc("test_program"), - cc("--i2=32"), - cc("--str1=abc"), - cc("--b2=1"), - cc("-b1=False"), - cc("--d2=.34"), - cc("--d1=0"), - cc("--l1=-12345678901234"), - cc("-ul2=3212")}; - int argc = sizeof(argv) / sizeof(char*); - paddle::ParseCommandLineFlags(&argc, argv); - ASSERT_EQ(argc, 1); - ASSERT_EQ(FLAGS_i2, 32); - ASSERT_EQ(FLAGS_str1, "abc"); - ASSERT_EQ(FLAGS_b2, true); - ASSERT_EQ(FLAGS_b1, false); - ASSERT_NEAR(FLAGS_d2, 0.34, EPSILON); - ASSERT_NEAR(FLAGS_d1, 0.0, EPSILON); - ASSERT_EQ(FLAGS_l1, -12345678901234); - ASSERT_EQ(FLAGS_ul2, 3212UL); -} - -TEST(CommandLineParser, printHelp) { - char* argv[] = {cc("test_program"), cc("--help")}; - int argc = sizeof(argv) / sizeof(char*); - - // Will Print Usage - ASSERT_DEATH(paddle::ParseCommandLineFlags(&argc, argv), ".*test_program.*"); -} - -TEST(CommandLineParser, parseError) { - char* argv[] = {cc("test_program"), cc("--i1=abc")}; - - int argc = sizeof(argv) / sizeof(char*); - ASSERT_DEATH( - paddle::ParseCommandLineFlags(&argc, argv), - "Parse command flag i1 error! User input is --i1=abc.*test_program.*"); -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -#else - -int main(int argc, char** argv) { return 0; } - -#endif diff --git a/paddle/utils/tests/test_CustomStackTrace.cpp b/paddle/utils/tests/test_CustomStackTrace.cpp index 292ed4619d8bb0c3f7069cbbea7e729d91dd126c..2ce199837601755ac018889c07c223ad34c4a45b 100644 --- a/paddle/utils/tests/test_CustomStackTrace.cpp +++ b/paddle/utils/tests/test_CustomStackTrace.cpp @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/utils/Locks.h" #include "paddle/utils/Util.h" -P_DEFINE_int32(test_thread_num, 10, "testing thread number"); +DEFINE_int32(test_thread_num, 10, "testing thread number"); void testNormalImpl( const std::function&, diff --git a/paddle/utils/tests/test_Logging.cpp b/paddle/utils/tests/test_Logging.cpp deleted file mode 100644 index fbfffcc65aeb9f5ebbe97dcb54ec44bd63f8d4bd..0000000000000000000000000000000000000000 --- a/paddle/utils/tests/test_Logging.cpp +++ /dev/null @@ -1,162 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -/* - * Basically from tensorflow/core/platform/default/logging.cc - * Used in embedded system where there is no glogs. - */ - -#include -#include -#include -#include -#include "paddle/utils/Logging.h" -#include "paddle/utils/Util.h" -#ifndef PADDLE_USE_GLOG -TEST(Logging, BasicalLog) { - auto pinfo = [] { - P_LOG(INFO) << "INFO"; - exit(1); - }; - ASSERT_DEATH(pinfo(), "I .*test_Logging.cpp:[0-9]+] INFO"); - - auto pwarn = [] { - P_LOG(WARNING) << "WARN"; - exit(1); - }; - ASSERT_DEATH(pwarn(), "W .*test_Logging.cpp:[0-9]+] WARN"); - - auto perr = [] { - P_LOG(ERROR) << "ERROR"; - exit(1); - }; - ASSERT_DEATH(perr(), "E .*test_Logging.cpp:[0-9]+] ERROR"); - - auto pfatal = [] { P_LOG(FATAL) << "FATAL"; }; - ASSERT_DEATH(pfatal(), "F .*test_Logging.cpp:[0-9]+] FATAL"); -} - -TEST(Logging, Check) { - int a = 1; - int b = 2; - P_CHECK(a != b); - - auto pcheckDown = [&] { P_CHECK(a == b); }; - ASSERT_DEATH(pcheckDown(), - "F .*test_Logging.cpp:[0-9]+] Check failed: a == b "); - - P_CHECK_LE(a, b); - P_CHECK_LT(a, b); - double t = 1.2; - P_CHECK_LE(a, t); - double* ptr = nullptr; - - auto pcheckDown2 = [&] { P_CHECK_NOTNULL(ptr); }; - ASSERT_DEATH(pcheckDown2(), "F"); -} - -#define cc(x) const_cast(x) - -TEST(Logging, LogToStderr) { - auto logToStderrCallback = [] { - setenv("PLOG_LOGTOSTDERR", "0", true); - char* argv[] = {cc("test")}; - paddle::initializeLogging(1, argv); - P_LOG(INFO) << "This output will not print to std error"; - exit(1); - }; - - ASSERT_DEATH(logToStderrCallback(), ""); -} - -constexpr char kLogDirName[] = "./test_log_dir"; -const std::vector kLevels = {"INFO", "WARNING", "ERROR", "FATAL"}; - -TEST(Logging, LogToDir) { - ASSERT_EQ(0, mkdir(kLogDirName, 0777)); - auto logToDirCallback = [] { - setenv("PLOG_LOGTOSTDERR", "0", true); - setenv("PLOG_LOGDIR", kLogDirName, true); - char* argv[] = {cc("test")}; - paddle::initializeLogging(1, argv); - - P_LOG(INFO) << "INFO"; - P_LOG(WARNING) << "WARNING"; - P_LOG(ERROR) << "ERROR"; - P_LOG(FATAL) << "FATAL"; - }; - ASSERT_DEATH(logToDirCallback(), ""); - - // There 4 file in logdir - auto dir = opendir(kLogDirName); - size_t fileCount = 0; - std::vector filenames; - for (auto dirContent = readdir(dir); dirContent != nullptr; - dirContent = readdir(dir)) { - std::string filename(dirContent->d_name); - if (filename == "." || filename == "..") { - continue; - } else { - ++fileCount; - for (size_t i = 0; i < kLevels.size(); ++i) { - const std::string& curLevel = kLevels[i]; - if (filename.size() > curLevel.length()) { - size_t diff = filename.size() - curLevel.length(); - size_t j = 0; - for (; j < curLevel.length(); ++j) { - if (filename[j + diff] != curLevel[j]) { - // File Suffix Not Same, then break. - break; - } - } - if (j == curLevel.length()) { // Same suffix. - std::ifstream fin; - auto fn = paddle::path::join(kLogDirName, filename); - fin.open(fn); - filenames.push_back(fn); - ASSERT_TRUE(fin.is_open()); - size_t lineCounter = 0; - for (std::string line; std::getline(fin, line); ++lineCounter) { - // Do Nothing, Just calc lineCounter. - } - - // For example. - // The info channel will have all log which level >= INFO - // So the info file's lineCounter should == 4. - ASSERT_EQ(kLevels.size() - i, lineCounter); - fin.close(); - } - } - } - } - } - closedir(dir); - ASSERT_EQ(4UL, fileCount); // 4 levels. - // Clean Unittest. - for (std::string& fn : filenames) { - ASSERT_EQ(remove(fn.c_str()), 0); - } - ASSERT_EQ(rmdir(kLogDirName), 0); -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -#else - -int main(int, char**) { return 0; } - -#endif diff --git a/paddle/utils/tests/test_SpinLock.cpp b/paddle/utils/tests/test_SpinLock.cpp index 22f8584ef559d78bb1ba01356b3361accf3093c4..8351e7e3acd1afe1c6507ffced32f27ce065e5ce 100644 --- a/paddle/utils/tests/test_SpinLock.cpp +++ b/paddle/utils/tests/test_SpinLock.cpp @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include "paddle/utils/Util.h" -P_DEFINE_int32(test_thread_num, 100, "testing thread number"); +DEFINE_int32(test_thread_num, 100, "testing thread number"); void testNormalImpl( size_t thread_num, diff --git a/paddle/utils/tests/test_ThreadBarrier.cpp b/paddle/utils/tests/test_ThreadBarrier.cpp index 4a8af5b97e3977961bce40a9aa9ad691113e342b..60c2214ffd1066ed4f7b95cd63dfe6a24fe66d67 100644 --- a/paddle/utils/tests/test_ThreadBarrier.cpp +++ b/paddle/utils/tests/test_ThreadBarrier.cpp @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include "paddle/utils/Util.h" -P_DEFINE_int32(test_thread_num, 100, "testing thread number"); +DEFINE_int32(test_thread_num, 100, "testing thread number"); void testNormalImpl( size_t thread_num, diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index b34e1ebdedab104f7c16dbf9e1a264f3665115ce..552af71e76e5adf27f35bb5ad6fd8a69c71df0f1 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -245,7 +245,7 @@ message ImageConfig { // The size of input feature map. required uint32 img_size = 8; - required uint32 img_size_y = 9; + optional uint32 img_size_y = 9; } message LayerInputConfig { diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index c62553f54cc304259015f1e8e6733eff99f476f2..0fcf993d57bb3e08c8711a20b16ad94d9c91db1e 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -186,7 +186,7 @@ def define_py_data_sources2(train_list, test_list, module, obj, args=None): obj="process", args={"dictionary": dict_name}) - The related data provider can refer to :ref:`api_pydataprovider2_en_sequential_model` . + The related data provider can refer to :ref:`api_pydataprovider2_sequential_model` . :param train_list: Train list name. :type train_list: basestring diff --git a/third_party/gflags.BUILD b/third_party/gflags.BUILD new file mode 100644 index 0000000000000000000000000000000000000000..85e8bd0bd74942102e5e9a9f817dc49383a745e7 --- /dev/null +++ b/third_party/gflags.BUILD @@ -0,0 +1,12 @@ +# Bazel (http://bazel.io/) BUILD file for gflags. +# +# See INSTALL.md for instructions for adding gflags to a Bazel workspace. + +licenses(["notice"]) + +exports_files(["src/gflags_complections.sh", "COPYING.txt"]) + +load(":bazel/gflags.bzl", "gflags_sources", "gflags_library") +(hdrs, srcs) = gflags_sources(namespace=["google", "gflags"]) +gflags_library(hdrs=hdrs, srcs=srcs, threads=0) +gflags_library(hdrs=hdrs, srcs=srcs, threads=1) diff --git a/third_party/gflags_test/BUILD b/third_party/gflags_test/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..b50615203ba17c74a4c7611b685f3d3210389bbf --- /dev/null +++ b/third_party/gflags_test/BUILD @@ -0,0 +1,10 @@ +licenses(["notice"]) # Apache 2.0 + +cc_test( + name="gflags_test", + srcs=["gflags_test.cc"], + copts=["-Iexternal/gtest/include"], + deps=[ + "@gtest//:gtest", + "@gflags//:gflags", + ], ) diff --git a/third_party/gflags_test/gflags_test.cc b/third_party/gflags_test/gflags_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..53286e7e5be062cf66b37d07047b173ea831e6c4 --- /dev/null +++ b/third_party/gflags_test/gflags_test.cc @@ -0,0 +1,33 @@ +#include +#include + +#include "gflags/gflags.h" +#include "gtest/gtest.h" + +DEFINE_bool(verbose, false, "Display program name before message"); +DEFINE_string(message, "Hello world!", "Message to print"); + +static bool IsNonEmptyMessage(const char *flagname, const std::string &value) { + return value[0] != '\0'; +} +DEFINE_validator(message, &IsNonEmptyMessage); + +namespace third_party { +namespace gflags_test { + +TEST(GflagsTest, ParseAndPrint) { + gflags::SetUsageMessage("some usage message"); + gflags::SetVersionString("1.0.0"); + int argc = 1; + char program_name[] = "gflags_test"; + char **argv = new char *[2]; + argv[0] = program_name; + argv[1] = NULL; + gflags::ParseCommandLineFlags(&argc, reinterpret_cast(&argv), true); + EXPECT_EQ("gflags_test", std::string(gflags::ProgramInvocationShortName())); + EXPECT_EQ("Hello world!", FLAGS_message); + gflags::ShutDownCommandLineFlags(); +} + +} // namespace gflags_test +} // namespace third_party diff --git a/third_party/glog.BUILD b/third_party/glog.BUILD new file mode 100644 index 0000000000000000000000000000000000000000..a0ff1d6b416c2217b62f64bceee3c6a611c11dfe --- /dev/null +++ b/third_party/glog.BUILD @@ -0,0 +1,128 @@ +licenses(["notice"]) + +cc_library( + visibility=["//visibility:public"], + name="glog", + includes=[ + ".", + "src", + ], + copts=[ + "-D_START_GOOGLE_NAMESPACE_='namespace google {'", + "-D_END_GOOGLE_NAMESPACE_='}'", + "-DGOOGLE_NAMESPACE='google'", + "-DGOOGLE_GLOG_DLL_DECL=''", + "-DHAVE_DLADDR", + "-DHAVE_SNPRINTF", + "-DHAVE_DLFCN_H", + "-DHAVE_FCNTL", + "-DHAVE_GLOB_H", + "-DHAVE_INTTYPES_H", + "-DHAVE_LIBPTHREAD", + "-DHAVE_SYS_SYSCALL_H", + "-DHAVE_MEMORY_H", + "-DHAVE_NAMESPACES", + "-DHAVE_PREAD", + "-DHAVE_PTHREAD", + "-DHAVE_PWD_H", + "-DHAVE_PWRITE", + "-DHAVE_RWLOCK", + "-DHAVE_SIGACTION", + "-DHAVE_SIGALTSTACK", + "-DHAVE_STDINT_H", + "-DHAVE_STRING_H", + "-DHAVE_SYS_TIME_H", + "-DHAVE_SYS_TYPES_H", + "-DHAVE_SYS_UCONTEXT_H", + "-DHAVE_SYS_UTSNAME_H", + "-DHAVE_UNISTD_H", + "-DHAVE_USING_OPERATOR", + "-DHAVE_HAVE___ATTRIBUTE___", + "-DHAVE_HAVE___BUILTIN_EXPECT", + #"-DNO_FRAME_POINTER", + "-D_GNU_SOURCE", + #"-fno-sanitize=thread", + #"-fno-sanitize=address", + "-Iexternal/glog/src", + ], + srcs=[ + "src/demangle.cc", + "src/logging.cc", + "src/raw_logging.cc", + "src/signalhandler.cc", + "src/symbolize.cc", + "src/utilities.cc", + "src/vlog_is_on.cc", + ":config_h", + ":logging_h", + ":raw_logging_h", + ":stl_logging_h", + ":vlog_is_on_h", + ], + hdrs=[ + "src/demangle.h", + "src/mock-log.h", + "src/stacktrace.h", + "src/symbolize.h", + "src/utilities.h", + "src/base/commandlineflags.h", + "src/base/googleinit.h", + "src/base/mutex.h", + "src/glog/log_severity.h", + ]) + +genrule( + name="config_h", + srcs=["src/config.h.cmake.in"], + outs=["config.h"], + cmd="awk '{ gsub(/^#cmakedefine/, \"//cmakedefine\"); print; }' $(<) > $(@)", +) + +genrule( + name="logging_h", + srcs=["src/glog/logging.h.in"], + outs=["glog/logging.h"], + cmd="$(location :gen_sh) < $(<) > $(@)", + tools=[":gen_sh"]) + +genrule( + name="raw_logging_h", + srcs=["src/glog/raw_logging.h.in"], + outs=["glog/raw_logging.h"], + cmd="$(location :gen_sh) < $(<) > $(@)", + tools=[":gen_sh"]) + +genrule( + name="stl_logging_h", + srcs=["src/glog/stl_logging.h.in"], + outs=["glog/stl_logging.h"], + cmd="$(location :gen_sh) < $(<) > $(@)", + tools=[":gen_sh"]) + +genrule( + name="vlog_is_on_h", + srcs=["src/glog/vlog_is_on.h.in"], + outs=["glog/vlog_is_on.h"], + cmd="$(location :gen_sh) < $(<) > $(@)", + tools=[":gen_sh"]) + +genrule( + name="gen_sh", + outs=["gen.sh"], + cmd=""" +cat > $@ <<"EOF" +#! /bin/sh +sed -e 's/@ac_cv_have_unistd_h@/1/g' \ + -e 's/@ac_cv_have_stdint_h@/1/g' \ + -e 's/@ac_cv_have_systypes_h@/1/g' \ + -e 's/@ac_cv_have_libgflags_h@/1/g' \ + -e 's/@ac_cv_have_uint16_t@/1/g' \ + -e 's/@ac_cv_have___builtin_expect@/1/g' \ + -e 's/@ac_cv_have_.*@/0/g' \ + -e 's/@ac_google_start_namespace@/namespace google {/g' \ + -e 's/@ac_google_end_namespace@/}/g' \ + -e 's/@ac_google_namespace@/google/g' \ + -e 's/@ac_cv___attribute___noinline@/__attribute__((noinline))/g' \ + -e 's/@ac_cv___attribute___noreturn@/__attribute__((noreturn))/g' \ + -e 's/@ac_cv___attribute___printf_4_5@/__attribute__((__format__ (__printf__, 4, 5)))/g' +EOF""") diff --git a/third_party/glog_test/BUILD b/third_party/glog_test/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..56d08e95f8e8f063829ae68586fa9ef53306fef6 --- /dev/null +++ b/third_party/glog_test/BUILD @@ -0,0 +1,10 @@ +licenses(["notice"]) # Apache 2.0 + +cc_test( + name="glog_test", + srcs=["glog_test.cc"], + copts=["-Iexternal/gtest/include"], + deps=[ + "@gtest//:gtest", + "@glog//:glog", + ], ) diff --git a/third_party/glog_test/glog_test.cc b/third_party/glog_test/glog_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..f1d737d625d25e8675f636075876903c42881a35 --- /dev/null +++ b/third_party/glog_test/glog_test.cc @@ -0,0 +1,7 @@ +#include +#include + +#include "glog/logging.h" +#include "gtest/gtest.h" + +TEST(GlogTest, Logging) { LOG(INFO) << "Hello world"; } diff --git a/third_party/gtest.BUILD b/third_party/gtest.BUILD index 71c74af513ca8c511dd273d089722bc7fbffd84c..9255b51d9aaa9c7ee5cbc1b2d537815c7ecbfcba 100644 --- a/third_party/gtest.BUILD +++ b/third_party/gtest.BUILD @@ -1,5 +1,5 @@ cc_library( - name="main", + name="gtest", srcs=glob( ["src/*.cc"], exclude=["src/gtest-all.cc"]), hdrs=glob(["include/**/*.h", "src/*.h"]), diff --git a/third_party/protobuf_test/BUILD b/third_party/protobuf_test/BUILD index 95a687a35629315398c7046ca7dc8d526892e227..67d4293c70eef081f6bb95de9774613a19ba91dd 100644 --- a/third_party/protobuf_test/BUILD +++ b/third_party/protobuf_test/BUILD @@ -19,6 +19,6 @@ cc_test( srcs=["example_lib_test.cc"], copts=["-Iexternal/gtest/include"], deps=[ - "@gtest//:main", + "@gtest//:gtest", ":example_lib", ], )