Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
提交 1de901ab 编写于 作者: L Luo Tao
浏览文件
操作

Merge branch 'develop' into dir

上级 f1e3ade0 8591e70b
展开全部 隐藏空白更改
内联 并排
Showing with 678 addition and 1460 deletion
CMakeLists.txt
浏览文件 @ 1de901ab
......@@ -25,8 +25,8 @@ find_package(ZLIB REQUIRED)
find_package(NumPy REQUIRED)
find_package(Threads REQUIRED)
find_package(AVX QUIET)
find_package(Glog)
find_package(Gflags QUIET)
find_package(Glog REQUIRED)
find_package(Gflags REQUIRED)
find_package(GTest)
find_package(Sphinx)
find_package(Doxygen)
......@@ -40,8 +40,6 @@ option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND})
option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
option(WITH_GLOG "Compile PaddlePaddle use glog, otherwise use a log implement internally" ${LIBGLOG_FOUND})
option(WITH_GFLAGS "Compile PaddlePaddle use gflags, otherwise use a flag implement internally" ${GFLAGS_FOUND})
option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND})
......@@ -136,16 +134,12 @@ else(WITH_RDMA)
add_definitions(-DPADDLE_DISABLE_RDMA)
endif(WITH_RDMA)
if(WITH_GLOG)
add_definitions(-DPADDLE_USE_GLOG)
include_directories(${LIBGLOG_INCLUDE_DIR})
endif()
# glog
include_directories(${LIBGLOG_INCLUDE_DIR})
if(WITH_GFLAGS)
add_definitions(-DPADDLE_USE_GFLAGS)
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
endif()
#gflags
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
if(WITH_TESTING)
enable_testing()
......
CONTRIBUTING.md 0 → 120000
浏览文件 @ 1de901ab
./doc/howto/contribute_to_paddle_en.md
\ No newline at end of file
WORKSPACE
浏览文件 @ 1de901ab
......@@ -3,7 +3,7 @@ http_archive(
name="protobuf",
url="http://github.com/google/protobuf/archive/v3.1.0.tar.gz",
sha256="0a0ae63cbffc274efb573bdde9a253e3f32e458c41261df51c5dbc5ad541e8f7",
strip_prefix="protobuf-3.1.0", )
strip_prefix="protobuf-3.1.0")
# External dependency to gtest 1.7.0. This method comes from
# https://www.bazel.io/versions/master/docs/tutorial/cpp.html.
......@@ -12,4 +12,20 @@ new_http_archive(
url="https://github.com/google/googletest/archive/release-1.7.0.zip",
sha256="b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0",
build_file="third_party/gtest.BUILD",
strip_prefix="googletest-release-1.7.0", )
strip_prefix="googletest-release-1.7.0")
# External dependency to gflags. This method comes from
# https://github.com/gflags/example/blob/master/WORKSPACE.
new_git_repository(
name="gflags",
tag="v2.2.0",
remote="https://github.com/gflags/gflags.git",
build_file="third_party/gflags.BUILD")
# External dependency to glog. This method comes from
# https://github.com/reyoung/bazel_playground/blob/master/WORKSPACE
new_git_repository(
name="glog",
remote="https://github.com/google/glog.git",
commit="b6a5e0524c28178985f0d228e9eaa43808dbec3c",
build_file="third_party/glog.BUILD")
cmake/check_packages.cmake
浏览文件 @ 1de901ab
......@@ -14,13 +14,9 @@ if(WITH_STYLE_CHECK)
find_package(PythonInterp REQUIRED)
endif()
if(WITH_GLOG)
find_package(Glog REQUIRED)
endif()
find_package(Glog REQUIRED)
if(WITH_GFLAGS)
find_package(Gflags REQUIRED)
endif()
find_package(Gflags REQUIRED)
if(WITH_TESTING)
find_package(GTest REQUIRED)
......
cmake/util.cmake
浏览文件 @ 1de901ab
......@@ -65,7 +65,7 @@ endmacro()
# link_paddle_exe
# add paddle library for a paddle executable, such as trainer, pserver.
#
# It will handle WITH_PYTHON/WITH_GLOG etc.
# It will handle WITH_PYTHON etc.
function(link_paddle_exe TARGET_NAME)
if(WITH_RDMA)
generate_rdma_links()
......@@ -108,6 +108,8 @@ function(link_paddle_exe TARGET_NAME)
paddle_cuda
${METRIC_LIBS}
${PROTOBUF_LIBRARY}
${LIBGLOG_LIBRARY}
${GFLAGS_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
${CBLAS_LIBS}
${ZLIB_LIBRARIES}
......@@ -125,16 +127,6 @@ function(link_paddle_exe TARGET_NAME)
${PYTHON_LIBRARIES})
endif()
if(WITH_GLOG)
target_link_libraries(${TARGET_NAME}
${LIBGLOG_LIBRARY})
endif()
if(WITH_GFLAGS)
target_link_libraries(${TARGET_NAME}
${GFLAGS_LIBRARIES})
endif()
if(WITH_GPU)
if(NOT WITH_DSO OR WITH_METRIC)
target_link_libraries(${TARGET_NAME}
......
demo/semantic_role_labeling/data/extract_dict_feature.py
浏览文件 @ 1de901ab
......@@ -43,13 +43,13 @@ def extract_dict_features(pair_file, feature_file):
mark[verb_index] = 1
ctx_0 = sentence_list[verb_index]
if verb_index < len(labels_list) - 2:
if verb_index < len(labels_list) - 1:
mark[verb_index + 1] = 1
ctx_p1 = sentence_list[verb_index + 1]
else:
ctx_p1 = 'eos'
if verb_index < len(labels_list) - 3:
if verb_index < len(labels_list) - 2:
mark[verb_index + 2] = 1
ctx_p2 = sentence_list[verb_index + 2]
else:
......
doc/getstarted/build_and_install/build_from_source_en.md
浏览文件 @ 1de901ab
......@@ -49,10 +49,8 @@ PaddlePaddle supports some build options. To enable it, first you need to instal
<tbody>
<tr><td class="left">WITH_GPU</td><td class="left">Compile with GPU mode.</td></tr>
<tr><td class="left">WITH_DOUBLE</td><td class="left">Compile with double precision floating-point, default: single precision.</td></tr>
<tr><td class="left">WITH_GLOG</td><td class="left">Compile with glog. If not found, default: an internal log implementation.</td></tr>
<tr><td class="left">WITH_GFLAGS</td><td class="left">Compile with gflags. If not found, default: an internal flag implementation.</td></tr>
<tr><td class="left">WITH_TESTING</td><td class="left">Compile with gtest for PaddlePaddle's unit testing.</td></tr>
<tr><td class="left">WITH_DOC</td><td class="left"> Compile to generate PaddlePaddle's docs, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_DOC</td><td class="left"> Compile to generate PaddlePaddle's docs, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_SWIG_PY</td><td class="left">Compile with python predict API, default: disabled (OFF).</td></tr>
<tr><td class="left">WITH_STYLE_CHECK</td><td class="left">Compile with code style check, default: enabled (ON).</td></tr>
</tbody>
......
doc/getstarted/build_and_install/cmake/compile_options.csv
浏览文件 @ 1de901ab
......@@ -6,8 +6,6 @@ WITH_AVX,是否编译含有AVX指令集的PaddlePaddle二进制文件,是
WITH_PYTHON,是否内嵌PYTHON解释器。方便今后的嵌入式移植工作。,是
WITH_STYLE_CHECK,是否编译时进行代码风格检查,是
WITH_RDMA,是否开启RDMA,否
WITH_GLOG,是否开启GLOG。如果不开启，则会使用一个简化版的日志，同时方便今后的嵌入式移植工作。,取决于是否寻找到GLOG
WITH_GFLAGS,是否使用GFLAGS。如果不开启，则会使用一个简化版的命令行参数解析器，同时方便今后的嵌入式移植工作。,取决于是否寻找到GFLAGS
WITH_TIMER,是否开启计时功能。如果开启会导致运行略慢，打印的日志变多，但是方便调试和测Benchmark,否
WITH_TESTING,是否开启单元测试,取决于是否寻找到GTEST
WITH_DOC,是否编译中英文文档,否
......
doc/getstarted/build_and_install/ubuntu_install_cn.rst
浏览文件 @ 1de901ab
......@@ -46,8 +46,6 @@ PaddlePaddle提供了ubuntu 14.04 deb安装包。
with_double: OFF
with_python: ON
with_rdma: OFF
with_glog: ON
with_gflags: ON
with_metric_learning:
with_timer: OFF
with_predict_sdk:
......
doc/howto/dev/contribute_to_paddle_en.md
浏览文件 @ 1de901ab
# Contribute Code
We sincerely appreciate your contributions. You can use fork and pull request
workflow to merge your code.
workflow to merge your code.
## Code Requirements
- Your code must be fully documented by
[doxygen](http://www.stack.nl/~dimitri/doxygen/) style.
......@@ -12,11 +12,11 @@ workflow to merge your code.
- Pass all unit tests.
The following tutorial guides you into submitting your contibution.
## [Creating a Fork](https://help.github.com/articles/fork-a-repo/)
Just head over to the GitHub page and click the "Fork" button.
It's just that simple.
It's just that simple.
## Clone
......@@ -25,7 +25,7 @@ The **develop** is the main branch, and other user's branches are feature branch
Once you've created a fork, you can use your favorite git client to clone your
repo or just head straight to the command line:
```shell
# Clone your fork to your local machine
git clone --branch develop https://github.com/USERNAME/Paddle.git
......@@ -47,6 +47,22 @@ Then you can start to develop by making a local developement branch
git checkout -b MY_COOL_STUFF_BRANCH
```
## Using `pre-commit` hook
Paddle developers use [pre-commit](http://pre-commit.com/) tool to manage git
pre-commit hooks. It can help us format source codes (cpp, python), check some
basic thing before commit (only one EOL for each file, do not add a huge file
in git). `pre-commit` tests is a part of unit tests in Travis-CI now, every
PR doesn't fit hook can not be merged into Paddle.
To use [pre-commit](http://pre-commit.com/), you should install it by
`pip install pre-commit`, and currently, Paddle uses `clang-format` to format
c/cpp sources. Please make sure clang-format 3.8+ installed.
Then just run `pre-commit install` in your Paddle clone directory. When you
commit your code, the pre-commit hook will check the local code if there is
anything not suitable to commit, and so on.
## Commit
Commit your changes by following command lines:
......@@ -83,7 +99,7 @@ git pull --rebase upstream develop
If there are no unique commits locally, git will simply perform a fast-forward.
However, if you have been making changes (in the vast majority of cases you
probably shouldn't be), you may have to deal with conflicts.
probably shouldn't be), you may have to deal with conflicts.
Now, your local master branch is up-to-date with everything modified upstream.
......
paddle/api/CMakeLists.txt
浏览文件 @ 1de901ab
......@@ -17,22 +17,18 @@ add_library(paddle_api STATIC
${API_SOURCES})
add_dependencies(paddle_api gen_proto_cpp)
list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH)
if(WITH_GFLAGS)
list(LENGTH "${GFLAGS_LIBRARIES}" GFLAGS_LIBRARIES_LENGTH)
if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}")
# Because gflags compiled by cmake, so it is imported by cmake target,
# not a real library path. Get the real library path here.
message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}")
get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION)
message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}")
else()
set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES})
endif()
if(${GFLAGS_LIBRARIES_LENGTH} EQUAL 0 AND TARGET "${GFLAGS_LIBRARIES}")
# Because gflags compiled by cmake, so it is imported by cmake target,
# not a real library path. Get the real library path here.
message(STATUS "GFLAGS Libraries is ${GFLAGS_LIBRARIES}")
get_target_property(GFLAGS_LOCATION ${GFLAGS_LIBRARIES} LOCATION)
message(STATUS "GFLAGS Target location is ${GFLAGS_LOCATION}")
else()
set(GFLAGS_LOCATION ${GFLAGS_LIBRARIES})
endif()
configure_file(
paddle_api_config.py.in
${PROJ_ROOT}/paddle/api/paddle_api_config.py
......@@ -57,7 +53,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp
paddle_trainer
paddle_api
paddle_cuda
${PY_PADDLE_PYTHON_FILES}
${PY_PADDLE_PYTHON_FILES}
)
install(DIRECTORY ${PROJ_ROOT}/paddle/dist/
......
paddle/api/Trainer.cpp
浏览文件 @ 1de901ab
......@@ -27,9 +27,9 @@ limitations under the License. */
using paddle::real;
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
struct TrainerPrivate : public paddle::Trainer {
bool _trainOneBatch(size_t batchSize);
......
paddle/api/paddle_api_config.py.in
浏览文件 @ 1de901ab
......@@ -8,9 +8,7 @@ CMAKE_DL_LIBS="@CMAKE_DL_LIBS@"
WITH_PYTHON="@WITH_PYTHON@"
PYTHON_LIBRARIES="@PYTHON_LIBRARIES@"
WITH_GLOG="@WITH_GLOG@"
LIBGLOG_LIBRARY="@LIBGLOG_LIBRARY@"
WITH_GFLAGS="@WITH_GFLAGS@"
GFLAGS_LIBRARIES="@GFLAGS_LIBRARIES@"
GFLAGS_LOCATION="@GFLAGS_LOCATION@"
CBLAS_LIBRARIES="@CBLAS_LIBS@"
......
paddle/api/paddle_ld_flags.py
浏览文件 @ 1de901ab
......@@ -47,10 +47,8 @@ try:
self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON)
self.python_libs = PYTHON_LIBRARIES
self.with_glog = PaddleLDFlag.cmake_bool(WITH_GLOG)
self.glog_libs = LIBGLOG_LIBRARY
self.with_gflags = PaddleLDFlag.cmake_bool(WITH_GFLAGS)
self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS)
self.gflags_libs = GFLAGS_LIBRARIES
self.gflags_location = GFLAGS_LOCATION
......@@ -88,6 +86,8 @@ try:
"-lpaddle_cuda",
"-lpaddle_api",
self.normalize_flag(self.protolib),
self.normalize_flag(self.glog_libs),
self.normalize_flag(self.gflags_libs),
self.normalize_flag(self.zlib),
self.normalize_flag(self.thread),
self.normalize_flag(self.dl_libs),
......@@ -96,10 +96,6 @@ try:
if self.with_python:
libs.append(self.normalize_flag(self.python_libs))
if self.with_glog:
libs.append(self.normalize_flag(self.glog_libs))
if self.with_gflags:
libs.append(self.normalize_flag(self.gflags_libs))
if self.with_gpu:
libs.append(self.normalize_flag(self.curt))
if self.with_coverage:
......
paddle/cuda/src/hl_cuda_cudnn.cc
浏览文件 @ 1de901ab
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
namespace dynload {
......
paddle/cuda/src/hl_cuda_device.cc
浏览文件 @ 1de901ab
......@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
// clang-format off
// Because clang-format 4.X and clang-format 3.8+ format
// Because clang-format 4.X and clang-format 3.8+ format
// following lines in different. So disable clang-format.
#include "hl_cuda.h"
#include <cuda_profiler_api.h>
......@@ -22,6 +22,7 @@ limitations under the License. */
#include <sys/time.h>
#include <unistd.h>
#include <mutex>
#include "hl_cuda.h"
#include "hl_cuda.ph"
#include "hl_dso_loader.h"
#include "hl_thread.ph"
......
paddle/cuda/src/hl_dso_loader.cc
浏览文件 @ 1de901ab
......@@ -16,21 +16,21 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
P_DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
P_DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
static inline std::string join(const std::string& part1,
const std::string& part2) {
......
paddle/gserver/dataproviders/ProtoDataProvider.cpp
浏览文件 @ 1de901ab
......@@ -22,9 +22,9 @@ limitations under the License. */
#include "DataProviderGroup.h"
#include "paddle/utils/Logging.h"
P_DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
namespace paddle {
......
paddle/gserver/evaluators/Evaluator.cpp
浏览文件 @ 1de901ab
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
浏览文件 @ 1de901ab
......@@ -21,11 +21,11 @@ limitations under the License. */
#include "NeuralNetwork.h"
#include "ParallelNeuralNetwork.h"
P_DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
#ifdef PADDLE_METRIC_LEARNING
P_DECLARE_bool(external);
DECLARE_bool(external);
#endif
namespace paddle {
......
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
浏览文件 @ 1de901ab
......@@ -24,7 +24,7 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
static const char* DIY_CALC_PROB_SYMBOL_NAME = "calc_prob";
static const char* DIY_START_CALC_PROB_SYMBOL_NAME = "start_calc_prob";
......
paddle/gserver/layers/DataLayer.cpp
浏览文件 @ 1de901ab
......@@ -54,7 +54,7 @@ void DataLayer::copyDataToOutput(Argument& output) {
output.setFrameWidth(config_.width());
} else {
output.setFrameHeight(data_.getFrameHeight());
output.setFrameHeight(data_.getFrameHeight());
output.setFrameWidth(data_.getFrameWidth());
}
output.cpuSequenceDims = data_.cpuSequenceDims;
output.sequenceStartPositions = data_.sequenceStartPositions;
......
paddle/gserver/layers/Layer.cpp
浏览文件 @ 1de901ab
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "TransLayer.h"
#include "ValidationLayer.h"
P_DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
namespace paddle {
......
paddle/gserver/layers/LstmLayer.cpp
浏览文件 @ 1de901ab
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/Matrix.h"
#include "paddle/utils/Stat.h"
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(prev_batch_state);
namespace paddle {
......
paddle/gserver/layers/RecurrentLayer.cpp
浏览文件 @ 1de901ab
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
namespace paddle {
......
paddle/gserver/layers/ValidationLayer.h
浏览文件 @ 1de901ab
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "Layer.h"
#include "paddle/gserver/evaluators/Evaluator.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/tests/LayerGradUtil.cpp
浏览文件 @ 1de901ab
......@@ -14,7 +14,7 @@ limitations under the License. */
#include "LayerGradUtil.h"
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(thread_local_rand_use_global_seed);
namespace paddle {
real getCostSum(LayerPtr& testLayer, MatrixPtr weights) {
......
paddle/gserver/tests/TestUtil.cpp
浏览文件 @ 1de901ab
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/CommandLineParser.h"
P_DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
namespace paddle {
......
paddle/gserver/tests/test_ActivationGrad.cpp
浏览文件 @ 1de901ab
......@@ -25,8 +25,8 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_bool(thread_local_rand_use_global_seed);
void testActivation(const string& act) {
LOG(INFO) << "test activation: " << act;
......
paddle/gserver/tests/test_BatchNorm.cpp
浏览文件 @ 1de901ab
......@@ -27,11 +27,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the batchNormLayer can be followed by a ConvLayer
TEST(Layer, batchNorm) {
......
paddle/gserver/tests/test_ConvTrans.cpp
浏览文件 @ 1de901ab
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the convTrans forward is the same as conv backward
TEST(Layer, convTransLayerFwd) {
......
paddle/gserver/tests/test_ConvUnify.cpp
浏览文件 @ 1de901ab
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Do one forward pass of convTrans layer and check to see if its output
// matches the given result
......
paddle/gserver/tests/test_Evaluator.cpp
浏览文件 @ 1de901ab
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_bool(thread_local_rand_use_global_seed);
enum InputType {
INPUT_DATA, // dense vector
......
paddle/gserver/tests/test_LayerGrad.cpp
浏览文件 @ 1de901ab
......@@ -26,11 +26,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
TEST(Operator, dot_mul) {
TestConfig config;
......
paddle/gserver/tests/test_NetworkCompare.cpp
浏览文件 @ 1de901ab
......@@ -25,10 +25,10 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DEFINE_bool(use_label, true, "input label or sequence label");
P_DEFINE_bool(static_para, false, "static parameter");
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DEFINE_bool(use_label, true, "input label or sequence label");
DEFINE_bool(static_para, false, "static parameter");
struct DataIn {
std::vector<Argument> inArgs;
......@@ -267,8 +267,8 @@ TEST(Compare, img_conv2) {
}
#endif
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
TEST(Compare, network) {
if (FLAGS_config_file_a != "" && FLAGS_config_file_b != "") {
compareNetwork(FLAGS_config_file_a, FLAGS_config_file_b);
......
paddle/gserver/tests/test_PyDataProvider2.cpp
浏览文件 @ 1de901ab
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/PythonUtil.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(train_list, "unittest.list", "file list for unittest");
DEFINE_string(train_list, "unittest.list", "file list for unittest");
namespace paddle {
namespace unittest {
......
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
浏览文件 @ 1de901ab
......@@ -20,7 +20,7 @@ limitations under the License. */
#include <paddle/utils/Util.h>
#include <paddle/utils/Version.h>
P_DECLARE_int32(seed);
DECLARE_int32(seed);
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/gserver/tests/test_RecurrentLayer.cpp
浏览文件 @ 1de901ab
......@@ -23,9 +23,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(rnn_use_batch);
P_DECLARE_int32(fixed_seq_length);
DECLARE_bool(use_gpu);
DECLARE_bool(rnn_use_batch);
DECLARE_int32(fixed_seq_length);
void checkError(const Matrix& matrix1, const Matrix& matrix2) {
CHECK(matrix1.getHeight() == matrix2.getHeight());
......
paddle/gserver/tests/test_SelectiveFCLayer.cpp
浏览文件 @ 1de901ab
......@@ -29,11 +29,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(num_passes);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(config_args);
DECLARE_bool(use_gpu);
DECLARE_int32(num_passes);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(config_args);
size_t fcLayerWidth = 1024;
......
paddle/gserver/tests/test_WarpCTCLayer.cpp
浏览文件 @ 1de901ab
......@@ -25,7 +25,7 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
DECLARE_bool(use_gpu);
const real* getData(const Matrix& matrix) {
if (matrix.useGpu()) {
......
paddle/math/SparseRowMatrix.cpp
浏览文件 @ 1de901ab
......@@ -24,9 +24,9 @@ limitations under the License. */
#include "paddle/utils/Thread.h"
#include "paddle/utils/Util.h"
P_DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
namespace paddle {
......
paddle/math/SparseRowMatrix.h
浏览文件 @ 1de901ab
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Util.h"
P_DECLARE_bool(allow_inefficient_sparse_update);
DECLARE_bool(allow_inefficient_sparse_update);
namespace paddle {
......
paddle/math/Storage.cpp
浏览文件 @ 1de901ab
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "Allocator.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
namespace paddle {
......
paddle/math/tests/test_TrainingAlgorithm.cpp
浏览文件 @ 1de901ab
......@@ -22,9 +22,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
#ifndef PADDLE_TYPE_DOUBLE
P_DEFINE_double(max_diff, 1e-5, "max diff allowed");
DEFINE_double(max_diff, 1e-5, "max diff allowed");
#else
P_DEFINE_double(max_diff, 1e-13, "max diff allowed");
DEFINE_double(max_diff, 1e-13, "max diff allowed");
#endif
class SetMaxDiff {
......
paddle/parameter/Argument.cpp
浏览文件 @ 1de901ab
......@@ -245,6 +245,8 @@ int32_t Argument::resizeAndCopyFrom(const Argument& src,
bool useGpu,
hl_stream_t stream) {
dataId = src.dataId;
frameWidth = src.frameWidth;
frameHeight = src.frameHeight;
if (!src.sequenceStartPositions) {
// non-sequence input, copy samples directly
......
paddle/parameter/FirstOrderOptimizer.cpp
浏览文件 @ 1de901ab
......@@ -19,7 +19,7 @@ limitations under the License. */
#include <cmath>
P_DEFINE_bool(log_clipping, false, "enable log clipping or not");
DEFINE_bool(log_clipping, false, "enable log clipping or not");
namespace paddle {
......
paddle/parameter/Parameter.cpp
浏览文件 @ 1de901ab
......@@ -26,11 +26,11 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
P_DEFINE_int32(
DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
DEFINE_int32(
grad_share_block_num,
64,
"block number of gradient parameter share for batch multi-cpu training");
......
paddle/pserver/BaseClient.cpp
浏览文件 @ 1de901ab
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DECLARE_string(pservers);
DECLARE_string(pservers);
namespace paddle {
......
paddle/pserver/LightNetwork.cpp
浏览文件 @ 1de901ab
......@@ -31,23 +31,23 @@ limitations under the License. */
#include "paddle/utils/Util.h"
/// quick ack can reduce the latency of small message
P_DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
/// reasonable size can hold bursted packets and reduce packets loss
P_DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
namespace paddle {
......
paddle/pserver/ParameterClient2.cpp
浏览文件 @ 1de901ab
......@@ -20,8 +20,8 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/StringUtil.h"
P_DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
P_DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
namespace paddle {
......
paddle/pserver/ParameterClient2.h
浏览文件 @ 1de901ab
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "ProtoServer.h"
#include "SparseParameterDistribution.h"
P_DECLARE_int32(parallel_thread_num);
DECLARE_int32(parallel_thread_num);
namespace paddle {
......
paddle/pserver/ParameterServer2.cpp
浏览文件 @ 1de901ab
......@@ -30,11 +30,11 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
P_DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
P_DEFINE_double(
DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
DEFINE_double(
async_lagged_ratio_default,
1.5,
"if async_lagged_grad_discard_ratio is not set in trainer_config.conf"
......
paddle/pserver/ParameterServer2.h
浏览文件 @ 1de901ab
......@@ -38,7 +38,7 @@ limitations under the License. */
#include "ProtoServer.h"
P_DECLARE_int32(port);
DECLARE_int32(port);
namespace paddle {
......
paddle/pserver/SparseParameterDistribution.cpp
浏览文件 @ 1de901ab
......@@ -20,26 +20,26 @@ limitations under the License. */
#include "SparseParameterDistribution.h"
P_DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
P_DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
P_DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
P_DEFINE_double(
DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
DEFINE_double(
check_sparse_distribution_ratio,
0.6,
"if parameters dispatched to different pservers exhibit unbalanced "
" distribution for check_sparse_distribution_ratio * "
" check_sparse_distribution_batches times, crash program");
P_DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
namespace paddle {
......
paddle/pserver/test/SocketTest.cpp
浏览文件 @ 1de901ab
......@@ -195,9 +195,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) {
channel_.reset(new SocketChannel(sockfd));
}
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 10000000, "Data size");
P_DEFINE_int32(loop_time, 100000, "test loop time");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 10000000, "Data size");
DEFINE_int32(loop_time, 100000, "test loop time");
using namespace paddle; // NOLINT
......
paddle/pserver/test/test_ParameterServer2.cpp
浏览文件 @ 1de901ab
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(num_gradient_servers);
P_DEFINE_string(server_addr, "127.0.0.1", "assign server address");
P_DEFINE_int32(server_cpu, 0, "assign server cpu");
DECLARE_int32(num_gradient_servers);
DEFINE_string(server_addr, "127.0.0.1", "assign server address");
DEFINE_int32(server_cpu, 0, "assign server cpu");
class ParameterServer2Tester : public ParameterServer2 {
public:
......
paddle/pserver/test/test_ProtoServer.cpp
浏览文件 @ 1de901ab
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/pserver/ProtoServer.h"
#include "paddle/utils/Stat.h"
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 50000000, "Data size");
P_DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
P_DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 50000000, "Data size");
DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
using namespace paddle; // NOLINT
......
paddle/scripts/submit_local.sh.in
浏览文件 @ 1de901ab
......@@ -21,8 +21,6 @@ function version(){
echo " with_double: @WITH_DOUBLE@"
echo " with_python: @WITH_PYTHON@"
echo " with_rdma: @WITH_RDMA@"
echo " with_glog: @WITH_GLOG@"
echo " with_gflags: @WITH_GFLAGS@"
echo " with_metric_learning: @WITH_METRIC@"
echo " with_timer: @WITH_TIMER@"
echo " with_predict_sdk: @WITH_PREDICT_SDK@"
......
paddle/trainer/MergeModel.cpp
浏览文件 @ 1de901ab
......@@ -19,8 +19,8 @@ limitations under the License. */
#include "paddle/pserver/ParameterServer2.h"
#include "paddle/utils/PythonUtil.h"
P_DEFINE_string(model_dir, "", "Directory for separated model files");
P_DEFINE_string(model_file, "", "File for merged model file");
DEFINE_string(model_dir, "", "Directory for separated model files");
DEFINE_string(model_file, "", "File for merged model file");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/RemoteParameterUpdater.cpp
浏览文件 @ 1de901ab
......@@ -17,8 +17,8 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DECLARE_int32(trainer_id);
P_DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_string(save_dir);
namespace paddle {
......
paddle/trainer/ThreadParameterUpdater.cpp
浏览文件 @ 1de901ab
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/math/SparseRowMatrix.h"
#include "paddle/utils/Thread.h"
P_DECLARE_int32(trainer_count);
DECLARE_int32(trainer_count);
namespace paddle {
......
paddle/trainer/Trainer.cpp
浏览文件 @ 1de901ab
......@@ -38,60 +38,56 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
#include "paddle/gserver/layers/ValidationLayer.h"
P_DEFINE_string(config, "", "Trainer config file");
P_DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
P_DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
P_DEFINE_bool(local, true, "Train in local mode or not");
P_DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
P_DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
P_DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
P_DEFINE_string(save_dir, "", "Directory for saving model parameter");
P_DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
P_DEFINE_int32(test_pass,
-1,
"Will load parameter start from this pass to test");
P_DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
P_DEFINE_bool(with_cost, true, "enable cost layer or not");
P_DEFINE_bool(distribute_test, false, "test in distribute mode");
P_DEFINE_int32(num_passes, 100, "train for so many passes");
P_DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
P_DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
P_DEFINE_string(feat_file, "", "File name of extracted feature.");
P_DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
P_DEFINE_string(model_list,
"",
"File that saves the model list when evaluation");
DEFINE_string(config, "", "Trainer config file");
DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
DEFINE_bool(local, true, "Train in local mode or not");
DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
DEFINE_string(save_dir, "", "Directory for saving model parameter");
DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
DEFINE_int32(test_pass, -1, "Will load parameter start from this pass to test");
DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
DEFINE_bool(with_cost, true, "enable cost layer or not");
DEFINE_bool(distribute_test, false, "test in distribute mode");
DEFINE_int32(num_passes, 100, "train for so many passes");
DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
DEFINE_string(feat_file, "", "File name of extracted feature.");
DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
DEFINE_string(model_list, "", "File that saves the model list when evaluation");
namespace paddle {
......
paddle/trainer/Trainer.h
浏览文件 @ 1de901ab
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "paddle/internals/metric_learning/MetricTrainer.h"
#endif
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
namespace paddle {
......
paddle/trainer/TrainerBenchmark.cpp
浏览文件 @ 1de901ab
......@@ -18,9 +18,9 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DECLARE_int32(test_period);
DECLARE_int32(test_period);
P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
namespace paddle {
......
paddle/trainer/TrainerConfigHelper.cpp
浏览文件 @ 1de901ab
......@@ -18,16 +18,16 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/PythonUtil.h"
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
P_DECLARE_string(save_dir);
P_DECLARE_int32(trainer_id);
P_DECLARE_bool(local);
P_DECLARE_bool(with_cost);
P_DECLARE_bool(with_gpu);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_bool(local);
DECLARE_bool(with_cost);
DECLARE_bool(with_gpu);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
const char *kConfigParserModuleName = "paddle.trainer.config_parser";
const char *kConfigParserFuncName = "parse_config_and_serialize";
......
paddle/trainer/TrainerInternalConfig.cpp
浏览文件 @ 1de901ab
......@@ -14,17 +14,17 @@ limitations under the License. */
#include "TrainerInternalConfig.h"
P_DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
P_DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
P_DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
P_DECLARE_bool(local);
DECLARE_bool(local);
namespace paddle {
......
paddle/trainer/TrainerMain.cpp
浏览文件 @ 1de901ab
......@@ -22,21 +22,20 @@ limitations under the License. */
#include "Trainer.h"
#include "paddle/pserver/RDMANetwork.h"
P_DEFINE_bool(start_pserver, false, "Whether to start pserver");
P_DECLARE_int32(gpu_id);
P_DEFINE_string(job, "train", "one of (train, test, checkgrad)");
P_DECLARE_int32(start_pass);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(rdma_tcp);
DEFINE_bool(start_pserver, false, "Whether to start pserver");
DECLARE_int32(gpu_id);
DEFINE_string(job, "train", "one of (train, test, checkgrad)");
DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(rdma_tcp);
using namespace paddle; // NOLINT
int main(int argc, char** argv) {
// write logs instantly (never buffer log messages)
#ifdef PADDLE_USE_GLOG
// write logs instantly (never buffer log messages)
FLAGS_logbuflevel = -1;
#endif
initMain(argc, argv);
initPython(argc, argv);
......
paddle/trainer/tests/test_Compare.cpp
浏览文件 @ 1de901ab
......@@ -24,10 +24,10 @@ using namespace std; // NOLINT
static const string& configFile = "trainer/tests/sample_trainer_config.conf";
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(config_args);
DECLARE_int32(gpu_id);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_string(config_args);
struct comData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareSparse.cpp
浏览文件 @ 1de901ab
......@@ -25,22 +25,22 @@ using namespace std; // NOLINT
static const string& configFile1 =
"trainer/tests/sample_trainer_config_qb_rnn.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
P_DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
int gNumDevices = 0;
......
paddle/trainer/tests/test_CompareTwoNets.cpp
浏览文件 @ 1de901ab
......@@ -22,25 +22,25 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_int32(seed);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_int32(seed);
struct ComData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareTwoOpts.cpp
浏览文件 @ 1de901ab
......@@ -22,20 +22,20 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
......
paddle/trainer/tests/test_Prediction.cpp
浏览文件 @ 1de901ab
......@@ -18,11 +18,11 @@ limitations under the License. */
#include <gtest/gtest.h>
P_DECLARE_string(config);
P_DECLARE_string(config_args);
P_DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
DECLARE_string(config);
DECLARE_string(config_args);
DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/tests/test_Trainer.cpp
浏览文件 @ 1de901ab
......@@ -28,10 +28,10 @@ static const string& configFile3 = "trainer/tests/chunking.conf";
static const string& configFile4 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(allow_only_one_model_on_one_gpu);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_bool(allow_only_one_model_on_one_gpu);
void checkGradientTest(const string& configFile,
bool useGpu,
......
paddle/trainer/tests/test_TrainerOnePass.cpp
浏览文件 @ 1de901ab
......@@ -27,12 +27,12 @@ static const string& configFile1 = "trainer/tests/sample_trainer_config.conf";
static const string& configFile2 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
class TrainerForTest : public paddle::Trainer {
public:
......@@ -122,10 +122,10 @@ TEST(average_window_cpu, gpu4) {
#endif
// 3. test trainer + pserver.
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
double checkRemoteParameterUpdater(TrainerForTest& trainer) {
auto gradientMachine = trainer.getGradientMachine();
......
paddle/trainer/tests/test_recurrent_machine_generation.cpp
浏览文件 @ 1de901ab
......@@ -30,7 +30,7 @@ static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT
static string expectFile = // NOLINT
"trainer/tests/rnn_gen_test_model_dir/r1.test"; // NOLINT
P_DECLARE_string(config_args);
DECLARE_string(config_args);
vector<float> readRetFile(const string& fname) {
ifstream inFile(fname);
......
paddle/utils/BarrierStat.cpp
浏览文件 @ 1de901ab
......@@ -20,15 +20,15 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
P_DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
P_DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
namespace paddle {
......
paddle/utils/CommandLineParser.cpp
浏览文件 @ 1de901ab
......@@ -13,220 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "CommandLineParser.h"
#ifndef PADDLE_USE_GFLAGS
#include <stdlib.h>
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "paddle/utils/StringUtil.h"
namespace paddle {
static constexpr int kStatusOK = 0;
static constexpr int kStatusInvalid = 1;
static constexpr int kStatusNotFound = 2;
/**
* \brief: Convert a string to any type value.
*
* \note: It will specialize by type T that is supported.
*/
template <typename T>
bool StringToValue(const std::string& content, T* value) {
bool ok;
*value = str::toWithStatus<T>(content, &ok);
return ok;
}
template <>
bool StringToValue<bool>(const std::string& content, bool* value) {
std::string tmp = content;
std::transform(tmp.begin(), tmp.end(), tmp.begin(), [](char in) -> char {
if (in <= 'Z' && in >= 'A') {
return in - ('Z' - 'z');
} else {
return in;
}
}); // tolower.
if (tmp == "true" || tmp == "1") {
*value = true;
return true;
} else if (tmp == "false" || tmp == "0") {
*value = false;
return true;
} else {
return false;
}
}
template <>
bool StringToValue<std::string>(const std::string& content,
std::string* value) {
*value = content;
return true;
}
/**
* \brief Parse argument "--blah=blah".
*
* \param argument: The command line argument string, such as "--blah=blah"
* \param [out] extraInfo: The details error message for parse argument.
* \return: kStatusOK, kStatusInvalid, kStatusNotFound
*/
template <typename T>
int ParseArgument(const std::string& argument, std::string* extraInfo) {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
T* value = command.value;
std::string prefix = "--";
prefix += name;
prefix += "=";
std::string content;
if (str::startsWith(argument, prefix)) {
content = argument.substr(prefix.size(), argument.size() - prefix.size());
} else {
prefix = "-";
prefix += name;
prefix += "=";
if (str::startsWith(argument, prefix)) {
content =
argument.substr(prefix.size(), argument.size() - prefix.size());
}
}
if (!content.empty()) {
if (StringToValue(content, value)) {
return kStatusOK;
} else {
*extraInfo = name;
return kStatusInvalid;
}
}
}
return kStatusNotFound;
}
/**
* @brief ParseBoolArgumentExtra
* parse '--flag_name', '-flag_name' as true; '--noflag_name', '-noflag_name' as
* false
*/
static int ParseBoolArgumentExtra(const std::string& argument,
std::string* extraInfo) {
(void)(extraInfo); // unused extraInfo, just make api same.
//! @warning: The order and content of prefixes is DESIGNED for parsing
//! command line. The length of prefixes are 1, 2, 3, 4. The parse logic takes
//! use of this fact. DO NOT CHANGE IT without reading how to parse command
//! below.
static const std::vector<std::pair<const char*, bool>> prefixes = {
{"-", true}, {"--", true}, {"-no", false}, {"--no", false}};
for (flags_internal::CommandLineFlagRegistry<bool>::Command& command :
flags_internal::CommandLineFlagRegistry<bool>::Instance()->commands) {
if (argument.size() > command.name.size()) {
//! Use the length of prefix is 1, 2, 3, 4.
size_t diff = argument.size() - command.name.size() - 1UL;
if (diff < prefixes.size()) {
const std::string& prefix = std::get<0>(prefixes[diff]);
if (argument == prefix + command.name) {
*command.value = std::get<1>(prefixes[diff]);
return kStatusOK;
}
}
}
}
return kStatusNotFound;
}
/**
* \brief: Print command line arguments' usage with type T.
*/
template <typename T>
static void PrintTypeUsage() {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
name = "--" + name; // Program will exit, so modify name is safe.
std::string& desc = command.text;
T& defaultValue = command.defaultValue;
std::cerr << std::setw(20) << name << ": " << desc
<< "[default:" << defaultValue << "]." << std::endl;
}
}
template <typename... TS>
static void PrintTypeUsages() {
int unused[] = {0, (PrintTypeUsage<TS>(), 0)...};
(void)(unused);
}
/**
* \brief: Print all usage, and exit(1)
*/
static void PrintUsageAndExit(const char* argv0) {
std::cerr << "Program " << argv0 << " Flags: " << std::endl;
PrintTypeUsages<bool, int32_t, std::string, double, int64_t, uint64_t>();
exit(1);
}
/**
* \brief: Print the error flags, usage, and exit.
*/
static void PrintParseError(const std::string& name,
const char* actualInput,
const char* arg0) {
std::cerr << "Parse command flag " << name << " error! User input is "
<< actualInput << std::endl;
PrintUsageAndExit(arg0);
}
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
int unused_argc = 1;
std::string extra;
for (int i = 1; i < *argc; ++i) {
std::string arg = argv[i];
int s = kStatusInvalid;
#define ParseArgumentWithType(type) \
s = ParseArgument<type>(arg, &extra); \
if (s == kStatusOK) { \
continue; \
} else if (s == kStatusInvalid) { \
PrintParseError(extra, argv[i], argv[0]); \
}
ParseArgumentWithType(bool); // NOLINT
ParseArgumentWithType(int32_t);
ParseArgumentWithType(double); // NOLINT
ParseArgumentWithType(int64_t);
ParseArgumentWithType(uint64_t);
ParseArgumentWithType(std::string);
#undef ParseArgumentWithType
s = ParseBoolArgumentExtra(arg, &extra);
if (s == kStatusOK) {
continue;
}
if (withHelp && (arg == "--help" || arg == "-h")) {
PrintUsageAndExit(argv[0]);
}
// NOT Found for all flags.
std::swap(argv[unused_argc++], argv[i]);
}
*argc = unused_argc;
}
} // namespace paddle
#else
namespace paddle {
#ifndef GFLAGS_NS
#define GFLAGS_NS google
......@@ -243,4 +30,3 @@ void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
}
} // namespace paddle
#endif
paddle/utils/CommandLineParser.h
浏览文件 @ 1de901ab
......@@ -13,167 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#ifndef PADDLE_USE_GFLAGS
#include <stdint.h>
#include <string>
#include <vector>
#include "DisableCopy.h"
namespace paddle {
namespace flags_internal {
/**
* Command line flag registry for special type T. It will store all command
* arguments settings. such as name, default value.
*/
template <typename T>
struct CommandLineFlagRegistry {
/**
* The factory method of CommandLineFlagRegistry
*
* \return: The singleton instance of CommandLineFlagRegistry.
*/
static CommandLineFlagRegistry* Instance() {
static CommandLineFlagRegistry instance_;
return &instance_;
}
struct Command {
/// name of argument.
std::string name;
/// address of actual variable. such as FLAGS_xxx.
T* value;
/// usage text.
std::string text;
/// default value of this command.
T defaultValue;
};
/// the command line arguments of type T.
std::vector<Command> commands;
DISABLE_COPY(CommandLineFlagRegistry);
private:
inline CommandLineFlagRegistry() {}
};
/**
*Helper class to register command line flag.
*/
template <typename T>
struct CommandLineFlagRegister {
/**
* \brief: Register a command line argument
*
* \param [in] name: The command line name.
* \param [inout] val: The command line argument instance, FLAGS_xxx.
* \param [in] desc: The command line helper message.
*/
CommandLineFlagRegister(const std::string& name,
T* val,
const std::string desc) {
CommandLineFlagRegistry<T>::Instance()->commands.push_back(
{name, val, desc, *val});
}
};
/**
* \brief: Define a command line arguments.
*
* \param type: The variable type, such as int, double, etc.
* \param name: The variable name. The command line argument is '--name', the
*variable
*is 'FLAGS_name'
* \param default_value: The default value of command line argument.
* \param text: The description in command line argument.
*/
#define PADDLE_DEFINE_variable(type, name, default_value, text) \
type FLAGS_##name = default_value; \
namespace paddle_flags_internal { \
paddle::flags_internal::CommandLineFlagRegister<type> \
flags_internal_var_##name(#name, &FLAGS_##name, text); \
} // namespace paddle_flags_internal
/**
* Declare a variable to use.
*/
#define PADDLE_DECLARE_variable(type, name) extern type FLAGS_##name;
// DEFINE macro for each types.
#define P_DEFINE_int32(name, default_value, text) \
PADDLE_DEFINE_variable(int32_t, name, default_value, text)
#define P_DEFINE_bool(name, default_value, text) \
PADDLE_DEFINE_variable(bool, name, default_value, text)
#define P_DEFINE_string(name, default_value, text) \
PADDLE_DEFINE_variable(std::string, name, default_value, text)
#define P_DEFINE_double(name, default_value, text) \
PADDLE_DEFINE_variable(double, name, default_value, text)
#define P_DEFINE_int64(name, default_value, text) \
PADDLE_DEFINE_variable(int64_t, name, default_value, text)
#define P_DEFINE_uint64(name, default_value, text) \
PADDLE_DEFINE_variable(uint64_t, name, default_value, text)
// Declare macro for each types.
#define P_DECLARE_int32(name) PADDLE_DECLARE_variable(int32_t, name)
#define P_DECLARE_bool(name) PADDLE_DECLARE_variable(bool, name)
#define P_DECLARE_string(name) PADDLE_DECLARE_variable(std::string, name)
#define P_DECLARE_double(name) PADDLE_DECLARE_variable(double, name)
#define P_DECLARE_int64(name) PADDLE_DECLARE_variable(int64_t, name)
#define P_DECLARE_uint64(name) PADDLE_DECLARE_variable(uint64_t, name)
} // namespace flags_internal
/**
* \brief Parse command line flags. If parse error, just failed and exit 1.
*
* \param [inout] argc: The command argument count. This method will modify
*argc, and left unused arguments.
* \param [inout] argv: The command argument values. This method will modify
*argv, and left unused arguments.
* \param [in] withHelp: True will parse '-h' and '--help' to print usage.
*
* \note: The Command line flags format basically as follow:
*
* * If the type of flag is not bool, then the follow format of command line
* will be parsed:
* * --flag_name=value
* * -flag_name=value
*
* * If the flag is bool, then:
* * --flag_name=value, -flag_name=value will be parsed.
* * if value.tolower() == "true"| "1" will be treated as true.
* * else if value.tolower() == "false" | "0" will be treated as false.
* * --flag_name will be parsed as true.
* * --noflag_name will be parsed as false.
*/
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#else // if use gflags.
#include <gflags/gflags.h>
#define P_DEFINE_int32 DEFINE_int32
#define P_DEFINE_bool DEFINE_bool
#define P_DEFINE_string DEFINE_string
#define P_DEFINE_double DEFINE_double
#define P_DEFINE_int64 DEFINE_int64
#define P_DEFINE_uint64 DEFINE_uint64
#define P_DECLARE_int32 DECLARE_int32
#define P_DECLARE_bool DECLARE_bool
#define P_DECLARE_string DECLARE_string
#define P_DECLARE_double DECLARE_double
#define P_DECLARE_int64 DECLARE_int64
#define P_DECLARE_uint64 DECLARE_uint64
namespace paddle {
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#endif
paddle/utils/CustomStackTrace.cpp
浏览文件 @ 1de901ab
......@@ -16,7 +16,7 @@ limitations under the License. */
#include <iostream>
#include "CommandLineParser.h"
P_DEFINE_bool(
DEFINE_bool(
layer_stack_error_only_current_thread,
true,
"Dump current thread or whole process layer stack when signal error "
......
paddle/utils/Flags.cpp
浏览文件 @ 1de901ab
......@@ -15,65 +15,61 @@ limitations under the License. */
#include "Flags.h"
#ifdef PADDLE_ONLY_CPU
P_DEFINE_bool(use_gpu, false, "Only support CPU training");
DEFINE_bool(use_gpu, false, "Only support CPU training");
#else
P_DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
#endif
P_DEFINE_bool(
parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
P_DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
P_DEFINE_int32(gpu_id, 0, "Which gpu core to use");
P_DEFINE_int32(port, 20134, "Listening port for pserver");
P_DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
P_DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
P_DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
P_DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
P_DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
P_DEFINE_int32(
trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
P_DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
P_DEFINE_string(comment, "", "A string for commenting this training task");
P_DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
P_DEFINE_int32(log_period, 100, "Log progress every so many batches");
P_DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
P_DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
P_DEFINE_int32(enable_parallel_vector,
0,
"threshold for enable parallel vector");
P_DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
P_DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
DEFINE_bool(parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
DEFINE_int32(gpu_id, 0, "Which gpu core to use");
DEFINE_int32(port, 20134, "Listening port for pserver");
DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
DEFINE_int32(trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
DEFINE_string(comment, "", "A string for commenting this training task");
DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
DEFINE_int32(log_period, 100, "Log progress every so many batches");
DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
DEFINE_int32(enable_parallel_vector, 0, "threshold for enable parallel vector");
DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
P_DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
P_DEFINE_string(predict_file, "", "File name for saving predict result");
P_DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
P_DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
DEFINE_string(predict_file, "", "File name for saving predict result");
DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
paddle/utils/Flags.h
浏览文件 @ 1de901ab
......@@ -16,28 +16,28 @@ limitations under the License. */
#include "CommandLineParser.h"
P_DECLARE_bool(parallel_nn);
P_DECLARE_int32(async_count);
P_DECLARE_int32(port);
P_DECLARE_int32(data_server_port);
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(trainer_count);
P_DECLARE_int32(ports_num);
P_DECLARE_int32(ports_num_for_sparse);
P_DECLARE_string(nics);
P_DECLARE_string(rdma_tcp);
P_DECLARE_int32(trainer_id);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_string(comment);
P_DECLARE_string(load_missing_parameter_strategy);
P_DECLARE_int32(log_period);
P_DECLARE_int32(log_period_server);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_int32(enable_parallel_vector);
P_DECLARE_bool(loadsave_parameters_in_pserver);
P_DECLARE_int32(beam_size);
P_DECLARE_bool(show_layer_stat);
P_DECLARE_string(predict_file);
P_DECLARE_bool(prev_batch_state);
P_DECLARE_string(init_model_path);
DECLARE_bool(parallel_nn);
DECLARE_int32(async_count);
DECLARE_int32(port);
DECLARE_int32(data_server_port);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_int32(trainer_count);
DECLARE_int32(ports_num);
DECLARE_int32(ports_num_for_sparse);
DECLARE_string(nics);
DECLARE_string(rdma_tcp);
DECLARE_int32(trainer_id);
DECLARE_int32(num_gradient_servers);
DECLARE_string(comment);
DECLARE_string(load_missing_parameter_strategy);
DECLARE_int32(log_period);
DECLARE_int32(log_period_server);
DECLARE_double(checkgrad_eps);
DECLARE_int32(enable_parallel_vector);
DECLARE_bool(loadsave_parameters_in_pserver);
DECLARE_int32(beam_size);
DECLARE_bool(show_layer_stat);
DECLARE_string(predict_file);
DECLARE_bool(prev_batch_state);
DECLARE_string(init_model_path);
paddle/utils/Logging.cpp
浏览文件 @ 1de901ab
......@@ -18,175 +18,9 @@ limitations under the License. */
*/
#include "Logging.h"
#ifndef PADDLE_USE_GLOG
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mutex>
#include <thread>
#include <vector>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
namespace paddle {
namespace internal {
std::string join(const std::string& part1, const std::string& part2) {
const char sep = '/';
if (!part2.empty() && part2.front() == sep) {
return part2;
}
std::string ret;
ret.reserve(part1.size() + part2.size() + 1);
ret = part1;
if (!ret.empty() && ret.back() != sep) {
ret += sep;
}
ret += part2;
return ret;
}
static inline bool env2bool(const char* envName, bool defaultValue = false) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
return memchr("tTyY1\0", envValue[0], 6) != nullptr;
}
}
static inline int env2int(const char* envName, int defaultValue = 0) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
int retValue = defaultValue;
try {
retValue = std::stoi(envValue);
} catch (...) {
// pass
}
return retValue;
}
}
static inline int env2index(const char* envName,
const std::vector<std::string>& options,
int defaultValue) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
for (size_t i = 0; i < options.size(); ++i) {
if (options[i] == envValue) {
return static_cast<int>(i);
}
}
return defaultValue;
}
}
static bool gLogToStderr = env2bool("PLOG_LOGTOSTDERR", true);
static const std::vector<std::string> gLevelName = {
"INFO", "WARNING", "ERROR", "FATAL"};
static int gMinLogLevel =
env2int("PLOG_MINLOGLEVEL", env2index("PLOG_MINLOGLEVEL", gLevelName, 0));
static std::vector<std::vector<int>> gLogFds;
static std::vector<int> gLogFileFds;
static bool gLogInited = false;
static void freeLogFileFds() {
for (auto fd : gLogFileFds) {
close(fd);
}
}
static void initializeLogFds(char* argv0) {
gLogFds.resize(NUM_SEVERITIES);
for (int i = gMinLogLevel; i < NUM_SEVERITIES && gLogToStderr;
++i) { // Add stderr
std::vector<int>& fds = gLogFds[i];
fds.push_back(STDERR_FILENO);
}
char* logDir = getenv("PLOG_LOGDIR");
for (int i = gMinLogLevel; i < NUM_SEVERITIES && logDir != nullptr; ++i) {
std::string filename =
join(logDir, std::string(argv0) + "." + gLevelName[i]);
int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644);
if (fd == -1) {
fprintf(stderr, "Open log file error!");
exit(1);
}
gLogFileFds.push_back(fd);
std::vector<int>& curFds = gLogFds[i];
curFds.insert(curFds.end(), gLogFileFds.begin(), gLogFileFds.end());
}
atexit(freeLogFileFds);
gLogInited = true;
}
static void (*gFailureFunctionPtr)() ATTR_NORETURN = abort;
LogMessage::LogMessage(const char* fname, int line, int severity)
: fname_(fname), line_(line), severity_(severity) {}
LogMessage::~LogMessage() { this->generateLogMessage(); }
void LogMessage::generateLogMessage() {
if (!gLogInited) {
fprintf(stderr,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
} else {
for (auto& fd : gLogFds[this->severity_]) {
dprintf(fd,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
}
}
}
LogMessageFatal::LogMessageFatal(const char* file, int line)
: LogMessage(file, line, FATAL) {}
LogMessageFatal::~LogMessageFatal() {
generateLogMessage();
gFailureFunctionPtr();
}
} // namespace internal
void initializeLogging(int argc, char** argv) {
internal::initializeLogFds(argv[0]);
}
namespace logging {
void setMinLogLevel(int level) { paddle::internal::gMinLogLevel = level; }
void installFailureFunction(void (*callback)() ATTR_NORETURN) {
paddle::internal::gFailureFunctionPtr = callback;
}
} // namespace logging
} // namespace paddle
#else
namespace paddle {
void initializeLogging(int argc, char** argv) {
(void)(argc);
if (!getenv("GLOG_logtostderr")) {
......@@ -197,13 +31,16 @@ void initializeLogging(int argc, char** argv) {
}
namespace logging {
void setMinLogLevel(int level) { FLAGS_minloglevel = level; }
void installFailureFunction(void (*callback)()) {
google::InstallFailureFunction(callback);
}
void installFailureWriter(void (*callback)(const char*, int)) {
google::InstallFailureWriter(callback);
}
} // namespace logging
} // namespace paddle
#endif
paddle/utils/Logging.h
浏览文件 @ 1de901ab
......@@ -22,175 +22,21 @@ limitations under the License. */
#include <sstream>
#include <string>
#ifndef PADDLE_USE_GLOG
#include "CompilerMacros.h"
//! TODO(yuyang18): Move this utility macro into some global header.
#define PP_CAT(a, b) PP_CAT_I(a, b)
#define PP_CAT_I(a, b) PP_CAT_II(~, a##b)
#define PP_CAT_II(p, res) res
/**
* Generate Unique Variable Name, Usefully in macro.
* @SEE
* http://stackoverflow.com/questions/1082192/how-to-generate-random-variable-names-in-c-using-macros
*/
#define UNIQUE_NAME(base) PP_CAT(base, __LINE__)
#include <glog/logging.h>
namespace paddle {
//! Log levels.
const int INFO = 0;
const int WARNING = 1;
const int ERROR = 2;
const int FATAL = 3;
const int NUM_SEVERITIES = 4;
namespace internal {
class LogMessage : public std::basic_ostringstream<char> {
public:
LogMessage(const char* fname, int line, int severity);
~LogMessage();
protected:
/**
* @brief Print log message to stderr, files, etc.
*/
void generateLogMessage();
private:
const char* fname_;
int line_;
int severity_;
};
// LogMessageFatal ensures the process will exit in failure after
// logging this message.
class LogMessageFatal : public LogMessage {
public:
LogMessageFatal(const char* file, int line) __attribute__((cold));
~LogMessageFatal() __attribute__((noreturn));
};
#define _P_LOG_INFO \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::INFO)
#define _P_LOG_WARNING \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::WARNING)
#define _P_LOG_ERROR \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::ERROR)
#define _P_LOG_FATAL ::paddle::internal::LogMessageFatal(__FILE__, __LINE__)
#define P_LOG(severity) _P_LOG_##severity
#define P_LOG_FIRST_N(severity, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) ++UNIQUE_NAME(LOG_OCCURRENCES); \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) P_LOG(severity)
#define P_LOG_IF_EVERY_N(severity, condition, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (condition && ((UNIQUE_NAME(LOG_OCCURRENCES) = \
(UNIQUE_NAME(LOG_OCCURRENCES) + 1) % n) == (1 % n))) \
P_LOG(severity)
#define P_LOG_EVERY_N(severity, n) P_LOG_IF_EVERY_N(severity, true, n)
// TODO(jeff): Define a proper implementation of VLOG_IS_ON
#define P_VLOG_IS_ON(lvl) ((lvl) <= 0)
#define P_LOG_IF(severity, condition) \
if (condition) P_LOG(severity)
#define P_VLOG(lvl) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl))
#define P_VLOG_IF(lvl, cond) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl) && cond)
#define P_VLOG_EVERY_N(lvl, n) P_LOG_IF_EVERY_N(INFO, P_VLOG_IS_ON(lvl), n)
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
// CHECK dies with a fatal error if condition is not true. It is *not*
// controlled by NDEBUG, so the check will be executed regardless of
// compilation mode. Therefore, it is safe to do things like:
// CHECK(fp->Write(x) == 4)
#define P_CHECK(condition) \
if (PREDICT_FALSE(!(condition))) \
P_LOG(FATAL) << "Check failed: " #condition " "
#define P_CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define P_CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define P_CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define P_CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define P_CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define P_CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define P_CHECK_NOTNULL(val) P_CHECK((val) != NULL)
//! GLOG compatible APIs
//! NOTE: only implement Paddle actually used APIs.
#define LOG(x) P_LOG(x)
#define VLOG(x) P_VLOG(x)
#define DLOG(x) P_VLOG(5)
#define CHECK(x) P_CHECK(x)
#define PCHECK(x) P_CHECK(x)
#define CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define CHECK_NOTNULL(val) P_CHECK((val) != NULL)
#define VLOG_IS_ON(x) P_VLOG_IS_ON(x)
#define LOG_FIRST_N(severity, n) P_LOG_FIRST_N(severity, n)
#define LOG_IF(severity, condition) P_LOG_IF(severity, condition)
#define VLOG_EVERY_N(lvl, n) P_VLOG_EVERY_N(lvl, n)
#define VLOG_IF(lvl, cond) P_VLOG_IF(lvl, cond)
#define LOG_EVERY_N(severity, n) P_LOG_EVERY_N(severity, n)
} // namespace internal
/**
* @brief initialize logging
* @note: Current implement of logging is lack of:
* PrintCallStack when fatal.
* VLOG_IS_ON
* But it is portable to multi-platform, and simple enough to modify.
*/
void initializeLogging(int argc, char** argv);
namespace logging {
/**
* @brief Set Min Log Level. if Log.level < minLogLevel, then will not print log
* to stream
* @param level. Any integer is OK, but only 0 <= x <= NUM_SEVERITIES is useful.
*/
void setMinLogLevel(int level);
/**
* @brief Install Log(Fatal) failure function. Default is abort();
* @param callback: The failure function.
*/
void installFailureFunction(void (*callback)() ATTR_NORETURN);
/**
* @brief installFailureWriter
* @note: not implemented currently.
*/
inline void installFailureWriter(void (*callback)(const char*, int)) {
(void)(callback); // unused callback.
}
} // namespace logging
} // namespace paddle
#else
#include <glog/logging.h>
namespace paddle {
void initializeLogging(int argc, char** argv);
namespace logging {
void setMinLogLevel(int level);
void installFailureFunction(void (*callback)());
void installFailureWriter(void (*callback)(const char*, int));
} // namespace logging
}
#endif // PADDLE_USE_GLOG
} // namespace logging
} // namespace paddle
#ifndef NDEBUG
#define DEBUG_LEVEL 5
......
paddle/utils/PythonUtil.cpp
浏览文件 @ 1de901ab
......@@ -20,8 +20,8 @@ namespace paddle {
#ifdef PADDLE_NO_PYTHON
P_DEFINE_string(python_path, "", "python path");
P_DEFINE_string(python_bin, "python2.7", "python bin");
DEFINE_string(python_path, "", "python path");
DEFINE_string(python_bin, "python2.7", "python bin");
constexpr int kExecuteCMDBufLength = 204800;
......
paddle/utils/ThreadLocal.cpp
浏览文件 @ 1de901ab
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "CommandLineParser.h"
#include "Util.h"
P_DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
namespace paddle {
......
paddle/utils/Util.cpp
浏览文件 @ 1de901ab
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "ThreadLocal.h"
#include "Version.h"
P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#ifdef WITH_GOOGLE_PERFTOOLS
/*
......@@ -52,10 +52,8 @@ P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#include <gperftools/profiler.h>
P_DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
P_DEFINE_string(profile_data_file,
"gperf.prof",
"file for storing profile data");
DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
DEFINE_string(profile_data_file, "gperf.prof", "file for storing profile data");
static void profilerSwitch(int signalNumber) {
bool static started = false;
......
paddle/utils/Version.cpp
浏览文件 @ 1de901ab
......@@ -18,13 +18,8 @@ limitations under the License. */
#include <numeric>
#include "Flags.h"
#include "Util.h"
//! TODO(yuyang18) in gflags, version has another define. Use another flag
//! instead.
#ifndef PADDLE_USE_GFLAGS
P_DEFINE_bool(version, false, "print version");
#else
P_DECLARE_bool(version);
#endif
DECLARE_bool(version);
namespace paddle {
namespace version {
......
paddle/utils/tests/CMakeLists.txt
浏览文件 @ 1de901ab
add_simple_unittest(test_CommandLineParser)
add_simple_unittest(test_Logging)
add_simple_unittest(test_Thread)
add_simple_unittest(test_StringUtils)
add_simple_unittest(test_CustomStackTrace)
......
paddle/utils/tests/test_CommandLineParser.cpp 已删除 100644 → 0
浏览文件 @ f1e3ade0
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#ifndef PADDLE_USE_GFLAGS
//! Test Command Line Parser for paddle internal implement.
#include <gtest/gtest.h>
#include <paddle/utils/CommandLineParser.h>
P_DEFINE_int32(i1, 1, "test int flag 1");
P_DEFINE_int32(i2, 2, "test int flag 2");
P_DEFINE_string(str1, "1", "test str flag 1");
P_DEFINE_string(str2, "2", "test str flag 2");
P_DEFINE_bool(b1, true, "test bool flag 1");
P_DEFINE_bool(b2, false, "test bool flag 2");
P_DEFINE_double(d1, 0.1, "test double flag 1");
P_DEFINE_double(d2, -42.3, "test double flag 2");
P_DEFINE_int64(l1, 1, "test int64 flag 1");
P_DEFINE_int64(l2, 2, "test int64 flag 2");
P_DEFINE_uint64(ul1, 32, "test uint64 flag 1");
P_DEFINE_uint64(ul2, 33, "test uint64 flag 2");
constexpr double EPSILON = 1e-5;
#define cc(x) const_cast<char*>((x))
TEST(CommandLineParser, defaultValue) {
char* argv[] = {cc("test_program"), cc("--unused_flag=134")};
int argc = sizeof(argv) / sizeof(char*);
paddle::ParseCommandLineFlags(&argc, argv);
// Check Default Value
ASSERT_EQ(argc, 2);
ASSERT_EQ(FLAGS_i1, 1);
ASSERT_EQ(FLAGS_i2, 2);
ASSERT_EQ(FLAGS_str1, "1");
ASSERT_EQ(FLAGS_str2, "2");
ASSERT_EQ(FLAGS_b1, true);
ASSERT_EQ(FLAGS_b2, false);
ASSERT_NEAR(FLAGS_d1, 0.1, EPSILON);
ASSERT_NEAR(FLAGS_d2, -42.3, EPSILON);
ASSERT_EQ(FLAGS_i1, 1);
ASSERT_EQ(FLAGS_i2, 2);
ASSERT_EQ(FLAGS_ul1, 32UL);
ASSERT_EQ(FLAGS_ul2, 33UL);
}
TEST(CommandLineParser, normal) {
char* argv[] = {cc("test_program"),
cc("--i2=32"),
cc("--str1=abc"),
cc("--b2=1"),
cc("-b1=False"),
cc("--d2=.34"),
cc("--d1=0"),
cc("--l1=-12345678901234"),
cc("-ul2=3212")};
int argc = sizeof(argv) / sizeof(char*);
paddle::ParseCommandLineFlags(&argc, argv);
ASSERT_EQ(argc, 1);
ASSERT_EQ(FLAGS_i2, 32);
ASSERT_EQ(FLAGS_str1, "abc");
ASSERT_EQ(FLAGS_b2, true);
ASSERT_EQ(FLAGS_b1, false);
ASSERT_NEAR(FLAGS_d2, 0.34, EPSILON);
ASSERT_NEAR(FLAGS_d1, 0.0, EPSILON);
ASSERT_EQ(FLAGS_l1, -12345678901234);
ASSERT_EQ(FLAGS_ul2, 3212UL);
}
TEST(CommandLineParser, printHelp) {
char* argv[] = {cc("test_program"), cc("--help")};
int argc = sizeof(argv) / sizeof(char*);
// Will Print Usage
ASSERT_DEATH(paddle::ParseCommandLineFlags(&argc, argv), ".*test_program.*");
}
TEST(CommandLineParser, parseError) {
char* argv[] = {cc("test_program"), cc("--i1=abc")};
int argc = sizeof(argv) / sizeof(char*);
ASSERT_DEATH(
paddle::ParseCommandLineFlags(&argc, argv),
"Parse command flag i1 error! User input is --i1=abc.*test_program.*");
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
int main(int argc, char** argv) { return 0; }
#endif
paddle/utils/tests/test_CustomStackTrace.cpp
浏览文件 @ 1de901ab
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Locks.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 10, "testing thread number");
DEFINE_int32(test_thread_num, 10, "testing thread number");
void testNormalImpl(
const std::function<void(paddle::CustomStackTrace<std::string>&,
......
paddle/utils/tests/test_Logging.cpp 已删除 100644 → 0
浏览文件 @ f1e3ade0
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
/*
* Basically from tensorflow/core/platform/default/logging.cc
* Used in embedded system where there is no glogs.
*/
#include <dirent.h>
#include <gtest/gtest.h>
#include <stdlib.h>
#include <fstream>
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
#ifndef PADDLE_USE_GLOG
TEST(Logging, BasicalLog) {
auto pinfo = [] {
P_LOG(INFO) << "INFO";
exit(1);
};
ASSERT_DEATH(pinfo(), "I .*test_Logging.cpp:[0-9]+] INFO");
auto pwarn = [] {
P_LOG(WARNING) << "WARN";
exit(1);
};
ASSERT_DEATH(pwarn(), "W .*test_Logging.cpp:[0-9]+] WARN");
auto perr = [] {
P_LOG(ERROR) << "ERROR";
exit(1);
};
ASSERT_DEATH(perr(), "E .*test_Logging.cpp:[0-9]+] ERROR");
auto pfatal = [] { P_LOG(FATAL) << "FATAL"; };
ASSERT_DEATH(pfatal(), "F .*test_Logging.cpp:[0-9]+] FATAL");
}
TEST(Logging, Check) {
int a = 1;
int b = 2;
P_CHECK(a != b);
auto pcheckDown = [&] { P_CHECK(a == b); };
ASSERT_DEATH(pcheckDown(),
"F .*test_Logging.cpp:[0-9]+] Check failed: a == b ");
P_CHECK_LE(a, b);
P_CHECK_LT(a, b);
double t = 1.2;
P_CHECK_LE(a, t);
double* ptr = nullptr;
auto pcheckDown2 = [&] { P_CHECK_NOTNULL(ptr); };
ASSERT_DEATH(pcheckDown2(), "F");
}
#define cc(x) const_cast<char*>(x)
TEST(Logging, LogToStderr) {
auto logToStderrCallback = [] {
setenv("PLOG_LOGTOSTDERR", "0", true);
char* argv[] = {cc("test")};
paddle::initializeLogging(1, argv);
P_LOG(INFO) << "This output will not print to std error";
exit(1);
};
ASSERT_DEATH(logToStderrCallback(), "");
}
constexpr char kLogDirName[] = "./test_log_dir";
const std::vector<std::string> kLevels = {"INFO", "WARNING", "ERROR", "FATAL"};
TEST(Logging, LogToDir) {
ASSERT_EQ(0, mkdir(kLogDirName, 0777));
auto logToDirCallback = [] {
setenv("PLOG_LOGTOSTDERR", "0", true);
setenv("PLOG_LOGDIR", kLogDirName, true);
char* argv[] = {cc("test")};
paddle::initializeLogging(1, argv);
P_LOG(INFO) << "INFO";
P_LOG(WARNING) << "WARNING";
P_LOG(ERROR) << "ERROR";
P_LOG(FATAL) << "FATAL";
};
ASSERT_DEATH(logToDirCallback(), "");
// There 4 file in logdir
auto dir = opendir(kLogDirName);
size_t fileCount = 0;
std::vector<std::string> filenames;
for (auto dirContent = readdir(dir); dirContent != nullptr;
dirContent = readdir(dir)) {
std::string filename(dirContent->d_name);
if (filename == "." || filename == "..") {
continue;
} else {
++fileCount;
for (size_t i = 0; i < kLevels.size(); ++i) {
const std::string& curLevel = kLevels[i];
if (filename.size() > curLevel.length()) {
size_t diff = filename.size() - curLevel.length();
size_t j = 0;
for (; j < curLevel.length(); ++j) {
if (filename[j + diff] != curLevel[j]) {
// File Suffix Not Same, then break.
break;
}
}
if (j == curLevel.length()) { // Same suffix.
std::ifstream fin;
auto fn = paddle::path::join(kLogDirName, filename);
fin.open(fn);
filenames.push_back(fn);
ASSERT_TRUE(fin.is_open());
size_t lineCounter = 0;
for (std::string line; std::getline(fin, line); ++lineCounter) {
// Do Nothing, Just calc lineCounter.
}
// For example.
// The info channel will have all log which level >= INFO
// So the info file's lineCounter should == 4.
ASSERT_EQ(kLevels.size() - i, lineCounter);
fin.close();
}
}
}
}
}
closedir(dir);
ASSERT_EQ(4UL, fileCount); // 4 levels.
// Clean Unittest.
for (std::string& fn : filenames) {
ASSERT_EQ(remove(fn.c_str()), 0);
}
ASSERT_EQ(rmdir(kLogDirName), 0);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
int main(int, char**) { return 0; }
#endif
paddle/utils/tests/test_SpinLock.cpp
浏览文件 @ 1de901ab
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 100, "testing thread number");
DEFINE_int32(test_thread_num, 100, "testing thread number");
void testNormalImpl(
size_t thread_num,
......
paddle/utils/tests/test_ThreadBarrier.cpp
浏览文件 @ 1de901ab
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 100, "testing thread number");
DEFINE_int32(test_thread_num, 100, "testing thread number");
void testNormalImpl(
size_t thread_num,
......
proto/ModelConfig.proto
浏览文件 @ 1de901ab
......@@ -245,7 +245,7 @@ message ImageConfig {
// The size of input feature map.
required uint32 img_size = 8;
required uint32 img_size_y = 9;
optional uint32 img_size_y = 9;
}
message LayerInputConfig {
......
third_party/gflags.BUILD 0 → 100644
浏览文件 @ 1de901ab
# Bazel (http://bazel.io/) BUILD file for gflags.
#
# See INSTALL.md for instructions for adding gflags to a Bazel workspace.
licenses(["notice"])
exports_files(["src/gflags_complections.sh", "COPYING.txt"])
load(":bazel/gflags.bzl", "gflags_sources", "gflags_library")
(hdrs, srcs) = gflags_sources(namespace=["google", "gflags"])
gflags_library(hdrs=hdrs, srcs=srcs, threads=0)
gflags_library(hdrs=hdrs, srcs=srcs, threads=1)
third_party/gflags_test/BUILD 0 → 100644
浏览文件 @ 1de901ab
licenses(["notice"]) # Apache 2.0
cc_test(
name="gflags_test",
srcs=["gflags_test.cc"],
copts=["-Iexternal/gtest/include"],
deps=[
"@gtest//:gtest",
"@gflags//:gflags",
], )
third_party/gflags_test/gflags_test.cc 0 → 100644
浏览文件 @ 1de901ab
#include <iostream>
#include <string>
#include "gflags/gflags.h"
#include "gtest/gtest.h"
DEFINE_bool(verbose, false, "Display program name before message");
DEFINE_string(message, "Hello world!", "Message to print");
static bool IsNonEmptyMessage(const char *flagname, const std::string &value) {
return value[0] != '\0';
}
DEFINE_validator(message, &IsNonEmptyMessage);
namespace third_party {
namespace gflags_test {
TEST(GflagsTest, ParseAndPrint) {
gflags::SetUsageMessage("some usage message");
gflags::SetVersionString("1.0.0");
int argc = 1;
char program_name[] = "gflags_test";
char **argv = new char *[2];
argv[0] = program_name;
argv[1] = NULL;
gflags::ParseCommandLineFlags(&argc, reinterpret_cast<char ***>(&argv), true);
EXPECT_EQ("gflags_test", std::string(gflags::ProgramInvocationShortName()));
EXPECT_EQ("Hello world!", FLAGS_message);
gflags::ShutDownCommandLineFlags();
}
} // namespace gflags_test
} // namespace third_party
third_party/glog.BUILD 0 → 100644
浏览文件 @ 1de901ab
licenses(["notice"])
cc_library(
visibility=["//visibility:public"],
name="glog",
includes=[
".",
"src",
],
copts=[
"-D_START_GOOGLE_NAMESPACE_='namespace google {'",
"-D_END_GOOGLE_NAMESPACE_='}'",
"-DGOOGLE_NAMESPACE='google'",
"-DGOOGLE_GLOG_DLL_DECL=''",
"-DHAVE_DLADDR",
"-DHAVE_SNPRINTF",
"-DHAVE_DLFCN_H",
"-DHAVE_FCNTL",
"-DHAVE_GLOB_H",
"-DHAVE_INTTYPES_H",
"-DHAVE_LIBPTHREAD",
"-DHAVE_SYS_SYSCALL_H",
"-DHAVE_MEMORY_H",
"-DHAVE_NAMESPACES",
"-DHAVE_PREAD",
"-DHAVE_PTHREAD",
"-DHAVE_PWD_H",
"-DHAVE_PWRITE",
"-DHAVE_RWLOCK",
"-DHAVE_SIGACTION",
"-DHAVE_SIGALTSTACK",
"-DHAVE_STDINT_H",
"-DHAVE_STRING_H",
"-DHAVE_SYS_TIME_H",
"-DHAVE_SYS_TYPES_H",
"-DHAVE_SYS_UCONTEXT_H",
"-DHAVE_SYS_UTSNAME_H",
"-DHAVE_UNISTD_H",
"-DHAVE_USING_OPERATOR",
"-DHAVE_HAVE___ATTRIBUTE___",
"-DHAVE_HAVE___BUILTIN_EXPECT",
#"-DNO_FRAME_POINTER",
"-D_GNU_SOURCE",
#"-fno-sanitize=thread",
#"-fno-sanitize=address",
"-Iexternal/glog/src",
],
srcs=[
"src/demangle.cc",
"src/logging.cc",
"src/raw_logging.cc",
"src/signalhandler.cc",
"src/symbolize.cc",
"src/utilities.cc",
"src/vlog_is_on.cc",
":config_h",
":logging_h",
":raw_logging_h",
":stl_logging_h",
":vlog_is_on_h",
],
hdrs=[
"src/demangle.h",
"src/mock-log.h",
"src/stacktrace.h",
"src/symbolize.h",
"src/utilities.h",
"src/base/commandlineflags.h",
"src/base/googleinit.h",
"src/base/mutex.h",
"src/glog/log_severity.h",
])
genrule(
name="config_h",
srcs=["src/config.h.cmake.in"],
outs=["config.h"],
cmd="awk '{ gsub(/^#cmakedefine/, \"//cmakedefine\"); print; }' $(<) > $(@)",
)
genrule(
name="logging_h",
srcs=["src/glog/logging.h.in"],
outs=["glog/logging.h"],
cmd="$(location :gen_sh) < $(<) > $(@)",
tools=[":gen_sh"])
genrule(
name="raw_logging_h",
srcs=["src/glog/raw_logging.h.in"],
outs=["glog/raw_logging.h"],
cmd="$(location :gen_sh) < $(<) > $(@)",
tools=[":gen_sh"])
genrule(
name="stl_logging_h",
srcs=["src/glog/stl_logging.h.in"],
outs=["glog/stl_logging.h"],
cmd="$(location :gen_sh) < $(<) > $(@)",
tools=[":gen_sh"])
genrule(
name="vlog_is_on_h",
srcs=["src/glog/vlog_is_on.h.in"],
outs=["glog/vlog_is_on.h"],
cmd="$(location :gen_sh) < $(<) > $(@)",
tools=[":gen_sh"])
genrule(
name="gen_sh",
outs=["gen.sh"],
cmd="""
cat > $@ <<"EOF"
#! /bin/sh
sed -e 's/@ac_cv_have_unistd_h@/1/g' \
-e 's/@ac_cv_have_stdint_h@/1/g' \
-e 's/@ac_cv_have_systypes_h@/1/g' \
-e 's/@ac_cv_have_libgflags_h@/1/g' \
-e 's/@ac_cv_have_uint16_t@/1/g' \
-e 's/@ac_cv_have___builtin_expect@/1/g' \
-e 's/@ac_cv_have_.*@/0/g' \
-e 's/@ac_google_start_namespace@/namespace google {/g' \
-e 's/@ac_google_end_namespace@/}/g' \
-e 's/@ac_google_namespace@/google/g' \
-e 's/@ac_cv___attribute___noinline@/__attribute__((noinline))/g' \
-e 's/@ac_cv___attribute___noreturn@/__attribute__((noreturn))/g' \
-e 's/@ac_cv___attribute___printf_4_5@/__attribute__((__format__ (__printf__, 4, 5)))/g'
EOF""")
third_party/glog_test/BUILD 0 → 100644
浏览文件 @ 1de901ab
licenses(["notice"]) # Apache 2.0
cc_test(
name="glog_test",
srcs=["glog_test.cc"],
copts=["-Iexternal/gtest/include"],
deps=[
"@gtest//:gtest",
"@glog//:glog",
], )
third_party/glog_test/glog_test.cc 0 → 100644
浏览文件 @ 1de901ab
#include <iostream>
#include <string>
#include "glog/logging.h"
#include "gtest/gtest.h"
TEST(GlogTest, Logging) { LOG(INFO) << "Hello world"; }
third_party/gtest.BUILD
浏览文件 @ 1de901ab
此差异已折叠。
third_party/protobuf_test/BUILD
浏览文件 @ 1de901ab
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或