Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
夏有凉风,冬有雪
Paddle
提交
3d0e73bd
P
Paddle
项目概览
夏有凉风,冬有雪
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3d0e73bd
编写于
12月 14, 2016
作者:
L
liaogang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Remove custom glog-like and gflags-like macros
上级
be734a60
变更
77
隐藏空白更改
内联
并排
Showing
77 changed file
with
408 addition
and
1396 deletion
+408
-1396
CMakeLists.txt
CMakeLists.txt
+7
-13
paddle/api/Trainer.cpp
paddle/api/Trainer.cpp
+3
-3
paddle/cuda/src/hl_cuda_cudnn.cc
paddle/cuda/src/hl_cuda_cudnn.cc
+4
-4
paddle/cuda/src/hl_dso_loader.cc
paddle/cuda/src/hl_dso_loader.cc
+15
-15
paddle/gserver/dataproviders/ProtoDataProvider.cpp
paddle/gserver/dataproviders/ProtoDataProvider.cpp
+3
-3
paddle/gserver/evaluators/Evaluator.cpp
paddle/gserver/evaluators/Evaluator.cpp
+1
-1
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
+4
-4
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
+1
-1
paddle/gserver/layers/Layer.cpp
paddle/gserver/layers/Layer.cpp
+1
-1
paddle/gserver/layers/LstmLayer.cpp
paddle/gserver/layers/LstmLayer.cpp
+1
-1
paddle/gserver/layers/RecurrentLayer.cpp
paddle/gserver/layers/RecurrentLayer.cpp
+1
-1
paddle/gserver/layers/ValidationLayer.h
paddle/gserver/layers/ValidationLayer.h
+1
-1
paddle/gserver/tests/LayerGradUtil.cpp
paddle/gserver/tests/LayerGradUtil.cpp
+1
-1
paddle/gserver/tests/TestUtil.cpp
paddle/gserver/tests/TestUtil.cpp
+1
-1
paddle/gserver/tests/test_ActivationGrad.cpp
paddle/gserver/tests/test_ActivationGrad.cpp
+2
-2
paddle/gserver/tests/test_BatchNorm.cpp
paddle/gserver/tests/test_BatchNorm.cpp
+5
-5
paddle/gserver/tests/test_ConvTrans.cpp
paddle/gserver/tests/test_ConvTrans.cpp
+5
-5
paddle/gserver/tests/test_ConvUnify.cpp
paddle/gserver/tests/test_ConvUnify.cpp
+5
-5
paddle/gserver/tests/test_Evaluator.cpp
paddle/gserver/tests/test_Evaluator.cpp
+3
-3
paddle/gserver/tests/test_LayerGrad.cpp
paddle/gserver/tests/test_LayerGrad.cpp
+5
-5
paddle/gserver/tests/test_NetworkCompare.cpp
paddle/gserver/tests/test_NetworkCompare.cpp
+6
-6
paddle/gserver/tests/test_PyDataProvider2.cpp
paddle/gserver/tests/test_PyDataProvider2.cpp
+1
-1
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
+1
-1
paddle/gserver/tests/test_RecurrentLayer.cpp
paddle/gserver/tests/test_RecurrentLayer.cpp
+3
-3
paddle/gserver/tests/test_SelectiveFCLayer.cpp
paddle/gserver/tests/test_SelectiveFCLayer.cpp
+5
-5
paddle/gserver/tests/test_WarpCTCLayer.cpp
paddle/gserver/tests/test_WarpCTCLayer.cpp
+1
-1
paddle/math/SparseRowMatrix.cpp
paddle/math/SparseRowMatrix.cpp
+3
-3
paddle/math/SparseRowMatrix.h
paddle/math/SparseRowMatrix.h
+1
-1
paddle/math/Storage.cpp
paddle/math/Storage.cpp
+3
-3
paddle/math/tests/test_TrainingAlgorithm.cpp
paddle/math/tests/test_TrainingAlgorithm.cpp
+2
-2
paddle/parameter/FirstOrderOptimizer.cpp
paddle/parameter/FirstOrderOptimizer.cpp
+1
-1
paddle/parameter/Parameter.cpp
paddle/parameter/Parameter.cpp
+5
-5
paddle/pserver/BaseClient.cpp
paddle/pserver/BaseClient.cpp
+1
-1
paddle/pserver/LightNetwork.cpp
paddle/pserver/LightNetwork.cpp
+11
-11
paddle/pserver/ParameterClient2.cpp
paddle/pserver/ParameterClient2.cpp
+2
-2
paddle/pserver/ParameterClient2.h
paddle/pserver/ParameterClient2.h
+1
-1
paddle/pserver/ParameterServer2.cpp
paddle/pserver/ParameterServer2.cpp
+5
-5
paddle/pserver/ParameterServer2.h
paddle/pserver/ParameterServer2.h
+1
-1
paddle/pserver/SparseParameterDistribution.cpp
paddle/pserver/SparseParameterDistribution.cpp
+15
-15
paddle/pserver/test/SocketTest.cpp
paddle/pserver/test/SocketTest.cpp
+3
-3
paddle/pserver/test/test_ParameterServer2.cpp
paddle/pserver/test/test_ParameterServer2.cpp
+3
-3
paddle/pserver/test/test_ProtoServer.cpp
paddle/pserver/test/test_ProtoServer.cpp
+4
-4
paddle/trainer/MergeModel.cpp
paddle/trainer/MergeModel.cpp
+2
-2
paddle/trainer/RemoteParameterUpdater.cpp
paddle/trainer/RemoteParameterUpdater.cpp
+2
-2
paddle/trainer/ThreadParameterUpdater.cpp
paddle/trainer/ThreadParameterUpdater.cpp
+1
-1
paddle/trainer/Trainer.cpp
paddle/trainer/Trainer.cpp
+50
-54
paddle/trainer/Trainer.h
paddle/trainer/Trainer.h
+1
-1
paddle/trainer/TrainerBenchmark.cpp
paddle/trainer/TrainerBenchmark.cpp
+2
-2
paddle/trainer/TrainerConfigHelper.cpp
paddle/trainer/TrainerConfigHelper.cpp
+10
-10
paddle/trainer/TrainerInternalConfig.cpp
paddle/trainer/TrainerInternalConfig.cpp
+7
-7
paddle/trainer/TrainerMain.cpp
paddle/trainer/TrainerMain.cpp
+9
-10
paddle/trainer/tests/test_Compare.cpp
paddle/trainer/tests/test_Compare.cpp
+4
-4
paddle/trainer/tests/test_CompareSparse.cpp
paddle/trainer/tests/test_CompareSparse.cpp
+16
-16
paddle/trainer/tests/test_CompareTwoNets.cpp
paddle/trainer/tests/test_CompareTwoNets.cpp
+13
-13
paddle/trainer/tests/test_CompareTwoOpts.cpp
paddle/trainer/tests/test_CompareTwoOpts.cpp
+11
-11
paddle/trainer/tests/test_Prediction.cpp
paddle/trainer/tests/test_Prediction.cpp
+5
-5
paddle/trainer/tests/test_Trainer.cpp
paddle/trainer/tests/test_Trainer.cpp
+4
-4
paddle/trainer/tests/test_TrainerOnePass.cpp
paddle/trainer/tests/test_TrainerOnePass.cpp
+10
-10
paddle/trainer/tests/test_recurrent_machine_generation.cpp
paddle/trainer/tests/test_recurrent_machine_generation.cpp
+1
-1
paddle/utils/BarrierStat.cpp
paddle/utils/BarrierStat.cpp
+9
-9
paddle/utils/CommandLineParser.cpp
paddle/utils/CommandLineParser.cpp
+0
-214
paddle/utils/CommandLineParser.h
paddle/utils/CommandLineParser.h
+0
-157
paddle/utils/CustomStackTrace.cpp
paddle/utils/CustomStackTrace.cpp
+1
-1
paddle/utils/Flags.cpp
paddle/utils/Flags.cpp
+54
-58
paddle/utils/Flags.h
paddle/utils/Flags.h
+25
-25
paddle/utils/Logging.cpp
paddle/utils/Logging.cpp
+4
-167
paddle/utils/Logging.h
paddle/utils/Logging.h
+7
-161
paddle/utils/PythonUtil.cpp
paddle/utils/PythonUtil.cpp
+2
-2
paddle/utils/ThreadLocal.cpp
paddle/utils/ThreadLocal.cpp
+3
-3
paddle/utils/Util.cpp
paddle/utils/Util.cpp
+3
-5
paddle/utils/Version.cpp
paddle/utils/Version.cpp
+2
-7
paddle/utils/tests/CMakeLists.txt
paddle/utils/tests/CMakeLists.txt
+0
-2
paddle/utils/tests/test_CommandLineParser.cpp
paddle/utils/tests/test_CommandLineParser.cpp
+0
-114
paddle/utils/tests/test_CustomStackTrace.cpp
paddle/utils/tests/test_CustomStackTrace.cpp
+1
-1
paddle/utils/tests/test_Logging.cpp
paddle/utils/tests/test_Logging.cpp
+0
-162
paddle/utils/tests/test_SpinLock.cpp
paddle/utils/tests/test_SpinLock.cpp
+1
-1
paddle/utils/tests/test_ThreadBarrier.cpp
paddle/utils/tests/test_ThreadBarrier.cpp
+1
-1
未找到文件。
CMakeLists.txt
浏览文件 @
3d0e73bd
...
...
@@ -25,8 +25,8 @@ find_package(ZLIB REQUIRED)
find_package
(
NumPy REQUIRED
)
find_package
(
Threads REQUIRED
)
find_package
(
AVX QUIET
)
find_package
(
Glog
)
find_package
(
Gflags
QUIET
)
find_package
(
Glog
REQUIRED
)
find_package
(
Gflags
REQUIRED
)
find_package
(
GTest
)
find_package
(
Sphinx
)
find_package
(
Doxygen
)
...
...
@@ -40,8 +40,6 @@ option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
option
(
WITH_PYTHON
"Compile PaddlePaddle with python interpreter"
ON
)
option
(
WITH_STYLE_CHECK
"Style Check for PaddlePaddle"
${
PYTHONINTERP_FOUND
}
)
option
(
WITH_RDMA
"Compile PaddlePaddle with rdma support"
OFF
)
option
(
WITH_GLOG
"Compile PaddlePaddle use glog, otherwise use a log implement internally"
${
LIBGLOG_FOUND
}
)
option
(
WITH_GFLAGS
"Compile PaddlePaddle use gflags, otherwise use a flag implement internally"
${
GFLAGS_FOUND
}
)
option
(
WITH_TIMER
"Compile PaddlePaddle use timer"
OFF
)
option
(
WITH_PROFILER
"Compile PaddlePaddle use gpu profiler"
OFF
)
option
(
WITH_TESTING
"Compile and run unittest for PaddlePaddle"
${
GTEST_FOUND
}
)
...
...
@@ -136,16 +134,12 @@ else(WITH_RDMA)
add_definitions
(
-DPADDLE_DISABLE_RDMA
)
endif
(
WITH_RDMA
)
if
(
WITH_GLOG
)
add_definitions
(
-DPADDLE_USE_GLOG
)
include_directories
(
${
LIBGLOG_INCLUDE_DIR
}
)
endif
()
# glog
include_directories
(
${
LIBGLOG_INCLUDE_DIR
}
)
if
(
WITH_GFLAGS
)
add_definitions
(
-DPADDLE_USE_GFLAGS
)
add_definitions
(
-DGFLAGS_NS=
${
GFLAGS_NAMESPACE
}
)
include_directories
(
${
GFLAGS_INCLUDE_DIRS
}
)
endif
()
#gflags
add_definitions
(
-DGFLAGS_NS=
${
GFLAGS_NAMESPACE
}
)
include_directories
(
${
GFLAGS_INCLUDE_DIRS
}
)
if
(
WITH_TESTING
)
enable_testing
()
...
...
paddle/api/Trainer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -27,9 +27,9 @@ limitations under the License. */
using
paddle
::
real
;
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
init_model_path
);
P_
DECLARE_int32
(
start_pass
);
DECLARE_string
(
config
);
DECLARE_string
(
init_model_path
);
DECLARE_int32
(
start_pass
);
struct
TrainerPrivate
:
public
paddle
::
Trainer
{
bool
_trainOneBatch
(
size_t
batchSize
);
...
...
paddle/cuda/src/hl_cuda_cudnn.cc
浏览文件 @
3d0e73bd
...
...
@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_
DEFINE_int32
(
cudnn_conv_workspace_limit_in_mb
,
4096
,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default."
);
DEFINE_int32
(
cudnn_conv_workspace_limit_in_mb
,
4096
,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default."
);
namespace
dynload
{
...
...
paddle/cuda/src/hl_dso_loader.cc
浏览文件 @
3d0e73bd
...
...
@@ -16,21 +16,21 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_
DEFINE_string
(
cudnn_dir
,
""
,
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH"
);
P_
DEFINE_string
(
cuda_dir
,
""
,
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH"
);
P_
DEFINE_string
(
warpctc_dir
,
""
,
"Specify path for loading libwarpctc.so."
);
DEFINE_string
(
cudnn_dir
,
""
,
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH"
);
DEFINE_string
(
cuda_dir
,
""
,
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH"
);
DEFINE_string
(
warpctc_dir
,
""
,
"Specify path for loading libwarpctc.so."
);
static
inline
std
::
string
join
(
const
std
::
string
&
part1
,
const
std
::
string
&
part2
)
{
...
...
paddle/gserver/dataproviders/ProtoDataProvider.cpp
浏览文件 @
3d0e73bd
...
...
@@ -22,9 +22,9 @@ limitations under the License. */
#include "DataProviderGroup.h"
#include "paddle/utils/Logging.h"
P_
DEFINE_double
(
memory_threshold_on_load_data
,
1.0
,
"stop loading data when memory is not sufficient"
);
DEFINE_double
(
memory_threshold_on_load_data
,
1.0
,
"stop loading data when memory is not sufficient"
);
namespace
paddle
{
...
...
paddle/gserver/evaluators/Evaluator.cpp
浏览文件 @
3d0e73bd
...
...
@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
P_
DECLARE_int32
(
trainer_id
);
DECLARE_int32
(
trainer_id
);
namespace
paddle
{
...
...
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
浏览文件 @
3d0e73bd
...
...
@@ -21,11 +21,11 @@ limitations under the License. */
#include "NeuralNetwork.h"
#include "ParallelNeuralNetwork.h"
P_
DEFINE_bool
(
allow_only_one_model_on_one_gpu
,
true
,
"If true, do not allow multiple models on one GPU device"
);
DEFINE_bool
(
allow_only_one_model_on_one_gpu
,
true
,
"If true, do not allow multiple models on one GPU device"
);
#ifdef PADDLE_METRIC_LEARNING
P_
DECLARE_bool
(
external
);
DECLARE_bool
(
external
);
#endif
namespace
paddle
{
...
...
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
浏览文件 @
3d0e73bd
...
...
@@ -24,7 +24,7 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_
DEFINE_string
(
diy_beam_search_prob_so
,
""
,
"the diy beam search cost so"
);
DEFINE_string
(
diy_beam_search_prob_so
,
""
,
"the diy beam search cost so"
);
static
const
char
*
DIY_CALC_PROB_SYMBOL_NAME
=
"calc_prob"
;
static
const
char
*
DIY_START_CALC_PROB_SYMBOL_NAME
=
"start_calc_prob"
;
...
...
paddle/gserver/layers/Layer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -33,7 +33,7 @@ limitations under the License. */
#include "TransLayer.h"
#include "ValidationLayer.h"
P_
DEFINE_bool
(
log_error_clipping
,
false
,
"enable log error clipping or not"
);
DEFINE_bool
(
log_error_clipping
,
false
,
"enable log error clipping or not"
);
namespace
paddle
{
...
...
paddle/gserver/layers/LstmLayer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/Matrix.h"
#include "paddle/utils/Stat.h"
P_
DECLARE_bool
(
prev_batch_state
);
DECLARE_bool
(
prev_batch_state
);
namespace
paddle
{
...
...
paddle/gserver/layers/RecurrentLayer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_
DEFINE_bool
(
rnn_use_batch
,
false
,
"Using the batch method for calculation."
);
DEFINE_bool
(
rnn_use_batch
,
false
,
"Using the batch method for calculation."
);
namespace
paddle
{
...
...
paddle/gserver/layers/ValidationLayer.h
浏览文件 @
3d0e73bd
...
...
@@ -18,7 +18,7 @@ limitations under the License. */
#include "Layer.h"
#include "paddle/gserver/evaluators/Evaluator.h"
P_
DECLARE_int32
(
trainer_id
);
DECLARE_int32
(
trainer_id
);
namespace
paddle
{
...
...
paddle/gserver/tests/LayerGradUtil.cpp
浏览文件 @
3d0e73bd
...
...
@@ -14,7 +14,7 @@ limitations under the License. */
#include "LayerGradUtil.h"
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
namespace
paddle
{
real
getCostSum
(
LayerPtr
&
testLayer
,
MatrixPtr
weights
)
{
...
...
paddle/gserver/tests/TestUtil.cpp
浏览文件 @
3d0e73bd
...
...
@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/CommandLineParser.h"
P_
DEFINE_int32
(
fixed_seq_length
,
0
,
"Produce some sequence of fixed length"
);
DEFINE_int32
(
fixed_seq_length
,
0
,
"Produce some sequence of fixed length"
);
namespace
paddle
{
...
...
paddle/gserver/tests/test_ActivationGrad.cpp
浏览文件 @
3d0e73bd
...
...
@@ -25,8 +25,8 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
use_gpu
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
void
testActivation
(
const
string
&
act
)
{
LOG
(
INFO
)
<<
"test activation: "
<<
act
;
...
...
paddle/gserver/tests/test_BatchNorm.cpp
浏览文件 @
3d0e73bd
...
...
@@ -27,11 +27,11 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
P_
DECLARE_bool
(
prev_batch_state
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_double
(
checkgrad_eps
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
prev_batch_state
);
// Test that the batchNormLayer can be followed by a ConvLayer
TEST
(
Layer
,
batchNorm
)
{
...
...
paddle/gserver/tests/test_ConvTrans.cpp
浏览文件 @
3d0e73bd
...
...
@@ -28,11 +28,11 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
P_
DECLARE_bool
(
prev_batch_state
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_double
(
checkgrad_eps
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
prev_batch_state
);
// Test that the convTrans forward is the same as conv backward
TEST
(
Layer
,
convTransLayerFwd
)
{
...
...
paddle/gserver/tests/test_ConvUnify.cpp
浏览文件 @
3d0e73bd
...
...
@@ -28,11 +28,11 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
P_
DECLARE_bool
(
prev_batch_state
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_double
(
checkgrad_eps
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
prev_batch_state
);
// Do one forward pass of convTrans layer and check to see if its output
// matches the given result
...
...
paddle/gserver/tests/test_Evaluator.cpp
浏览文件 @
3d0e73bd
...
...
@@ -21,9 +21,9 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
enum
InputType
{
INPUT_DATA
,
// dense vector
...
...
paddle/gserver/tests/test_LayerGrad.cpp
浏览文件 @
3d0e73bd
...
...
@@ -26,11 +26,11 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
P_
DECLARE_bool
(
prev_batch_state
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_double
(
checkgrad_eps
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_bool
(
prev_batch_state
);
TEST
(
Operator
,
dot_mul
)
{
TestConfig
config
;
...
...
paddle/gserver/tests/test_NetworkCompare.cpp
浏览文件 @
3d0e73bd
...
...
@@ -25,10 +25,10 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DEFINE_bool
(
use_label
,
true
,
"input label or sequence label"
);
P_
DEFINE_bool
(
static_para
,
false
,
"static parameter"
);
DECLARE_int32
(
gpu_id
);
DECLARE_double
(
checkgrad_eps
);
DEFINE_bool
(
use_label
,
true
,
"input label or sequence label"
);
DEFINE_bool
(
static_para
,
false
,
"static parameter"
);
struct
DataIn
{
std
::
vector
<
Argument
>
inArgs
;
...
...
@@ -267,8 +267,8 @@ TEST(Compare, img_conv2) {
}
#endif
P_
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
P_
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
TEST
(
Compare
,
network
)
{
if
(
FLAGS_config_file_a
!=
""
&&
FLAGS_config_file_b
!=
""
)
{
compareNetwork
(
FLAGS_config_file_a
,
FLAGS_config_file_b
);
...
...
paddle/gserver/tests/test_PyDataProvider2.cpp
浏览文件 @
3d0e73bd
...
...
@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/PythonUtil.h"
#include "paddle/utils/Util.h"
P_
DEFINE_string
(
train_list
,
"unittest.list"
,
"file list for unittest"
);
DEFINE_string
(
train_list
,
"unittest.list"
,
"file list for unittest"
);
namespace
paddle
{
namespace
unittest
{
...
...
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,7 +20,7 @@ limitations under the License. */
#include <paddle/utils/Util.h>
#include <paddle/utils/Version.h>
P_
DECLARE_int32
(
seed
);
DECLARE_int32
(
seed
);
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
...
...
paddle/gserver/tests/test_RecurrentLayer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -23,9 +23,9 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_bool
(
rnn_use_batch
);
P_
DECLARE_int32
(
fixed_seq_length
);
DECLARE_bool
(
use_gpu
);
DECLARE_bool
(
rnn_use_batch
);
DECLARE_int32
(
fixed_seq_length
);
void
checkError
(
const
Matrix
&
matrix1
,
const
Matrix
&
matrix2
)
{
CHECK
(
matrix1
.
getHeight
()
==
matrix2
.
getHeight
());
...
...
paddle/gserver/tests/test_SelectiveFCLayer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -29,11 +29,11 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
num_passes
);
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
init_model_path
);
P_
DECLARE_string
(
config_args
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
num_passes
);
DECLARE_string
(
config
);
DECLARE_string
(
init_model_path
);
DECLARE_string
(
config_args
);
size_t
fcLayerWidth
=
1024
;
...
...
paddle/gserver/tests/test_WarpCTCLayer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -25,7 +25,7 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_bool
(
use_gpu
);
DECLARE_bool
(
use_gpu
);
const
real
*
getData
(
const
Matrix
&
matrix
)
{
if
(
matrix
.
useGpu
())
{
...
...
paddle/math/SparseRowMatrix.cpp
浏览文件 @
3d0e73bd
...
...
@@ -24,9 +24,9 @@ limitations under the License. */
#include "paddle/utils/Thread.h"
#include "paddle/utils/Util.h"
P_
DEFINE_bool
(
allow_inefficient_sparse_update
,
false
,
"Whether to allow inefficient sparse update"
);
DEFINE_bool
(
allow_inefficient_sparse_update
,
false
,
"Whether to allow inefficient sparse update"
);
namespace
paddle
{
...
...
paddle/math/SparseRowMatrix.h
浏览文件 @
3d0e73bd
...
...
@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Util.h"
P_
DECLARE_bool
(
allow_inefficient_sparse_update
);
DECLARE_bool
(
allow_inefficient_sparse_update
);
namespace
paddle
{
...
...
paddle/math/Storage.cpp
浏览文件 @
3d0e73bd
...
...
@@ -16,9 +16,9 @@ limitations under the License. */
#include "Allocator.h"
#include "paddle/utils/Util.h"
P_
DEFINE_int32
(
pool_limit_size
,
536870912
,
"maximum memory size managed by a memory pool, default is 512M"
);
DEFINE_int32
(
pool_limit_size
,
536870912
,
"maximum memory size managed by a memory pool, default is 512M"
);
namespace
paddle
{
...
...
paddle/math/tests/test_TrainingAlgorithm.cpp
浏览文件 @
3d0e73bd
...
...
@@ -22,9 +22,9 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
#ifndef PADDLE_TYPE_DOUBLE
P_
DEFINE_double
(
max_diff
,
1e-5
,
"max diff allowed"
);
DEFINE_double
(
max_diff
,
1e-5
,
"max diff allowed"
);
#else
P_
DEFINE_double
(
max_diff
,
1e-13
,
"max diff allowed"
);
DEFINE_double
(
max_diff
,
1e-13
,
"max diff allowed"
);
#endif
class
SetMaxDiff
{
...
...
paddle/parameter/FirstOrderOptimizer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -19,7 +19,7 @@ limitations under the License. */
#include <cmath>
P_
DEFINE_bool
(
log_clipping
,
false
,
"enable log clipping or not"
);
DEFINE_bool
(
log_clipping
,
false
,
"enable log clipping or not"
);
namespace
paddle
{
...
...
paddle/parameter/Parameter.cpp
浏览文件 @
3d0e73bd
...
...
@@ -26,11 +26,11 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_
DEFINE_int32
(
enable_grad_share
,
(
100
*
1024
*
1024
),
"threshold for enable gradient parameter share for batch "
"multi-cpu training"
);
P_
DEFINE_int32
(
DEFINE_int32
(
enable_grad_share
,
(
100
*
1024
*
1024
),
"threshold for enable gradient parameter share for batch "
"multi-cpu training"
);
DEFINE_int32
(
grad_share_block_num
,
64
,
"block number of gradient parameter share for batch multi-cpu training"
);
...
...
paddle/pserver/BaseClient.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,7 +18,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_
DECLARE_string
(
pservers
);
DECLARE_string
(
pservers
);
namespace
paddle
{
...
...
paddle/pserver/LightNetwork.cpp
浏览文件 @
3d0e73bd
...
...
@@ -31,23 +31,23 @@ limitations under the License. */
#include "paddle/utils/Util.h"
/// quick ack can reduce the latency of small message
P_
DEFINE_bool
(
small_messages
,
false
,
"if message size is small, recommend set it True to enable quick "
"ack and no delay"
);
DEFINE_bool
(
small_messages
,
false
,
"if message size is small, recommend set it True to enable quick "
"ack and no delay"
);
/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_
DEFINE_int32
(
sock_send_buf_size
,
1024
*
1024
*
40
,
"restrict sock send buff size, can reduce network congestion if "
"set carefully"
);
DEFINE_int32
(
sock_send_buf_size
,
1024
*
1024
*
40
,
"restrict sock send buff size, can reduce network congestion if "
"set carefully"
);
/// reasonable size can hold bursted packets and reduce packets loss
P_
DEFINE_int32
(
sock_recv_buf_size
,
1024
*
1024
*
40
,
"restrict sock recv buff size"
);
DEFINE_int32
(
sock_recv_buf_size
,
1024
*
1024
*
40
,
"restrict sock recv buff size"
);
namespace
paddle
{
...
...
paddle/pserver/ParameterClient2.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,8 +20,8 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/StringUtil.h"
P_
DEFINE_string
(
pservers
,
"127.0.0.1"
,
"Comma separated addresses of pservers"
);
P_
DEFINE_int32
(
parallel_thread_num
,
1
,
"Thread number for parameter send"
);
DEFINE_string
(
pservers
,
"127.0.0.1"
,
"Comma separated addresses of pservers"
);
DEFINE_int32
(
parallel_thread_num
,
1
,
"Thread number for parameter send"
);
namespace
paddle
{
...
...
paddle/pserver/ParameterClient2.h
浏览文件 @
3d0e73bd
...
...
@@ -34,7 +34,7 @@ limitations under the License. */
#include "ProtoServer.h"
#include "SparseParameterDistribution.h"
P_
DECLARE_int32
(
parallel_thread_num
);
DECLARE_int32
(
parallel_thread_num
);
namespace
paddle
{
...
...
paddle/pserver/ParameterServer2.cpp
浏览文件 @
3d0e73bd
...
...
@@ -30,11 +30,11 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_
DEFINE_int32
(
pserver_num_threads
,
1
,
"number of threads for sync op exec"
);
P_
DEFINE_double
(
async_lagged_ratio_min
,
1.0
,
"control config_.async_lagged_grad_discard_ratio() min value"
);
P_
DEFINE_double
(
DEFINE_int32
(
pserver_num_threads
,
1
,
"number of threads for sync op exec"
);
DEFINE_double
(
async_lagged_ratio_min
,
1.0
,
"control config_.async_lagged_grad_discard_ratio() min value"
);
DEFINE_double
(
async_lagged_ratio_default
,
1.5
,
"if async_lagged_grad_discard_ratio is not set in trainer_config.conf"
...
...
paddle/pserver/ParameterServer2.h
浏览文件 @
3d0e73bd
...
...
@@ -38,7 +38,7 @@ limitations under the License. */
#include "ProtoServer.h"
P_
DECLARE_int32
(
port
);
DECLARE_int32
(
port
);
namespace
paddle
{
...
...
paddle/pserver/SparseParameterDistribution.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,26 +20,26 @@ limitations under the License. */
#include "SparseParameterDistribution.h"
P_
DEFINE_bool
(
check_sparse_distribution_in_pserver
,
false
,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers"
);
P_
DEFINE_bool
(
show_check_sparse_distribution_log
,
false
,
"show logs details for sparse parameter distribution in pserver"
);
P_
DEFINE_int32
(
check_sparse_distribution_batches
,
100
,
"run sparse parameter distribution check for N batches"
);
P_
DEFINE_double
(
DEFINE_bool
(
check_sparse_distribution_in_pserver
,
false
,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers"
);
DEFINE_bool
(
show_check_sparse_distribution_log
,
false
,
"show logs details for sparse parameter distribution in pserver"
);
DEFINE_int32
(
check_sparse_distribution_batches
,
100
,
"run sparse parameter distribution check for N batches"
);
DEFINE_double
(
check_sparse_distribution_ratio
,
0.6
,
"if parameters dispatched to different pservers exhibit unbalanced "
" distribution for check_sparse_distribution_ratio * "
" check_sparse_distribution_batches times, crash program"
);
P_
DEFINE_double
(
check_sparse_distribution_unbalance_degree
,
2.0
,
"the ratio of maximum data size and minimun data size for "
"different pserver"
);
DEFINE_double
(
check_sparse_distribution_unbalance_degree
,
2.0
,
"the ratio of maximum data size and minimun data size for "
"different pserver"
);
namespace
paddle
{
...
...
paddle/pserver/test/SocketTest.cpp
浏览文件 @
3d0e73bd
...
...
@@ -195,9 +195,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) {
channel_
.
reset
(
new
SocketChannel
(
sockfd
));
}
P_
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"Server address"
);
P_
DEFINE_int64
(
dim
,
10000000
,
"Data size"
);
P_
DEFINE_int32
(
loop_time
,
100000
,
"test loop time"
);
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"Server address"
);
DEFINE_int64
(
dim
,
10000000
,
"Data size"
);
DEFINE_int32
(
loop_time
,
100000
,
"test loop time"
);
using
namespace
paddle
;
// NOLINT
...
...
paddle/pserver/test/test_ParameterServer2.cpp
浏览文件 @
3d0e73bd
...
...
@@ -21,9 +21,9 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_int32
(
num_gradient_servers
);
P_
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"assign server address"
);
P_
DEFINE_int32
(
server_cpu
,
0
,
"assign server cpu"
);
DECLARE_int32
(
num_gradient_servers
);
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"assign server address"
);
DEFINE_int32
(
server_cpu
,
0
,
"assign server cpu"
);
class
ParameterServer2Tester
:
public
ParameterServer2
{
public:
...
...
paddle/pserver/test/test_ProtoServer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/pserver/ProtoServer.h"
#include "paddle/utils/Stat.h"
P_
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"Server address"
);
P_
DEFINE_int64
(
dim
,
50000000
,
"Data size"
);
P_
DEFINE_bool
(
test_proto_server
,
true
,
"whether to test ProtoServer"
);
P_
DEFINE_bool
(
benchmark
,
false
,
"Do benchmark. Skip some tests"
);
DEFINE_string
(
server_addr
,
"127.0.0.1"
,
"Server address"
);
DEFINE_int64
(
dim
,
50000000
,
"Data size"
);
DEFINE_bool
(
test_proto_server
,
true
,
"whether to test ProtoServer"
);
DEFINE_bool
(
benchmark
,
false
,
"Do benchmark. Skip some tests"
);
using
namespace
paddle
;
// NOLINT
...
...
paddle/trainer/MergeModel.cpp
浏览文件 @
3d0e73bd
...
...
@@ -19,8 +19,8 @@ limitations under the License. */
#include "paddle/pserver/ParameterServer2.h"
#include "paddle/utils/PythonUtil.h"
P_
DEFINE_string
(
model_dir
,
""
,
"Directory for separated model files"
);
P_
DEFINE_string
(
model_file
,
""
,
"File for merged model file"
);
DEFINE_string
(
model_dir
,
""
,
"Directory for separated model files"
);
DEFINE_string
(
model_file
,
""
,
"File for merged model file"
);
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
...
...
paddle/trainer/RemoteParameterUpdater.cpp
浏览文件 @
3d0e73bd
...
...
@@ -17,8 +17,8 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_
DECLARE_int32
(
trainer_id
);
P_
DECLARE_string
(
save_dir
);
DECLARE_int32
(
trainer_id
);
DECLARE_string
(
save_dir
);
namespace
paddle
{
...
...
paddle/trainer/ThreadParameterUpdater.cpp
浏览文件 @
3d0e73bd
...
...
@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/math/SparseRowMatrix.h"
#include "paddle/utils/Thread.h"
P_
DECLARE_int32
(
trainer_count
);
DECLARE_int32
(
trainer_count
);
namespace
paddle
{
...
...
paddle/trainer/Trainer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -38,60 +38,56 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
#include "paddle/gserver/layers/ValidationLayer.h"
P_DEFINE_string
(
config
,
""
,
"Trainer config file"
);
P_DEFINE_int32
(
test_period
,
0
,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches"
);
P_DEFINE_bool
(
test_all_data_in_one_period
,
false
,
"This option was deprecated, since we will always do "
"test on all test set "
);
P_DEFINE_bool
(
local
,
true
,
"Train in local mode or not"
);
P_DEFINE_int32
(
average_test_period
,
0
,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter"
);
P_DEFINE_int32
(
saving_period
,
1
,
"Save parameteres every so many passes"
);
P_DEFINE_int64
(
saving_period_by_batches
,
0
,
"Save parameters every so many batches in one pass"
);
P_DEFINE_string
(
save_dir
,
""
,
"Directory for saving model parameter"
);
P_DEFINE_int32
(
start_pass
,
0
,
"Start training from this pass. "
"Will load parameter from the previous pass"
);
P_DEFINE_int32
(
test_pass
,
-
1
,
"Will load parameter start from this pass to test"
);
P_DEFINE_int32
(
test_wait
,
0
,
"Waiting for pass parameter if not exist"
);
P_DEFINE_bool
(
with_cost
,
true
,
"enable cost layer or not"
);
P_DEFINE_bool
(
distribute_test
,
false
,
"test in distribute mode"
);
P_DEFINE_int32
(
num_passes
,
100
,
"train for so many passes"
);
P_DEFINE_string
(
config_args
,
""
,
"arguments passed to config file."
"Format: key1=value1,key2=value2"
);
P_DEFINE_bool
(
save_only_one
,
false
,
"Save only parameters in last pass, remove previous."
);
P_DEFINE_string
(
feat_file
,
""
,
"File name of extracted feature."
);
P_DEFINE_string
(
predict_output_dir
,
""
,
"Directory that saves the predicted results of output layers"
);
P_DEFINE_string
(
model_list
,
""
,
"File that saves the model list when evaluation"
);
DEFINE_string
(
config
,
""
,
"Trainer config file"
);
DEFINE_int32
(
test_period
,
0
,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches"
);
DEFINE_bool
(
test_all_data_in_one_period
,
false
,
"This option was deprecated, since we will always do "
"test on all test set "
);
DEFINE_bool
(
local
,
true
,
"Train in local mode or not"
);
DEFINE_int32
(
average_test_period
,
0
,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter"
);
DEFINE_int32
(
saving_period
,
1
,
"Save parameteres every so many passes"
);
DEFINE_int64
(
saving_period_by_batches
,
0
,
"Save parameters every so many batches in one pass"
);
DEFINE_string
(
save_dir
,
""
,
"Directory for saving model parameter"
);
DEFINE_int32
(
start_pass
,
0
,
"Start training from this pass. "
"Will load parameter from the previous pass"
);
DEFINE_int32
(
test_pass
,
-
1
,
"Will load parameter start from this pass to test"
);
DEFINE_int32
(
test_wait
,
0
,
"Waiting for pass parameter if not exist"
);
DEFINE_bool
(
with_cost
,
true
,
"enable cost layer or not"
);
DEFINE_bool
(
distribute_test
,
false
,
"test in distribute mode"
);
DEFINE_int32
(
num_passes
,
100
,
"train for so many passes"
);
DEFINE_string
(
config_args
,
""
,
"arguments passed to config file."
"Format: key1=value1,key2=value2"
);
DEFINE_bool
(
save_only_one
,
false
,
"Save only parameters in last pass, remove previous."
);
DEFINE_string
(
feat_file
,
""
,
"File name of extracted feature."
);
DEFINE_string
(
predict_output_dir
,
""
,
"Directory that saves the predicted results of output layers"
);
DEFINE_string
(
model_list
,
""
,
"File that saves the model list when evaluation"
);
namespace
paddle
{
...
...
paddle/trainer/Trainer.h
浏览文件 @
3d0e73bd
...
...
@@ -34,7 +34,7 @@ limitations under the License. */
#include "paddle/internals/metric_learning/MetricTrainer.h"
#endif
P_
DECLARE_int32
(
num_passes
);
DECLARE_int32
(
num_passes
);
namespace
paddle
{
...
...
paddle/trainer/TrainerBenchmark.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,9 +18,9 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_
DECLARE_int32
(
test_period
);
DECLARE_int32
(
test_period
);
P_
DEFINE_bool
(
feed_data
,
false
,
"Wether to read data from DataProvider."
);
DEFINE_bool
(
feed_data
,
false
,
"Wether to read data from DataProvider."
);
namespace
paddle
{
...
...
paddle/trainer/TrainerConfigHelper.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,16 +18,16 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/PythonUtil.h"
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
init_model_path
);
P_
DECLARE_int32
(
start_pass
);
P_
DECLARE_string
(
save_dir
);
P_
DECLARE_int32
(
trainer_id
);
P_
DECLARE_bool
(
local
);
P_
DECLARE_bool
(
with_cost
);
P_
DECLARE_bool
(
with_gpu
);
P_
DECLARE_bool
(
parallel_nn
);
P_
DECLARE_string
(
config_args
);
DECLARE_string
(
config
);
DECLARE_string
(
init_model_path
);
DECLARE_int32
(
start_pass
);
DECLARE_string
(
save_dir
);
DECLARE_int32
(
trainer_id
);
DECLARE_bool
(
local
);
DECLARE_bool
(
with_cost
);
DECLARE_bool
(
with_gpu
);
DECLARE_bool
(
parallel_nn
);
DECLARE_string
(
config_args
);
const
char
*
kConfigParserModuleName
=
"paddle.trainer.config_parser"
;
const
char
*
kConfigParserFuncName
=
"parse_config_and_serialize"
;
...
...
paddle/trainer/TrainerInternalConfig.cpp
浏览文件 @
3d0e73bd
...
...
@@ -14,17 +14,17 @@ limitations under the License. */
#include "TrainerInternalConfig.h"
P_
DEFINE_int32
(
show_parameter_stats_period
,
0
,
"Whether to show parameter stats during training"
);
DEFINE_int32
(
show_parameter_stats_period
,
0
,
"Whether to show parameter stats during training"
);
P_
DEFINE_int32
(
dot_period
,
1
,
"Print '.' every so many batches"
);
DEFINE_int32
(
dot_period
,
1
,
"Print '.' every so many batches"
);
P_
DEFINE_bool
(
use_old_updater
,
false
,
"Use the old RemoteParameterUpdater"
);
DEFINE_bool
(
use_old_updater
,
false
,
"Use the old RemoteParameterUpdater"
);
P_
DECLARE_int32
(
num_passes
);
DECLARE_int32
(
num_passes
);
P_
DECLARE_bool
(
local
);
DECLARE_bool
(
local
);
namespace
paddle
{
...
...
paddle/trainer/TrainerMain.cpp
浏览文件 @
3d0e73bd
...
...
@@ -22,21 +22,20 @@ limitations under the License. */
#include "Trainer.h"
#include "paddle/pserver/RDMANetwork.h"
P_
DEFINE_bool
(
start_pserver
,
false
,
"Whether to start pserver"
);
P_
DECLARE_int32
(
gpu_id
);
P_
DEFINE_string
(
job
,
"train"
,
"one of (train, test, checkgrad)"
);
P_
DECLARE_int32
(
start_pass
);
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
init_model_path
);
P_
DECLARE_string
(
rdma_tcp
);
DEFINE_bool
(
start_pserver
,
false
,
"Whether to start pserver"
);
DECLARE_int32
(
gpu_id
);
DEFINE_string
(
job
,
"train"
,
"one of (train, test, checkgrad)"
);
DECLARE_int32
(
start_pass
);
DECLARE_string
(
config
);
DECLARE_string
(
init_model_path
);
DECLARE_string
(
rdma_tcp
);
using
namespace
paddle
;
// NOLINT
int
main
(
int
argc
,
char
**
argv
)
{
// write logs instantly (never buffer log messages)
#ifdef PADDLE_USE_GLOG
// write logs instantly (never buffer log messages)
FLAGS_logbuflevel
=
-
1
;
#endif
initMain
(
argc
,
argv
);
initPython
(
argc
,
argv
);
...
...
paddle/trainer/tests/test_Compare.cpp
浏览文件 @
3d0e73bd
...
...
@@ -24,10 +24,10 @@ using namespace std; // NOLINT
static
const
string
&
configFile
=
"trainer/tests/sample_trainer_config.conf"
;
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
config_args
);
DECLARE_int32
(
gpu_id
);
DECLARE_bool
(
use_gpu
);
DECLARE_string
(
config
);
DECLARE_string
(
config_args
);
struct
comData
{
vector
<
Argument
>
outArgs
;
...
...
paddle/trainer/tests/test_CompareSparse.cpp
浏览文件 @
3d0e73bd
...
...
@@ -25,22 +25,22 @@ using namespace std; // NOLINT
static
const
string
&
configFile1
=
"trainer/tests/sample_trainer_config_qb_rnn.conf"
;
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_int32
(
seed
);
P_
DECLARE_int32
(
num_passes
);
P_
DECLARE_int32
(
saving_period
);
P_
DECLARE_int32
(
num_gradient_servers
);
P_
DECLARE_int32
(
port
);
P_
DECLARE_bool
(
local
);
P_
DECLARE_bool
(
use_old_updater
);
P_
DECLARE_bool
(
parallel_nn
);
P_
DECLARE_string
(
config_args
);
P_
DEFINE_double
(
max_diff_ratio
,
0.0
f
,
"max diff ratio allowed for parameters value"
);
DECLARE_bool
(
use_gpu
);
DECLARE_string
(
config
);
DECLARE_int32
(
gpu_id
);
DECLARE_int32
(
seed
);
DECLARE_int32
(
num_passes
);
DECLARE_int32
(
saving_period
);
DECLARE_int32
(
num_gradient_servers
);
DECLARE_int32
(
port
);
DECLARE_bool
(
local
);
DECLARE_bool
(
use_old_updater
);
DECLARE_bool
(
parallel_nn
);
DECLARE_string
(
config_args
);
DEFINE_double
(
max_diff_ratio
,
0.0
f
,
"max diff ratio allowed for parameters value"
);
int
gNumDevices
=
0
;
...
...
paddle/trainer/tests/test_CompareTwoNets.cpp
浏览文件 @
3d0e73bd
...
...
@@ -22,25 +22,25 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_int32
(
gpu_id
);
DECLARE_int32
(
gpu_id
);
P_
DECLARE_bool
(
local
);
P_
DECLARE_bool
(
use_gpu
);
DECLARE_bool
(
local
);
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
nics
);
DECLARE_string
(
config
);
DECLARE_string
(
nics
);
P_
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
P_
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
P_
DEFINE_bool
(
need_high_accuracy
,
false
,
"whether need to run in double accuracy"
);
P_
DEFINE_double
(
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
DEFINE_bool
(
need_high_accuracy
,
false
,
"whether need to run in double accuracy"
);
DEFINE_double
(
max_diff_ratio
,
0.0
f
,
"max diff ratio allowed for outputs and parameters (value/gradient)"
);
P_
DECLARE_bool
(
thread_local_rand_use_global_seed
);
P_
DECLARE_int32
(
seed
);
DECLARE_bool
(
thread_local_rand_use_global_seed
);
DECLARE_int32
(
seed
);
struct
ComData
{
vector
<
Argument
>
outArgs
;
...
...
paddle/trainer/tests/test_CompareTwoOpts.cpp
浏览文件 @
3d0e73bd
...
...
@@ -22,20 +22,20 @@ limitations under the License. */
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
P_
DECLARE_int32
(
gpu_id
);
DECLARE_int32
(
gpu_id
);
P_
DECLARE_bool
(
local
);
P_
DECLARE_bool
(
use_gpu
);
DECLARE_bool
(
local
);
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
nics
);
DECLARE_string
(
config
);
DECLARE_string
(
nics
);
P_
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
P_
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
P_
DEFINE_bool
(
need_high_accuracy
,
true
,
"whether need to run in double accuracy (recommended)"
);
P_
DEFINE_double
(
DEFINE_string
(
config_file_a
,
""
,
"config of one network to compare"
);
DEFINE_string
(
config_file_b
,
""
,
"config of another network to compare"
);
DEFINE_bool
(
need_high_accuracy
,
true
,
"whether need to run in double accuracy (recommended)"
);
DEFINE_double
(
max_diff_ratio
,
0.0
f
,
"max diff ratio allowed for outputs and parameters (value/gradient)"
);
...
...
paddle/trainer/tests/test_Prediction.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,11 +18,11 @@ limitations under the License. */
#include <gtest/gtest.h>
P_
DECLARE_string
(
config
);
P_
DECLARE_string
(
config_args
);
P_
DEFINE_string
(
merger
,
"./paddle_merge_model"
,
"path to paddle_merge_model binary"
);
DECLARE_string
(
config
);
DECLARE_string
(
config_args
);
DEFINE_string
(
merger
,
"./paddle_merge_model"
,
"path to paddle_merge_model binary"
);
using
namespace
paddle
;
// NOLINT
using
namespace
std
;
// NOLINT
...
...
paddle/trainer/tests/test_Trainer.cpp
浏览文件 @
3d0e73bd
...
...
@@ -28,10 +28,10 @@ static const string& configFile3 = "trainer/tests/chunking.conf";
static
const
string
&
configFile4
=
"trainer/tests/sample_trainer_config_parallel.conf"
;
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_bool
(
allow_only_one_model_on_one_gpu
);
DECLARE_bool
(
use_gpu
);
DECLARE_string
(
config
);
DECLARE_int32
(
gpu_id
);
DECLARE_bool
(
allow_only_one_model_on_one_gpu
);
void
checkGradientTest
(
const
string
&
configFile
,
bool
useGpu
,
...
...
paddle/trainer/tests/test_TrainerOnePass.cpp
浏览文件 @
3d0e73bd
...
...
@@ -27,12 +27,12 @@ static const string& configFile1 = "trainer/tests/sample_trainer_config.conf";
static
const
string
&
configFile2
=
"trainer/tests/sample_trainer_config_parallel.conf"
;
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_string
(
config
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_int32
(
seed
);
P_
DECLARE_int32
(
num_passes
);
P_
DECLARE_int32
(
saving_period
);
DECLARE_bool
(
use_gpu
);
DECLARE_string
(
config
);
DECLARE_int32
(
gpu_id
);
DECLARE_int32
(
seed
);
DECLARE_int32
(
num_passes
);
DECLARE_int32
(
saving_period
);
class
TrainerForTest
:
public
paddle
::
Trainer
{
public:
...
...
@@ -122,10 +122,10 @@ TEST(average_window_cpu, gpu4) {
#endif
// 3. test trainer + pserver.
P_
DECLARE_int32
(
num_gradient_servers
);
P_
DECLARE_int32
(
port
);
P_
DECLARE_bool
(
local
);
P_
DECLARE_bool
(
use_old_updater
);
DECLARE_int32
(
num_gradient_servers
);
DECLARE_int32
(
port
);
DECLARE_bool
(
local
);
DECLARE_bool
(
use_old_updater
);
double
checkRemoteParameterUpdater
(
TrainerForTest
&
trainer
)
{
auto
gradientMachine
=
trainer
.
getGradientMachine
();
...
...
paddle/trainer/tests/test_recurrent_machine_generation.cpp
浏览文件 @
3d0e73bd
...
...
@@ -30,7 +30,7 @@ static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT
static
string
expectFile
=
// NOLINT
"trainer/tests/rnn_gen_test_model_dir/r1.test"
;
// NOLINT
P_
DECLARE_string
(
config_args
);
DECLARE_string
(
config_args
);
vector
<
float
>
readRetFile
(
const
string
&
fname
)
{
ifstream
inFile
(
fname
);
...
...
paddle/utils/BarrierStat.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,15 +20,15 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/Stat.h"
P_
DEFINE_bool
(
log_barrier_abstract
,
true
,
"if true, show abstract of barrier performance"
);
P_
DEFINE_int32
(
log_barrier_lowest_nodes
,
5
,
"how many lowest node will be logged"
);
P_
DEFINE_bool
(
log_barrier_show_log
,
false
,
// for performance tuning insight
"if true, always show barrier abstract even with little gap"
);
DEFINE_bool
(
log_barrier_abstract
,
true
,
"if true, show abstract of barrier performance"
);
DEFINE_int32
(
log_barrier_lowest_nodes
,
5
,
"how many lowest node will be logged"
);
DEFINE_bool
(
log_barrier_show_log
,
false
,
// for performance tuning insight
"if true, always show barrier abstract even with little gap"
);
namespace
paddle
{
...
...
paddle/utils/CommandLineParser.cpp
浏览文件 @
3d0e73bd
...
...
@@ -13,220 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "CommandLineParser.h"
#ifndef PADDLE_USE_GFLAGS
#include <stdlib.h>
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "paddle/utils/StringUtil.h"
namespace
paddle
{
static
constexpr
int
kStatusOK
=
0
;
static
constexpr
int
kStatusInvalid
=
1
;
static
constexpr
int
kStatusNotFound
=
2
;
/**
* \brief: Convert a string to any type value.
*
* \note: It will specialize by type T that is supported.
*/
template
<
typename
T
>
bool
StringToValue
(
const
std
::
string
&
content
,
T
*
value
)
{
bool
ok
;
*
value
=
str
::
toWithStatus
<
T
>
(
content
,
&
ok
);
return
ok
;
}
template
<
>
bool
StringToValue
<
bool
>
(
const
std
::
string
&
content
,
bool
*
value
)
{
std
::
string
tmp
=
content
;
std
::
transform
(
tmp
.
begin
(),
tmp
.
end
(),
tmp
.
begin
(),
[](
char
in
)
->
char
{
if
(
in
<=
'Z'
&&
in
>=
'A'
)
{
return
in
-
(
'Z'
-
'z'
);
}
else
{
return
in
;
}
});
// tolower.
if
(
tmp
==
"true"
||
tmp
==
"1"
)
{
*
value
=
true
;
return
true
;
}
else
if
(
tmp
==
"false"
||
tmp
==
"0"
)
{
*
value
=
false
;
return
true
;
}
else
{
return
false
;
}
}
template
<
>
bool
StringToValue
<
std
::
string
>
(
const
std
::
string
&
content
,
std
::
string
*
value
)
{
*
value
=
content
;
return
true
;
}
/**
* \brief Parse argument "--blah=blah".
*
* \param argument: The command line argument string, such as "--blah=blah"
* \param [out] extraInfo: The details error message for parse argument.
* \return: kStatusOK, kStatusInvalid, kStatusNotFound
*/
template
<
typename
T
>
int
ParseArgument
(
const
std
::
string
&
argument
,
std
::
string
*
extraInfo
)
{
for
(
auto
&
command
:
flags_internal
::
CommandLineFlagRegistry
<
T
>::
Instance
()
->
commands
)
{
std
::
string
&
name
=
command
.
name
;
T
*
value
=
command
.
value
;
std
::
string
prefix
=
"--"
;
prefix
+=
name
;
prefix
+=
"="
;
std
::
string
content
;
if
(
str
::
startsWith
(
argument
,
prefix
))
{
content
=
argument
.
substr
(
prefix
.
size
(),
argument
.
size
()
-
prefix
.
size
());
}
else
{
prefix
=
"-"
;
prefix
+=
name
;
prefix
+=
"="
;
if
(
str
::
startsWith
(
argument
,
prefix
))
{
content
=
argument
.
substr
(
prefix
.
size
(),
argument
.
size
()
-
prefix
.
size
());
}
}
if
(
!
content
.
empty
())
{
if
(
StringToValue
(
content
,
value
))
{
return
kStatusOK
;
}
else
{
*
extraInfo
=
name
;
return
kStatusInvalid
;
}
}
}
return
kStatusNotFound
;
}
/**
* @brief ParseBoolArgumentExtra
* parse '--flag_name', '-flag_name' as true; '--noflag_name', '-noflag_name' as
* false
*/
static
int
ParseBoolArgumentExtra
(
const
std
::
string
&
argument
,
std
::
string
*
extraInfo
)
{
(
void
)(
extraInfo
);
// unused extraInfo, just make api same.
//! @warning: The order and content of prefixes is DESIGNED for parsing
//! command line. The length of prefixes are 1, 2, 3, 4. The parse logic takes
//! use of this fact. DO NOT CHANGE IT without reading how to parse command
//! below.
static
const
std
::
vector
<
std
::
pair
<
const
char
*
,
bool
>>
prefixes
=
{
{
"-"
,
true
},
{
"--"
,
true
},
{
"-no"
,
false
},
{
"--no"
,
false
}};
for
(
flags_internal
::
CommandLineFlagRegistry
<
bool
>::
Command
&
command
:
flags_internal
::
CommandLineFlagRegistry
<
bool
>::
Instance
()
->
commands
)
{
if
(
argument
.
size
()
>
command
.
name
.
size
())
{
//! Use the length of prefix is 1, 2, 3, 4.
size_t
diff
=
argument
.
size
()
-
command
.
name
.
size
()
-
1UL
;
if
(
diff
<
prefixes
.
size
())
{
const
std
::
string
&
prefix
=
std
::
get
<
0
>
(
prefixes
[
diff
]);
if
(
argument
==
prefix
+
command
.
name
)
{
*
command
.
value
=
std
::
get
<
1
>
(
prefixes
[
diff
]);
return
kStatusOK
;
}
}
}
}
return
kStatusNotFound
;
}
/**
* \brief: Print command line arguments' usage with type T.
*/
template
<
typename
T
>
static
void
PrintTypeUsage
()
{
for
(
auto
&
command
:
flags_internal
::
CommandLineFlagRegistry
<
T
>::
Instance
()
->
commands
)
{
std
::
string
&
name
=
command
.
name
;
name
=
"--"
+
name
;
// Program will exit, so modify name is safe.
std
::
string
&
desc
=
command
.
text
;
T
&
defaultValue
=
command
.
defaultValue
;
std
::
cerr
<<
std
::
setw
(
20
)
<<
name
<<
": "
<<
desc
<<
"[default:"
<<
defaultValue
<<
"]."
<<
std
::
endl
;
}
}
template
<
typename
...
TS
>
static
void
PrintTypeUsages
()
{
int
unused
[]
=
{
0
,
(
PrintTypeUsage
<
TS
>
(),
0
)...};
(
void
)(
unused
);
}
/**
* \brief: Print all usage, and exit(1)
*/
static
void
PrintUsageAndExit
(
const
char
*
argv0
)
{
std
::
cerr
<<
"Program "
<<
argv0
<<
" Flags: "
<<
std
::
endl
;
PrintTypeUsages
<
bool
,
int32_t
,
std
::
string
,
double
,
int64_t
,
uint64_t
>
();
exit
(
1
);
}
/**
* \brief: Print the error flags, usage, and exit.
*/
static
void
PrintParseError
(
const
std
::
string
&
name
,
const
char
*
actualInput
,
const
char
*
arg0
)
{
std
::
cerr
<<
"Parse command flag "
<<
name
<<
" error! User input is "
<<
actualInput
<<
std
::
endl
;
PrintUsageAndExit
(
arg0
);
}
void
ParseCommandLineFlags
(
int
*
argc
,
char
**
argv
,
bool
withHelp
)
{
int
unused_argc
=
1
;
std
::
string
extra
;
for
(
int
i
=
1
;
i
<
*
argc
;
++
i
)
{
std
::
string
arg
=
argv
[
i
];
int
s
=
kStatusInvalid
;
#define ParseArgumentWithType(type) \
s = ParseArgument<type>(arg, &extra); \
if (s == kStatusOK) { \
continue; \
} else if (s == kStatusInvalid) { \
PrintParseError(extra, argv[i], argv[0]); \
}
ParseArgumentWithType
(
bool
);
// NOLINT
ParseArgumentWithType
(
int32_t
);
ParseArgumentWithType
(
double
);
// NOLINT
ParseArgumentWithType
(
int64_t
);
ParseArgumentWithType
(
uint64_t
);
ParseArgumentWithType
(
std
::
string
);
#undef ParseArgumentWithType
s
=
ParseBoolArgumentExtra
(
arg
,
&
extra
);
if
(
s
==
kStatusOK
)
{
continue
;
}
if
(
withHelp
&&
(
arg
==
"--help"
||
arg
==
"-h"
))
{
PrintUsageAndExit
(
argv
[
0
]);
}
// NOT Found for all flags.
std
::
swap
(
argv
[
unused_argc
++
],
argv
[
i
]);
}
*
argc
=
unused_argc
;
}
}
// namespace paddle
#else
namespace
paddle
{
#ifndef GFLAGS_NS
#define GFLAGS_NS google
...
...
@@ -243,4 +30,3 @@ void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
}
}
// namespace paddle
#endif
paddle/utils/CommandLineParser.h
浏览文件 @
3d0e73bd
...
...
@@ -13,167 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#ifndef PADDLE_USE_GFLAGS
#include <stdint.h>
#include <string>
#include <vector>
#include "DisableCopy.h"
namespace
paddle
{
namespace
flags_internal
{
/**
* Command line flag registry for special type T. It will store all command
* arguments settings. such as name, default value.
*/
template
<
typename
T
>
struct
CommandLineFlagRegistry
{
/**
* The factory method of CommandLineFlagRegistry
*
* \return: The singleton instance of CommandLineFlagRegistry.
*/
static
CommandLineFlagRegistry
*
Instance
()
{
static
CommandLineFlagRegistry
instance_
;
return
&
instance_
;
}
struct
Command
{
/// name of argument.
std
::
string
name
;
/// address of actual variable. such as FLAGS_xxx.
T
*
value
;
/// usage text.
std
::
string
text
;
/// default value of this command.
T
defaultValue
;
};
/// the command line arguments of type T.
std
::
vector
<
Command
>
commands
;
DISABLE_COPY
(
CommandLineFlagRegistry
);
private:
inline
CommandLineFlagRegistry
()
{}
};
/**
*Helper class to register command line flag.
*/
template
<
typename
T
>
struct
CommandLineFlagRegister
{
/**
* \brief: Register a command line argument
*
* \param [in] name: The command line name.
* \param [inout] val: The command line argument instance, FLAGS_xxx.
* \param [in] desc: The command line helper message.
*/
CommandLineFlagRegister
(
const
std
::
string
&
name
,
T
*
val
,
const
std
::
string
desc
)
{
CommandLineFlagRegistry
<
T
>::
Instance
()
->
commands
.
push_back
(
{
name
,
val
,
desc
,
*
val
});
}
};
/**
* \brief: Define a command line arguments.
*
* \param type: The variable type, such as int, double, etc.
* \param name: The variable name. The command line argument is '--name', the
*variable
*is 'FLAGS_name'
* \param default_value: The default value of command line argument.
* \param text: The description in command line argument.
*/
#define PADDLE_DEFINE_variable(type, name, default_value, text) \
type FLAGS_##name = default_value; \
namespace paddle_flags_internal { \
paddle::flags_internal::CommandLineFlagRegister<type> \
flags_internal_var_##name(#name, &FLAGS_##name, text); \
} // namespace paddle_flags_internal
/**
* Declare a variable to use.
*/
#define PADDLE_DECLARE_variable(type, name) extern type FLAGS_##name;
// DEFINE macro for each types.
#define P_DEFINE_int32(name, default_value, text) \
PADDLE_DEFINE_variable(int32_t, name, default_value, text)
#define P_DEFINE_bool(name, default_value, text) \
PADDLE_DEFINE_variable(bool, name, default_value, text)
#define P_DEFINE_string(name, default_value, text) \
PADDLE_DEFINE_variable(std::string, name, default_value, text)
#define P_DEFINE_double(name, default_value, text) \
PADDLE_DEFINE_variable(double, name, default_value, text)
#define P_DEFINE_int64(name, default_value, text) \
PADDLE_DEFINE_variable(int64_t, name, default_value, text)
#define P_DEFINE_uint64(name, default_value, text) \
PADDLE_DEFINE_variable(uint64_t, name, default_value, text)
// Declare macro for each types.
#define P_DECLARE_int32(name) PADDLE_DECLARE_variable(int32_t, name)
#define P_DECLARE_bool(name) PADDLE_DECLARE_variable(bool, name)
#define P_DECLARE_string(name) PADDLE_DECLARE_variable(std::string, name)
#define P_DECLARE_double(name) PADDLE_DECLARE_variable(double, name)
#define P_DECLARE_int64(name) PADDLE_DECLARE_variable(int64_t, name)
#define P_DECLARE_uint64(name) PADDLE_DECLARE_variable(uint64_t, name)
}
// namespace flags_internal
/**
* \brief Parse command line flags. If parse error, just failed and exit 1.
*
* \param [inout] argc: The command argument count. This method will modify
*argc, and left unused arguments.
* \param [inout] argv: The command argument values. This method will modify
*argv, and left unused arguments.
* \param [in] withHelp: True will parse '-h' and '--help' to print usage.
*
* \note: The Command line flags format basically as follow:
*
* * If the type of flag is not bool, then the follow format of command line
* will be parsed:
* * --flag_name=value
* * -flag_name=value
*
* * If the flag is bool, then:
* * --flag_name=value, -flag_name=value will be parsed.
* * if value.tolower() == "true"| "1" will be treated as true.
* * else if value.tolower() == "false" | "0" will be treated as false.
* * --flag_name will be parsed as true.
* * --noflag_name will be parsed as false.
*/
void
ParseCommandLineFlags
(
int
*
argc
,
char
**
argv
,
bool
withHelp
=
true
);
}
// namespace paddle
#else // if use gflags.
#include <gflags/gflags.h>
#define P_DEFINE_int32 DEFINE_int32
#define P_DEFINE_bool DEFINE_bool
#define P_DEFINE_string DEFINE_string
#define P_DEFINE_double DEFINE_double
#define P_DEFINE_int64 DEFINE_int64
#define P_DEFINE_uint64 DEFINE_uint64
#define P_DECLARE_int32 DECLARE_int32
#define P_DECLARE_bool DECLARE_bool
#define P_DECLARE_string DECLARE_string
#define P_DECLARE_double DECLARE_double
#define P_DECLARE_int64 DECLARE_int64
#define P_DECLARE_uint64 DECLARE_uint64
namespace
paddle
{
void
ParseCommandLineFlags
(
int
*
argc
,
char
**
argv
,
bool
withHelp
=
true
);
}
// namespace paddle
#endif
paddle/utils/CustomStackTrace.cpp
浏览文件 @
3d0e73bd
...
...
@@ -16,7 +16,7 @@ limitations under the License. */
#include <iostream>
#include "CommandLineParser.h"
P_
DEFINE_bool
(
DEFINE_bool
(
layer_stack_error_only_current_thread
,
true
,
"Dump current thread or whole process layer stack when signal error "
...
...
paddle/utils/Flags.cpp
浏览文件 @
3d0e73bd
...
...
@@ -15,65 +15,61 @@ limitations under the License. */
#include "Flags.h"
#ifdef PADDLE_ONLY_CPU
P_
DEFINE_bool
(
use_gpu
,
false
,
"Only support CPU training"
);
DEFINE_bool
(
use_gpu
,
false
,
"Only support CPU training"
);
#else
P_
DEFINE_bool
(
use_gpu
,
true
,
"Whether to use GPU for training"
);
DEFINE_bool
(
use_gpu
,
true
,
"Whether to use GPU for training"
);
#endif
P_DEFINE_bool
(
parallel_nn
,
false
,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored)."
);
P_DEFINE_int32
(
trainer_count
,
1
,
"Defined how many trainers to train"
);
P_DEFINE_int32
(
gpu_id
,
0
,
"Which gpu core to use"
);
P_DEFINE_int32
(
port
,
20134
,
"Listening port for pserver"
);
P_DEFINE_int32
(
data_server_port
,
21134
,
"Listening port for dserver"
);
P_DEFINE_int32
(
ports_num
,
1
,
"The ports number for parameter send,"
" increment based on default port number"
);
P_DEFINE_int32
(
ports_num_for_sparse
,
0
,
"The ports number for parameter send,"
" increment based on default (port + ports_num)"
);
P_DEFINE_string
(
nics
,
"xgbe0,xgbe1"
,
"network device name for pservers"
);
P_DEFINE_string
(
rdma_tcp
,
"tcp"
,
"use rdma or tcp rdma transport protocol"
);
P_DEFINE_int32
(
trainer_id
,
0
,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer"
);
P_DEFINE_int32
(
num_gradient_servers
,
1
,
"number of gradient servers"
);
P_DEFINE_string
(
comment
,
""
,
"A string for commenting this training task"
);
P_DEFINE_string
(
load_missing_parameter_strategy
,
"fail"
,
"which operation to take on load model fails. support "
"fail/rand/zero only."
);
P_DEFINE_int32
(
log_period
,
100
,
"Log progress every so many batches"
);
P_DEFINE_int32
(
log_period_server
,
500
,
"Log progress every so many batches at pserver end"
);
P_DEFINE_double
(
checkgrad_eps
,
1e-5
,
"parameter change size for checkgrad"
);
P_DEFINE_int32
(
enable_parallel_vector
,
0
,
"threshold for enable parallel vector"
);
P_DEFINE_bool
(
loadsave_parameters_in_pserver
,
false
,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update."
);
P_DEFINE_int32
(
beam_size
,
1
,
"Beam size used in generating most probable output sequences."
);
DEFINE_bool
(
parallel_nn
,
false
,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored)."
);
DEFINE_int32
(
trainer_count
,
1
,
"Defined how many trainers to train"
);
DEFINE_int32
(
gpu_id
,
0
,
"Which gpu core to use"
);
DEFINE_int32
(
port
,
20134
,
"Listening port for pserver"
);
DEFINE_int32
(
data_server_port
,
21134
,
"Listening port for dserver"
);
DEFINE_int32
(
ports_num
,
1
,
"The ports number for parameter send,"
" increment based on default port number"
);
DEFINE_int32
(
ports_num_for_sparse
,
0
,
"The ports number for parameter send,"
" increment based on default (port + ports_num)"
);
DEFINE_string
(
nics
,
"xgbe0,xgbe1"
,
"network device name for pservers"
);
DEFINE_string
(
rdma_tcp
,
"tcp"
,
"use rdma or tcp rdma transport protocol"
);
DEFINE_int32
(
trainer_id
,
0
,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer"
);
DEFINE_int32
(
num_gradient_servers
,
1
,
"number of gradient servers"
);
DEFINE_string
(
comment
,
""
,
"A string for commenting this training task"
);
DEFINE_string
(
load_missing_parameter_strategy
,
"fail"
,
"which operation to take on load model fails. support "
"fail/rand/zero only."
);
DEFINE_int32
(
log_period
,
100
,
"Log progress every so many batches"
);
DEFINE_int32
(
log_period_server
,
500
,
"Log progress every so many batches at pserver end"
);
DEFINE_double
(
checkgrad_eps
,
1e-5
,
"parameter change size for checkgrad"
);
DEFINE_int32
(
enable_parallel_vector
,
0
,
"threshold for enable parallel vector"
);
DEFINE_bool
(
loadsave_parameters_in_pserver
,
false
,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update."
);
DEFINE_int32
(
beam_size
,
1
,
"Beam size used in generating most probable output sequences."
);
P_
DEFINE_bool
(
show_layer_stat
,
false
,
"show the statistics of each layer"
);
P_
DEFINE_string
(
predict_file
,
""
,
"File name for saving predict result"
);
P_
DEFINE_bool
(
prev_batch_state
,
false
,
"batch is continue with next batch"
);
P_
DEFINE_string
(
init_model_path
,
""
,
"Path of the initial model parameters."
"If it was set, start_pass will be ignored."
);
DEFINE_bool
(
show_layer_stat
,
false
,
"show the statistics of each layer"
);
DEFINE_string
(
predict_file
,
""
,
"File name for saving predict result"
);
DEFINE_bool
(
prev_batch_state
,
false
,
"batch is continue with next batch"
);
DEFINE_string
(
init_model_path
,
""
,
"Path of the initial model parameters."
"If it was set, start_pass will be ignored."
);
paddle/utils/Flags.h
浏览文件 @
3d0e73bd
...
...
@@ -16,28 +16,28 @@ limitations under the License. */
#include "CommandLineParser.h"
P_
DECLARE_bool
(
parallel_nn
);
P_
DECLARE_int32
(
async_count
);
P_
DECLARE_int32
(
port
);
P_
DECLARE_int32
(
data_server_port
);
P_
DECLARE_bool
(
use_gpu
);
P_
DECLARE_int32
(
gpu_id
);
P_
DECLARE_int32
(
trainer_count
);
P_
DECLARE_int32
(
ports_num
);
P_
DECLARE_int32
(
ports_num_for_sparse
);
P_
DECLARE_string
(
nics
);
P_
DECLARE_string
(
rdma_tcp
);
P_
DECLARE_int32
(
trainer_id
);
P_
DECLARE_int32
(
num_gradient_servers
);
P_
DECLARE_string
(
comment
);
P_
DECLARE_string
(
load_missing_parameter_strategy
);
P_
DECLARE_int32
(
log_period
);
P_
DECLARE_int32
(
log_period_server
);
P_
DECLARE_double
(
checkgrad_eps
);
P_
DECLARE_int32
(
enable_parallel_vector
);
P_
DECLARE_bool
(
loadsave_parameters_in_pserver
);
P_
DECLARE_int32
(
beam_size
);
P_
DECLARE_bool
(
show_layer_stat
);
P_
DECLARE_string
(
predict_file
);
P_
DECLARE_bool
(
prev_batch_state
);
P_
DECLARE_string
(
init_model_path
);
DECLARE_bool
(
parallel_nn
);
DECLARE_int32
(
async_count
);
DECLARE_int32
(
port
);
DECLARE_int32
(
data_server_port
);
DECLARE_bool
(
use_gpu
);
DECLARE_int32
(
gpu_id
);
DECLARE_int32
(
trainer_count
);
DECLARE_int32
(
ports_num
);
DECLARE_int32
(
ports_num_for_sparse
);
DECLARE_string
(
nics
);
DECLARE_string
(
rdma_tcp
);
DECLARE_int32
(
trainer_id
);
DECLARE_int32
(
num_gradient_servers
);
DECLARE_string
(
comment
);
DECLARE_string
(
load_missing_parameter_strategy
);
DECLARE_int32
(
log_period
);
DECLARE_int32
(
log_period_server
);
DECLARE_double
(
checkgrad_eps
);
DECLARE_int32
(
enable_parallel_vector
);
DECLARE_bool
(
loadsave_parameters_in_pserver
);
DECLARE_int32
(
beam_size
);
DECLARE_bool
(
show_layer_stat
);
DECLARE_string
(
predict_file
);
DECLARE_bool
(
prev_batch_state
);
DECLARE_string
(
init_model_path
);
paddle/utils/Logging.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,175 +18,9 @@ limitations under the License. */
*/
#include "Logging.h"
#ifndef PADDLE_USE_GLOG
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mutex>
#include <thread>
#include <vector>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
namespace
paddle
{
namespace
internal
{
std
::
string
join
(
const
std
::
string
&
part1
,
const
std
::
string
&
part2
)
{
const
char
sep
=
'/'
;
if
(
!
part2
.
empty
()
&&
part2
.
front
()
==
sep
)
{
return
part2
;
}
std
::
string
ret
;
ret
.
reserve
(
part1
.
size
()
+
part2
.
size
()
+
1
);
ret
=
part1
;
if
(
!
ret
.
empty
()
&&
ret
.
back
()
!=
sep
)
{
ret
+=
sep
;
}
ret
+=
part2
;
return
ret
;
}
static
inline
bool
env2bool
(
const
char
*
envName
,
bool
defaultValue
=
false
)
{
char
*
envValue
=
getenv
(
envName
);
if
(
envValue
==
nullptr
)
{
return
defaultValue
;
}
else
{
return
memchr
(
"tTyY1
\0
"
,
envValue
[
0
],
6
)
!=
nullptr
;
}
}
static
inline
int
env2int
(
const
char
*
envName
,
int
defaultValue
=
0
)
{
char
*
envValue
=
getenv
(
envName
);
if
(
envValue
==
nullptr
)
{
return
defaultValue
;
}
else
{
int
retValue
=
defaultValue
;
try
{
retValue
=
std
::
stoi
(
envValue
);
}
catch
(...)
{
// pass
}
return
retValue
;
}
}
static
inline
int
env2index
(
const
char
*
envName
,
const
std
::
vector
<
std
::
string
>&
options
,
int
defaultValue
)
{
char
*
envValue
=
getenv
(
envName
);
if
(
envValue
==
nullptr
)
{
return
defaultValue
;
}
else
{
for
(
size_t
i
=
0
;
i
<
options
.
size
();
++
i
)
{
if
(
options
[
i
]
==
envValue
)
{
return
static_cast
<
int
>
(
i
);
}
}
return
defaultValue
;
}
}
static
bool
gLogToStderr
=
env2bool
(
"PLOG_LOGTOSTDERR"
,
true
);
static
const
std
::
vector
<
std
::
string
>
gLevelName
=
{
"INFO"
,
"WARNING"
,
"ERROR"
,
"FATAL"
};
static
int
gMinLogLevel
=
env2int
(
"PLOG_MINLOGLEVEL"
,
env2index
(
"PLOG_MINLOGLEVEL"
,
gLevelName
,
0
));
static
std
::
vector
<
std
::
vector
<
int
>>
gLogFds
;
static
std
::
vector
<
int
>
gLogFileFds
;
static
bool
gLogInited
=
false
;
static
void
freeLogFileFds
()
{
for
(
auto
fd
:
gLogFileFds
)
{
close
(
fd
);
}
}
static
void
initializeLogFds
(
char
*
argv0
)
{
gLogFds
.
resize
(
NUM_SEVERITIES
);
for
(
int
i
=
gMinLogLevel
;
i
<
NUM_SEVERITIES
&&
gLogToStderr
;
++
i
)
{
// Add stderr
std
::
vector
<
int
>&
fds
=
gLogFds
[
i
];
fds
.
push_back
(
STDERR_FILENO
);
}
char
*
logDir
=
getenv
(
"PLOG_LOGDIR"
);
for
(
int
i
=
gMinLogLevel
;
i
<
NUM_SEVERITIES
&&
logDir
!=
nullptr
;
++
i
)
{
std
::
string
filename
=
join
(
logDir
,
std
::
string
(
argv0
)
+
"."
+
gLevelName
[
i
]);
int
fd
=
open
(
filename
.
c_str
(),
O_CREAT
|
O_WRONLY
,
0644
);
if
(
fd
==
-
1
)
{
fprintf
(
stderr
,
"Open log file error!"
);
exit
(
1
);
}
gLogFileFds
.
push_back
(
fd
);
std
::
vector
<
int
>&
curFds
=
gLogFds
[
i
];
curFds
.
insert
(
curFds
.
end
(),
gLogFileFds
.
begin
(),
gLogFileFds
.
end
());
}
atexit
(
freeLogFileFds
);
gLogInited
=
true
;
}
static
void
(
*
gFailureFunctionPtr
)()
ATTR_NORETURN
=
abort
;
LogMessage
::
LogMessage
(
const
char
*
fname
,
int
line
,
int
severity
)
:
fname_
(
fname
),
line_
(
line
),
severity_
(
severity
)
{}
LogMessage
::~
LogMessage
()
{
this
->
generateLogMessage
();
}
void
LogMessage
::
generateLogMessage
()
{
if
(
!
gLogInited
)
{
fprintf
(
stderr
,
"%c %s:%d] %s
\n
"
,
"IWEF"
[
severity_
],
fname_
,
line_
,
str
().
c_str
());
}
else
{
for
(
auto
&
fd
:
gLogFds
[
this
->
severity_
])
{
dprintf
(
fd
,
"%c %s:%d] %s
\n
"
,
"IWEF"
[
severity_
],
fname_
,
line_
,
str
().
c_str
());
}
}
}
LogMessageFatal
::
LogMessageFatal
(
const
char
*
file
,
int
line
)
:
LogMessage
(
file
,
line
,
FATAL
)
{}
LogMessageFatal
::~
LogMessageFatal
()
{
generateLogMessage
();
gFailureFunctionPtr
();
}
}
// namespace internal
void
initializeLogging
(
int
argc
,
char
**
argv
)
{
internal
::
initializeLogFds
(
argv
[
0
]);
}
namespace
logging
{
void
setMinLogLevel
(
int
level
)
{
paddle
::
internal
::
gMinLogLevel
=
level
;
}
void
installFailureFunction
(
void
(
*
callback
)()
ATTR_NORETURN
)
{
paddle
::
internal
::
gFailureFunctionPtr
=
callback
;
}
}
// namespace logging
}
// namespace paddle
#else
namespace
paddle
{
void
initializeLogging
(
int
argc
,
char
**
argv
)
{
(
void
)(
argc
);
if
(
!
getenv
(
"GLOG_logtostderr"
))
{
...
...
@@ -197,13 +31,16 @@ void initializeLogging(int argc, char** argv) {
}
namespace
logging
{
void
setMinLogLevel
(
int
level
)
{
FLAGS_minloglevel
=
level
;
}
void
installFailureFunction
(
void
(
*
callback
)())
{
google
::
InstallFailureFunction
(
callback
);
}
void
installFailureWriter
(
void
(
*
callback
)(
const
char
*
,
int
))
{
google
::
InstallFailureWriter
(
callback
);
}
}
// namespace logging
}
// namespace paddle
#endif
paddle/utils/Logging.h
浏览文件 @
3d0e73bd
...
...
@@ -22,175 +22,21 @@ limitations under the License. */
#include <sstream>
#include <string>
#ifndef PADDLE_USE_GLOG
#include "CompilerMacros.h"
//! TODO(yuyang18): Move this utility macro into some global header.
#define PP_CAT(a, b) PP_CAT_I(a, b)
#define PP_CAT_I(a, b) PP_CAT_II(~, a##b)
#define PP_CAT_II(p, res) res
/**
* Generate Unique Variable Name, Usefully in macro.
* @SEE
* http://stackoverflow.com/questions/1082192/how-to-generate-random-variable-names-in-c-using-macros
*/
#define UNIQUE_NAME(base) PP_CAT(base, __LINE__)
#include <glog/logging.h>
namespace
paddle
{
//! Log levels.
const
int
INFO
=
0
;
const
int
WARNING
=
1
;
const
int
ERROR
=
2
;
const
int
FATAL
=
3
;
const
int
NUM_SEVERITIES
=
4
;
namespace
internal
{
class
LogMessage
:
public
std
::
basic_ostringstream
<
char
>
{
public:
LogMessage
(
const
char
*
fname
,
int
line
,
int
severity
);
~
LogMessage
();
protected:
/**
* @brief Print log message to stderr, files, etc.
*/
void
generateLogMessage
();
private:
const
char
*
fname_
;
int
line_
;
int
severity_
;
};
// LogMessageFatal ensures the process will exit in failure after
// logging this message.
class
LogMessageFatal
:
public
LogMessage
{
public:
LogMessageFatal
(
const
char
*
file
,
int
line
)
__attribute__
((
cold
));
~
LogMessageFatal
()
__attribute__
((
noreturn
));
};
#define _P_LOG_INFO \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::INFO)
#define _P_LOG_WARNING \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::WARNING)
#define _P_LOG_ERROR \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::ERROR)
#define _P_LOG_FATAL ::paddle::internal::LogMessageFatal(__FILE__, __LINE__)
#define P_LOG(severity) _P_LOG_##severity
#define P_LOG_FIRST_N(severity, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) ++UNIQUE_NAME(LOG_OCCURRENCES); \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) P_LOG(severity)
#define P_LOG_IF_EVERY_N(severity, condition, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (condition && ((UNIQUE_NAME(LOG_OCCURRENCES) = \
(UNIQUE_NAME(LOG_OCCURRENCES) + 1) % n) == (1 % n))) \
P_LOG(severity)
#define P_LOG_EVERY_N(severity, n) P_LOG_IF_EVERY_N(severity, true, n)
// TODO(jeff): Define a proper implementation of VLOG_IS_ON
#define P_VLOG_IS_ON(lvl) ((lvl) <= 0)
#define P_LOG_IF(severity, condition) \
if (condition) P_LOG(severity)
#define P_VLOG(lvl) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl))
#define P_VLOG_IF(lvl, cond) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl) && cond)
#define P_VLOG_EVERY_N(lvl, n) P_LOG_IF_EVERY_N(INFO, P_VLOG_IS_ON(lvl), n)
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
// CHECK dies with a fatal error if condition is not true. It is *not*
// controlled by NDEBUG, so the check will be executed regardless of
// compilation mode. Therefore, it is safe to do things like:
// CHECK(fp->Write(x) == 4)
#define P_CHECK(condition) \
if (PREDICT_FALSE(!(condition))) \
P_LOG(FATAL) << "Check failed: " #condition " "
#define P_CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define P_CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define P_CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define P_CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define P_CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define P_CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define P_CHECK_NOTNULL(val) P_CHECK((val) != NULL)
//! GLOG compatible APIs
//! NOTE: only implement Paddle actually used APIs.
#define LOG(x) P_LOG(x)
#define VLOG(x) P_VLOG(x)
#define DLOG(x) P_VLOG(5)
#define CHECK(x) P_CHECK(x)
#define PCHECK(x) P_CHECK(x)
#define CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define CHECK_NOTNULL(val) P_CHECK((val) != NULL)
#define VLOG_IS_ON(x) P_VLOG_IS_ON(x)
#define LOG_FIRST_N(severity, n) P_LOG_FIRST_N(severity, n)
#define LOG_IF(severity, condition) P_LOG_IF(severity, condition)
#define VLOG_EVERY_N(lvl, n) P_VLOG_EVERY_N(lvl, n)
#define VLOG_IF(lvl, cond) P_VLOG_IF(lvl, cond)
#define LOG_EVERY_N(severity, n) P_LOG_EVERY_N(severity, n)
}
// namespace internal
/**
* @brief initialize logging
* @note: Current implement of logging is lack of:
* PrintCallStack when fatal.
* VLOG_IS_ON
* But it is portable to multi-platform, and simple enough to modify.
*/
void
initializeLogging
(
int
argc
,
char
**
argv
);
namespace
logging
{
/**
* @brief Set Min Log Level. if Log.level < minLogLevel, then will not print log
* to stream
* @param level. Any integer is OK, but only 0 <= x <= NUM_SEVERITIES is useful.
*/
void
setMinLogLevel
(
int
level
);
/**
* @brief Install Log(Fatal) failure function. Default is abort();
* @param callback: The failure function.
*/
void
installFailureFunction
(
void
(
*
callback
)()
ATTR_NORETURN
);
/**
* @brief installFailureWriter
* @note: not implemented currently.
*/
inline
void
installFailureWriter
(
void
(
*
callback
)(
const
char
*
,
int
))
{
(
void
)(
callback
);
// unused callback.
}
}
// namespace logging
}
// namespace paddle
#else
#include <glog/logging.h>
namespace
paddle
{
void
initializeLogging
(
int
argc
,
char
**
argv
);
namespace
logging
{
void
setMinLogLevel
(
int
level
);
void
installFailureFunction
(
void
(
*
callback
)());
void
installFailureWriter
(
void
(
*
callback
)(
const
char
*
,
int
));
}
// namespace logging
}
#endif // PADDLE_USE_GLOG
}
// namespace logging
}
// namespace paddle
#ifndef NDEBUG
#define DEBUG_LEVEL 5
...
...
paddle/utils/PythonUtil.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,8 +20,8 @@ namespace paddle {
#ifdef PADDLE_NO_PYTHON
P_
DEFINE_string
(
python_path
,
""
,
"python path"
);
P_
DEFINE_string
(
python_bin
,
"python2.7"
,
"python bin"
);
DEFINE_string
(
python_path
,
""
,
"python path"
);
DEFINE_string
(
python_bin
,
"python2.7"
,
"python bin"
);
constexpr
int
kExecuteCMDBufLength
=
204800
;
...
...
paddle/utils/ThreadLocal.cpp
浏览文件 @
3d0e73bd
...
...
@@ -16,9 +16,9 @@ limitations under the License. */
#include "CommandLineParser.h"
#include "Util.h"
P_
DEFINE_bool
(
thread_local_rand_use_global_seed
,
false
,
"Whether to use global seed in thread local rand."
);
DEFINE_bool
(
thread_local_rand_use_global_seed
,
false
,
"Whether to use global seed in thread local rand."
);
namespace
paddle
{
...
...
paddle/utils/Util.cpp
浏览文件 @
3d0e73bd
...
...
@@ -33,7 +33,7 @@ limitations under the License. */
#include "ThreadLocal.h"
#include "Version.h"
P_
DEFINE_int32
(
seed
,
1
,
"random number seed. 0 for srand(time)"
);
DEFINE_int32
(
seed
,
1
,
"random number seed. 0 for srand(time)"
);
#ifdef WITH_GOOGLE_PERFTOOLS
/*
...
...
@@ -52,10 +52,8 @@ P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#include <gperftools/profiler.h>
P_DEFINE_int32
(
profile_signal
,
12
,
"signal for switch google profiler"
);
P_DEFINE_string
(
profile_data_file
,
"gperf.prof"
,
"file for storing profile data"
);
DEFINE_int32
(
profile_signal
,
12
,
"signal for switch google profiler"
);
DEFINE_string
(
profile_data_file
,
"gperf.prof"
,
"file for storing profile data"
);
static
void
profilerSwitch
(
int
signalNumber
)
{
bool
static
started
=
false
;
...
...
paddle/utils/Version.cpp
浏览文件 @
3d0e73bd
...
...
@@ -18,13 +18,8 @@ limitations under the License. */
#include <numeric>
#include "Flags.h"
#include "Util.h"
//! TODO(yuyang18) in gflags, version has another define. Use another flag
//! instead.
#ifndef PADDLE_USE_GFLAGS
P_DEFINE_bool
(
version
,
false
,
"print version"
);
#else
P_DECLARE_bool
(
version
);
#endif
DECLARE_bool
(
version
);
namespace
paddle
{
namespace
version
{
...
...
paddle/utils/tests/CMakeLists.txt
浏览文件 @
3d0e73bd
add_simple_unittest
(
test_CommandLineParser
)
add_simple_unittest
(
test_Logging
)
add_simple_unittest
(
test_Thread
)
add_simple_unittest
(
test_StringUtils
)
add_simple_unittest
(
test_CustomStackTrace
)
...
...
paddle/utils/tests/test_CommandLineParser.cpp
已删除
100644 → 0
浏览文件 @
be734a60
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#ifndef PADDLE_USE_GFLAGS
//! Test Command Line Parser for paddle internal implement.
#include <gtest/gtest.h>
#include <paddle/utils/CommandLineParser.h>
P_DEFINE_int32
(
i1
,
1
,
"test int flag 1"
);
P_DEFINE_int32
(
i2
,
2
,
"test int flag 2"
);
P_DEFINE_string
(
str1
,
"1"
,
"test str flag 1"
);
P_DEFINE_string
(
str2
,
"2"
,
"test str flag 2"
);
P_DEFINE_bool
(
b1
,
true
,
"test bool flag 1"
);
P_DEFINE_bool
(
b2
,
false
,
"test bool flag 2"
);
P_DEFINE_double
(
d1
,
0.1
,
"test double flag 1"
);
P_DEFINE_double
(
d2
,
-
42.3
,
"test double flag 2"
);
P_DEFINE_int64
(
l1
,
1
,
"test int64 flag 1"
);
P_DEFINE_int64
(
l2
,
2
,
"test int64 flag 2"
);
P_DEFINE_uint64
(
ul1
,
32
,
"test uint64 flag 1"
);
P_DEFINE_uint64
(
ul2
,
33
,
"test uint64 flag 2"
);
constexpr
double
EPSILON
=
1e-5
;
#define cc(x) const_cast<char*>((x))
TEST
(
CommandLineParser
,
defaultValue
)
{
char
*
argv
[]
=
{
cc
(
"test_program"
),
cc
(
"--unused_flag=134"
)};
int
argc
=
sizeof
(
argv
)
/
sizeof
(
char
*
);
paddle
::
ParseCommandLineFlags
(
&
argc
,
argv
);
// Check Default Value
ASSERT_EQ
(
argc
,
2
);
ASSERT_EQ
(
FLAGS_i1
,
1
);
ASSERT_EQ
(
FLAGS_i2
,
2
);
ASSERT_EQ
(
FLAGS_str1
,
"1"
);
ASSERT_EQ
(
FLAGS_str2
,
"2"
);
ASSERT_EQ
(
FLAGS_b1
,
true
);
ASSERT_EQ
(
FLAGS_b2
,
false
);
ASSERT_NEAR
(
FLAGS_d1
,
0.1
,
EPSILON
);
ASSERT_NEAR
(
FLAGS_d2
,
-
42.3
,
EPSILON
);
ASSERT_EQ
(
FLAGS_i1
,
1
);
ASSERT_EQ
(
FLAGS_i2
,
2
);
ASSERT_EQ
(
FLAGS_ul1
,
32UL
);
ASSERT_EQ
(
FLAGS_ul2
,
33UL
);
}
TEST
(
CommandLineParser
,
normal
)
{
char
*
argv
[]
=
{
cc
(
"test_program"
),
cc
(
"--i2=32"
),
cc
(
"--str1=abc"
),
cc
(
"--b2=1"
),
cc
(
"-b1=False"
),
cc
(
"--d2=.34"
),
cc
(
"--d1=0"
),
cc
(
"--l1=-12345678901234"
),
cc
(
"-ul2=3212"
)};
int
argc
=
sizeof
(
argv
)
/
sizeof
(
char
*
);
paddle
::
ParseCommandLineFlags
(
&
argc
,
argv
);
ASSERT_EQ
(
argc
,
1
);
ASSERT_EQ
(
FLAGS_i2
,
32
);
ASSERT_EQ
(
FLAGS_str1
,
"abc"
);
ASSERT_EQ
(
FLAGS_b2
,
true
);
ASSERT_EQ
(
FLAGS_b1
,
false
);
ASSERT_NEAR
(
FLAGS_d2
,
0.34
,
EPSILON
);
ASSERT_NEAR
(
FLAGS_d1
,
0.0
,
EPSILON
);
ASSERT_EQ
(
FLAGS_l1
,
-
12345678901234
);
ASSERT_EQ
(
FLAGS_ul2
,
3212UL
);
}
TEST
(
CommandLineParser
,
printHelp
)
{
char
*
argv
[]
=
{
cc
(
"test_program"
),
cc
(
"--help"
)};
int
argc
=
sizeof
(
argv
)
/
sizeof
(
char
*
);
// Will Print Usage
ASSERT_DEATH
(
paddle
::
ParseCommandLineFlags
(
&
argc
,
argv
),
".*test_program.*"
);
}
TEST
(
CommandLineParser
,
parseError
)
{
char
*
argv
[]
=
{
cc
(
"test_program"
),
cc
(
"--i1=abc"
)};
int
argc
=
sizeof
(
argv
)
/
sizeof
(
char
*
);
ASSERT_DEATH
(
paddle
::
ParseCommandLineFlags
(
&
argc
,
argv
),
"Parse command flag i1 error! User input is --i1=abc.*test_program.*"
);
}
int
main
(
int
argc
,
char
**
argv
)
{
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
}
#else
int
main
(
int
argc
,
char
**
argv
)
{
return
0
;
}
#endif
paddle/utils/tests/test_CustomStackTrace.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Locks.h"
#include "paddle/utils/Util.h"
P_
DEFINE_int32
(
test_thread_num
,
10
,
"testing thread number"
);
DEFINE_int32
(
test_thread_num
,
10
,
"testing thread number"
);
void
testNormalImpl
(
const
std
::
function
<
void
(
paddle
::
CustomStackTrace
<
std
::
string
>&
,
...
...
paddle/utils/tests/test_Logging.cpp
已删除
100644 → 0
浏览文件 @
be734a60
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
/*
* Basically from tensorflow/core/platform/default/logging.cc
* Used in embedded system where there is no glogs.
*/
#include <dirent.h>
#include <gtest/gtest.h>
#include <stdlib.h>
#include <fstream>
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
#ifndef PADDLE_USE_GLOG
TEST
(
Logging
,
BasicalLog
)
{
auto
pinfo
=
[]
{
P_LOG
(
INFO
)
<<
"INFO"
;
exit
(
1
);
};
ASSERT_DEATH
(
pinfo
(),
"I .*test_Logging.cpp:[0-9]+] INFO"
);
auto
pwarn
=
[]
{
P_LOG
(
WARNING
)
<<
"WARN"
;
exit
(
1
);
};
ASSERT_DEATH
(
pwarn
(),
"W .*test_Logging.cpp:[0-9]+] WARN"
);
auto
perr
=
[]
{
P_LOG
(
ERROR
)
<<
"ERROR"
;
exit
(
1
);
};
ASSERT_DEATH
(
perr
(),
"E .*test_Logging.cpp:[0-9]+] ERROR"
);
auto
pfatal
=
[]
{
P_LOG
(
FATAL
)
<<
"FATAL"
;
};
ASSERT_DEATH
(
pfatal
(),
"F .*test_Logging.cpp:[0-9]+] FATAL"
);
}
TEST
(
Logging
,
Check
)
{
int
a
=
1
;
int
b
=
2
;
P_CHECK
(
a
!=
b
);
auto
pcheckDown
=
[
&
]
{
P_CHECK
(
a
==
b
);
};
ASSERT_DEATH
(
pcheckDown
(),
"F .*test_Logging.cpp:[0-9]+] Check failed: a == b "
);
P_CHECK_LE
(
a
,
b
);
P_CHECK_LT
(
a
,
b
);
double
t
=
1.2
;
P_CHECK_LE
(
a
,
t
);
double
*
ptr
=
nullptr
;
auto
pcheckDown2
=
[
&
]
{
P_CHECK_NOTNULL
(
ptr
);
};
ASSERT_DEATH
(
pcheckDown2
(),
"F"
);
}
#define cc(x) const_cast<char*>(x)
TEST
(
Logging
,
LogToStderr
)
{
auto
logToStderrCallback
=
[]
{
setenv
(
"PLOG_LOGTOSTDERR"
,
"0"
,
true
);
char
*
argv
[]
=
{
cc
(
"test"
)};
paddle
::
initializeLogging
(
1
,
argv
);
P_LOG
(
INFO
)
<<
"This output will not print to std error"
;
exit
(
1
);
};
ASSERT_DEATH
(
logToStderrCallback
(),
""
);
}
constexpr
char
kLogDirName
[]
=
"./test_log_dir"
;
const
std
::
vector
<
std
::
string
>
kLevels
=
{
"INFO"
,
"WARNING"
,
"ERROR"
,
"FATAL"
};
TEST
(
Logging
,
LogToDir
)
{
ASSERT_EQ
(
0
,
mkdir
(
kLogDirName
,
0777
));
auto
logToDirCallback
=
[]
{
setenv
(
"PLOG_LOGTOSTDERR"
,
"0"
,
true
);
setenv
(
"PLOG_LOGDIR"
,
kLogDirName
,
true
);
char
*
argv
[]
=
{
cc
(
"test"
)};
paddle
::
initializeLogging
(
1
,
argv
);
P_LOG
(
INFO
)
<<
"INFO"
;
P_LOG
(
WARNING
)
<<
"WARNING"
;
P_LOG
(
ERROR
)
<<
"ERROR"
;
P_LOG
(
FATAL
)
<<
"FATAL"
;
};
ASSERT_DEATH
(
logToDirCallback
(),
""
);
// There 4 file in logdir
auto
dir
=
opendir
(
kLogDirName
);
size_t
fileCount
=
0
;
std
::
vector
<
std
::
string
>
filenames
;
for
(
auto
dirContent
=
readdir
(
dir
);
dirContent
!=
nullptr
;
dirContent
=
readdir
(
dir
))
{
std
::
string
filename
(
dirContent
->
d_name
);
if
(
filename
==
"."
||
filename
==
".."
)
{
continue
;
}
else
{
++
fileCount
;
for
(
size_t
i
=
0
;
i
<
kLevels
.
size
();
++
i
)
{
const
std
::
string
&
curLevel
=
kLevels
[
i
];
if
(
filename
.
size
()
>
curLevel
.
length
())
{
size_t
diff
=
filename
.
size
()
-
curLevel
.
length
();
size_t
j
=
0
;
for
(;
j
<
curLevel
.
length
();
++
j
)
{
if
(
filename
[
j
+
diff
]
!=
curLevel
[
j
])
{
// File Suffix Not Same, then break.
break
;
}
}
if
(
j
==
curLevel
.
length
())
{
// Same suffix.
std
::
ifstream
fin
;
auto
fn
=
paddle
::
path
::
join
(
kLogDirName
,
filename
);
fin
.
open
(
fn
);
filenames
.
push_back
(
fn
);
ASSERT_TRUE
(
fin
.
is_open
());
size_t
lineCounter
=
0
;
for
(
std
::
string
line
;
std
::
getline
(
fin
,
line
);
++
lineCounter
)
{
// Do Nothing, Just calc lineCounter.
}
// For example.
// The info channel will have all log which level >= INFO
// So the info file's lineCounter should == 4.
ASSERT_EQ
(
kLevels
.
size
()
-
i
,
lineCounter
);
fin
.
close
();
}
}
}
}
}
closedir
(
dir
);
ASSERT_EQ
(
4UL
,
fileCount
);
// 4 levels.
// Clean Unittest.
for
(
std
::
string
&
fn
:
filenames
)
{
ASSERT_EQ
(
remove
(
fn
.
c_str
()),
0
);
}
ASSERT_EQ
(
rmdir
(
kLogDirName
),
0
);
}
int
main
(
int
argc
,
char
**
argv
)
{
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
}
#else
int
main
(
int
,
char
**
)
{
return
0
;
}
#endif
paddle/utils/tests/test_SpinLock.cpp
浏览文件 @
3d0e73bd
...
...
@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_
DEFINE_int32
(
test_thread_num
,
100
,
"testing thread number"
);
DEFINE_int32
(
test_thread_num
,
100
,
"testing thread number"
);
void
testNormalImpl
(
size_t
thread_num
,
...
...
paddle/utils/tests/test_ThreadBarrier.cpp
浏览文件 @
3d0e73bd
...
...
@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_
DEFINE_int32
(
test_thread_num
,
100
,
"testing thread number"
);
DEFINE_int32
(
test_thread_num
,
100
,
"testing thread number"
);
void
testNormalImpl
(
size_t
thread_num
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录