Skip to content

P
Paddle

csdn_franckjun / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle

体验新版 GitCode，发现更多精彩内容 >>

提交 3d0e73bd 编写于 作者: L liaogang
浏览文件
操作

Remove custom glog-like and gflags-like macros

上级 be734a60
隐藏空白更改
内联 并排
Showing with 408 addition and 1396 deletion
CMakeLists.txt
浏览文件 @ 3d0e73bd
......@@ -25,8 +25,8 @@ find_package(ZLIB REQUIRED)
find_package(NumPy REQUIRED)
find_package(Threads REQUIRED)
find_package(AVX QUIET)
find_package(Glog)
find_package(Gflags QUIET)
find_package(Glog REQUIRED)
find_package(Gflags REQUIRED)
find_package(GTest)
find_package(Sphinx)
find_package(Doxygen)
......@@ -40,8 +40,6 @@ option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND})
option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
option(WITH_GLOG "Compile PaddlePaddle use glog, otherwise use a log implement internally" ${LIBGLOG_FOUND})
option(WITH_GFLAGS "Compile PaddlePaddle use gflags, otherwise use a flag implement internally" ${GFLAGS_FOUND})
option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND})
......@@ -136,16 +134,12 @@ else(WITH_RDMA)
add_definitions(-DPADDLE_DISABLE_RDMA)
endif(WITH_RDMA)
if(WITH_GLOG)
add_definitions(-DPADDLE_USE_GLOG)
include_directories(${LIBGLOG_INCLUDE_DIR})
endif()
# glog
include_directories(${LIBGLOG_INCLUDE_DIR})
if(WITH_GFLAGS)
add_definitions(-DPADDLE_USE_GFLAGS)
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
endif()
#gflags
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
include_directories(${GFLAGS_INCLUDE_DIRS})
if(WITH_TESTING)
enable_testing()
......
paddle/api/Trainer.cpp
浏览文件 @ 3d0e73bd
......@@ -27,9 +27,9 @@ limitations under the License. */
using paddle::real;
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
struct TrainerPrivate : public paddle::Trainer {
bool _trainOneBatch(size_t batchSize);
......
paddle/cuda/src/hl_cuda_cudnn.cc
浏览文件 @ 3d0e73bd
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
DEFINE_int32(cudnn_conv_workspace_limit_in_mb,
4096,
"Specify cuDNN max workspace limit, in units MB, "
"4096MB=4GB by default.");
namespace dynload {
......
paddle/cuda/src/hl_dso_loader.cc
浏览文件 @ 3d0e73bd
......@@ -16,21 +16,21 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
P_DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
P_DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
DEFINE_string(cudnn_dir,
"",
"Specify path for loading libcudnn.so. For instance, "
"/usr/local/cudnn/lib. If empty [default], dlopen "
"will search cudnn from LD_LIBRARY_PATH");
DEFINE_string(cuda_dir,
"",
"Specify path for loading cuda library, such as libcublas, "
"libcurand. For instance, /usr/local/cuda/lib64. (Note: "
"libcudart can not be specified by cuda_dir, since some "
"build-in function in cudart already ran before main entry). "
"If default, dlopen will search cuda from LD_LIBRARY_PATH");
DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so.");
static inline std::string join(const std::string& part1,
const std::string& part2) {
......
paddle/gserver/dataproviders/ProtoDataProvider.cpp
浏览文件 @ 3d0e73bd
......@@ -22,9 +22,9 @@ limitations under the License. */
#include "DataProviderGroup.h"
#include "paddle/utils/Logging.h"
P_DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
DEFINE_double(memory_threshold_on_load_data,
1.0,
"stop loading data when memory is not sufficient");
namespace paddle {
......
paddle/gserver/evaluators/Evaluator.cpp
浏览文件 @ 3d0e73bd
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/gradientmachines/MultiGradientMachine.cpp
浏览文件 @ 3d0e73bd
......@@ -21,11 +21,11 @@ limitations under the License. */
#include "NeuralNetwork.h"
#include "ParallelNeuralNetwork.h"
P_DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
DEFINE_bool(allow_only_one_model_on_one_gpu,
true,
"If true, do not allow multiple models on one GPU device");
#ifdef PADDLE_METRIC_LEARNING
P_DECLARE_bool(external);
DECLARE_bool(external);
#endif
namespace paddle {
......
paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp
浏览文件 @ 3d0e73bd
......@@ -24,7 +24,7 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
DEFINE_string(diy_beam_search_prob_so, "", "the diy beam search cost so");
static const char* DIY_CALC_PROB_SYMBOL_NAME = "calc_prob";
static const char* DIY_START_CALC_PROB_SYMBOL_NAME = "start_calc_prob";
......
paddle/gserver/layers/Layer.cpp
浏览文件 @ 3d0e73bd
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "TransLayer.h"
#include "ValidationLayer.h"
P_DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
DEFINE_bool(log_error_clipping, false, "enable log error clipping or not");
namespace paddle {
......
paddle/gserver/layers/LstmLayer.cpp
浏览文件 @ 3d0e73bd
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/Matrix.h"
#include "paddle/utils/Stat.h"
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(prev_batch_state);
namespace paddle {
......
paddle/gserver/layers/RecurrentLayer.cpp
浏览文件 @ 3d0e73bd
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
DEFINE_bool(rnn_use_batch, false, "Using the batch method for calculation.");
namespace paddle {
......
paddle/gserver/layers/ValidationLayer.h
浏览文件 @ 3d0e73bd
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "Layer.h"
#include "paddle/gserver/evaluators/Evaluator.h"
P_DECLARE_int32(trainer_id);
DECLARE_int32(trainer_id);
namespace paddle {
......
paddle/gserver/tests/LayerGradUtil.cpp
浏览文件 @ 3d0e73bd
......@@ -14,7 +14,7 @@ limitations under the License. */
#include "LayerGradUtil.h"
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(thread_local_rand_use_global_seed);
namespace paddle {
real getCostSum(LayerPtr& testLayer, MatrixPtr weights) {
......
paddle/gserver/tests/TestUtil.cpp
浏览文件 @ 3d0e73bd
......@@ -17,7 +17,7 @@ limitations under the License. */
#include "paddle/math/SparseMatrix.h"
#include "paddle/utils/CommandLineParser.h"
P_DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
DEFINE_int32(fixed_seq_length, 0, "Produce some sequence of fixed length");
namespace paddle {
......
paddle/gserver/tests/test_ActivationGrad.cpp
浏览文件 @ 3d0e73bd
......@@ -25,8 +25,8 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_bool(thread_local_rand_use_global_seed);
void testActivation(const string& act) {
LOG(INFO) << "test activation: " << act;
......
paddle/gserver/tests/test_BatchNorm.cpp
浏览文件 @ 3d0e73bd
......@@ -27,11 +27,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the batchNormLayer can be followed by a ConvLayer
TEST(Layer, batchNorm) {
......
paddle/gserver/tests/test_ConvTrans.cpp
浏览文件 @ 3d0e73bd
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Test that the convTrans forward is the same as conv backward
TEST(Layer, convTransLayerFwd) {
......
paddle/gserver/tests/test_ConvUnify.cpp
浏览文件 @ 3d0e73bd
......@@ -28,11 +28,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
// Do one forward pass of convTrans layer and check to see if its output
// matches the given result
......
paddle/gserver/tests/test_Evaluator.cpp
浏览文件 @ 3d0e73bd
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_bool(thread_local_rand_use_global_seed);
enum InputType {
INPUT_DATA, // dense vector
......
paddle/gserver/tests/test_LayerGrad.cpp
浏览文件 @ 3d0e73bd
......@@ -26,11 +26,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_bool(prev_batch_state);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_bool(prev_batch_state);
TEST(Operator, dot_mul) {
TestConfig config;
......
paddle/gserver/tests/test_NetworkCompare.cpp
浏览文件 @ 3d0e73bd
......@@ -25,10 +25,10 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
P_DECLARE_double(checkgrad_eps);
P_DEFINE_bool(use_label, true, "input label or sequence label");
P_DEFINE_bool(static_para, false, "static parameter");
DECLARE_int32(gpu_id);
DECLARE_double(checkgrad_eps);
DEFINE_bool(use_label, true, "input label or sequence label");
DEFINE_bool(static_para, false, "static parameter");
struct DataIn {
std::vector<Argument> inArgs;
......@@ -267,8 +267,8 @@ TEST(Compare, img_conv2) {
}
#endif
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
TEST(Compare, network) {
if (FLAGS_config_file_a != "" && FLAGS_config_file_b != "") {
compareNetwork(FLAGS_config_file_a, FLAGS_config_file_b);
......
paddle/gserver/tests/test_PyDataProvider2.cpp
浏览文件 @ 3d0e73bd
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/PythonUtil.h"
#include "paddle/utils/Util.h"
P_DEFINE_string(train_list, "unittest.list", "file list for unittest");
DEFINE_string(train_list, "unittest.list", "file list for unittest");
namespace paddle {
namespace unittest {
......
paddle/gserver/tests/test_RecurrentGradientMachine.cpp
浏览文件 @ 3d0e73bd
......@@ -20,7 +20,7 @@ limitations under the License. */
#include <paddle/utils/Util.h>
#include <paddle/utils/Version.h>
P_DECLARE_int32(seed);
DECLARE_int32(seed);
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/gserver/tests/test_RecurrentLayer.cpp
浏览文件 @ 3d0e73bd
......@@ -23,9 +23,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_bool(rnn_use_batch);
P_DECLARE_int32(fixed_seq_length);
DECLARE_bool(use_gpu);
DECLARE_bool(rnn_use_batch);
DECLARE_int32(fixed_seq_length);
void checkError(const Matrix& matrix1, const Matrix& matrix2) {
CHECK(matrix1.getHeight() == matrix2.getHeight());
......
paddle/gserver/tests/test_SelectiveFCLayer.cpp
浏览文件 @ 3d0e73bd
......@@ -29,11 +29,11 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(num_passes);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(config_args);
DECLARE_bool(use_gpu);
DECLARE_int32(num_passes);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(config_args);
size_t fcLayerWidth = 1024;
......
paddle/gserver/tests/test_WarpCTCLayer.cpp
浏览文件 @ 3d0e73bd
......@@ -25,7 +25,7 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_bool(use_gpu);
DECLARE_bool(use_gpu);
const real* getData(const Matrix& matrix) {
if (matrix.useGpu()) {
......
paddle/math/SparseRowMatrix.cpp
浏览文件 @ 3d0e73bd
......@@ -24,9 +24,9 @@ limitations under the License. */
#include "paddle/utils/Thread.h"
#include "paddle/utils/Util.h"
P_DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
DEFINE_bool(allow_inefficient_sparse_update,
false,
"Whether to allow inefficient sparse update");
namespace paddle {
......
paddle/math/SparseRowMatrix.h
浏览文件 @ 3d0e73bd
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Util.h"
P_DECLARE_bool(allow_inefficient_sparse_update);
DECLARE_bool(allow_inefficient_sparse_update);
namespace paddle {
......
paddle/math/Storage.cpp
浏览文件 @ 3d0e73bd
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "Allocator.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
DEFINE_int32(pool_limit_size,
536870912,
"maximum memory size managed by a memory pool, default is 512M");
namespace paddle {
......
paddle/math/tests/test_TrainingAlgorithm.cpp
浏览文件 @ 3d0e73bd
......@@ -22,9 +22,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
#ifndef PADDLE_TYPE_DOUBLE
P_DEFINE_double(max_diff, 1e-5, "max diff allowed");
DEFINE_double(max_diff, 1e-5, "max diff allowed");
#else
P_DEFINE_double(max_diff, 1e-13, "max diff allowed");
DEFINE_double(max_diff, 1e-13, "max diff allowed");
#endif
class SetMaxDiff {
......
paddle/parameter/FirstOrderOptimizer.cpp
浏览文件 @ 3d0e73bd
......@@ -19,7 +19,7 @@ limitations under the License. */
#include <cmath>
P_DEFINE_bool(log_clipping, false, "enable log clipping or not");
DEFINE_bool(log_clipping, false, "enable log clipping or not");
namespace paddle {
......
paddle/parameter/Parameter.cpp
浏览文件 @ 3d0e73bd
......@@ -26,11 +26,11 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Logging.h"
P_DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
P_DEFINE_int32(
DEFINE_int32(enable_grad_share,
(100 * 1024 * 1024),
"threshold for enable gradient parameter share for batch "
"multi-cpu training");
DEFINE_int32(
grad_share_block_num,
64,
"block number of gradient parameter share for batch multi-cpu training");
......
paddle/pserver/BaseClient.cpp
浏览文件 @ 3d0e73bd
......@@ -18,7 +18,7 @@ limitations under the License. */
#include "paddle/utils/CommandLineParser.h"
#include "paddle/utils/Stat.h"
P_DECLARE_string(pservers);
DECLARE_string(pservers);
namespace paddle {
......
paddle/pserver/LightNetwork.cpp
浏览文件 @ 3d0e73bd
......@@ -31,23 +31,23 @@ limitations under the License. */
#include "paddle/utils/Util.h"
/// quick ack can reduce the latency of small message
P_DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
DEFINE_bool(small_messages,
false,
"if message size is small, recommend set it True to enable quick "
"ack and no delay");
/// reasonable sock_send_buf_size can control the traffic injected into switch
/// network. Injecting too many data into traffic could cause packets loss which
/// cause long latency and degrade the efficiency of communication.
P_DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
DEFINE_int32(sock_send_buf_size,
1024 * 1024 * 40,
"restrict sock send buff size, can reduce network congestion if "
"set carefully");
/// reasonable size can hold bursted packets and reduce packets loss
P_DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
DEFINE_int32(sock_recv_buf_size,
1024 * 1024 * 40,
"restrict sock recv buff size");
namespace paddle {
......
paddle/pserver/ParameterClient2.cpp
浏览文件 @ 3d0e73bd
......@@ -20,8 +20,8 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/StringUtil.h"
P_DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
P_DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
DEFINE_string(pservers, "127.0.0.1", "Comma separated addresses of pservers");
DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
namespace paddle {
......
paddle/pserver/ParameterClient2.h
浏览文件 @ 3d0e73bd
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "ProtoServer.h"
#include "SparseParameterDistribution.h"
P_DECLARE_int32(parallel_thread_num);
DECLARE_int32(parallel_thread_num);
namespace paddle {
......
paddle/pserver/ParameterServer2.cpp
浏览文件 @ 3d0e73bd
......@@ -30,11 +30,11 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
P_DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
P_DEFINE_double(
DEFINE_int32(pserver_num_threads, 1, "number of threads for sync op exec");
DEFINE_double(async_lagged_ratio_min,
1.0,
"control config_.async_lagged_grad_discard_ratio() min value");
DEFINE_double(
async_lagged_ratio_default,
1.5,
"if async_lagged_grad_discard_ratio is not set in trainer_config.conf"
......
paddle/pserver/ParameterServer2.h
浏览文件 @ 3d0e73bd
......@@ -38,7 +38,7 @@ limitations under the License. */
#include "ProtoServer.h"
P_DECLARE_int32(port);
DECLARE_int32(port);
namespace paddle {
......
paddle/pserver/SparseParameterDistribution.cpp
浏览文件 @ 3d0e73bd
......@@ -20,26 +20,26 @@ limitations under the License. */
#include "SparseParameterDistribution.h"
P_DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
P_DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
P_DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
P_DEFINE_double(
DEFINE_bool(check_sparse_distribution_in_pserver,
false,
"check whether sparse parameter exhibts balanced distribution at "
"all pservers");
DEFINE_bool(show_check_sparse_distribution_log,
false,
"show logs details for sparse parameter distribution in pserver");
DEFINE_int32(check_sparse_distribution_batches,
100,
"run sparse parameter distribution check for N batches");
DEFINE_double(
check_sparse_distribution_ratio,
0.6,
"if parameters dispatched to different pservers exhibit unbalanced "
" distribution for check_sparse_distribution_ratio * "
" check_sparse_distribution_batches times, crash program");
P_DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
DEFINE_double(check_sparse_distribution_unbalance_degree,
2.0,
"the ratio of maximum data size and minimun data size for "
"different pserver");
namespace paddle {
......
paddle/pserver/test/SocketTest.cpp
浏览文件 @ 3d0e73bd
......@@ -195,9 +195,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) {
channel_.reset(new SocketChannel(sockfd));
}
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 10000000, "Data size");
P_DEFINE_int32(loop_time, 100000, "test loop time");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 10000000, "Data size");
DEFINE_int32(loop_time, 100000, "test loop time");
using namespace paddle; // NOLINT
......
paddle/pserver/test/test_ParameterServer2.cpp
浏览文件 @ 3d0e73bd
......@@ -21,9 +21,9 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(num_gradient_servers);
P_DEFINE_string(server_addr, "127.0.0.1", "assign server address");
P_DEFINE_int32(server_cpu, 0, "assign server cpu");
DECLARE_int32(num_gradient_servers);
DEFINE_string(server_addr, "127.0.0.1", "assign server address");
DEFINE_int32(server_cpu, 0, "assign server cpu");
class ParameterServer2Tester : public ParameterServer2 {
public:
......
paddle/pserver/test/test_ProtoServer.cpp
浏览文件 @ 3d0e73bd
......@@ -21,10 +21,10 @@ limitations under the License. */
#include "paddle/pserver/ProtoServer.h"
#include "paddle/utils/Stat.h"
P_DEFINE_string(server_addr, "127.0.0.1", "Server address");
P_DEFINE_int64(dim, 50000000, "Data size");
P_DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
P_DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
DEFINE_string(server_addr, "127.0.0.1", "Server address");
DEFINE_int64(dim, 50000000, "Data size");
DEFINE_bool(test_proto_server, true, "whether to test ProtoServer");
DEFINE_bool(benchmark, false, "Do benchmark. Skip some tests");
using namespace paddle; // NOLINT
......
paddle/trainer/MergeModel.cpp
浏览文件 @ 3d0e73bd
......@@ -19,8 +19,8 @@ limitations under the License. */
#include "paddle/pserver/ParameterServer2.h"
#include "paddle/utils/PythonUtil.h"
P_DEFINE_string(model_dir, "", "Directory for separated model files");
P_DEFINE_string(model_file, "", "File for merged model file");
DEFINE_string(model_dir, "", "Directory for separated model files");
DEFINE_string(model_file, "", "File for merged model file");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/RemoteParameterUpdater.cpp
浏览文件 @ 3d0e73bd
......@@ -17,8 +17,8 @@ limitations under the License. */
#include "paddle/utils/GlobalConstants.h"
#include "paddle/utils/Stat.h"
P_DECLARE_int32(trainer_id);
P_DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_string(save_dir);
namespace paddle {
......
paddle/trainer/ThreadParameterUpdater.cpp
浏览文件 @ 3d0e73bd
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/math/SparseRowMatrix.h"
#include "paddle/utils/Thread.h"
P_DECLARE_int32(trainer_count);
DECLARE_int32(trainer_count);
namespace paddle {
......
paddle/trainer/Trainer.cpp
浏览文件 @ 3d0e73bd
......@@ -38,60 +38,56 @@ limitations under the License. */
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
#include "paddle/gserver/layers/ValidationLayer.h"
P_DEFINE_string(config, "", "Trainer config file");
P_DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
P_DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
P_DEFINE_bool(local, true, "Train in local mode or not");
P_DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
P_DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
P_DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
P_DEFINE_string(save_dir, "", "Directory for saving model parameter");
P_DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
P_DEFINE_int32(test_pass,
-1,
"Will load parameter start from this pass to test");
P_DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
P_DEFINE_bool(with_cost, true, "enable cost layer or not");
P_DEFINE_bool(distribute_test, false, "test in distribute mode");
P_DEFINE_int32(num_passes, 100, "train for so many passes");
P_DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
P_DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
P_DEFINE_string(feat_file, "", "File name of extracted feature.");
P_DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
P_DEFINE_string(model_list,
"",
"File that saves the model list when evaluation");
DEFINE_string(config, "", "Trainer config file");
DEFINE_int32(test_period,
0,
"if equal 0, do test on all test data at the end of "
"each pass. While if equal non-zero, do test on all test "
"data every test_period batches");
DEFINE_bool(test_all_data_in_one_period,
false,
"This option was deprecated, since we will always do "
"test on all test set ");
DEFINE_bool(local, true, "Train in local mode or not");
DEFINE_int32(average_test_period,
0,
"Do test on average parameter every so"
" many batches. MUST be devided by FLAGS_log_period."
" Default 0 means do not test average parameter");
DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
DEFINE_int64(saving_period_by_batches,
0,
"Save parameters every so many batches in one pass");
DEFINE_string(save_dir, "", "Directory for saving model parameter");
DEFINE_int32(start_pass,
0,
"Start training from this pass. "
"Will load parameter from the previous pass");
DEFINE_int32(test_pass, -1, "Will load parameter start from this pass to test");
DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
DEFINE_bool(with_cost, true, "enable cost layer or not");
DEFINE_bool(distribute_test, false, "test in distribute mode");
DEFINE_int32(num_passes, 100, "train for so many passes");
DEFINE_string(config_args,
"",
"arguments passed to config file."
"Format: key1=value1,key2=value2");
DEFINE_bool(save_only_one,
false,
"Save only parameters in last pass, remove previous.");
DEFINE_string(feat_file, "", "File name of extracted feature.");
DEFINE_string(predict_output_dir,
"",
"Directory that saves the predicted results of output layers");
DEFINE_string(model_list, "", "File that saves the model list when evaluation");
namespace paddle {
......
paddle/trainer/Trainer.h
浏览文件 @ 3d0e73bd
......@@ -34,7 +34,7 @@ limitations under the License. */
#include "paddle/internals/metric_learning/MetricTrainer.h"
#endif
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
namespace paddle {
......
paddle/trainer/TrainerBenchmark.cpp
浏览文件 @ 3d0e73bd
......@@ -18,9 +18,9 @@ limitations under the License. */
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"
P_DECLARE_int32(test_period);
DECLARE_int32(test_period);
P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
DEFINE_bool(feed_data, false, "Wether to read data from DataProvider.");
namespace paddle {
......
paddle/trainer/TrainerConfigHelper.cpp
浏览文件 @ 3d0e73bd
......@@ -18,16 +18,16 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/PythonUtil.h"
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_int32(start_pass);
P_DECLARE_string(save_dir);
P_DECLARE_int32(trainer_id);
P_DECLARE_bool(local);
P_DECLARE_bool(with_cost);
P_DECLARE_bool(with_gpu);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_int32(start_pass);
DECLARE_string(save_dir);
DECLARE_int32(trainer_id);
DECLARE_bool(local);
DECLARE_bool(with_cost);
DECLARE_bool(with_gpu);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
const char *kConfigParserModuleName = "paddle.trainer.config_parser";
const char *kConfigParserFuncName = "parse_config_and_serialize";
......
paddle/trainer/TrainerInternalConfig.cpp
浏览文件 @ 3d0e73bd
......@@ -14,17 +14,17 @@ limitations under the License. */
#include "TrainerInternalConfig.h"
P_DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
DEFINE_int32(show_parameter_stats_period,
0,
"Whether to show parameter stats during training");
P_DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
DEFINE_int32(dot_period, 1, "Print '.' every so many batches");
P_DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
DEFINE_bool(use_old_updater, false, "Use the old RemoteParameterUpdater");
P_DECLARE_int32(num_passes);
DECLARE_int32(num_passes);
P_DECLARE_bool(local);
DECLARE_bool(local);
namespace paddle {
......
paddle/trainer/TrainerMain.cpp
浏览文件 @ 3d0e73bd
......@@ -22,21 +22,20 @@ limitations under the License. */
#include "Trainer.h"
#include "paddle/pserver/RDMANetwork.h"
P_DEFINE_bool(start_pserver, false, "Whether to start pserver");
P_DECLARE_int32(gpu_id);
P_DEFINE_string(job, "train", "one of (train, test, checkgrad)");
P_DECLARE_int32(start_pass);
P_DECLARE_string(config);
P_DECLARE_string(init_model_path);
P_DECLARE_string(rdma_tcp);
DEFINE_bool(start_pserver, false, "Whether to start pserver");
DECLARE_int32(gpu_id);
DEFINE_string(job, "train", "one of (train, test, checkgrad)");
DECLARE_int32(start_pass);
DECLARE_string(config);
DECLARE_string(init_model_path);
DECLARE_string(rdma_tcp);
using namespace paddle; // NOLINT
int main(int argc, char** argv) {
// write logs instantly (never buffer log messages)
#ifdef PADDLE_USE_GLOG
// write logs instantly (never buffer log messages)
FLAGS_logbuflevel = -1;
#endif
initMain(argc, argv);
initPython(argc, argv);
......
paddle/trainer/tests/test_Compare.cpp
浏览文件 @ 3d0e73bd
......@@ -24,10 +24,10 @@ using namespace std; // NOLINT
static const string& configFile = "trainer/tests/sample_trainer_config.conf";
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(config_args);
DECLARE_int32(gpu_id);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_string(config_args);
struct comData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareSparse.cpp
浏览文件 @ 3d0e73bd
......@@ -25,22 +25,22 @@ using namespace std; // NOLINT
static const string& configFile1 =
"trainer/tests/sample_trainer_config_qb_rnn.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
P_DECLARE_bool(parallel_nn);
P_DECLARE_string(config_args);
P_DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
DECLARE_bool(parallel_nn);
DECLARE_string(config_args);
DEFINE_double(max_diff_ratio,
0.0f,
"max diff ratio allowed for parameters value");
int gNumDevices = 0;
......
paddle/trainer/tests/test_CompareTwoNets.cpp
浏览文件 @ 3d0e73bd
......@@ -22,25 +22,25 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
false,
"whether need to run in double accuracy");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
P_DECLARE_bool(thread_local_rand_use_global_seed);
P_DECLARE_int32(seed);
DECLARE_bool(thread_local_rand_use_global_seed);
DECLARE_int32(seed);
struct ComData {
vector<Argument> outArgs;
......
paddle/trainer/tests/test_CompareTwoOpts.cpp
浏览文件 @ 3d0e73bd
......@@ -22,20 +22,20 @@ limitations under the License. */
using namespace paddle; // NOLINT
using namespace std; // NOLINT
P_DECLARE_int32(gpu_id);
DECLARE_int32(gpu_id);
P_DECLARE_bool(local);
P_DECLARE_bool(use_gpu);
DECLARE_bool(local);
DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_string(nics);
DECLARE_string(config);
DECLARE_string(nics);
P_DEFINE_string(config_file_a, "", "config of one network to compare");
P_DEFINE_string(config_file_b, "", "config of another network to compare");
P_DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
P_DEFINE_double(
DEFINE_string(config_file_a, "", "config of one network to compare");
DEFINE_string(config_file_b, "", "config of another network to compare");
DEFINE_bool(need_high_accuracy,
true,
"whether need to run in double accuracy (recommended)");
DEFINE_double(
max_diff_ratio,
0.0f,
"max diff ratio allowed for outputs and parameters (value/gradient)");
......
paddle/trainer/tests/test_Prediction.cpp
浏览文件 @ 3d0e73bd
......@@ -18,11 +18,11 @@ limitations under the License. */
#include <gtest/gtest.h>
P_DECLARE_string(config);
P_DECLARE_string(config_args);
P_DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
DECLARE_string(config);
DECLARE_string(config_args);
DEFINE_string(merger,
"./paddle_merge_model",
"path to paddle_merge_model binary");
using namespace paddle; // NOLINT
using namespace std; // NOLINT
......
paddle/trainer/tests/test_Trainer.cpp
浏览文件 @ 3d0e73bd
......@@ -28,10 +28,10 @@ static const string& configFile3 = "trainer/tests/chunking.conf";
static const string& configFile4 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_bool(allow_only_one_model_on_one_gpu);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_bool(allow_only_one_model_on_one_gpu);
void checkGradientTest(const string& configFile,
bool useGpu,
......
paddle/trainer/tests/test_TrainerOnePass.cpp
浏览文件 @ 3d0e73bd
......@@ -27,12 +27,12 @@ static const string& configFile1 = "trainer/tests/sample_trainer_config.conf";
static const string& configFile2 =
"trainer/tests/sample_trainer_config_parallel.conf";
P_DECLARE_bool(use_gpu);
P_DECLARE_string(config);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(seed);
P_DECLARE_int32(num_passes);
P_DECLARE_int32(saving_period);
DECLARE_bool(use_gpu);
DECLARE_string(config);
DECLARE_int32(gpu_id);
DECLARE_int32(seed);
DECLARE_int32(num_passes);
DECLARE_int32(saving_period);
class TrainerForTest : public paddle::Trainer {
public:
......@@ -122,10 +122,10 @@ TEST(average_window_cpu, gpu4) {
#endif
// 3. test trainer + pserver.
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_int32(port);
P_DECLARE_bool(local);
P_DECLARE_bool(use_old_updater);
DECLARE_int32(num_gradient_servers);
DECLARE_int32(port);
DECLARE_bool(local);
DECLARE_bool(use_old_updater);
double checkRemoteParameterUpdater(TrainerForTest& trainer) {
auto gradientMachine = trainer.getGradientMachine();
......
paddle/trainer/tests/test_recurrent_machine_generation.cpp
浏览文件 @ 3d0e73bd
......@@ -30,7 +30,7 @@ static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT
static string expectFile = // NOLINT
"trainer/tests/rnn_gen_test_model_dir/r1.test"; // NOLINT
P_DECLARE_string(config_args);
DECLARE_string(config_args);
vector<float> readRetFile(const string& fname) {
ifstream inFile(fname);
......
paddle/utils/BarrierStat.cpp
浏览文件 @ 3d0e73bd
......@@ -20,15 +20,15 @@ limitations under the License. */
#include "paddle/utils/Flags.h"
#include "paddle/utils/Stat.h"
P_DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
P_DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
P_DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
DEFINE_bool(log_barrier_abstract,
true,
"if true, show abstract of barrier performance");
DEFINE_int32(log_barrier_lowest_nodes,
5,
"how many lowest node will be logged");
DEFINE_bool(log_barrier_show_log,
false, // for performance tuning insight
"if true, always show barrier abstract even with little gap");
namespace paddle {
......
paddle/utils/CommandLineParser.cpp
浏览文件 @ 3d0e73bd
......@@ -13,220 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "CommandLineParser.h"
#ifndef PADDLE_USE_GFLAGS
#include <stdlib.h>
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "paddle/utils/StringUtil.h"
namespace paddle {
static constexpr int kStatusOK = 0;
static constexpr int kStatusInvalid = 1;
static constexpr int kStatusNotFound = 2;
/**
* \brief: Convert a string to any type value.
*
* \note: It will specialize by type T that is supported.
*/
template <typename T>
bool StringToValue(const std::string& content, T* value) {
bool ok;
*value = str::toWithStatus<T>(content, &ok);
return ok;
}
template <>
bool StringToValue<bool>(const std::string& content, bool* value) {
std::string tmp = content;
std::transform(tmp.begin(), tmp.end(), tmp.begin(), [](char in) -> char {
if (in <= 'Z' && in >= 'A') {
return in - ('Z' - 'z');
} else {
return in;
}
}); // tolower.
if (tmp == "true" || tmp == "1") {
*value = true;
return true;
} else if (tmp == "false" || tmp == "0") {
*value = false;
return true;
} else {
return false;
}
}
template <>
bool StringToValue<std::string>(const std::string& content,
std::string* value) {
*value = content;
return true;
}
/**
* \brief Parse argument "--blah=blah".
*
* \param argument: The command line argument string, such as "--blah=blah"
* \param [out] extraInfo: The details error message for parse argument.
* \return: kStatusOK, kStatusInvalid, kStatusNotFound
*/
template <typename T>
int ParseArgument(const std::string& argument, std::string* extraInfo) {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
T* value = command.value;
std::string prefix = "--";
prefix += name;
prefix += "=";
std::string content;
if (str::startsWith(argument, prefix)) {
content = argument.substr(prefix.size(), argument.size() - prefix.size());
} else {
prefix = "-";
prefix += name;
prefix += "=";
if (str::startsWith(argument, prefix)) {
content =
argument.substr(prefix.size(), argument.size() - prefix.size());
}
}
if (!content.empty()) {
if (StringToValue(content, value)) {
return kStatusOK;
} else {
*extraInfo = name;
return kStatusInvalid;
}
}
}
return kStatusNotFound;
}
/**
* @brief ParseBoolArgumentExtra
* parse '--flag_name', '-flag_name' as true; '--noflag_name', '-noflag_name' as
* false
*/
static int ParseBoolArgumentExtra(const std::string& argument,
std::string* extraInfo) {
(void)(extraInfo); // unused extraInfo, just make api same.
//! @warning: The order and content of prefixes is DESIGNED for parsing
//! command line. The length of prefixes are 1, 2, 3, 4. The parse logic takes
//! use of this fact. DO NOT CHANGE IT without reading how to parse command
//! below.
static const std::vector<std::pair<const char*, bool>> prefixes = {
{"-", true}, {"--", true}, {"-no", false}, {"--no", false}};
for (flags_internal::CommandLineFlagRegistry<bool>::Command& command :
flags_internal::CommandLineFlagRegistry<bool>::Instance()->commands) {
if (argument.size() > command.name.size()) {
//! Use the length of prefix is 1, 2, 3, 4.
size_t diff = argument.size() - command.name.size() - 1UL;
if (diff < prefixes.size()) {
const std::string& prefix = std::get<0>(prefixes[diff]);
if (argument == prefix + command.name) {
*command.value = std::get<1>(prefixes[diff]);
return kStatusOK;
}
}
}
}
return kStatusNotFound;
}
/**
* \brief: Print command line arguments' usage with type T.
*/
template <typename T>
static void PrintTypeUsage() {
for (auto& command :
flags_internal::CommandLineFlagRegistry<T>::Instance()->commands) {
std::string& name = command.name;
name = "--" + name; // Program will exit, so modify name is safe.
std::string& desc = command.text;
T& defaultValue = command.defaultValue;
std::cerr << std::setw(20) << name << ": " << desc
<< "[default:" << defaultValue << "]." << std::endl;
}
}
template <typename... TS>
static void PrintTypeUsages() {
int unused[] = {0, (PrintTypeUsage<TS>(), 0)...};
(void)(unused);
}
/**
* \brief: Print all usage, and exit(1)
*/
static void PrintUsageAndExit(const char* argv0) {
std::cerr << "Program " << argv0 << " Flags: " << std::endl;
PrintTypeUsages<bool, int32_t, std::string, double, int64_t, uint64_t>();
exit(1);
}
/**
* \brief: Print the error flags, usage, and exit.
*/
static void PrintParseError(const std::string& name,
const char* actualInput,
const char* arg0) {
std::cerr << "Parse command flag " << name << " error! User input is "
<< actualInput << std::endl;
PrintUsageAndExit(arg0);
}
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
int unused_argc = 1;
std::string extra;
for (int i = 1; i < *argc; ++i) {
std::string arg = argv[i];
int s = kStatusInvalid;
#define ParseArgumentWithType(type) \
s = ParseArgument<type>(arg, &extra); \
if (s == kStatusOK) { \
continue; \
} else if (s == kStatusInvalid) { \
PrintParseError(extra, argv[i], argv[0]); \
}
ParseArgumentWithType(bool); // NOLINT
ParseArgumentWithType(int32_t);
ParseArgumentWithType(double); // NOLINT
ParseArgumentWithType(int64_t);
ParseArgumentWithType(uint64_t);
ParseArgumentWithType(std::string);
#undef ParseArgumentWithType
s = ParseBoolArgumentExtra(arg, &extra);
if (s == kStatusOK) {
continue;
}
if (withHelp && (arg == "--help" || arg == "-h")) {
PrintUsageAndExit(argv[0]);
}
// NOT Found for all flags.
std::swap(argv[unused_argc++], argv[i]);
}
*argc = unused_argc;
}
} // namespace paddle
#else
namespace paddle {
#ifndef GFLAGS_NS
#define GFLAGS_NS google
......@@ -243,4 +30,3 @@ void ParseCommandLineFlags(int* argc, char** argv, bool withHelp) {
}
} // namespace paddle
#endif
paddle/utils/CommandLineParser.h
浏览文件 @ 3d0e73bd
......@@ -13,167 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#ifndef PADDLE_USE_GFLAGS
#include <stdint.h>
#include <string>
#include <vector>
#include "DisableCopy.h"
namespace paddle {
namespace flags_internal {
/**
* Command line flag registry for special type T. It will store all command
* arguments settings. such as name, default value.
*/
template <typename T>
struct CommandLineFlagRegistry {
/**
* The factory method of CommandLineFlagRegistry
*
* \return: The singleton instance of CommandLineFlagRegistry.
*/
static CommandLineFlagRegistry* Instance() {
static CommandLineFlagRegistry instance_;
return &instance_;
}
struct Command {
/// name of argument.
std::string name;
/// address of actual variable. such as FLAGS_xxx.
T* value;
/// usage text.
std::string text;
/// default value of this command.
T defaultValue;
};
/// the command line arguments of type T.
std::vector<Command> commands;
DISABLE_COPY(CommandLineFlagRegistry);
private:
inline CommandLineFlagRegistry() {}
};
/**
*Helper class to register command line flag.
*/
template <typename T>
struct CommandLineFlagRegister {
/**
* \brief: Register a command line argument
*
* \param [in] name: The command line name.
* \param [inout] val: The command line argument instance, FLAGS_xxx.
* \param [in] desc: The command line helper message.
*/
CommandLineFlagRegister(const std::string& name,
T* val,
const std::string desc) {
CommandLineFlagRegistry<T>::Instance()->commands.push_back(
{name, val, desc, *val});
}
};
/**
* \brief: Define a command line arguments.
*
* \param type: The variable type, such as int, double, etc.
* \param name: The variable name. The command line argument is '--name', the
*variable
*is 'FLAGS_name'
* \param default_value: The default value of command line argument.
* \param text: The description in command line argument.
*/
#define PADDLE_DEFINE_variable(type, name, default_value, text) \
type FLAGS_##name = default_value; \
namespace paddle_flags_internal { \
paddle::flags_internal::CommandLineFlagRegister<type> \
flags_internal_var_##name(#name, &FLAGS_##name, text); \
} // namespace paddle_flags_internal
/**
* Declare a variable to use.
*/
#define PADDLE_DECLARE_variable(type, name) extern type FLAGS_##name;
// DEFINE macro for each types.
#define P_DEFINE_int32(name, default_value, text) \
PADDLE_DEFINE_variable(int32_t, name, default_value, text)
#define P_DEFINE_bool(name, default_value, text) \
PADDLE_DEFINE_variable(bool, name, default_value, text)
#define P_DEFINE_string(name, default_value, text) \
PADDLE_DEFINE_variable(std::string, name, default_value, text)
#define P_DEFINE_double(name, default_value, text) \
PADDLE_DEFINE_variable(double, name, default_value, text)
#define P_DEFINE_int64(name, default_value, text) \
PADDLE_DEFINE_variable(int64_t, name, default_value, text)
#define P_DEFINE_uint64(name, default_value, text) \
PADDLE_DEFINE_variable(uint64_t, name, default_value, text)
// Declare macro for each types.
#define P_DECLARE_int32(name) PADDLE_DECLARE_variable(int32_t, name)
#define P_DECLARE_bool(name) PADDLE_DECLARE_variable(bool, name)
#define P_DECLARE_string(name) PADDLE_DECLARE_variable(std::string, name)
#define P_DECLARE_double(name) PADDLE_DECLARE_variable(double, name)
#define P_DECLARE_int64(name) PADDLE_DECLARE_variable(int64_t, name)
#define P_DECLARE_uint64(name) PADDLE_DECLARE_variable(uint64_t, name)
} // namespace flags_internal
/**
* \brief Parse command line flags. If parse error, just failed and exit 1.
*
* \param [inout] argc: The command argument count. This method will modify
*argc, and left unused arguments.
* \param [inout] argv: The command argument values. This method will modify
*argv, and left unused arguments.
* \param [in] withHelp: True will parse '-h' and '--help' to print usage.
*
* \note: The Command line flags format basically as follow:
*
* * If the type of flag is not bool, then the follow format of command line
* will be parsed:
* * --flag_name=value
* * -flag_name=value
*
* * If the flag is bool, then:
* * --flag_name=value, -flag_name=value will be parsed.
* * if value.tolower() == "true"| "1" will be treated as true.
* * else if value.tolower() == "false" | "0" will be treated as false.
* * --flag_name will be parsed as true.
* * --noflag_name will be parsed as false.
*/
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#else // if use gflags.
#include <gflags/gflags.h>
#define P_DEFINE_int32 DEFINE_int32
#define P_DEFINE_bool DEFINE_bool
#define P_DEFINE_string DEFINE_string
#define P_DEFINE_double DEFINE_double
#define P_DEFINE_int64 DEFINE_int64
#define P_DEFINE_uint64 DEFINE_uint64
#define P_DECLARE_int32 DECLARE_int32
#define P_DECLARE_bool DECLARE_bool
#define P_DECLARE_string DECLARE_string
#define P_DECLARE_double DECLARE_double
#define P_DECLARE_int64 DECLARE_int64
#define P_DECLARE_uint64 DECLARE_uint64
namespace paddle {
void ParseCommandLineFlags(int* argc, char** argv, bool withHelp = true);
} // namespace paddle
#endif
paddle/utils/CustomStackTrace.cpp
浏览文件 @ 3d0e73bd
......@@ -16,7 +16,7 @@ limitations under the License. */
#include <iostream>
#include "CommandLineParser.h"
P_DEFINE_bool(
DEFINE_bool(
layer_stack_error_only_current_thread,
true,
"Dump current thread or whole process layer stack when signal error "
......
paddle/utils/Flags.cpp
浏览文件 @ 3d0e73bd
......@@ -15,65 +15,61 @@ limitations under the License. */
#include "Flags.h"
#ifdef PADDLE_ONLY_CPU
P_DEFINE_bool(use_gpu, false, "Only support CPU training");
DEFINE_bool(use_gpu, false, "Only support CPU training");
#else
P_DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
DEFINE_bool(use_gpu, true, "Whether to use GPU for training");
#endif
P_DEFINE_bool(
parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
P_DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
P_DEFINE_int32(gpu_id, 0, "Which gpu core to use");
P_DEFINE_int32(port, 20134, "Listening port for pserver");
P_DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
P_DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
P_DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
P_DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
P_DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
P_DEFINE_int32(
trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
P_DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
P_DEFINE_string(comment, "", "A string for commenting this training task");
P_DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
P_DEFINE_int32(log_period, 100, "Log progress every so many batches");
P_DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
P_DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
P_DEFINE_int32(enable_parallel_vector,
0,
"threshold for enable parallel vector");
P_DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
P_DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
DEFINE_bool(parallel_nn,
false,
"Whether to use multi-threads to calculate one neural network."
"If it was set false, use gpu_id specify which gpu core to use"
"(the device property in the trainer config file will be ingored)."
"If it was set true, the gpu core is specified by the trainer"
" config file(gpu_id will be ignored).");
DEFINE_int32(trainer_count, 1, "Defined how many trainers to train");
DEFINE_int32(gpu_id, 0, "Which gpu core to use");
DEFINE_int32(port, 20134, "Listening port for pserver");
DEFINE_int32(data_server_port, 21134, "Listening port for dserver");
DEFINE_int32(ports_num,
1,
"The ports number for parameter send,"
" increment based on default port number");
DEFINE_int32(ports_num_for_sparse,
0,
"The ports number for parameter send,"
" increment based on default (port + ports_num)");
DEFINE_string(nics, "xgbe0,xgbe1", "network device name for pservers");
DEFINE_string(rdma_tcp, "tcp", "use rdma or tcp rdma transport protocol");
DEFINE_int32(trainer_id,
0,
"For distributed training, each trainer must be given an unique id"
" ranging from 0 to num_trainers-1. Trainer 0 is the master"
" trainer");
DEFINE_int32(num_gradient_servers, 1, "number of gradient servers");
DEFINE_string(comment, "", "A string for commenting this training task");
DEFINE_string(load_missing_parameter_strategy,
"fail",
"which operation to take on load model fails. support "
"fail/rand/zero only.");
DEFINE_int32(log_period, 100, "Log progress every so many batches");
DEFINE_int32(log_period_server,
500,
"Log progress every so many batches at pserver end");
DEFINE_double(checkgrad_eps, 1e-5, "parameter change size for checkgrad");
DEFINE_int32(enable_parallel_vector, 0, "threshold for enable parallel vector");
DEFINE_bool(loadsave_parameters_in_pserver,
false,
"load and save parameters in pserver. "
"only work while parameter set sparse_remote_update.");
DEFINE_int32(beam_size,
1,
"Beam size used in generating most probable output sequences.");
P_DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
P_DEFINE_string(predict_file, "", "File name for saving predict result");
P_DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
P_DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
DEFINE_bool(show_layer_stat, false, "show the statistics of each layer");
DEFINE_string(predict_file, "", "File name for saving predict result");
DEFINE_bool(prev_batch_state, false, "batch is continue with next batch");
DEFINE_string(init_model_path,
"",
"Path of the initial model parameters."
"If it was set, start_pass will be ignored.");
paddle/utils/Flags.h
浏览文件 @ 3d0e73bd
......@@ -16,28 +16,28 @@ limitations under the License. */
#include "CommandLineParser.h"
P_DECLARE_bool(parallel_nn);
P_DECLARE_int32(async_count);
P_DECLARE_int32(port);
P_DECLARE_int32(data_server_port);
P_DECLARE_bool(use_gpu);
P_DECLARE_int32(gpu_id);
P_DECLARE_int32(trainer_count);
P_DECLARE_int32(ports_num);
P_DECLARE_int32(ports_num_for_sparse);
P_DECLARE_string(nics);
P_DECLARE_string(rdma_tcp);
P_DECLARE_int32(trainer_id);
P_DECLARE_int32(num_gradient_servers);
P_DECLARE_string(comment);
P_DECLARE_string(load_missing_parameter_strategy);
P_DECLARE_int32(log_period);
P_DECLARE_int32(log_period_server);
P_DECLARE_double(checkgrad_eps);
P_DECLARE_int32(enable_parallel_vector);
P_DECLARE_bool(loadsave_parameters_in_pserver);
P_DECLARE_int32(beam_size);
P_DECLARE_bool(show_layer_stat);
P_DECLARE_string(predict_file);
P_DECLARE_bool(prev_batch_state);
P_DECLARE_string(init_model_path);
DECLARE_bool(parallel_nn);
DECLARE_int32(async_count);
DECLARE_int32(port);
DECLARE_int32(data_server_port);
DECLARE_bool(use_gpu);
DECLARE_int32(gpu_id);
DECLARE_int32(trainer_count);
DECLARE_int32(ports_num);
DECLARE_int32(ports_num_for_sparse);
DECLARE_string(nics);
DECLARE_string(rdma_tcp);
DECLARE_int32(trainer_id);
DECLARE_int32(num_gradient_servers);
DECLARE_string(comment);
DECLARE_string(load_missing_parameter_strategy);
DECLARE_int32(log_period);
DECLARE_int32(log_period_server);
DECLARE_double(checkgrad_eps);
DECLARE_int32(enable_parallel_vector);
DECLARE_bool(loadsave_parameters_in_pserver);
DECLARE_int32(beam_size);
DECLARE_bool(show_layer_stat);
DECLARE_string(predict_file);
DECLARE_bool(prev_batch_state);
DECLARE_string(init_model_path);
paddle/utils/Logging.cpp
浏览文件 @ 3d0e73bd
......@@ -18,175 +18,9 @@ limitations under the License. */
*/
#include "Logging.h"
#ifndef PADDLE_USE_GLOG
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mutex>
#include <thread>
#include <vector>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
namespace paddle {
namespace internal {
std::string join(const std::string& part1, const std::string& part2) {
const char sep = '/';
if (!part2.empty() && part2.front() == sep) {
return part2;
}
std::string ret;
ret.reserve(part1.size() + part2.size() + 1);
ret = part1;
if (!ret.empty() && ret.back() != sep) {
ret += sep;
}
ret += part2;
return ret;
}
static inline bool env2bool(const char* envName, bool defaultValue = false) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
return memchr("tTyY1\0", envValue[0], 6) != nullptr;
}
}
static inline int env2int(const char* envName, int defaultValue = 0) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
int retValue = defaultValue;
try {
retValue = std::stoi(envValue);
} catch (...) {
// pass
}
return retValue;
}
}
static inline int env2index(const char* envName,
const std::vector<std::string>& options,
int defaultValue) {
char* envValue = getenv(envName);
if (envValue == nullptr) {
return defaultValue;
} else {
for (size_t i = 0; i < options.size(); ++i) {
if (options[i] == envValue) {
return static_cast<int>(i);
}
}
return defaultValue;
}
}
static bool gLogToStderr = env2bool("PLOG_LOGTOSTDERR", true);
static const std::vector<std::string> gLevelName = {
"INFO", "WARNING", "ERROR", "FATAL"};
static int gMinLogLevel =
env2int("PLOG_MINLOGLEVEL", env2index("PLOG_MINLOGLEVEL", gLevelName, 0));
static std::vector<std::vector<int>> gLogFds;
static std::vector<int> gLogFileFds;
static bool gLogInited = false;
static void freeLogFileFds() {
for (auto fd : gLogFileFds) {
close(fd);
}
}
static void initializeLogFds(char* argv0) {
gLogFds.resize(NUM_SEVERITIES);
for (int i = gMinLogLevel; i < NUM_SEVERITIES && gLogToStderr;
++i) { // Add stderr
std::vector<int>& fds = gLogFds[i];
fds.push_back(STDERR_FILENO);
}
char* logDir = getenv("PLOG_LOGDIR");
for (int i = gMinLogLevel; i < NUM_SEVERITIES && logDir != nullptr; ++i) {
std::string filename =
join(logDir, std::string(argv0) + "." + gLevelName[i]);
int fd = open(filename.c_str(), O_CREAT | O_WRONLY, 0644);
if (fd == -1) {
fprintf(stderr, "Open log file error!");
exit(1);
}
gLogFileFds.push_back(fd);
std::vector<int>& curFds = gLogFds[i];
curFds.insert(curFds.end(), gLogFileFds.begin(), gLogFileFds.end());
}
atexit(freeLogFileFds);
gLogInited = true;
}
static void (*gFailureFunctionPtr)() ATTR_NORETURN = abort;
LogMessage::LogMessage(const char* fname, int line, int severity)
: fname_(fname), line_(line), severity_(severity) {}
LogMessage::~LogMessage() { this->generateLogMessage(); }
void LogMessage::generateLogMessage() {
if (!gLogInited) {
fprintf(stderr,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
} else {
for (auto& fd : gLogFds[this->severity_]) {
dprintf(fd,
"%c %s:%d] %s\n",
"IWEF"[severity_],
fname_,
line_,
str().c_str());
}
}
}
LogMessageFatal::LogMessageFatal(const char* file, int line)
: LogMessage(file, line, FATAL) {}
LogMessageFatal::~LogMessageFatal() {
generateLogMessage();
gFailureFunctionPtr();
}
} // namespace internal
void initializeLogging(int argc, char** argv) {
internal::initializeLogFds(argv[0]);
}
namespace logging {
void setMinLogLevel(int level) { paddle::internal::gMinLogLevel = level; }
void installFailureFunction(void (*callback)() ATTR_NORETURN) {
paddle::internal::gFailureFunctionPtr = callback;
}
} // namespace logging
} // namespace paddle
#else
namespace paddle {
void initializeLogging(int argc, char** argv) {
(void)(argc);
if (!getenv("GLOG_logtostderr")) {
......@@ -197,13 +31,16 @@ void initializeLogging(int argc, char** argv) {
}
namespace logging {
void setMinLogLevel(int level) { FLAGS_minloglevel = level; }
void installFailureFunction(void (*callback)()) {
google::InstallFailureFunction(callback);
}
void installFailureWriter(void (*callback)(const char*, int)) {
google::InstallFailureWriter(callback);
}
} // namespace logging
} // namespace paddle
#endif
paddle/utils/Logging.h
浏览文件 @ 3d0e73bd
......@@ -22,175 +22,21 @@ limitations under the License. */
#include <sstream>
#include <string>
#ifndef PADDLE_USE_GLOG
#include "CompilerMacros.h"
//! TODO(yuyang18): Move this utility macro into some global header.
#define PP_CAT(a, b) PP_CAT_I(a, b)
#define PP_CAT_I(a, b) PP_CAT_II(~, a##b)
#define PP_CAT_II(p, res) res
/**
* Generate Unique Variable Name, Usefully in macro.
* @SEE
* http://stackoverflow.com/questions/1082192/how-to-generate-random-variable-names-in-c-using-macros
*/
#define UNIQUE_NAME(base) PP_CAT(base, __LINE__)
#include <glog/logging.h>
namespace paddle {
//! Log levels.
const int INFO = 0;
const int WARNING = 1;
const int ERROR = 2;
const int FATAL = 3;
const int NUM_SEVERITIES = 4;
namespace internal {
class LogMessage : public std::basic_ostringstream<char> {
public:
LogMessage(const char* fname, int line, int severity);
~LogMessage();
protected:
/**
* @brief Print log message to stderr, files, etc.
*/
void generateLogMessage();
private:
const char* fname_;
int line_;
int severity_;
};
// LogMessageFatal ensures the process will exit in failure after
// logging this message.
class LogMessageFatal : public LogMessage {
public:
LogMessageFatal(const char* file, int line) __attribute__((cold));
~LogMessageFatal() __attribute__((noreturn));
};
#define _P_LOG_INFO \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::INFO)
#define _P_LOG_WARNING \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::WARNING)
#define _P_LOG_ERROR \
::paddle::internal::LogMessage(__FILE__, __LINE__, paddle::ERROR)
#define _P_LOG_FATAL ::paddle::internal::LogMessageFatal(__FILE__, __LINE__)
#define P_LOG(severity) _P_LOG_##severity
#define P_LOG_FIRST_N(severity, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) ++UNIQUE_NAME(LOG_OCCURRENCES); \
if (UNIQUE_NAME(LOG_OCCURRENCES) <= n) P_LOG(severity)
#define P_LOG_IF_EVERY_N(severity, condition, n) \
static int UNIQUE_NAME(LOG_OCCURRENCES) = 0; \
if (condition && ((UNIQUE_NAME(LOG_OCCURRENCES) = \
(UNIQUE_NAME(LOG_OCCURRENCES) + 1) % n) == (1 % n))) \
P_LOG(severity)
#define P_LOG_EVERY_N(severity, n) P_LOG_IF_EVERY_N(severity, true, n)
// TODO(jeff): Define a proper implementation of VLOG_IS_ON
#define P_VLOG_IS_ON(lvl) ((lvl) <= 0)
#define P_LOG_IF(severity, condition) \
if (condition) P_LOG(severity)
#define P_VLOG(lvl) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl))
#define P_VLOG_IF(lvl, cond) P_LOG_IF(INFO, P_VLOG_IS_ON(lvl) && cond)
#define P_VLOG_EVERY_N(lvl, n) P_LOG_IF_EVERY_N(INFO, P_VLOG_IS_ON(lvl), n)
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
// CHECK dies with a fatal error if condition is not true. It is *not*
// controlled by NDEBUG, so the check will be executed regardless of
// compilation mode. Therefore, it is safe to do things like:
// CHECK(fp->Write(x) == 4)
#define P_CHECK(condition) \
if (PREDICT_FALSE(!(condition))) \
P_LOG(FATAL) << "Check failed: " #condition " "
#define P_CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define P_CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define P_CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define P_CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define P_CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define P_CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define P_CHECK_NOTNULL(val) P_CHECK((val) != NULL)
//! GLOG compatible APIs
//! NOTE: only implement Paddle actually used APIs.
#define LOG(x) P_LOG(x)
#define VLOG(x) P_VLOG(x)
#define DLOG(x) P_VLOG(5)
#define CHECK(x) P_CHECK(x)
#define PCHECK(x) P_CHECK(x)
#define CHECK_EQ(val1, val2) P_CHECK((val1) == (val2))
#define CHECK_NE(val1, val2) P_CHECK((val1) != (val2))
#define CHECK_LE(val1, val2) P_CHECK((val1) <= (val2))
#define CHECK_LT(val1, val2) P_CHECK((val1) < (val2))
#define CHECK_GE(val1, val2) P_CHECK((val1) >= (val2))
#define CHECK_GT(val1, val2) P_CHECK((val1) > (val2))
#define CHECK_NOTNULL(val) P_CHECK((val) != NULL)
#define VLOG_IS_ON(x) P_VLOG_IS_ON(x)
#define LOG_FIRST_N(severity, n) P_LOG_FIRST_N(severity, n)
#define LOG_IF(severity, condition) P_LOG_IF(severity, condition)
#define VLOG_EVERY_N(lvl, n) P_VLOG_EVERY_N(lvl, n)
#define VLOG_IF(lvl, cond) P_VLOG_IF(lvl, cond)
#define LOG_EVERY_N(severity, n) P_LOG_EVERY_N(severity, n)
} // namespace internal
/**
* @brief initialize logging
* @note: Current implement of logging is lack of:
* PrintCallStack when fatal.
* VLOG_IS_ON
* But it is portable to multi-platform, and simple enough to modify.
*/
void initializeLogging(int argc, char** argv);
namespace logging {
/**
* @brief Set Min Log Level. if Log.level < minLogLevel, then will not print log
* to stream
* @param level. Any integer is OK, but only 0 <= x <= NUM_SEVERITIES is useful.
*/
void setMinLogLevel(int level);
/**
* @brief Install Log(Fatal) failure function. Default is abort();
* @param callback: The failure function.
*/
void installFailureFunction(void (*callback)() ATTR_NORETURN);
/**
* @brief installFailureWriter
* @note: not implemented currently.
*/
inline void installFailureWriter(void (*callback)(const char*, int)) {
(void)(callback); // unused callback.
}
} // namespace logging
} // namespace paddle
#else
#include <glog/logging.h>
namespace paddle {
void initializeLogging(int argc, char** argv);
namespace logging {
void setMinLogLevel(int level);
void installFailureFunction(void (*callback)());
void installFailureWriter(void (*callback)(const char*, int));
} // namespace logging
}
#endif // PADDLE_USE_GLOG
} // namespace logging
} // namespace paddle
#ifndef NDEBUG
#define DEBUG_LEVEL 5
......
paddle/utils/PythonUtil.cpp
浏览文件 @ 3d0e73bd
......@@ -20,8 +20,8 @@ namespace paddle {
#ifdef PADDLE_NO_PYTHON
P_DEFINE_string(python_path, "", "python path");
P_DEFINE_string(python_bin, "python2.7", "python bin");
DEFINE_string(python_path, "", "python path");
DEFINE_string(python_bin, "python2.7", "python bin");
constexpr int kExecuteCMDBufLength = 204800;
......
paddle/utils/ThreadLocal.cpp
浏览文件 @ 3d0e73bd
......@@ -16,9 +16,9 @@ limitations under the License. */
#include "CommandLineParser.h"
#include "Util.h"
P_DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
DEFINE_bool(thread_local_rand_use_global_seed,
false,
"Whether to use global seed in thread local rand.");
namespace paddle {
......
paddle/utils/Util.cpp
浏览文件 @ 3d0e73bd
......@@ -33,7 +33,7 @@ limitations under the License. */
#include "ThreadLocal.h"
#include "Version.h"
P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#ifdef WITH_GOOGLE_PERFTOOLS
/*
......@@ -52,10 +52,8 @@ P_DEFINE_int32(seed, 1, "random number seed. 0 for srand(time)");
#include <gperftools/profiler.h>
P_DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
P_DEFINE_string(profile_data_file,
"gperf.prof",
"file for storing profile data");
DEFINE_int32(profile_signal, 12, "signal for switch google profiler");
DEFINE_string(profile_data_file, "gperf.prof", "file for storing profile data");
static void profilerSwitch(int signalNumber) {
bool static started = false;
......
paddle/utils/Version.cpp
浏览文件 @ 3d0e73bd
......@@ -18,13 +18,8 @@ limitations under the License. */
#include <numeric>
#include "Flags.h"
#include "Util.h"
//! TODO(yuyang18) in gflags, version has another define. Use another flag
//! instead.
#ifndef PADDLE_USE_GFLAGS
P_DEFINE_bool(version, false, "print version");
#else
P_DECLARE_bool(version);
#endif
DECLARE_bool(version);
namespace paddle {
namespace version {
......
paddle/utils/tests/CMakeLists.txt
浏览文件 @ 3d0e73bd
add_simple_unittest(test_CommandLineParser)
add_simple_unittest(test_Logging)
add_simple_unittest(test_Thread)
add_simple_unittest(test_StringUtils)
add_simple_unittest(test_CustomStackTrace)
......
paddle/utils/tests/test_CommandLineParser.cpp 已删除 100644 → 0
浏览文件 @ be734a60
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#ifndef PADDLE_USE_GFLAGS
//! Test Command Line Parser for paddle internal implement.
#include <gtest/gtest.h>
#include <paddle/utils/CommandLineParser.h>
P_DEFINE_int32(i1, 1, "test int flag 1");
P_DEFINE_int32(i2, 2, "test int flag 2");
P_DEFINE_string(str1, "1", "test str flag 1");
P_DEFINE_string(str2, "2", "test str flag 2");
P_DEFINE_bool(b1, true, "test bool flag 1");
P_DEFINE_bool(b2, false, "test bool flag 2");
P_DEFINE_double(d1, 0.1, "test double flag 1");
P_DEFINE_double(d2, -42.3, "test double flag 2");
P_DEFINE_int64(l1, 1, "test int64 flag 1");
P_DEFINE_int64(l2, 2, "test int64 flag 2");
P_DEFINE_uint64(ul1, 32, "test uint64 flag 1");
P_DEFINE_uint64(ul2, 33, "test uint64 flag 2");
constexpr double EPSILON = 1e-5;
#define cc(x) const_cast<char*>((x))
TEST(CommandLineParser, defaultValue) {
char* argv[] = {cc("test_program"), cc("--unused_flag=134")};
int argc = sizeof(argv) / sizeof(char*);
paddle::ParseCommandLineFlags(&argc, argv);
// Check Default Value
ASSERT_EQ(argc, 2);
ASSERT_EQ(FLAGS_i1, 1);
ASSERT_EQ(FLAGS_i2, 2);
ASSERT_EQ(FLAGS_str1, "1");
ASSERT_EQ(FLAGS_str2, "2");
ASSERT_EQ(FLAGS_b1, true);
ASSERT_EQ(FLAGS_b2, false);
ASSERT_NEAR(FLAGS_d1, 0.1, EPSILON);
ASSERT_NEAR(FLAGS_d2, -42.3, EPSILON);
ASSERT_EQ(FLAGS_i1, 1);
ASSERT_EQ(FLAGS_i2, 2);
ASSERT_EQ(FLAGS_ul1, 32UL);
ASSERT_EQ(FLAGS_ul2, 33UL);
}
TEST(CommandLineParser, normal) {
char* argv[] = {cc("test_program"),
cc("--i2=32"),
cc("--str1=abc"),
cc("--b2=1"),
cc("-b1=False"),
cc("--d2=.34"),
cc("--d1=0"),
cc("--l1=-12345678901234"),
cc("-ul2=3212")};
int argc = sizeof(argv) / sizeof(char*);
paddle::ParseCommandLineFlags(&argc, argv);
ASSERT_EQ(argc, 1);
ASSERT_EQ(FLAGS_i2, 32);
ASSERT_EQ(FLAGS_str1, "abc");
ASSERT_EQ(FLAGS_b2, true);
ASSERT_EQ(FLAGS_b1, false);
ASSERT_NEAR(FLAGS_d2, 0.34, EPSILON);
ASSERT_NEAR(FLAGS_d1, 0.0, EPSILON);
ASSERT_EQ(FLAGS_l1, -12345678901234);
ASSERT_EQ(FLAGS_ul2, 3212UL);
}
TEST(CommandLineParser, printHelp) {
char* argv[] = {cc("test_program"), cc("--help")};
int argc = sizeof(argv) / sizeof(char*);
// Will Print Usage
ASSERT_DEATH(paddle::ParseCommandLineFlags(&argc, argv), ".*test_program.*");
}
TEST(CommandLineParser, parseError) {
char* argv[] = {cc("test_program"), cc("--i1=abc")};
int argc = sizeof(argv) / sizeof(char*);
ASSERT_DEATH(
paddle::ParseCommandLineFlags(&argc, argv),
"Parse command flag i1 error! User input is --i1=abc.*test_program.*");
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
int main(int argc, char** argv) { return 0; }
#endif
paddle/utils/tests/test_CustomStackTrace.cpp
浏览文件 @ 3d0e73bd
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Locks.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 10, "testing thread number");
DEFINE_int32(test_thread_num, 10, "testing thread number");
void testNormalImpl(
const std::function<void(paddle::CustomStackTrace<std::string>&,
......
paddle/utils/tests/test_Logging.cpp 已删除 100644 → 0
浏览文件 @ be734a60
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
/*
* Basically from tensorflow/core/platform/default/logging.cc
* Used in embedded system where there is no glogs.
*/
#include <dirent.h>
#include <gtest/gtest.h>
#include <stdlib.h>
#include <fstream>
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
#ifndef PADDLE_USE_GLOG
TEST(Logging, BasicalLog) {
auto pinfo = [] {
P_LOG(INFO) << "INFO";
exit(1);
};
ASSERT_DEATH(pinfo(), "I .*test_Logging.cpp:[0-9]+] INFO");
auto pwarn = [] {
P_LOG(WARNING) << "WARN";
exit(1);
};
ASSERT_DEATH(pwarn(), "W .*test_Logging.cpp:[0-9]+] WARN");
auto perr = [] {
P_LOG(ERROR) << "ERROR";
exit(1);
};
ASSERT_DEATH(perr(), "E .*test_Logging.cpp:[0-9]+] ERROR");
auto pfatal = [] { P_LOG(FATAL) << "FATAL"; };
ASSERT_DEATH(pfatal(), "F .*test_Logging.cpp:[0-9]+] FATAL");
}
TEST(Logging, Check) {
int a = 1;
int b = 2;
P_CHECK(a != b);
auto pcheckDown = [&] { P_CHECK(a == b); };
ASSERT_DEATH(pcheckDown(),
"F .*test_Logging.cpp:[0-9]+] Check failed: a == b ");
P_CHECK_LE(a, b);
P_CHECK_LT(a, b);
double t = 1.2;
P_CHECK_LE(a, t);
double* ptr = nullptr;
auto pcheckDown2 = [&] { P_CHECK_NOTNULL(ptr); };
ASSERT_DEATH(pcheckDown2(), "F");
}
#define cc(x) const_cast<char*>(x)
TEST(Logging, LogToStderr) {
auto logToStderrCallback = [] {
setenv("PLOG_LOGTOSTDERR", "0", true);
char* argv[] = {cc("test")};
paddle::initializeLogging(1, argv);
P_LOG(INFO) << "This output will not print to std error";
exit(1);
};
ASSERT_DEATH(logToStderrCallback(), "");
}
constexpr char kLogDirName[] = "./test_log_dir";
const std::vector<std::string> kLevels = {"INFO", "WARNING", "ERROR", "FATAL"};
TEST(Logging, LogToDir) {
ASSERT_EQ(0, mkdir(kLogDirName, 0777));
auto logToDirCallback = [] {
setenv("PLOG_LOGTOSTDERR", "0", true);
setenv("PLOG_LOGDIR", kLogDirName, true);
char* argv[] = {cc("test")};
paddle::initializeLogging(1, argv);
P_LOG(INFO) << "INFO";
P_LOG(WARNING) << "WARNING";
P_LOG(ERROR) << "ERROR";
P_LOG(FATAL) << "FATAL";
};
ASSERT_DEATH(logToDirCallback(), "");
// There 4 file in logdir
auto dir = opendir(kLogDirName);
size_t fileCount = 0;
std::vector<std::string> filenames;
for (auto dirContent = readdir(dir); dirContent != nullptr;
dirContent = readdir(dir)) {
std::string filename(dirContent->d_name);
if (filename == "." || filename == "..") {
continue;
} else {
++fileCount;
for (size_t i = 0; i < kLevels.size(); ++i) {
const std::string& curLevel = kLevels[i];
if (filename.size() > curLevel.length()) {
size_t diff = filename.size() - curLevel.length();
size_t j = 0;
for (; j < curLevel.length(); ++j) {
if (filename[j + diff] != curLevel[j]) {
// File Suffix Not Same, then break.
break;
}
}
if (j == curLevel.length()) { // Same suffix.
std::ifstream fin;
auto fn = paddle::path::join(kLogDirName, filename);
fin.open(fn);
filenames.push_back(fn);
ASSERT_TRUE(fin.is_open());
size_t lineCounter = 0;
for (std::string line; std::getline(fin, line); ++lineCounter) {
// Do Nothing, Just calc lineCounter.
}
// For example.
// The info channel will have all log which level >= INFO
// So the info file's lineCounter should == 4.
ASSERT_EQ(kLevels.size() - i, lineCounter);
fin.close();
}
}
}
}
}
closedir(dir);
ASSERT_EQ(4UL, fileCount); // 4 levels.
// Clean Unittest.
for (std::string& fn : filenames) {
ASSERT_EQ(remove(fn.c_str()), 0);
}
ASSERT_EQ(rmdir(kLogDirName), 0);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
#else
int main(int, char**) { return 0; }
#endif
paddle/utils/tests/test_SpinLock.cpp
浏览文件 @ 3d0e73bd
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 100, "testing thread number");
DEFINE_int32(test_thread_num, 100, "testing thread number");
void testNormalImpl(
size_t thread_num,
......
paddle/utils/tests/test_ThreadBarrier.cpp
浏览文件 @ 3d0e73bd
......@@ -20,7 +20,7 @@ limitations under the License. */
#include "paddle/utils/Logging.h"
#include "paddle/utils/Util.h"
P_DEFINE_int32(test_thread_num, 100, "testing thread number");
DEFINE_int32(test_thread_num, 100, "testing thread number");
void testNormalImpl(
size_t thread_num,
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或