diff --git a/BCLOUD b/BCLOUD old mode 100644 new mode 100755 index 4687106f33f74c5405c2a99c1ef7fe4707142d6c..7b7aae714aa0a8f6cdbc8e49527f2ea27bb251bf --- a/BCLOUD +++ b/BCLOUD @@ -1,11 +1,10 @@ WORKROOT('../../../') COMPILER('gcc482') CPPFLAGS('-D_GNU_SOURCE -DNDEBUG') -GLOBAL_CFLAGS_STR = '-g -O0 -pipe -fopenmp ' +GLOBAL_CFLAGS_STR = '-g -O3 -pipe -fopenmp ' CFLAGS(GLOBAL_CFLAGS_STR) GLOBAL_CXXFLAGS_STR = GLOBAL_CFLAGS_STR + ' -std=c++11 ' CXXFLAGS(GLOBAL_CXXFLAGS_STR) - INCPATHS('./') INCPATHS('$OUT/../') INCPATHS('../../third-party') @@ -36,9 +35,8 @@ CONFIGS('baidu/third-party/pybind11@v2.2.4@git_branch') CONFIGS('baidu/third-party/python@gcc482output@git_branch') CONFIGS('baidu/third-party/yaml-cpp@yaml-cpp_0-6-2-0_GEN_PD_BL@git_tag') CONFIGS('baidu/third-party/openmpi@openmpi_1-4-5-0-feed_mlarch@git_branch') -CONFIGS('baidu/paddlepaddle/pslib@master@git_branch') +CONFIGS('baidu/paddlepaddle/pslib@no_abacus_in_proto@git_branch') CONFIGS('third-64/gtest@base') - HEADERS('paddle/fluid/memory/*.h', '$INC/paddle/fluid/memory/') HEADERS('paddle/fluid/memory/detail/*.h', '$INC/paddle/fluid/memory/detail/') HEADERS('paddle/fluid/memory/allocation/*.h', '$INC/paddle/fluid/memory/allocation/') @@ -59,12 +57,9 @@ HEADERS('paddle/fluid/pybind/pybind.h', '$INC/paddle/fluid/pybind') HEADERS('paddle/fluid/inference/api/*.h', '$INC/paddle/fluid/inference/api/') HEADERS(GLOB_GEN_SRCS('paddle/fluid/framework/*pb.h'), '$INC/paddle/fluid/framework') HEADERS(GLOB_GEN_SRCS('paddle/fluid/platform/*pb.h'), '$INC/paddle/fluid/platform') - PROTOC('../../third-party/protobuf/bin/protoc') - #proto StaticLibrary("fake_paddle_proto", Sources(GLOB("paddle/fluid/framework/*.proto"), GLOB("paddle/fluid/platform/*.proto"))) - #feed HEADERS('paddle/fluid/train/custom_trainer/feed/*.h', '$INC/paddle/fluid/train/custom_trainer/feed/') HEADERS('paddle/fluid/train/custom_trainer/feed/common/*.h', '$INC/paddle/fluid/train/custom_trainer/feed/common/') @@ -80,16 +75,15 @@ NEED_OUTPUT("baidu/third-party/openmpi") OUTPUT('paddle/fluid/train/custom_trainer/feed/conf', '$OUT') OUTPUT('paddle/fluid/train/custom_trainer/feed/scripts', '$OUT') + def UT_FILE(filename): UT_DIR = 'paddle/fluid/train/custom_trainer/feed/unit_test' import os return os.path.join(UT_DIR, filename) - custom_trainer_src = GLOB('paddle/fluid/train/custom_trainer/feed/*/*.cc', Exclude(UT_FILE('*'))) - CPPFLAGS_STR = '-DHPPL_STUB_FUNC -DLAPACK_FOUND -DPADDLE_DISABLE_PROFILER -DPADDLE_NO_PYTHON -DCUSTOM_TRAINER -DPADDLE_ON_INFERENCE -DPADDLE_USE_DSO -DPADDLE_USE_PTHREAD_BARRIER -DPADDLE_USE_PTHREAD_SPINLOCK -DPADDLE_VERSION=0.0.0 -DPADDLE_WITH_AVX -DPADDLE_WITH_MKLML -DPADDLE_WITH_XBYAK -DXBYAK64 -DXBYAK_NO_OP_NAMES -D_GNU_SOURCE -D__STDC_LIMIT_MACROS -DPYBIND_AVX_MKLML' + r" -DPADDLE_REVISION=\"%s@%s@%s\"" % (REPO_URL(), REPO_BRANCH(), REPO_REVISION()) -CFLAGS_STR = '-m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -fopenmp -mavx -O0 -DNDEBUG ' +CFLAGS_STR = '-m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -Wno-narrowing -Wnarrowing -fopenmp -mavx -O3 -DNDEBUG ' CXXFLAGS_STR = '-std=c++11 ' + CFLAGS_STR SharedLibrary("paddle_fluid_avx_mklml", PreBuilt(True)) diff --git a/paddle/fluid/framework/archive.h b/paddle/fluid/framework/archive.h old mode 100644 new mode 100755 index 100eb9518f71e76134e1baf4da9d1c569880a2db..30b5e0e83d6cba0ff5698f9af43b4cf41e6d5a96 --- a/paddle/fluid/framework/archive.h +++ b/paddle/fluid/framework/archive.h @@ -518,11 +518,11 @@ Archive& operator>>(Archive& ar, std::tuple& x) { } \ template \ Archive& operator>>(Archive& ar, MAP_TYPE& p) { \ - size_t size = ar.template get(); \ + size_t size = ar.template Get(); \ p.clear(); \ RESERVE_STATEMENT; \ for (size_t i = 0; i < size; i++) { \ - p.insert(ar.template get>()); \ + p.insert(ar.template Get>()); \ } \ return ar; \ } @@ -539,11 +539,11 @@ Archive& operator>>(Archive& ar, std::tuple& x) { } \ template \ Archive& operator>>(Archive& ar, MAP_TYPE& p) { \ - size_t size = ar.template get(); \ + size_t size = ar.template Get(); \ p.clear(); \ RESERVE_STATEMENT; \ for (size_t i = 0; i < size; i++) { \ - p.insert(ar.template get>()); \ + p.insert(ar.template Get>()); \ } \ return ar; \ } @@ -568,11 +568,11 @@ ARCHIVE_REPEAT(std::unordered_multimap, p.reserve(size)) } \ template \ Archive& operator>>(Archive& ar, SET_TYPE& p) { \ - size_t size = ar.template get(); \ + size_t size = ar.template Get(); \ p.clear(); \ RESERVE_STATEMENT; \ for (size_t i = 0; i < size; i++) { \ - p.insert(ar.template get()); \ + p.insert(ar.template Get()); \ } \ return ar; \ } @@ -588,11 +588,11 @@ ARCHIVE_REPEAT(std::unordered_multimap, p.reserve(size)) } \ template \ Archive& operator>>(Archive& ar, SET_TYPE& p) { \ - size_t size = ar.template get(); \ + size_t size = ar.template Get(); \ p.clear(); \ RESERVE_STATEMENT; \ for (size_t i = 0; i < size; i++) { \ - p.insert(ar.template get()); \ + p.insert(ar.template Get()); \ } \ return ar; \ } diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc old mode 100644 new mode 100755 diff --git a/paddle/fluid/string/string_helper.cc b/paddle/fluid/string/string_helper.cc old mode 100644 new mode 100755 index 27708b8eebd2131ebadcc310fd3521ad5ab824f3..8feff87b8dc0daa17e4f2d2e095398f343dfd06b --- a/paddle/fluid/string/string_helper.cc +++ b/paddle/fluid/string/string_helper.cc @@ -24,26 +24,6 @@ namespace paddle { namespace string { -inline size_t count_spaces(const char* s) { - size_t count = 0; - - while (*s != 0 && isspace(*s++)) { - count++; - } - - return count; -} - -inline size_t count_nonspaces(const char* s) { - size_t count = 0; - - while (*s != 0 && !isspace(*s++)) { - count++; - } - - return count; -} - // remove leading and tailing spaces std::string trim_spaces(const std::string& str) { const char* p = str.c_str(); diff --git a/paddle/fluid/string/string_helper.h b/paddle/fluid/string/string_helper.h old mode 100644 new mode 100755 index cbf7fd2a7ba8787ac56d9a5d112490c07f55c9ca..3f5c893b98985c14b9b130e31cb57de8b5d2075f --- a/paddle/fluid/string/string_helper.h +++ b/paddle/fluid/string/string_helper.h @@ -26,9 +26,25 @@ namespace paddle { namespace string { -inline size_t count_spaces(const char* s); +inline size_t count_spaces(const char* s) { + size_t count = 0; -inline size_t count_nonspaces(const char* s); + while (*s != 0 && isspace(*s++)) { + count++; + } + + return count; +} + +inline size_t count_nonspaces(const char* s) { + size_t count = 0; + + while (*s != 0 && !isspace(*s++)) { + count++; + } + + return count; +} template void format_string_append(std::string& str, const char* fmt, // NOLINT diff --git a/paddle/fluid/string/to_string.h b/paddle/fluid/string/to_string.h index 8caf149420393ec81131389d7787bee925f4a27d..d1496c00bbedfd6f07e30f529b55c488d32170b0 100644 --- a/paddle/fluid/string/to_string.h +++ b/paddle/fluid/string/to_string.h @@ -31,6 +31,15 @@ inline std::string to_string(T v) { return sout.str(); } +template +inline std::string to_string(const std::vector& v_list) { + std::ostringstream sout; + for (const auto& v : v_list) { + sout << v << " "; + } + return sout.str(); +} + template <> inline std::string to_string(std::type_index t) { return t.name(); diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/dense_input_accessor.cc b/paddle/fluid/train/custom_trainer/feed/accessor/dense_input_accessor.cc index c74a679c9f3a9334615a66f99a4618e931bbdfaf..f1ca59ecd0fa5582f4d83a11c842473f1538e4bd 100644 --- a/paddle/fluid/train/custom_trainer/feed/accessor/dense_input_accessor.cc +++ b/paddle/fluid/train/custom_trainer/feed/accessor/dense_input_accessor.cc @@ -7,14 +7,13 @@ namespace feed { int DenseInputAccessor::initialize(YAML::Node config, std::shared_ptr context_ptr) { CHECK(DataInputAccessor::initialize(config, context_ptr) == 0); - CHECK(config["input"] && config["input"].Type() == YAML::NodeType::Map); _total_dim = 0; _pull_request_num.store(0); for (const auto& input : config["input"]) { DenseInputVariable variable; - variable.name = input.first.as(); + variable.name = input["name"].as(); variable.gradient_name = paddle::framework::GradVarName(variable.name); - variable.shape = input.second["shape"].as>(); + variable.shape = input["shape"].as>(); variable.dim = 1; for (int i = 0; i < variable.shape.size(); ++i) { if (variable.shape[i] <= 0) { @@ -23,6 +22,7 @@ int DenseInputAccessor::initialize(YAML::Node config, variable.dim *= variable.shape[i]; } _total_dim += variable.dim; + _x_variables.emplace_back(variable); } if (config["async_pull"] && config["async_pull"].as()) { _need_async_pull = true; @@ -30,14 +30,28 @@ int DenseInputAccessor::initialize(YAML::Node config, return 0; } +int32_t DenseInputAccessor::create(::paddle::framework::Scope* scope) { + size_t data_buffer_idx = 0; + std::vector regions; + for (auto& variable : _x_variables) { + auto* tensor = scope->Var(variable.name)-> + GetMutable(); + auto* data = tensor->data(); + regions.emplace_back(data, variable.dim); + } + auto* ps_client = _trainer_context->pslib->ps_client(); + auto push_status = ps_client->push_dense_param(regions.data(), regions.size(), _table_id); + return push_status.get(); +} + // rpc拉取数据，需保证单线程运行 int32_t DenseInputAccessor::pull_dense(size_t table_id) { float* data_buffer = NULL; - if (_data_buffer != nullptr) { - data_buffer = _data_buffer; - } else { - data_buffer = new float[_total_dim]; + if (_data_buffer == nullptr) { + _data_buffer = new float[_total_dim]; } + // TODO 使用双buffer DataBuffer,避免训练期改写，当前异步SGD下，问题不大 + data_buffer = _data_buffer; size_t data_buffer_idx = 0; std::vector regions; @@ -77,6 +91,8 @@ int32_t DenseInputAccessor::forward(SampleInstance* samples, size_t num, auto* shape_ptr = &(variable.shape[0]); paddle::framework::DDim ddim(shape_ptr, variable.shape.size()); auto* tensor = ScopeHelper::resize_lod_tensor(scope, variable.name, ddim); + auto* grad_tensor = ScopeHelper::resize_lod_tensor(scope, variable.gradient_name, ddim); + VLOG(5) << "fill scope variable:" << variable.name << ", " << variable.gradient_name; auto* var_data = tensor->mutable_data(_trainer_context->cpu_place); memcpy(var_data, _data_buffer + data_buffer_idx, variable.dim * sizeof(float)); data_buffer_idx += variable.dim; @@ -102,7 +118,8 @@ int32_t DenseInputAccessor::backward(SampleInstance* samples, size_t num, } auto* ps_client = _trainer_context->pslib->ps_client(); auto push_status = ps_client->push_dense(regions.data(), regions.size(), _table_id); - return push_status.get(); + //return push_status.get(); + return 0; } int32_t EbdVariableInputAccessor::forward(SampleInstance* samples, size_t num, diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.cc b/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.cc index 36e0b9fd7920ad44d3cf4b3f0e284276339528ce..6ef4c398d389a727a86a1241c1155fcc5fec55ab 100644 --- a/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.cc +++ b/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.cc @@ -7,88 +7,149 @@ namespace custom_trainer { namespace feed { int EpochAccessor::initialize(YAML::Node config, std::shared_ptr context_ptr) { - _model_root_path = config["model_root_path"].as() + "/"; - - _done_file_path = _model_root_path; + _model_root_path = config["model_root_path"].as(); + + _trainer_context = context_ptr.get(); + if (context_ptr->file_system == nullptr) { + VLOG(0) << "file_system is not initialized"; + return -1; + } + auto fs = _trainer_context->file_system.get(); if (config["donefile"]) { - _done_file_path.append(config["donefile"].as()); + _done_file_path = fs->path_join(_model_root_path, config["donefile"].as()); } else { - _done_file_path.append("epoch_donefile.txt"); + _done_file_path = fs->path_join(_model_root_path, "epoch_donefile.txt"); } - if (!context_ptr->file_system->exists(_done_file_path)) { + if (!fs->exists(_done_file_path)) { VLOG(0) << "missing done file, path:" << _done_file_path; + return -1; } - std::string done_text = context_ptr->file_system->tail(_done_file_path); + std::string done_text = fs->tail(_done_file_path); _done_status = paddle::string::split_string(done_text, std::string("\t")); _current_epoch_id = get_status(EpochStatusFiled::EpochIdField); _last_checkpoint_epoch_id = get_status(EpochStatusFiled::CheckpointIdField); _last_checkpoint_path = get_status(EpochStatusFiled::CheckpointPathField); return 0; } + + int32_t EpochAccessor::epoch_done(uint64_t epoch_id) { + struct timeval now; + gettimeofday(&now, NULL); + if (need_save_model(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint)) { + _last_checkpoint_epoch_id = epoch_id; + _last_checkpoint_path = model_save_path(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint); + } + set_status(EpochStatusFiled::EpochIdField, epoch_id); + set_status(EpochStatusFiled::TimestampField, now.tv_sec); + set_status(EpochStatusFiled::CheckpointIdField, _last_checkpoint_epoch_id); + set_status(EpochStatusFiled::CheckpointPathField, _last_checkpoint_path); + set_status(EpochStatusFiled::DateField, format_timestamp(epoch_id, "%Y%m%d")); - int HourlyEpochAccessor::initialize(YAML::Node config, - std::shared_ptr context_ptr) { - EpochAccessor::initialize(config, context_ptr); + // 非主节点不做状态持久化 + if (!_trainer_context->environment->is_master_node(EnvironmentRole::WORKER)) { + return 0; + } + auto fs = _trainer_context->file_system.get(); + std::string done_str = paddle::string::join_strings(_done_status, '\t'); + // 保留末尾1000数据 + std::string tail_done_info = paddle::string::trim_spaces(fs->tail(_done_file_path, 1000)); + if (tail_done_info.size() > 0) { + tail_done_info = tail_done_info + "\n" + done_str; + } else { + tail_done_info = done_str; + } + VLOG(2) << "Write epoch donefile to " << _done_file_path << ", str:" << done_str; + bool write_success = false; + while (true) { + fs->remove(_done_file_path); + auto fp = fs->open_write(_done_file_path, ""); + if (fwrite(tail_done_info.c_str(), tail_done_info.length(), 1, &*fp) == 1) { + break; + } + sleep(10); + } + VLOG(2) << "Write epoch donefile success"; return 0; } - void HourlyEpochAccessor::next_epoch() { + int TimelyEpochAccessor::initialize(YAML::Node config, + std::shared_ptr context_ptr) { + _time_zone_seconds = config["time_zone_seconds"].as(); + _train_time_interval = config["train_time_interval"].as(); + CHECK(_train_time_interval > 0 && (_train_time_interval % SecondsPerMin) == 0); + _train_num_per_day = SecondsPerDay / _train_time_interval; + return EpochAccessor::initialize(config, context_ptr); + } + + void TimelyEpochAccessor::next_epoch() { _current_epoch_id = next_epoch_id(_current_epoch_id); } - std::string HourlyEpochAccessor::text(uint64_t epoch_id) { - return format_timestamp(epoch_id, "%Y%m%d delta-%H"); + std::string TimelyEpochAccessor::text(uint64_t epoch_id) { + auto delta = delta_id(epoch_id); + std::string date = format_timestamp(epoch_id, "%Y%m%d%H%M"); + return string::format_string("%s delta-%d", date.c_str(), delta); } - uint64_t HourlyEpochAccessor::next_epoch_id(uint64_t epoch_id) { + uint64_t TimelyEpochAccessor::next_epoch_id(uint64_t epoch_id) { if (epoch_id == 0) { struct timeval now; gettimeofday(&now, NULL); - return now.tv_sec / (24 * 3600) * (24 * 3600); + // 归整到零点 + return now.tv_sec / SecondsPerDay * SecondsPerDay; } - return epoch_id + 3600; + return epoch_id + _train_time_interval; } - bool HourlyEpochAccessor::is_last_epoch(uint64_t epoch_id) { - return ((epoch_id / 3600) % 24) == 23; + bool TimelyEpochAccessor::is_last_epoch(uint64_t epoch_id) { + auto delta = delta_id(epoch_id); + return delta == _train_num_per_day; } - uint64_t HourlyEpochAccessor::epoch_time_interval() { - return 3600; + uint64_t TimelyEpochAccessor::epoch_time_interval() { + return _train_time_interval; } - uint64_t HourlyEpochAccessor::epoch_timestamp(uint64_t epoch_id) { + uint64_t TimelyEpochAccessor::epoch_timestamp(uint64_t epoch_id) { return epoch_id; } - bool HourlyEpochAccessor::need_save_model(uint64_t epoch_id, ModelSaveWay save_way) { + bool TimelyEpochAccessor::need_save_model(uint64_t epoch_id, ModelSaveWay save_way) { if (epoch_id == 0) { return false; } - if (save_way == ModelSaveWay::ModelSaveInferenceDelta) { - return true; - } else if (save_way == ModelSaveWay::ModelSaveInferenceBase) { - return is_last_epoch(epoch_id); - } else if (save_way == ModelSaveWay::ModelSaveTrainCheckpoint) { - return ((epoch_id / 3600) % 8) == 0; + switch (save_way) { + case ModelSaveWay::ModelSaveInferenceDelta: + return true; + case ModelSaveWay::ModelSaveInferenceBase: + return is_last_epoch(epoch_id); + case ModelSaveWay::ModelSaveTrainCheckpoint: + return ((epoch_id / SecondsPerHour) % 8) == 0; } return false; } - std::string HourlyEpochAccessor::model_save_path(uint64_t epoch_id, ModelSaveWay save_way) { - if (save_way == ModelSaveWay::ModelSaveInferenceDelta) { - return _model_root_path + "/xbox/delta-" + std::to_string(epoch_id); - } else if (save_way == ModelSaveWay::ModelSaveInferenceBase) { - return _model_root_path + "/xbox/base"; - } else if (save_way == ModelSaveWay::ModelSaveTrainCheckpoint) { - return _model_root_path + "/xbox/checkpoint"; + std::string TimelyEpochAccessor::model_save_path(uint64_t epoch_id, ModelSaveWay save_way) { + int32_t delta = delta_id(epoch_id); + std::string date = format_timestamp(epoch_id, "%Y%m%d"); + std::string date_with_hour = format_timestamp(epoch_id, "%Y%m%d%H"); + switch (save_way) { + case ModelSaveWay::ModelSaveInferenceDelta: + return _trainer_context->file_system->path_join(_model_root_path, + string::format_string("xbox/%s/delta-%d", date.c_str(), delta)); + case ModelSaveWay::ModelSaveInferenceBase: + return _trainer_context->file_system->path_join(_model_root_path, + string::format_string("xbox/%s/base", date.c_str())); + case ModelSaveWay::ModelSaveTrainCheckpoint: + return _trainer_context->file_system->path_join(_model_root_path, + string::format_string("batch_model/%s", date_with_hour.c_str())); } return ""; } - REGIST_CLASS(EpochAccessor, HourlyEpochAccessor); + REGIST_CLASS(EpochAccessor, TimelyEpochAccessor); } // namespace feed } // namespace custom_trainer diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h b/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h index 069abff31d162fbf09f0793530bb2322a6ac3915..07b15c62c59228b3c6532550b4554542648f91e5 100644 --- a/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h +++ b/paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h @@ -31,27 +31,39 @@ public: virtual const std::string& checkpoint_path() { return _last_checkpoint_path; } - + + virtual int32_t epoch_done(uint64_t epoch_id); + template T get_status(EpochStatusFiled field) { auto status = paddle::string::trim_spaces(_done_status[static_cast(field)]); return boost::lexical_cast(status.c_str()); } - - virtual void next_epoch() = 0; + template + void set_status(EpochStatusFiled field, const T& status) { + auto str_status = paddle::string::to_string(status); + _done_status[static_cast(field)] = str_status; + return; + } virtual std::string model_root_path() { return _model_root_path; } + + virtual void next_epoch() = 0; + virtual std::string text(uint64_t epoch_id) = 0; virtual uint64_t next_epoch_id(uint64_t epoch_id) = 0; virtual bool is_last_epoch(uint64_t epoch_id) = 0; + //epoch间的数据时间间隔（秒） virtual uint64_t epoch_time_interval() = 0; //获取epoch的样本数据时间 virtual uint64_t epoch_timestamp(uint64_t epoch_id) = 0; + virtual bool need_save_model(uint64_t epoch_id, ModelSaveWay save_way) = 0; virtual std::string model_save_path(uint64_t epoch_id, ModelSaveWay save_way) = 0; protected: + TrainerContext* _trainer_context; std::string _done_file_path; std::string _model_root_path; uint64_t _current_epoch_id = 0; @@ -62,10 +74,10 @@ protected: }; REGIST_REGISTERER(EpochAccessor); -class HourlyEpochAccessor : public EpochAccessor { +class TimelyEpochAccessor : public EpochAccessor { public: - HourlyEpochAccessor() {} - virtual ~HourlyEpochAccessor() {} + TimelyEpochAccessor() {} + virtual ~TimelyEpochAccessor() {} virtual int initialize(YAML::Node config, std::shared_ptr context_ptr); virtual void next_epoch(); @@ -76,6 +88,14 @@ public: virtual uint64_t epoch_timestamp(uint64_t epoch_id); virtual bool need_save_model(uint64_t epoch_id, ModelSaveWay save_way); virtual std::string model_save_path(uint64_t epoch_id, ModelSaveWay save_way); + +private: + inline size_t delta_id(uint64_t epoch_id) { + return ((epoch_id + _time_zone_seconds) % SecondsPerDay) / _train_time_interval; + } + uint32_t _time_zone_seconds; // 相对UTC时差(秒) + uint32_t _train_time_interval; // 训练时间间隔(秒) + uint32_t _train_num_per_day; // 天级训练总轮数 }; } // namespace feed diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h b/paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h index 1cca4b1d5d76ea12b50f8028fe3f28d27c38f9ac..bc2063c4288b2623bfd477c65afc5ad0be1dac19 100644 --- a/paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h +++ b/paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h @@ -22,6 +22,11 @@ public: return 0; } + // 创建，一般用于模型冷启的随机初始化 + virtual int32_t create(::paddle::framework::Scope* scope) { + return 0; + } + // 前向， 一般用于填充输入，在训练网络执行前调用 virtual int32_t forward(SampleInstance* samples, size_t num, ::paddle::framework::Scope* scope) = 0; @@ -30,12 +35,34 @@ public: virtual int32_t backward(SampleInstance* samples, size_t num, ::paddle::framework::Scope* scope) = 0; protected: - size_t _table_id; + size_t _table_id = 0; bool _need_gradient = false; TrainerContext* _trainer_context = nullptr; }; REGIST_REGISTERER(DataInputAccessor); +struct LabelInputVariable { + std::string label_name; + std::string output_name; + size_t label_dim = 0; +}; +class LabelInputAccessor : public DataInputAccessor { +public: + LabelInputAccessor() {} + virtual ~LabelInputAccessor() {} + + virtual int initialize(YAML::Node config, + std::shared_ptr context_ptr); + + virtual int32_t forward(SampleInstance* samples, size_t num, + ::paddle::framework::Scope* scope); + + virtual int32_t backward(SampleInstance* samples, size_t num, + ::paddle::framework::Scope* scope); +protected: + size_t _label_total_dim = 0; + std::vector _labels; +}; struct SparseInputVariable { size_t slot_dim; @@ -104,6 +131,8 @@ public: virtual int initialize(YAML::Node config, std::shared_ptr context_ptr); + + virtual int32_t create(::paddle::framework::Scope* scope); virtual int32_t forward(SampleInstance* samples, size_t num, paddle::framework::Scope* scope); diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/label_input_accessor.cc b/paddle/fluid/train/custom_trainer/feed/accessor/label_input_accessor.cc new file mode 100644 index 0000000000000000000000000000000000000000..1a18dfea48c03029bd45d3bfb83e21a42eb38b0b --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/accessor/label_input_accessor.cc @@ -0,0 +1,74 @@ +#include "paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h" + +namespace paddle { +namespace custom_trainer { +namespace feed { + +int LabelInputAccessor::initialize(YAML::Node config, + std::shared_ptr context_ptr) { + _trainer_context = context_ptr.get(); + _label_total_dim = 0; + for (const auto& input : config["input"]) { + LabelInputVariable variable; + variable.label_name = input["label_name"].as(); + variable.output_name = input["output_name"].as(); + auto shape = input["shape"].as>(); + variable.label_dim = 0; + for (auto dim : shape) { + variable.label_dim += (dim > 0 ? dim : 0); + } + _label_total_dim += variable.label_dim; + _labels.emplace_back(variable); + } + return 0; +} + +int32_t LabelInputAccessor::forward(SampleInstance* samples, size_t num, + paddle::framework::Scope* scope) { + if (num < 1) { + return 0; + } + size_t sample_label_data_idx = 0; + for (auto& label : _labels) { + auto* tensor = ScopeHelper::resize_lod_tensor(scope, label.label_name, {num, label.label_dim}); + auto* res_tens = ScopeHelper::resize_lod_tensor(scope, label.output_name, {num, label.label_dim}); + auto* var_data = tensor->mutable_data(_trainer_context->cpu_place); + for (size_t i = 0; i < num; ++i) { + auto& sample = samples[i]; + CHECK(sample.labels.size() > sample_label_data_idx); + float* sample_label_buffer = sample.labels.data(); + memcpy(var_data + i * label.label_dim, + sample_label_buffer + sample_label_data_idx, label.label_dim * sizeof(float)); + } + sample_label_data_idx += label.label_dim; + } + return 0; +} + +int32_t LabelInputAccessor::backward(SampleInstance* samples, size_t num, + paddle::framework::Scope* scope) { + if (num < 1) { + return 0; + } + for (size_t i = 0; i < num; ++i) { + auto& sample = samples[i]; + sample.predicts.resize(_label_total_dim); + size_t sample_predict_data_idx = 0; + float* sample_predict_buffer = sample.predicts.data(); + for (auto& label : _labels) { + auto* tensor = scope->Var(label.output_name)-> + GetMutable(); + auto* var_data = tensor->mutable_data(_trainer_context->cpu_place); + memcpy(sample_predict_buffer + sample_predict_data_idx, + var_data + i * label.label_dim, label.label_dim * sizeof(float)); + sample_predict_data_idx += label.label_dim; + } + } + return 0; +} + +REGIST_CLASS(DataInputAccessor, LabelInputAccessor); + +} // namespace feed +} // namespace custom_trainer +} // namespace paddle diff --git a/paddle/fluid/train/custom_trainer/feed/accessor/sparse_input_accessor.cc b/paddle/fluid/train/custom_trainer/feed/accessor/sparse_input_accessor.cc index 2ff68d9fcdb2cc95ba4b188b8389e520759eb370..e51fd1b59b52e8d4a39e655bd3a8a7abdc5de5f6 100644 --- a/paddle/fluid/train/custom_trainer/feed/accessor/sparse_input_accessor.cc +++ b/paddle/fluid/train/custom_trainer/feed/accessor/sparse_input_accessor.cc @@ -12,20 +12,18 @@ namespace feed { int BaseSparseInputAccessor::initialize(YAML::Node config, std::shared_ptr context_ptr) { CHECK(DataInputAccessor::initialize(config, context_ptr) == 0); - CHECK(config["input"] && config["input"].Type() == YAML::NodeType::Map); for (const auto& input : config["input"]) { SparseInputVariable variable; - variable.name = input.first.as(); + variable.name = input["name"].as(); variable.gradient_name = paddle::framework::GradVarName(variable.name); - std::string slots_str = input.second["slots"].as(); - std::vector slots = paddle::string::split_string(slots_str, ","); + auto slots = input["slots"].as>(); variable.slot_idx.resize(UINT16_MAX, -1); for (int i = 0; i < slots.size(); ++i) { - auto slot = (uint16_t)atoi(slots[i].c_str()); + uint16_t slot = (uint16_t)slots[i]; variable.slot_idx[slot] = i; variable.slot_list.push_back(slot); } - variable.slot_dim = input.second["slot_dim"].as(); + variable.slot_dim = input["slot_dim"].as(); variable.total_dim = variable.slot_list.size() * variable.slot_dim; _x_variables.push_back(variable); } @@ -52,7 +50,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples, auto& features = samples[i].features; for (auto& feature_item : features) { feature_item.weights.resize(pull_value_dim, 0.0); - keys[key_idx] = feature_item.feature_sign; + keys[key_idx] = feature_item.sign(); pull_values[key_idx++] = &(feature_item.weights[0]); } } @@ -60,6 +58,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples, auto ret = pull_status.get(); delete[] pull_values; if (ret != 0) { + VLOG(0) << "pull sparse failed, table_id:" << _table_id << ", key_num:" << key_num << ", ret:" << ret; return ret; } @@ -78,6 +77,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples, scope, variable.name, {num, variable.total_dim}); auto* grad_tensor = ScopeHelper::resize_lod_tensor( scope, variable.gradient_name, {num, variable.total_dim}); + VLOG(5) << "fill scope variable:" << variable.name << ", " << variable.gradient_name; var_runtime_data[i].variable_data = tensor->mutable_data(_trainer_context->cpu_place); var_runtime_data[i].gradient_data = grad_tensor->mutable_data(_trainer_context->cpu_place); memset((void*) var_runtime_data[i].variable_data, 0, var_runtime_data[i].total_size * sizeof(float)); @@ -89,12 +89,12 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples, for (auto& feature_item : features) { for (size_t i = 0; i < _x_variables.size(); ++i) { auto& variable = _x_variables[i]; - auto slot_idx = variable.slot_idx[feature_item.slot_id]; + auto slot_idx = variable.slot_idx[feature_item.slot()]; if (slot_idx < 0) { continue; } float* item_data = var_runtime_data[i].variable_data + - samp_idx * var_runtime_data[i].total_size + variable.slot_dim * slot_idx; + samp_idx * variable.total_dim + variable.slot_dim * slot_idx; fill_input(item_data, &(feature_item.weights[0]), *value_accessor, variable, samples[samp_idx]); } } @@ -138,24 +138,24 @@ int32_t BaseSparseInputAccessor::backward(SampleInstance* samples, feature_item.gradients.resize(push_value_dim, 0.0); for (size_t i = 0; i < _x_variables.size(); ++i) { auto& variable = _x_variables[i]; - auto slot_idx = variable.slot_idx[feature_item.slot_id]; + auto slot_idx = variable.slot_idx[feature_item.slot()]; if (slot_idx < 0) { continue; } const float* grad_data = var_runtime_data[i].gradient_data + - samp_idx * var_runtime_data[i].total_size + variable.slot_dim * slot_idx; + samp_idx * variable.total_dim + variable.slot_dim * slot_idx; fill_gradient(&(feature_item.gradients[0]), grad_data, *value_accessor, variable, samples[samp_idx]); - keys[key_idx] = feature_item.feature_sign; + keys[key_idx] = feature_item.sign(); push_values[key_idx++] = &(feature_item.gradients[0]); } } } auto push_status = ps_client->push_sparse(_table_id, - keys.data(), (const float**)push_values, key_num); - auto ret = push_status.get(); + keys.data(), (const float**)push_values, key_idx); + //auto ret = push_status.get(); delete[] push_values; - return ret; + return 0; } class AbacusSparseJoinAccessor : public BaseSparseInputAccessor { diff --git a/paddle/fluid/train/custom_trainer/feed/common/pipeline.h b/paddle/fluid/train/custom_trainer/feed/common/pipeline.h index 1db754903b9feaecfa654ba985b180a524075a43..52ff8e4685cc6fcc41a2d016b0142c35a853abd0 100644 --- a/paddle/fluid/train/custom_trainer/feed/common/pipeline.h +++ b/paddle/fluid/train/custom_trainer/feed/common/pipeline.h @@ -59,6 +59,10 @@ public: _input_data_buffer.resize(input_batch_size * options.buffer_batch_count); _output_data_buffer.resize(options.batch_size * options.buffer_batch_count); _output_channel->SetCapacity(_output_data_buffer.size()); + if (_options.need_hold_input_data) { + _input_channel_backup = ::paddle::framework::MakeChannel(); + _input_channel_backup->SetBlockSize(input_batch_size); + } CHECK(_input_channel != nullptr) << " Input Channel is null"; _convert_thread = std::make_shared([this](){ async_convert_data(); @@ -114,12 +118,16 @@ private: CHECK(_converter(&_input_data_buffer[0], read_size, &_output_data_buffer[0], &write_size, 0) == 0) << "Data Converter Do Failed"; _output_channel->WriteMove(write_size, &_output_data_buffer[0]); - if (_options.need_hold_input_data) { + if (_input_channel_backup) { _input_channel_backup->WriteMove(read_size, &_input_data_buffer[0]); } } sleep(1); } + _output_channel->Close(); + if (_input_channel_backup) { + _input_channel_backup->Close(); + } } diff --git a/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.cc b/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.cc index 7f8ee5e3aeabb6acf78a0d62b4702ff2d5d6d710..fd7bd2d659b0ad0e87376517ee4ba16122630359 100644 --- a/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.cc +++ b/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.cc @@ -12,7 +12,8 @@ namespace custom_trainer { namespace feed { int PSlib::initialize(const std::string& conf_path, - RuntimeEnvironment* environment, EnvironmentRole role) { + RuntimeEnvironment* environment) { + _environment = environment; init_gflag(); int file_descriptor = open(conf_path.c_str(), O_RDONLY); if (file_descriptor == -1){ @@ -25,26 +26,26 @@ int PSlib::initialize(const std::string& conf_path, return -1; } close(file_descriptor); - init_server(role); - init_client(EnvironmentRole::ALL); + init_server(); + init_client(); return 0; } -int PSlib::init_server(EnvironmentRole role) { - if (role == EnvironmentRole::PSERVER) { +int PSlib::init_server() { + if (_environment->is_role(EnvironmentRole::PSERVER)) { _server_ptr.reset(paddle::ps::PSServerFactory::create(_ps_param)); _server_ptr->configure(_ps_param, *(_environment->ps_environment()), - _environment->rank_id(role)); + _environment->rank_id(EnvironmentRole::PSERVER)); _server_ptr->start(); } _environment->ps_environment()->gather_ps_servers(); return 0; } -int PSlib::init_client(EnvironmentRole role) { +int PSlib::init_client() { _client_ptr.reset(paddle::ps::PSClientFactory::create(_ps_param)); _client_ptr->configure(_ps_param, *(_environment->ps_environment()), - _environment->rank_id(role)); + _environment->rank_id(EnvironmentRole::ALL)); return 0; } diff --git a/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h b/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h index 0180a8e2ec479491948ed8a7b3e6cd304f74e168..d35aa9a70173163e5a55461966922b0efb9a57e6 100644 --- a/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h +++ b/paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h @@ -14,6 +14,9 @@ limitations under the License. */ #pragma once +// Hide BLOG +#define BUTIL_LOGGING_H_ +#define COMPACT_GOOGLE_LOG_NOTICE COMPACT_GOOGLE_LOG_INFO #include "communicate/ps_server.h" #include "communicate/ps_client.h" @@ -22,21 +25,20 @@ namespace custom_trainer { namespace feed { class RuntimeEnvironment; -enum class EnvironmentRole; class PSlib { public: PSlib() {} virtual ~PSlib() {} int initialize(const std::string& conf_path, - RuntimeEnvironment* environment, EnvironmentRole role); + RuntimeEnvironment* environment); virtual paddle::ps::PSServer* ps_server(); virtual paddle::ps::PSClient* ps_client(); virtual paddle::PSParameter* get_param(); private: void init_gflag(); - virtual int init_server(EnvironmentRole role); - virtual int init_client(EnvironmentRole role); + virtual int init_server(); + virtual int init_client(); paddle::PSParameter _ps_param; RuntimeEnvironment* _environment; diff --git a/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.cc b/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.cc index 66f59b9cf5e0a97cdf95d4aeae1b701373290ce6..36d536112f16683f0780a4e6fe5a20ff5139859a 100644 --- a/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.cc +++ b/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.cc @@ -41,7 +41,7 @@ public: return -1; } _roles_node_info.resize(static_cast(EnvironmentRole::ALL) + 1); - set_role(EnvironmentRole::ALL); + add_role(EnvironmentRole::ALL); return 0; } @@ -56,7 +56,7 @@ public: virtual uint32_t node_num(EnvironmentRole role) { return mpi_node_info(role).node_num; } - virtual int set_role(EnvironmentRole role) { + virtual int add_role(EnvironmentRole role) { auto& node_info = mpi_node_info(role); if (node_info.rank_id < 0) { if (role == EnvironmentRole::ALL) { @@ -68,8 +68,12 @@ public: MPI_Comm_rank(node_info.mpi_comm, &(node_info.rank_id)); MPI_Comm_size(node_info.mpi_comm, &(node_info.node_num)); } + _role_set.insert(role); return 0; } + virtual bool is_role(EnvironmentRole role) { + return _role_set.count(role) > 0; + } virtual void barrier(EnvironmentRole role) { MPI_Barrier(mpi_node_info(role).mpi_comm); @@ -98,6 +102,7 @@ protected: } private: + std::set _role_set; std::vector _roles_node_info; }; REGIST_CLASS(RuntimeEnvironment, MPIRuntimeEnvironment); @@ -123,9 +128,12 @@ public: virtual uint32_t node_num(EnvironmentRole role) { return 1; } - virtual int set_role(EnvironmentRole role) { + virtual int add_role(EnvironmentRole role) { return 0; } + virtual bool is_role(EnvironmentRole role) { + return true; + } virtual void barrier(EnvironmentRole role) { return; } diff --git a/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h b/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h index e4ac68a90dd1cd11d722d2e7bf16bb64ce0d25f9..2e2cec876ae7ff2e75cb268ac17d78e29adbede8 100644 --- a/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h +++ b/paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h @@ -42,7 +42,9 @@ public: // 配置初始化 virtual int initialize(YAML::Node config) = 0; // 设置role - virtual int set_role(EnvironmentRole role) = 0; + virtual int add_role(EnvironmentRole role) = 0; + // 判断role + virtual bool is_role(EnvironmentRole role) = 0; // 环境初始化，会在所有依赖模块initialize后调用 virtual int wireup() = 0; diff --git a/paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h b/paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h new file mode 100644 index 0000000000000000000000000000000000000000..71c38a29746b15ae0ab4c11d0234271a94ec8f61 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include + +namespace paddle { +namespace custom_trainer { +namespace feed { + +class YamlHelper { +public: + // 直接使用node["key"]判断，会导致node数据被加入key键 + static bool has_key(const YAML::Node& node, const std::string& key) { + CHECK(node.Type() == YAML::NodeType::Map); + for (const auto& itr : node) { + if (key == itr.first.as()) { + return true; + } + } + return false; + } + template + static T get_with_default(YAML::Node node, const std::string& key, const T& default_v) { + if (has_key(node, key)) { + return node[key].as(); + } + return default_v; + } +}; + +} // namespace feed +} // namespace custom_trainer +} // namespace paddle diff --git a/paddle/fluid/train/custom_trainer/feed/conf/gflags.conf b/paddle/fluid/train/custom_trainer/feed/conf/gflags.conf index f7ac928362ce0a2ebb6469fb074c99d6de575595..c7b8a66344993b10337eec12ad77a6f2b4560c7b 100644 --- a/paddle/fluid/train/custom_trainer/feed/conf/gflags.conf +++ b/paddle/fluid/train/custom_trainer/feed/conf/gflags.conf @@ -1 +1,5 @@ --v=10 +-log_dir=log +-v=4 +-logbufsecs=0 +-pslib_push_dense_merge_limit=1 +-pslib_push_sparse_merge_limit=1 diff --git a/paddle/fluid/train/custom_trainer/feed/conf/ps_table_config b/paddle/fluid/train/custom_trainer/feed/conf/ps_table_config new file mode 100644 index 0000000000000000000000000000000000000000..d21b1e1127c2972d548a67fc429cd37745810d95 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/conf/ps_table_config @@ -0,0 +1,120 @@ +server_param { + downpour_server_param { + downpour_table_param { + table_id: 0 + table_class: "DownpourSparseTable" + shard_num: 1950 + accessor { + accessor_class: "DownpourCtrAccessor" + sparse_sgd_param { + learning_rate: 0.05 + initial_g2sum: 3.0 + initial_range: 0.0001 + weight_bounds: -10.0 + weight_bounds: 10.0 + } + fea_dim: 11 + embedx_dim: 8 + embedx_threshold: 10 + downpour_accessor_param { + nonclk_coeff: 0.1 + click_coeff: 1 + base_threshold: 1.5 + delta_threshold: 0.25 + delta_keep_days: 16 + delete_after_unseen_days: 30 + show_click_decay_rate: 0.98 + delete_threshold: 0.8 + } + table_accessor_save_param { + param: 1 + converter: "(tool/xbox_compressor_mf.py | tool/xbox_pb_converter)" + deconverter: "(tool/xbox_pb_deconverter | tool/xbox_decompressor_mf.awk)" + } + table_accessor_save_param { + param: 2 + converter: "(tool/xbox_compressor_mf.py | tool/xbox_pb_converter)" + deconverter: "(tool/xbox_pb_deconverter | tool/xbox_decompressor_mf.awk)" + } + } + type: PS_SPARSE_TABLE + compress_in_save: true + } + downpour_table_param { + table_id: 1 + table_class: "DownpourDenseTable" + accessor { + accessor_class: "DownpourDenseValueAccessor" + dense_sgd_param { + name: "adam" + adam { + learning_rate: 5e-06 + avg_decay_rate: 0.999993 + ada_decay_rate: 0.9999 + ada_epsilon: 1e-08 + mom_decay_rate: 0.99 + } + naive { + learning_rate: 0.0002 + } + } + fea_dim: 2571127 + } + type: PS_DENSE_TABLE + compress_in_save: true + } + downpour_table_param { + table_id: 2 + table_class: "DownpourDenseDoubleTable" + accessor { + accessor_class: "DownpourDenseValueDoubleAccessor" + dense_sgd_param { + name: "summarydouble" + summary { + summary_decay_rate: 0.999999 + } + } + fea_dim: 13464 + } + type: PS_DENSE_TABLE + compress_in_save: true + } + downpour_table_param { + table_id: 3 + table_class: "DownpourDenseTable" + accessor { + accessor_class: "DownpourDenseValueAccessor" + dense_sgd_param { + name: "adam" + adam { + learning_rate: 5e-06 + avg_decay_rate: 0.999993 + ada_decay_rate: 0.9999 + ada_epsilon: 1e-08 + mom_decay_rate: 0.99 + } + naive { + learning_rate: 0.0002 + } + } + fea_dim: 2072615 + } + type: PS_DENSE_TABLE + compress_in_save: true + } + service_param { + server_class: "DownpourBrpcPsServer" + client_class: "DownpourBrpcPsClient" + service_class: "DownpourPsService" + start_server_port: 0 + server_thread_num: 12 + } + } +} + +fs_client_param { + uri: "afs://xingtian.afs.baidu.com:9902" + user: "mlarch" + passwd: "Fv1M87" + hadoop_bin: "$HADOOP_HOME/bin/hadoop" +} diff --git a/paddle/fluid/train/custom_trainer/feed/conf/trainer.yaml b/paddle/fluid/train/custom_trainer/feed/conf/trainer.yaml index 2b97e6e3a42650173661136cd3a8d66328ee4b34..c71205b2468b16226143f660fa806ecd959f9080 100644 --- a/paddle/fluid/train/custom_trainer/feed/conf/trainer.yaml +++ b/paddle/fluid/train/custom_trainer/feed/conf/trainer.yaml @@ -1,34 +1,56 @@ -train_thread_num : 10 +train_thread_num: 10 -environment : - environment_class : LocalRuntimeEnvironment +environment: + environment_class: LocalRuntimeEnvironment + ps: ./conf/ps_table_config + + -io : - file_systems : - afs : - class : HadoopFileSystem - buffer_size : 1024000 - ugis : +io: + file_systems: + afs: + class: HadoopFileSystem + buffer_size: 1024000 + ugis: 'default': 'feed_video,D3a0z8' 'xingtian.afs.baidu.com:9902': 'feed_video,D3a0z8' - local : - class : LocalFileSystem - buffer_size : 1024000 -dataset : - data_list : - train_sample : - prefetch_num : 2 - root_path : ./sample - data_spit_interval : 300 - data_path_formater : '%Y%m%d/%H%M' - data_reader : LineDataReader - done_file : to.hadoop.done - filename_prefix : part - pipeline_cmd : cat - parser : - class : LineDataParser + default: + class: LocalFileSystem + buffer_size: 1024000 +dataset: + data_list: + train_sample: + prefetch_num: 2 + root_path : [./sample] + data_spit_interval: 300 + data_path_formater: '%Y%m%d/%H%M' + data_reader: LineDataReader + done_file: to.hadoop.done + filename_prefix: part + pipeline_cmd: './tool/ins_weight.py | awk -f ./tool/format_newcate_hotnews.awk' + parser: + class: AbacusTextDataParser epoch: - epoch_class : HourlyEpochAccessor - model_root_path : ./model/ - + epoch_class: TimelyEpochAccessor + model_root_path: ./model/ + train_time_interval: 600 + time_zone_seconds: 28800 + +executor: +- name: join + class: SimpleExecutor + train_data_name: train_sample + train_batch_size: 32 + input_parse_thread_num: 10 + push_gradient_thread_num: 16 + train_thread_num: 16 + need_dump_all_model: true +- name: update + class: SimpleExecutor + train_data_name: train_sample + train_batch_size: 32 + input_parse_thread_num: 10 + push_gradient_thread_num: 16 + train_thread_num: 16 + need_dump_all_model: false diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/abacus_data_reader.cc b/paddle/fluid/train/custom_trainer/feed/dataset/abacus_data_reader.cc new file mode 100755 index 0000000000000000000000000000000000000000..55ce639888ec5d4442a30ba1f704fd7a5d422b18 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/dataset/abacus_data_reader.cc @@ -0,0 +1,76 @@ +#include "paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h" + +#include +#include + +#include +#include + +#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h" + +namespace paddle { +namespace custom_trainer { +namespace feed { + +/*解析Abacus格式明文Feasign + */ +class AbacusTextDataParser : public LineDataParser { +public: + AbacusTextDataParser() {} + virtual ~AbacusTextDataParser() {} + + virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const { + instance.id = data.id; + instance.labels.resize(1); + size_t len = data.data.size(); + const char* str = data.data.c_str(); + const char* line_end = str + len; + + char* cursor = NULL; + int show = (int)strtol(str, &cursor, 10); + str = cursor; + instance.labels[0] = (float)strtol(str, &cursor, 10);// click + str = cursor; + + while (*(str += paddle::string::count_nonspaces(str)) != 0) { + if (*str == '*') { + str++; + size_t len = paddle::string::count_nonspaces(str); + str += len; + } else if (*str == '$') { + str++; + CHECK(((int)strtol(str, &cursor, 10), cursor != str))<<" sample type parse err:" << str; + str = cursor; + } else if (*str == '#') { + str++; + break; + } else if (*str == '@') { + str++; + size_t len = paddle::string::count_nonspaces(str); + std::string all_str(str, str + len); + str += len; + } else { + FeatureItem feature_item; + feature_item.sign() = (uint64_t)strtoull(str, &cursor, 10); + if (cursor == str) { //FIXME abacus没有这种情况 + str++; + continue; + } + str = cursor; + CHECK(*str++ == ':'); + CHECK(!isspace(*str)); + CHECK((feature_item.slot() = (int) strtol(str, &cursor, 10), cursor != str)) << " format error: " << str; + str = cursor; + instance.features.emplace_back(feature_item); + } + } + VLOG(5) << "parse sample success, id:" << instance.id << ", fea_sum:" + << instance.features.size() << ", label:" << instance.labels[0]; + return 0; + } +}; +REGIST_CLASS(DataParser, AbacusTextDataParser); + +} // namespace feed +} // namespace custom_trainer +} // namespace paddle diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.cc b/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.cc old mode 100644 new mode 100755 index 46794a84d39b7b19aa4d61ffbe5d66427d18032a..3898583d82ccc3f08787d5eaddd260c3e83a2aa3 --- a/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.cc +++ b/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.cc @@ -12,51 +12,389 @@ namespace paddle { namespace custom_trainer { namespace feed { -class LineDataParser : public DataParser { +int LineDataParser::parse(const char* str, size_t len, DataItem& data) const { + size_t pos = 0; + while (pos < len && str[pos] != ' ') { + ++pos; + } + if (pos >= len) { + VLOG(2) << "fail to parse line: " << std::string(str, len) << ", strlen: " << len; + return -1; + } + VLOG(5) << "getline: " << str << " , pos: " << pos << ", len: " << len; + data.id.assign(str, pos); + data.data.assign(str + pos + 1, len - pos - 1); + return 0; +} +REGIST_CLASS(DataParser, LineDataParser); + +/******************************** + * feasign压缩格式 + * 情形1：slot:hot + * |4b|4b|4b|4b|4b| 28b | + * |slot |0 |sign | + * 情形2：slot:hot*n + * |4b|4b|4b|4b|4b|4b|4b|4b|32b*n| + * |slot |1 |0 |len |sign | + * 情形3：slot:cold + * |4b|4b|4b|4b|4b|4b| 64b | + * |slot |2 |0 |sign | + * 情形4：slot:cold*n + * |4b|4b|4b|4b|4b|4b|4b|4b|64b*n| + * |slot |3 |0 |len |sign | + ********************************/ +class ArchiveDataParse : public DataParser { +public: + static const uint8_t HOT_SIGN_SIZE = 4; + static const uint8_t COLD_SIGN_SIZE = 8; + public: - LineDataParser() {} + ArchiveDataParse() {} + virtual ~ArchiveDataParse() {} + + struct Record { + int show, clk; + std::string tags; + std::map> vec_feas; + int sample_type; + std::map> auc_category_info_map; //为细维度计算auc准备的数据 + std::vector hot_feas, cold_feas; //冷(int32_t)热(uint64_t)feasign + + void clear() { + show = 0; + clk = 0; + tags.clear(); + vec_feas.clear(); + sample_type = 0; + auc_category_info_map.clear(); + hot_feas.clear(); + cold_feas.clear(); + } + + uint32_t calc_compress_feas_lens() const { + uint32_t hot_len = hot_feas.size(); + uint32_t cold_len = cold_feas.size(); + uint32_t cursor = 0; + int32_t pre_slot = -1; + uint32_t k = 0; + //热编码 + if (hot_len > 0) { + pre_slot = hot_feas[0].slot(); - virtual ~LineDataParser() {} + for (uint32_t i = 0; i < hot_len + 1; ++i) { + if (i == hot_len || pre_slot != hot_feas[i].slot()) { + cursor += 2; + //情形2 + if (i - k > 1) { + cursor += 2; + } + //情形1/2 + cursor += (HOT_SIGN_SIZE * (i - k)); + k = i; + } + pre_slot = hot_feas[i].slot(); + } + } + //冷编码 + if (cold_len > 0) { + pre_slot = cold_feas[0].slot(); + k = 0; + + for (uint32_t i = 0; i < cold_len + 1; ++i) { + if (i == cold_len || pre_slot != cold_feas[i].slot()) { + cursor += 2; + //情形4 + if (i - k > 1) { + cursor += 2; + } else { //情形3 + cursor++; + } + //情形3/4 + cursor += (COLD_SIGN_SIZE * (i - k)); + k = i; + } + pre_slot = cold_feas[i].slot(); + } + } + return cursor; + } + + void serialize_to_compress_feas(char* buffer) const { + if (buffer == nullptr) { + return ; + } + uint32_t cursor = 0; + uint32_t hot_len = hot_feas.size(); + uint32_t cold_len = cold_feas.size(); + int32_t pre_slot = -1; + int32_t hot_sign; + uint16_t slot; + uint8_t flag = 0, len = 0; + uint32_t k = 0; + //热编码 + if (hot_len > 0) { + pre_slot = hot_feas[0].slot(); + + for (uint32_t i = 0; i < hot_len + 1; ++i) { + if (i == hot_len || pre_slot != hot_feas[i].slot()) { + memcpy(buffer + cursor, &pre_slot, 2); + cursor += 2; + //情形2 + if (i - k > 1) { + flag = 0x10; + memcpy(buffer + cursor, &flag, 1); + cursor++; + len = i - k; + memcpy(buffer + cursor, &len, 1); + cursor++; + } + //情形1/2 + for (uint32_t j = k; j < i; ++j) { + hot_sign = (int32_t) hot_feas[j].sign(); + for (uint8_t b = 0; b < HOT_SIGN_SIZE; ++b) { + flag = (hot_sign >> ((HOT_SIGN_SIZE - b - 1) * 8)) & 0xFF; + memcpy(buffer + cursor, &flag, 1); + cursor++; + } + } + k = i; + } + pre_slot = hot_feas[i].slot(); + } + } + //冷编码 + if (cold_len > 0) { + pre_slot = cold_feas[0].slot(); + k = 0; + + for (uint32_t i = 0; i < cold_len + 1; ++i) { + if (i == cold_len || pre_slot != cold_feas[i].slot()) { + memcpy(buffer + cursor, &pre_slot, 2); + cursor += 2; + //情形4 + if (i - k > 1) { + flag = 0x30; + memcpy(buffer + cursor, &flag, 1); + cursor++; + len = i - k; + memcpy(buffer + cursor, &len, 1); + cursor++; + } + //情形3/4 + for (uint32_t j = k; j < i; ++j) { + if (i - k == 1) { + flag = 0x20; + memcpy(buffer + cursor, &flag, 1); + cursor++; + } + memcpy(buffer + cursor, &cold_feas[j].sign(), COLD_SIGN_SIZE); + cursor += COLD_SIGN_SIZE; + } + k = i; + } + pre_slot = cold_feas[i].slot(); + } + } + } + }; virtual int initialize(const YAML::Node& config, std::shared_ptr context) { + _index = context->cache_dict; + return 0; } virtual int parse(const char* str, size_t len, DataItem& data) const { - size_t pos = 0; - while (pos < len && str[pos] != ' ') { - ++pos; - } + size_t pos = paddle::string::count_nonspaces(str); if (pos >= len) { VLOG(2) << "fail to parse line: " << std::string(str, len) << ", strlen: " << len; return -1; } VLOG(5) << "getline: " << str << " , pos: " << pos << ", len: " << len; data.id.assign(str, pos); - data.data.assign(str + pos + 1, len - pos - 1); + str += pos; + + static thread_local std::vector vec_feas; + static thread_local Record rec; + rec.clear(); + + const char* line_end = str + len; + char* cursor = NULL; + CHECK((rec.show = (int)strtol(str, &cursor, 10), cursor != str)); + str = cursor; + CHECK((rec.clk = (int)strtol(str, &cursor, 10), cursor != str)); + str = cursor; + CHECK(rec.show >= 1 && rec.clk >= 0 && rec.clk <= rec.show); + + while (*(str += paddle::string::count_nonspaces(str)) != 0) { + if (*str == '*') { + str++; + size_t len = paddle::string::count_nonspaces(str); + std::string tag(str, str + len); + rec.tags = tag; + str += len; + } else if (*str == '$') { + str++; + CHECK((rec.sample_type = (int)strtol(str, &cursor, 10), cursor != str))<<" sample type parse err:" << str; + str = cursor; + } else if (*str == '#') { + str++; + size_t len = std::find_if_not(str, line_end, + [](char c) { return std::isalnum(c) != 0 || c == '_';}) - str; + CHECK(len > 0 && *(str + len) == ':'); + std::string name(str, len); + str += len; + vec_feas.clear(); + while (*str == ':') { + float val = 0; + CHECK((val = strtof(str + 1, &cursor), cursor > str)); + vec_feas.push_back(val); + str = cursor; + } + CHECK(rec.vec_feas.insert({name, vec_feas}).second); + } else if (*str == '@') { + str++; + size_t len = paddle::string::count_nonspaces(str); + std::string all_str(str, str + len); + str += len; + //category_name1=value1,value2,value3|category_name2=value1,value2|.... + std::vector all_category_vec = paddle::string::split_string(all_str, "|"); + for (size_t i = 0; i < all_category_vec.size(); ++i) { + std::string& single_category_str = all_category_vec[i]; + std::vector str_vec = paddle::string::split_string(single_category_str, "="); + CHECK(str_vec.size() == 2); + std::string category_name = str_vec[0]; + std::vector category_info_vec = paddle::string::split_string(str_vec[1], ","); + CHECK(category_info_vec.size() > 0); + + CHECK(rec.auc_category_info_map.insert({category_name, category_info_vec}).second); + } + } else { + uint64_t sign = 0; + int slot = -1; + sign = (uint64_t)strtoull(str, &cursor, 10); + if (cursor == str) { //FIXME abacus没有这种情况 + str++; + continue; + } + //CHECK((sign = (uint64_t)strtoull(str, &cursor, 10), cursor != str)); + str = cursor; + CHECK(*str++ == ':'); + CHECK(!isspace(*str)); + CHECK((slot = (int) strtol(str, &cursor, 10), cursor != str)) << " format error: " << str; + CHECK((uint16_t) slot == slot); + str = cursor; + + int32_t compress_sign = _index->sign2index(sign); + if (compress_sign < 0) { + rec.cold_feas.emplace_back(sign, (uint16_t)slot); + } else { + rec.hot_feas.emplace_back(compress_sign, (uint16_t)slot); + } + } + } + + paddle::framework::BinaryArchive bar; + bar << rec.show << rec.clk << rec.tags << rec.vec_feas << rec.sample_type << rec.auc_category_info_map; + uint32_t feas_len = rec.calc_compress_feas_lens(); //事先计算好压缩后feasign的空间 + bar << feas_len; + bar.Resize(bar.Length() + feas_len); + rec.serialize_to_compress_feas(bar.Finish() - feas_len); //直接在archive内部buffer进行压缩，避免不必要的拷贝 + data.data.assign(bar.Buffer(), bar.Length()); //TODO 这一步拷贝是否也能避免 + return 0; } - virtual int parse(const char* str, DataItem& data) const { - size_t pos = 0; - while (str[pos] != '\0' && str[pos] != ' ') { - ++pos; - } - if (str[pos] == '\0') { - VLOG(2) << "fail to parse line: " << str << ", get '\\0' at pos: " << pos; + virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const { + instance.id = data.id; + if (data.data.empty()) { return -1; } - VLOG(5) << "getline: " << str << " , pos: " << pos; - data.id.assign(str, pos); - data.data.assign(str + pos + 1); + + //FIXME temp + int show = 0, clk = 0; + std::string tags; + std::map> vec_feas; + int sample_type = 0; + std::map> auc_category_info_map; + uint32_t feas_len = 0; + + paddle::framework::BinaryArchive bar; + bar.SetReadBuffer(const_cast(&data.data[0]), data.data.size(), nullptr); + bar >> show; + bar >> clk; + bar >> tags; + bar >> vec_feas; + bar >> sample_type; + bar >> auc_category_info_map; + bar >> feas_len; + CHECK((bar.Finish() - bar.Cursor()) == feas_len); + deserialize_feas_to_ins(bar.Cursor(), feas_len, instance.features); + return 0; } - virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const { - return 0; +private: + void deserialize_feas_to_ins(char* buffer, uint32_t len, std::vector& ins) const { + if (buffer == nullptr) { + return ; + } + + uint32_t cursor = 0; + uint16_t slot; + uint8_t flag; + while (cursor < len) { + memcpy(&slot, buffer + cursor, 2); + cursor += 2; + + memcpy(&flag, buffer + cursor, 1); + flag &= 0xF0; + + CHECK(flag == 0x00 || flag == 0x10|| flag == 0x20 || flag == 0x30); + + if (flag == 0x00 || flag == 0x10) { + uint8_t len = 1; + if (flag == 0x10) { + cursor++; + memcpy(&len, buffer + cursor, 1); + cursor++; + } + for (uint8_t i = 0; i < len; ++i) { + int32_t sign; + for (uint8_t j = 0; j < HOT_SIGN_SIZE; ++j) { + memcpy((char*)&sign + HOT_SIGN_SIZE-j-1, buffer + cursor, 1); + cursor++; + } + + uint64_t sign64 = sign & 0x0FFFFFFF; + sign64 = _index->index2sign((int32_t)sign64); + ins.emplace_back(sign64, slot); + } + } + + if (flag == 0x20 || flag == 0x30) { + uint8_t len = 1; + cursor++; + if (flag == 0x30) { + memcpy(&len, buffer + cursor, 1); + cursor++; + } + + for (uint8_t i = 0; i < len; ++i) { + uint64_t sign64; + memcpy(&sign64, buffer + cursor, COLD_SIGN_SIZE); + cursor += COLD_SIGN_SIZE; + ins.emplace_back(sign64, slot); + } + } + } } + +private: + std::shared_ptr _index; + }; -REGIST_CLASS(DataParser, LineDataParser); +REGIST_CLASS(DataParser, ArchiveDataParse); int DataReader::initialize(const YAML::Node& config, std::shared_ptr context) { _parser.reset(CREATE_INSTANCE(DataParser, config["parser"]["class"].as())); @@ -162,7 +500,7 @@ public: size_t buffer_size = 0; ssize_t line_len = 0; while ((line_len = getline(&buffer, &buffer_size, fin.get())) != -1) { - // 去掉行位回车 + // 去掉行尾回车 if (line_len > 0 && buffer[line_len - 1] == '\n') { buffer[--line_len] = '\0'; } @@ -175,7 +513,8 @@ public: VLOG(5) << "parse data: " << data_item.id << " " << data_item.data << ", filename: " << filepath << ", thread_num: " << thread_num << ", max_threads: " << max_threads; if (writer == nullptr) { if (!data_channel->Put(std::move(data_item))) { - VLOG(2) << "fail to put data, thread_num: " << thread_num; + LOG(WARNING) << "fail to put data, thread_num: " << thread_num; + is_failed = true; } } else { (*writer) << std::move(data_item); @@ -192,11 +531,6 @@ public: is_failed = true; continue; } - if (_file_system->err_no() != 0) { - _file_system->reset_err_no(); - is_failed = true; - continue; - } } // omp end diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h b/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h old mode 100644 new mode 100755 index f588fa4af8c35f0d76e4a5a3d64cb8a70396d15d..9bd824f6c92e607a34284b0efabac7bdf1eefa5a --- a/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h +++ b/paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h @@ -9,6 +9,8 @@ #include #include "paddle/fluid/framework/channel.h" #include "paddle/fluid/train/custom_trainer/feed/common/pipeline.h" +#include "paddle/fluid/framework/archive.h" +#include "paddle/fluid/string/string_helper.h" #include "paddle/fluid/train/custom_trainer/feed/common/registerer.h" namespace paddle { @@ -18,14 +20,40 @@ namespace feed { class TrainerContext; struct FeatureItem { - uint64_t feature_sign; - uint16_t slot_id; std::vector weights; std::vector gradients; +public: + FeatureItem() { + } + FeatureItem(uint64_t sign_, uint16_t slot_) { + sign() = sign_; + slot() = slot_; + } + uint64_t& sign() { + return *(uint64_t*)sign_buffer(); + } + const uint64_t& sign() const { + return *(const uint64_t*)sign_buffer(); + } + uint16_t& slot() { + return _slot; + } + const uint16_t& slot() const { + return _slot; + } + +private: + char _sign[sizeof(uint64_t)]; + uint16_t _slot; + + char* sign_buffer() const { + return (char*)_sign; + } }; struct SampleInstance { std::string id; + std::vector predicts; std::vector labels; std::vector features; std::vector embedx; @@ -50,10 +78,9 @@ public: virtual ~DataParser() {} virtual int initialize(const YAML::Node& config, std::shared_ptr context) = 0; virtual int parse(const std::string& str, DataItem& data) const { - return parse(str.c_str(), data); + return parse(str.c_str(), str.size(), data); } virtual int parse(const char* str, size_t len, DataItem& data) const = 0; - virtual int parse(const char* str, DataItem& data) const = 0; virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const = 0; }; REGIST_REGISTERER(DataParser); @@ -80,6 +107,23 @@ protected: }; REGIST_REGISTERER(DataReader); +class LineDataParser : public DataParser { +public: + LineDataParser() {} + + virtual ~LineDataParser() {} + + virtual int initialize(const YAML::Node& config, std::shared_ptr context) { + return 0; + } + + virtual int parse(const char* str, size_t len, DataItem& data) const; + + virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const { + return 0; + } +}; + }//namespace feed }//namespace custom_trainer }//namespace paddle diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/dataset.cc b/paddle/fluid/train/custom_trainer/feed/dataset/dataset.cc index cb9bd06e4bacff520643ac0cde675a6233e84b3e..a4081c0f5296730209afb237a790aa43246c0a16 100644 --- a/paddle/fluid/train/custom_trainer/feed/dataset/dataset.cc +++ b/paddle/fluid/train/custom_trainer/feed/dataset/dataset.cc @@ -7,14 +7,14 @@ namespace feed { int Dataset::initialize( const YAML::Node& config, std::shared_ptr context) { if (config["data_list"].Type() != YAML::NodeType::Map) { - VLOG(0) << "miss data_list config in dataset, or type error please check"; + LOG(FATAL) << "miss data_list config in dataset, or type error please check"; return -1; } for (auto& data_config : config["data_list"]) { std::string name = data_config.first.as(); auto data_ptr = std::make_shared(); if (data_ptr->initialize(data_config.second, context) != 0) { - VLOG(0) << "dataset initialize failed, name:" << name; + LOG(FATAL) << "dataset initialize failed, name:" << name; return -1; } _data_containers[name] = data_ptr; @@ -48,6 +48,22 @@ inline DatasetStatus Dataset::epoch_data_status( return _data_containers[data_name]->epoch_data_status(epoch_id); } +inline std::vector Dataset::epoch_data_path( + const std::string& data_name, uint64_t epoch_id) { + return _data_containers[data_name]->epoch_data_path(epoch_id); +} + +inline std::vector Dataset::epoch_data_path(uint64_t epoch_id) { + std::vector results; + for (auto it = _data_containers.begin(); it != _data_containers.end(); ++it) { + auto items = std::move(it->second->epoch_data_path(epoch_id)); + for (auto& item : items) { + results.emplace_back(item); + } + } + return results; +} + inline ::paddle::framework::Channel Dataset::fetch_data( const std::string& data_name, uint64_t epoch_id) { return _data_containers[data_name]->fetch(epoch_id); diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/dataset.h b/paddle/fluid/train/custom_trainer/feed/dataset/dataset.h index 23a786e023fbad9821d4ad11f4bd7f54c41ebcb7..eeda375204dd4a731996ba46d91ffd16ea3eb8d2 100644 --- a/paddle/fluid/train/custom_trainer/feed/dataset/dataset.h +++ b/paddle/fluid/train/custom_trainer/feed/dataset/dataset.h @@ -29,6 +29,10 @@ public: virtual DatasetStatus epoch_data_status(uint64_t epoch_id); virtual DatasetStatus epoch_data_status(const std::string& data_name, uint64_t epoch_id); + //获取数据路径 + virtual std::vector epoch_data_path(uint64_t epoch_id); + virtual std::vector epoch_data_path(const std::string& data_name, uint64_t epoch_id); + //返回各DataContainer内的原始数据(maybe 压缩格式) virtual ::paddle::framework::Channel fetch_data( const std::string& data_name, uint64_t epoch_id); diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.cc b/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.cc old mode 100644 new mode 100755 index 5fcfb9d616627511a16f315cae16be19a3b67ee6..f702511b678a7046bee7ea827c53fe7da3d8a321 --- a/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.cc +++ b/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.cc @@ -6,10 +6,10 @@ #include #include #include -#include "paddle/fluid/framework/io/shell.h" #include "paddle/fluid/string/string_helper.h" #include "paddle/fluid/train/custom_trainer/feed/trainer_context.h" #include "paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h" +#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h" #include "paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h" namespace paddle { @@ -27,8 +27,7 @@ int DatasetContainer::initialize( _dataset_list[i].reset(new DatasetInfo); } - _data_root_paths = paddle::string::split_string( - config["root_path"].as(), " "); + _data_root_paths = config["root_path"].as>(); _data_split_interval = config["data_spit_interval"].as(); _data_path_formater = config["data_path_formater"].as(); std::string data_reader_class = config["data_reader"].as(); @@ -42,6 +41,21 @@ std::shared_ptr DatasetContainer::dataset(uint64_t timestamp) { auto data_idx = timestamp / epoch_accessor->epoch_time_interval(); return _dataset_list[data_idx % _prefetch_num]; } +std::vector DatasetContainer::epoch_data_path(uint64_t epoch_id) { + std::vector results; + auto* epoch_accessor = _trainer_context->epoch_accessor.get(); + time_t timestamp = epoch_accessor->epoch_timestamp(epoch_id); + size_t data_num = data_num_for_train(timestamp, epoch_accessor->epoch_time_interval(), _data_split_interval); + uint64_t data_timestamp = timestamp % _data_split_interval == 0 ? timestamp : (timestamp / _data_split_interval + 1) * _data_split_interval; + for (int i = 0; i < _data_root_paths.size(); ++i) { + for (int j = 0; j < data_num; ++j) { + std::string path_suffix = format_timestamp(data_timestamp + j * _data_split_interval, _data_path_formater); + std::string data_dir = _trainer_context->file_system->path_join(_data_root_paths[i], path_suffix); + results.emplace_back(data_dir); + } + } + return results; +} void DatasetContainer::pre_detect_data(uint64_t epoch_id) { int status = 0; @@ -56,7 +70,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) { async_download_data(timestamp); })); } - for (int detect_idx = 0 ; detect_idx < _prefetch_num; ++detect_idx) { + for (int detect_idx = 0 ; detect_idx < _prefetch_num; ++detect_idx, ++epoch_id) { if (DatasetStatus::Empty != data_status(timestamp)) { continue; } @@ -66,7 +80,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) { for (int i = 0; i < _data_root_paths.size() && status == 0; ++i) { for (int j = 0; j < data_num && status == 0; ++j) { std::string path_suffix = format_timestamp(data_timestamp + j * _data_split_interval, _data_path_formater); - std::string data_dir = _data_root_paths[i] + "/" + path_suffix; + std::string data_dir = _trainer_context->file_system->path_join(_data_root_paths[i], path_suffix); status = read_data_list(data_dir, data_path_list); } } @@ -75,6 +89,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) { dataset_info->timestamp = timestamp; dataset_info->file_path_list = std::move(data_path_list); dataset_info->status = DatasetStatus::Detected; + VLOG(2) << epoch_accessor->text(epoch_id) << ", data is detected"; } timestamp += epoch_accessor->epoch_time_interval(); } @@ -150,16 +165,25 @@ void DatasetContainer::async_download_data(uint64_t start_timestamp) { } while (!_stop_download) { auto dataset_info = dataset(start_timestamp); - while (data_status(start_timestamp) != DatasetStatus::Detected) { + while (data_status(start_timestamp) == DatasetStatus::Empty) { sleep(30); } + dataset_info->status = DatasetStatus::Downloding; + + VLOG(2) << "Start download data, data_timestap:" << start_timestamp + << ", for epoch:" << epoch_accessor->text(start_timestamp); const auto& file_list = dataset_info->file_path_list; dataset_info->data_channel->Clear(); while (_data_reader->read_all(file_list, dataset_info->data_channel) != 0) { dataset_info->data_channel->Clear(); - VLOG(0) << "timestamp:" << start_timestamp << " data read failed, retry"; + VLOG(0) << "Failed download data, data_timestap:" << start_timestamp + << ", for epoch:" << epoch_accessor->text(start_timestamp) << ", Retry it"; sleep(30); } + VLOG(2) << "End download data num:" << dataset_info->data_channel->Size() + << ", data_timestap:" << start_timestamp + << ", for epoch:" << epoch_accessor->text(start_timestamp); + dataset_info->status = DatasetStatus::Ready; start_timestamp += epoch_accessor->epoch_time_interval(); } } diff --git a/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h b/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h index 7ed455a4d81fc257aecca2627f3853936ca97514..0215a3563f0d519085c9ec84a13c4eddd8ab631f 100644 --- a/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h +++ b/paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h @@ -49,20 +49,22 @@ public: } virtual int initialize( const YAML::Node& config, std::shared_ptr context); - //触发可预取的数据判断 + // 触发可预取的数据判断 virtual void pre_detect_data(uint64_t epoch_id); - //获取数据状态 + // 获取epoch对应的样本数据目录 + std::vector epoch_data_path(uint64_t epoch_id); + // 获取数据状态 virtual DatasetStatus epoch_data_status(uint64_t epoch_id); - //获取特定epoch_i样本，如果数据未ready，Channel内为空指针 + // 获取特定epoch_i样本，如果数据未ready，Channel内为空指针 virtual ::paddle::framework::Channel fetch(uint64_t epoch_id); - //获取DataItem解析器 + // 获取DataItem解析器 virtual const DataParser* data_parser() { return _data_reader->get_parser(); } protected: virtual DatasetStatus data_status(uint64_t timestamp); virtual int read_data_list(const std::string& data_dir, std::vector& data_list); - //异步样本download + // 异步样本download virtual void async_download_data(uint64_t start_timestamp); virtual std::shared_ptr dataset(uint64_t timestamp); diff --git a/paddle/fluid/train/custom_trainer/feed/executor/executor.cc b/paddle/fluid/train/custom_trainer/feed/executor/executor.cc index 0db6b9e887f8108a3320c3466202f15b3399cedf..246d4a36a2243cb9f888c81afaf30f4bad7f0a5f 100644 --- a/paddle/fluid/train/custom_trainer/feed/executor/executor.cc +++ b/paddle/fluid/train/custom_trainer/feed/executor/executor.cc @@ -1,3 +1,4 @@ +#include #include "paddle/fluid/train/custom_trainer/feed/executor/executor.h" #include "paddle/fluid/framework/program_desc.h" @@ -17,7 +18,7 @@ namespace { int ReadBinaryFile(const std::string& filename, std::string* contents) { std::ifstream fin(filename, std::ios::in | std::ios::binary); if (!fin) { - VLOG(2) << "Cannot open file " << filename; + LOG(FATAL) << "Cannot open file " << filename; return -1; } fin.seekg(0, std::ios::end); @@ -31,7 +32,7 @@ int ReadBinaryFile(const std::string& filename, std::string* contents) { std::unique_ptr Load( paddle::framework::Executor* /*executor*/, const std::string& model_filename) { - VLOG(3) << "loading model from " << model_filename; + LOG(INFO) << "loading model from " << model_filename; std::string program_desc_str; if (ReadBinaryFile(model_filename, &program_desc_str) != 0) { return nullptr; @@ -50,38 +51,34 @@ public: virtual ~SimpleExecutor() {}; virtual int initialize(YAML::Node exe_config, std::shared_ptr context_ptr) { - paddle::framework::InitDevices(false); - if (exe_config["num_threads"]) { - paddle::platform::SetNumThreads(exe_config["num_threads"].as()); - } - - if (!exe_config["startup_program"] || - !exe_config["main_program"]) { - VLOG(2) << "fail to load config"; - return -1; - } - + //if (exe_config["num_threads"]) { + + //} + paddle::platform::SetNumThreads(1); + std::string name = exe_config["name"].as(); + std::string main_program = YamlHelper::get_with_default(exe_config, "main_program", + string::format_string("./model/%s/main_program", name.c_str())); + std::string startup_program = YamlHelper::get_with_default(exe_config, "startup_program", + string::format_string("./model/%s/startup_program", name.c_str())); try { _context.reset(new SimpleExecutor::Context(context_ptr->cpu_place)); - _context->startup_program = Load(&_context->executor, exe_config["startup_program"].as()); + _context->startup_program = Load(&_context->executor, startup_program); if (_context->startup_program == nullptr) { - VLOG(2) << "fail to load startup_program: " << exe_config["startup_program"].as(); + VLOG(0) << "fail to load startup_program: " << startup_program; return -1; } - - _context->main_program = Load(&_context->executor, exe_config["main_program"].as()); + _context->main_program = Load(&_context->executor, main_program); if (_context->main_program == nullptr) { - VLOG(2) << "fail to load main_program: " << exe_config["main_program"].as(); + VLOG(0) << "fail to load main_program: " << main_program; return -1; } _context->prepare_context = _context->executor.Prepare(*_context->main_program, 0); } catch (::paddle::platform::EnforceNotMet& err) { - VLOG(2) << err.what(); + VLOG(0) << err.what(); _context.reset(nullptr); return -1; } - return 0; } virtual int initialize_scope(::paddle::framework::Scope* scope) { @@ -96,8 +93,8 @@ public: } try { _context->executor.RunPreparedContext(_context->prepare_context.get(), scope, - false, /* don't create local scope each time*/ - false /* don't create variable each time */); + false, /* don't create local scope each time*/ + false /* don't create variable each time */); // For some other vector like containers not cleaned after each batch. _context->tensor_array_batch_cleaner.CollectNoTensorVars(scope); diff --git a/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.cc b/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.cc index c0ecb05dc16fbd4fd0e8dcf874f5a8ef35dd01a8..27c7127b6f73ea9367ce3516930bee810fad018a 100644 --- a/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.cc +++ b/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.cc @@ -12,7 +12,7 @@ int MultiThreadExecutor::initialize(YAML::Node exe_config, _train_data_name = exe_config["train_data_name"].as(); _train_batch_size = exe_config["train_batch_size"].as(); _input_parse_thread_num = exe_config["input_parse_thread_num"].as(); - _push_gradient_thread_num = exe_config["push_gradient_thread_num "].as(); + _push_gradient_thread_num = exe_config["push_gradient_thread_num"].as(); _train_thread_num = exe_config["train_thread_num"].as(); _need_dump_all_model = exe_config["need_dump_all_model"].as(); CHECK(_train_thread_num > 0 && _train_batch_size > 0); @@ -26,28 +26,41 @@ int MultiThreadExecutor::initialize(YAML::Node exe_config, auto* e_ptr = CREATE_INSTANCE(Executor, e_class); _thread_executors[i].reset(e_ptr); if (e_ptr->initialize(exe_config, context_ptr) != 0) { + VLOG(0) << "executor initialize failed, name:" << _train_exe_name + << " class:" << e_class; ret = -1; } } CHECK(ret == 0); - _scope_obj_pool.config( + + // buffer + _scope_obj_pool.reset(new paddle::ps::ObjectPool<::paddle::framework::Scope>( [this]() -> ::paddle::framework::Scope* { auto* scope = new ::paddle::framework::Scope(); _thread_executors[0]->initialize_scope(scope); return scope; - }, _train_thread_num * 8); + }, _train_thread_num * 8, 0, _train_thread_num * 8)); + // 模型网络加载 std::string model_config_path = _trainer_context->file_system->path_join( - "./model", string::format_string("%s.yaml", _train_exe_name.c_str())); + "./model", string::format_string("%s/model.yaml", _train_exe_name.c_str())); CHECK(_trainer_context->file_system->exists(model_config_path)) << "miss model config file:" << model_config_path; _model_config = YAML::LoadFile(model_config_path); - _input_accessors.resize(_model_config["input_accessor"].size()); for (const auto& accessor_config : _model_config["input_accessor"]) { auto accessor_class = accessor_config["class"].as(); - _input_accessors.emplace_back(CREATE_INSTANCE(DataInputAccessor, accessor_class)); - CHECK(_input_accessors.back()->initialize(accessor_config, context_ptr) == 0) + auto* accessor_ptr = CREATE_INSTANCE(DataInputAccessor, accessor_class); + _input_accessors.emplace_back(accessor_ptr); + CHECK(accessor_ptr->initialize(accessor_config, context_ptr) == 0) << "InputAccessor init Failed, class:" << accessor_class; + if (accessor_config["table_id"]) { + auto table_id = accessor_config["table_id"].as(); + if (_table_to_accessors.count(table_id) > 0) { + _table_to_accessors[table_id].push_back(accessor_ptr); + } else { + _table_to_accessors[table_id] = {accessor_ptr}; + } + } } return ret; @@ -57,7 +70,7 @@ paddle::framework::Channel MultiThreadExecutor::run( paddle::framework::Channel input, const DataParser* parser) { PipelineOptions input_pipe_option; input_pipe_option.need_hold_input_data = true; - input_pipe_option.batch_size = _train_batch_size; + input_pipe_option.batch_size = 1; input_pipe_option.thread_num = _input_parse_thread_num; input_pipe_option.input_output_rate = _train_batch_size; input_pipe_option.buffer_batch_count = _train_thread_num; @@ -66,7 +79,7 @@ paddle::framework::Channel MultiThreadExecutor::run( [this, parser](DataItem* item, size_t item_num, ScopePoolObj* scope, size_t* scope_num, size_t thread_idx) -> int { *scope_num = 1; - auto scope_obj = _scope_obj_pool.get(); + auto scope_obj = _scope_obj_pool->get(); auto* samples = new SampleInstance[item_num]; for (size_t i = 0; i parse_to_sample(item[i], samples[i]) == 0); @@ -95,7 +108,7 @@ paddle::framework::Channel MultiThreadExecutor::run( auto* executor = _thread_executors[thread_idx].get(); size_t& out_idx = *out_num; for (out_idx = 0; out_idx < in_num; ++out_idx) { - executor->run(in_items[out_idx].get()); + //CHECK(executor->run(in_items[out_idx].get()) == 0); out_items[out_idx] = std::move(in_items[out_idx]); } return 0; @@ -123,7 +136,6 @@ paddle::framework::Channel MultiThreadExecutor::run( } delete[] samples; // 所有pipe完成后，再回收sample } - return 0; }); diff --git a/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.h b/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.h index 77f07c4d3113c85de0c29fe9a263d720527277ba..68c9f88fe3505634e3a429066250cada227a5e04 100644 --- a/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.h +++ b/paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.h @@ -8,6 +8,8 @@ namespace paddle { namespace custom_trainer { namespace feed { +typedef paddle::ps::ObjectPool<::paddle::framework::Scope>::PooledObject ScopePoolObj; + class MultiThreadExecutor { public: MultiThreadExecutor() {} @@ -24,10 +26,19 @@ public: virtual bool is_dump_all_model() { return _need_dump_all_model; } - + virtual const std::string& train_exe_name() { + return _train_exe_name; + } virtual const std::string& train_data_name() { return _train_data_name; } + virtual const std::map>& table_accessors() { + return _table_to_accessors; + } + virtual ScopePoolObj fetch_scope() { + ScopePoolObj scope_obj(_scope_obj_pool->get()); + return scope_obj; + } protected: std::string _train_data_name; size_t _train_batch_size = 32; @@ -41,8 +52,8 @@ protected: TrainerContext* _trainer_context = nullptr; std::vector> _thread_executors; std::vector> _input_accessors; - paddle::ps::ObjectPool<::paddle::framework::Scope> _scope_obj_pool; - typedef paddle::ps::ObjectPool<::paddle::framework::Scope>::PooledObject ScopePoolObj; + std::map> _table_to_accessors; + std::shared_ptr> _scope_obj_pool; }; } // namespace feed diff --git a/paddle/fluid/train/custom_trainer/feed/io/auto_file_system.cc b/paddle/fluid/train/custom_trainer/feed/io/auto_file_system.cc index a64a73efebb97a6025e7965c0800de15ba9ac531..48476a145097ab0cb71a8c885bcad29ec85bc16c 100644 --- a/paddle/fluid/train/custom_trainer/feed/io/auto_file_system.cc +++ b/paddle/fluid/train/custom_trainer/feed/io/auto_file_system.cc @@ -19,17 +19,18 @@ public: for (auto& prefix_fs: config["file_systems"]) { std::unique_ptr fs(CREATE_INSTANCE(FileSystem, prefix_fs.second["class"].as(""))); if (fs == nullptr) { - VLOG(2) << "fail to create class: " << prefix_fs.second["class"].as(""); + LOG(FATAL) << "fail to create class: " << prefix_fs.second["class"].as(""); return -1; } if (fs->initialize(prefix_fs.second, context) != 0) { - VLOG(2) << "fail to initialize class: " << prefix_fs.second["class"].as(""); - return 0; + LOG(FATAL) << "fail to initialize class: " << prefix_fs.second["class"].as(""); + return -1; } _file_system.emplace(prefix_fs.first.as(""), std::move(fs)); } } if (_file_system.find("default") == _file_system.end()) { + LOG(WARNING) << "miss default file_system, use LocalFileSystem as default"; std::unique_ptr fs(CREATE_INSTANCE(FileSystem, "LocalFileSystem")); if (fs == nullptr || fs->initialize(YAML::Load(""), context) != 0) { return -1; @@ -61,8 +62,8 @@ public: return get_file_system(path)->list(path); } - std::string tail(const std::string& path) override { - return get_file_system(path)->tail(path); + std::string tail(const std::string& path, size_t tail_num = 1) override { + return get_file_system(path)->tail(path, tail_num); } bool exists(const std::string& path) override { @@ -76,35 +77,15 @@ public: FileSystem* get_file_system(const std::string& path) { auto pos = path.find_first_of(":"); if (pos != std::string::npos) { - auto substr = path.substr(0, pos + 1); + auto substr = path.substr(0, pos); // example: afs:/xxx -> afs auto fs_it = _file_system.find(substr); if (fs_it != _file_system.end()) { return fs_it->second.get(); } } - VLOG(5) << "path: " << path << ", select default file system"; return _file_system["default"].get(); } - int err_no() const override { - if (_err_no == 0) { - for (const auto& file_system : _file_system) { - if (file_system.second->err_no() != 0) { - const_cast(_err_no) = -1; - break; - } - } - } - return FileSystem::err_no(); - } - - void reset_err_no() override { - _err_no = 0; - for (auto& file_system : _file_system) { - file_system.second->reset_err_no(); - } - } - private: std::unordered_map> _file_system; }; diff --git a/paddle/fluid/train/custom_trainer/feed/io/file_system.h b/paddle/fluid/train/custom_trainer/feed/io/file_system.h index e3b813349ef6691a12d71a26aaa5607f4390076b..0ef5a37b0c0a3e04d2f20d2b036ff2541b4f0f48 100644 --- a/paddle/fluid/train/custom_trainer/feed/io/file_system.h +++ b/paddle/fluid/train/custom_trainer/feed/io/file_system.h @@ -21,22 +21,12 @@ public: virtual int64_t file_size(const std::string& path) = 0; virtual void remove(const std::string& path) = 0; virtual std::vector list(const std::string& path) = 0; - virtual std::string tail(const std::string& path) = 0; + virtual std::string tail(const std::string& path, size_t tail_num = 1) = 0; virtual bool exists(const std::string& path) = 0; virtual void mkdir(const std::string& path) = 0; virtual std::string path_join(const std::string& dir, const std::string& path); virtual std::pair path_split(const std::string& path); - virtual int err_no() const { - return _err_no; - } - inline operator bool() { - return err_no() == 0; - } - virtual void reset_err_no() { - _err_no = 0; - } protected: - int _err_no = 0; }; REGIST_REGISTERER(FileSystem); diff --git a/paddle/fluid/train/custom_trainer/feed/io/hadoop_file_system.cc b/paddle/fluid/train/custom_trainer/feed/io/hadoop_file_system.cc index 97c12968d061d2c9040fefe69694769aa42b4d1e..2af8e08231f114166ac39808079528f981101a2d 100644 --- a/paddle/fluid/train/custom_trainer/feed/io/hadoop_file_system.cc +++ b/paddle/fluid/train/custom_trainer/feed/io/hadoop_file_system.cc @@ -25,7 +25,7 @@ public: } } if (_ugi.find("default") == _ugi.end()) { - VLOG(2) << "fail to load default ugi"; + LOG(FATAL) << "fail to load default ugi"; return -1; } return 0; @@ -33,6 +33,7 @@ public: std::shared_ptr open_read(const std::string& path, const std::string& converter) override { + int err_no = 0; std::string cmd; if (string::end_with(path, ".gz")) { cmd = string::format_string( @@ -43,11 +44,12 @@ public: bool is_pipe = true; shell_add_read_converter(cmd, is_pipe, converter); - return shell_open(cmd, is_pipe, "r", _buffer_size, &_err_no); + return shell_open(cmd, is_pipe, "r", _buffer_size, &err_no); } std::shared_ptr open_write(const std::string& path, const std::string& converter) override { + int err_no = 0; std::string cmd = string::format_string("%s -put - \"%s\"", hdfs_command(path).c_str(), path.c_str()); bool is_pipe = true; @@ -57,12 +59,11 @@ public: } shell_add_write_converter(cmd, is_pipe, converter); - return shell_open(cmd, is_pipe, "w", _buffer_size, &_err_no); + return shell_open(cmd, is_pipe, "w", _buffer_size, &err_no); } int64_t file_size(const std::string& path) override { - _err_no = -1; - VLOG(2) << "not support"; + LOG(FATAL) << "not support"; return 0; } @@ -107,13 +108,13 @@ public: return list; } - std::string tail(const std::string& path) override { + std::string tail(const std::string& path, size_t tail_num = 1) override { if (path == "") { return ""; } return shell_get_command_output(string::format_string( - "%s -text %s | tail -1 ", hdfs_command(path).c_str(), path.c_str())); + "%s -text %s | tail -%u", hdfs_command(path).c_str(), path.c_str(), tail_num)); } bool exists(const std::string& path) override { diff --git a/paddle/fluid/train/custom_trainer/feed/io/local_file_system.cc b/paddle/fluid/train/custom_trainer/feed/io/local_file_system.cc index 2da715061280119dde1155aeb015d6eb0844feea..0b5e5cce0ab62000deee36da92b898e00431195f 100644 --- a/paddle/fluid/train/custom_trainer/feed/io/local_file_system.cc +++ b/paddle/fluid/train/custom_trainer/feed/io/local_file_system.cc @@ -64,10 +64,10 @@ public: if (path == "") { return {}; } - + int err_no; std::shared_ptr pipe; pipe = shell_popen( - string::format_string("find %s -maxdepth 1 -type f", path.c_str()), "r", &_err_no); + string::format_string("find %s -maxdepth 1 -type f", path.c_str()), "r", &err_no); string::LineFileReader reader; std::vector list; @@ -78,12 +78,12 @@ public: return list; } - std::string tail(const std::string& path) override { + std::string tail(const std::string& path, size_t tail_num = 1) override { if (path == "") { return ""; } - return shell_get_command_output(string::format_string("tail -1 %s ", path.c_str())); + return shell_get_command_output(string::format_string("tail -%u %s ", tail_num, path.c_str())); } bool exists(const std::string& path) override { diff --git a/paddle/fluid/train/custom_trainer/feed/main.cc b/paddle/fluid/train/custom_trainer/feed/main.cc index f14985215369f5a48ba433b07a83e0204ed7cd28..8e8c9851db6560c9bf9a29b322dc163ad49410cd 100644 --- a/paddle/fluid/train/custom_trainer/feed/main.cc +++ b/paddle/fluid/train/custom_trainer/feed/main.cc @@ -13,6 +13,7 @@ using namespace paddle::custom_trainer::feed; DEFINE_string(feed_trainer_conf_path, "./conf/trainer.yaml", "path of trainer conf"); int main(int argc, char* argv[]) { + google::InitGoogleLogging(argv[0]); //gflags google::ParseCommandLineFlags(&argc, &argv, true); std::string gflag_conf = "./conf/gflags.conf"; @@ -29,26 +30,27 @@ int main(int argc, char* argv[]) { if (trainer_context_ptr->environment->initialize(config["environment"]) != 0) { return -1; } - EnvironmentRole role; auto* environment = trainer_context_ptr->environment.get(); environment->wireup(); - if (environment->rank_id(EnvironmentRole::ALL) % 2 == 0) { - role = EnvironmentRole::WORKER; + if (environment->node_num(EnvironmentRole::ALL) == 1) { + environment->add_role(EnvironmentRole::WORKER); + environment->add_role(EnvironmentRole::PSERVER); + } else if (environment->rank_id(EnvironmentRole::ALL) % 2 == 0) { + environment->add_role(EnvironmentRole::WORKER); } else { - role = EnvironmentRole::PSERVER; + environment->add_role(EnvironmentRole::PSERVER); } - environment->set_role(role); trainer_context_ptr->pslib.reset(new PSlib()); std::string ps_config = config["environment"]["ps"].as(); - trainer_context_ptr->pslib->initialize(ps_config, environment, role); + trainer_context_ptr->pslib->initialize(ps_config, environment); //VLOG(3) << "Node Start With Role:" << role; - std::vector process_name_list = { - "InitEnvProcess", - "LearnerProcess" - }; - switch (role) { - case EnvironmentRole::WORKER: + + if (environment->is_role(EnvironmentRole::WORKER)) { + std::vector process_name_list = { + "InitEnvProcess", + "LearnerProcess" + }; for (const auto& process_name : process_name_list) { Process* process = CREATE_INSTANCE(Process, process_name); if (process == NULL) { @@ -64,14 +66,13 @@ int main(int argc, char* argv[]) { for (auto& process : trainer_context_ptr->process_list) { process->run(); } - break; - case EnvironmentRole::PSERVER: - //wait server done - while (true) { - sleep(10000); - } - break; + + } + + //TODO exit control + bool running = true; + while (running) { + sleep(10000); } - return 0; } diff --git a/paddle/fluid/train/custom_trainer/feed/process/init_env_process.cc b/paddle/fluid/train/custom_trainer/feed/process/init_env_process.cc index a45320a685fbeffc9c68b4a7e99ddf8a3a3bf123..05b398d3de16778a37c3e2316333db9e6730529d 100644 --- a/paddle/fluid/train/custom_trainer/feed/process/init_env_process.cc +++ b/paddle/fluid/train/custom_trainer/feed/process/init_env_process.cc @@ -40,7 +40,7 @@ int InitEnvProcess::initialize(std::shared_ptr context_ptr) { return -1; } - VLOG(3) << "Env initialize success"; + VLOG(3) << "Env initialize success"; return 0; } @@ -49,14 +49,6 @@ int InitEnvProcess::run() { VLOG(3) << "Trainer Resume From epoch:" << epoch_accessor->current_epoch_id(); auto next_epoch_id = epoch_accessor->next_epoch_id(epoch_accessor->current_epoch_id()); _context_ptr->dataset->pre_detect_data(next_epoch_id); - - if (epoch_accessor->checkpoint_path().size() > 0) { - //Load Model - } else { - //Random Init Model - } - //context_ptr->pslib_client()->load_model(); - VLOG(3) << "Psserver Load Model Success"; return 0; } diff --git a/paddle/fluid/train/custom_trainer/feed/process/learner_process.cc b/paddle/fluid/train/custom_trainer/feed/process/learner_process.cc old mode 100644 new mode 100755 index e54c87233388976aa4bc37b96c341048714036cd..6eb8fb653e1c310271a98a43d0c342587389f64e --- a/paddle/fluid/train/custom_trainer/feed/process/learner_process.cc +++ b/paddle/fluid/train/custom_trainer/feed/process/learner_process.cc @@ -3,6 +3,7 @@ *Train样本 */ #include +#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h" #include "paddle/fluid/train/custom_trainer/feed/dataset/dataset.h" #include "paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h" #include "paddle/fluid/train/custom_trainer/feed/process/learner_process.h" @@ -27,8 +28,12 @@ int LearnerProcess::initialize(std::shared_ptr context_ptr) { std::future LearnerProcess::save_model(uint64_t epoch_id, int table_id, ModelSaveWay way) { std::promise p; auto ret = p.get_future(); - if (_context_ptr->epoch_accessor->need_save_model(epoch_id, way)) { - LOG(NOTICE) << "save table, table_id:" << table_id; + auto* ps_client = _context_ptr->pslib->ps_client(); + auto* epoch_accessor = _context_ptr->epoch_accessor.get(); + if (epoch_accessor->need_save_model(epoch_id, way)) { + VLOG(2) << "Start save model, table_id:" << table_id; + auto model_dir = epoch_accessor->model_save_path(epoch_id, way); + return ps_client->save(table_id, model_dir, std::to_string((int)way)); } else { p.set_value(0); } @@ -40,14 +45,19 @@ int LearnerProcess::wait_save_model(uint64_t epoch_id, ModelSaveWay way) { if (!environment->is_master_node(EnvironmentRole::WORKER)) { return 0; } + std::set table_set; + for (auto& executor : _executors) { + const auto& table_accessors = executor->table_accessors(); + for (auto& itr : table_accessors) { + table_set.insert(itr.first); + } + } int ret_size = 0; - auto table_num = _context_ptr->params_table_list.size(); + auto table_num = table_set.size(); std::future rets[table_num]; - for (int i = 0; i < table_num; ++i) { - auto table_id = _context_ptr->params_table_list[i].table_id(); + for (auto table_id : table_set) { rets[ret_size++] = save_model(epoch_id, table_id, way); } - int all_ret = 0; for (int i = 0; i < ret_size; ++i) { rets[i].wait(); @@ -56,6 +66,36 @@ int LearnerProcess::wait_save_model(uint64_t epoch_id, ModelSaveWay way) { return all_ret; } +int LearnerProcess::load_model(uint64_t epoch_id) { + auto* environment = _context_ptr->environment.get(); + if (!environment->is_master_node(EnvironmentRole::WORKER)) { + return 0; + } + std::set loaded_table_set; + auto model_dir = _context_ptr->epoch_accessor->checkpoint_path(); + for (auto& executor : _executors) { + const auto& table_accessors = executor->table_accessors(); + for (auto& itr : table_accessors) { + if (loaded_table_set.count(itr.first)) { + continue; + } + auto table_model_path = _context_ptr->file_system->path_join( + model_dir, string::format_string("%03d", itr.first)); + if (_context_ptr->file_system->list(table_model_path).size() == 0) { + VLOG(2) << "miss table_model:" << table_model_path << ", initialize by default"; + auto scope = std::move(executor->fetch_scope()); + CHECK(itr.second[0]->create(scope.get()) == 0); + } else { + auto status = _context_ptr->ps_client()->load(itr.first, + model_dir, std::to_string((int)ModelSaveWay::ModelSaveTrainCheckpoint)); + CHECK(status.get() == 0) << "table load failed, id:" << itr.first; + } + loaded_table_set.insert(itr.first); + } + } + return 0; +} + int LearnerProcess::run() { auto* dataset = _context_ptr->dataset.get(); auto* environment = _context_ptr->environment.get(); @@ -63,8 +103,12 @@ int LearnerProcess::run() { uint64_t epoch_id = epoch_accessor->current_epoch_id(); environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, - "Resume traine with epoch_id:%d label:%s", epoch_id, _context_ptr->epoch_accessor->text(epoch_id).c_str()); + "Resume train with epoch_id:%d %s", epoch_id, _context_ptr->epoch_accessor->text(epoch_id).c_str()); + //尝试加载模型 or 初始化 + CHECK(load_model(epoch_id) == 0); + environment->barrier(EnvironmentRole::WORKER); + //判断是否先dump出base wait_save_model(epoch_id, ModelSaveWay::ModelSaveInferenceBase); environment->barrier(EnvironmentRole::WORKER); @@ -73,21 +117,22 @@ int LearnerProcess::run() { epoch_accessor->next_epoch(); bool already_dump_inference_model = false; epoch_id = epoch_accessor->current_epoch_id(); - std::string epoch_log_title= paddle::string::format_string( + std::string epoch_log_title = paddle::string::format_string( "train epoch_id:%d label:%s", epoch_id, epoch_accessor->text(epoch_id).c_str()); + std::string data_path = paddle::string::to_string(dataset->epoch_data_path(epoch_id)); //Step1. 等待样本ready { environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, - "Start %s, wait data ready", epoch_log_title.c_str()); + "%s, wait data ready:%s", epoch_log_title.c_str(), data_path.c_str()); while (dataset->epoch_data_status(epoch_id) != DatasetStatus::Ready) { sleep(30); dataset->pre_detect_data(epoch_id); environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, - "%s, data not ready, wait 30s", epoch_log_title.c_str()); + "data not ready, wait 30s"); } environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, - "%s, data is ready, start traning", epoch_log_title.c_str()); + "Start %s, data is ready", epoch_log_title.c_str()); environment->barrier(EnvironmentRole::WORKER); } @@ -96,6 +141,7 @@ int LearnerProcess::run() { std::map> backup_input_map; for (auto& executor : _executors) { environment->barrier(EnvironmentRole::WORKER); + VLOG(2) << "Start executor:" << executor->train_exe_name(); auto data_name = executor->train_data_name(); paddle::framework::Channel input_channel; if (backup_input_map.count(data_name)) { @@ -104,6 +150,12 @@ int LearnerProcess::run() { input_channel = dataset->fetch_data(data_name, epoch_id); } input_channel = executor->run(input_channel, dataset->data_parser(data_name)); + VLOG(2) << "End executor:" << executor->train_exe_name(); + + // 等待异步梯度完成 + _context_ptr->ps_client()->flush(); + environment->barrier(EnvironmentRole::WORKER); + if (executor->is_dump_all_model()) { already_dump_inference_model = true; wait_save_model(epoch_id, ModelSaveWay::ModelSaveInferenceDelta); @@ -121,6 +173,10 @@ int LearnerProcess::run() { } wait_save_model(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint); environment->barrier(EnvironmentRole::WORKER); + + epoch_accessor->epoch_done(epoch_id); + environment->barrier(EnvironmentRole::WORKER); + } //Step4. Output Monitor && RunStatus diff --git a/paddle/fluid/train/custom_trainer/feed/process/learner_process.h b/paddle/fluid/train/custom_trainer/feed/process/learner_process.h index b7e50cc98b65ce0b50615e56944bdd4bf2e7a255..86f0378a305d2604082a4f1b03be9e7f1e93ab19 100644 --- a/paddle/fluid/train/custom_trainer/feed/process/learner_process.h +++ b/paddle/fluid/train/custom_trainer/feed/process/learner_process.h @@ -18,6 +18,8 @@ public: virtual int initialize(std::shared_ptr context_ptr); protected: +// 加载所有模型 +virtual int load_model(uint64_t epoch_id); // 同步保存所有模型 virtual int wait_save_model(uint64_t epoch_id, ModelSaveWay way); // 异步保存指定模型 diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/create_programs.py b/paddle/fluid/train/custom_trainer/feed/scripts/create_programs.py index 6f3e16e4f04e9557fc499e2d4f6afd7d6652355a..af0de70ddf070b2a66f2a47423066b59ba02b168 100644 --- a/paddle/fluid/train/custom_trainer/feed/scripts/create_programs.py +++ b/paddle/fluid/train/custom_trainer/feed/scripts/create_programs.py @@ -12,87 +12,163 @@ def print_help(this_name): """Print help """ dirname = os.path.dirname(this_name) - print("Usage: {} [model_dir]\n".format(this_name)) - print(" example: {} {}".format(this_name, os.path.join(dirname, 'example.py'))) + print('Usage: {} [model_dir]\n'.format(this_name)) + print(' example: {} {}'.format(this_name, os.path.join(dirname, 'example.py'))) +class ModelBuilder: + """ + Attributes: + _save_path: Save path of programs + + def _inference(): + Build inference network(without loss and optimizer) + **This function is declared in the network_desc_path file, and will be set in initialize()** + + Returns: + list: inputs + and + list: outputs + pass + + def _loss_function(*outputs): + **This function is declared in the network_desc_path file, and will be set in initialize()** + Args: + *outputs: the second result of inference() + + Returns: + Variable: loss + and + list: labels + pass + """ + def initialize(self, network_desc_path, save_path=None): + """compile the network description module + Args: + network_desc_path: path + save_path: model save path, default is ./model// + + Returns: + bool: True if succeed else False + """ + if not isinstance(network_desc_path, str): + print('network_desc_path must be str') + return False + + if not network_desc_path.endswith('.py'): + print('network_desc_path must be end with .py') + return False + + if not os.path.exists(network_desc_path): + print('file not exists:', network_desc_path) + return False + + scope = dict() + with open(network_desc_path, 'r') as f: + code = f.read() + compiled = compile(code, network_desc_path, 'exec') + exec(compiled, scope) + + if not 'inference' in scope: + print('inference not defined') + return False + + if not 'loss_function' in scope: + print('loss_function not defined') + return False + + if save_path is None: + # example /a/b/c.d -> ./model/c + save_path = os.path.join('./model', os.path.splitext(os.path.split(network_desc_path)[1])[0]) + print('save in the default path:', save_path) + + self._save_path = save_path + + self._inference = scope['inference'] + self._loss_function = scope['loss_function'] + + return True + + def build_and_save(self): + """Build programs and save to _save_path + """ + main_program = fluid.Program() + startup_program = fluid.Program() + with fluid.program_guard(main_program, startup_program): + input_accessor, sparses, inputs, outputs = self._inference() + test_program = main_program.clone(for_test=True) + loss, labels = self._loss_function(*outputs) + + optimizer = fluid.optimizer.SGD(learning_rate=1.0) + params_grads = optimizer.backward(loss) + + if not os.path.exists(self._save_path): + os.makedirs(self._save_path) + + programs = { + 'startup_program': startup_program, + 'main_program': main_program, + 'test_program': test_program, + } + for name, program in programs.items(): + with open(os.path.join(self._save_path, name), 'w') as f: + f.write(program.desc.serialize_to_string()) + + params = filter(fluid.io.is_parameter, main_program.list_vars()) + vars = [] + sums=[] + for param in params: + if param.name.find("bn") == 0: + sums.append({"name": param.name, "shape": param.shape}); + else: + vars.append({"name": param.name, "shape": param.shape}); + + for accessor in input_accessor: + if (accessor["input"] == "sparses"): + accessor["input"] = sparses + if (accessor["input"] == "vars"): + accessor["input"] = vars + if (accessor["input"] == "sums"): + accessor["input"] = sums + if (accessor["input"] == "labels"): + accessor["input"] = [ + {"label_name": label.name, "shape": label.shape, "output_name": output.name } + for (label, output) in zip(labels, outputs) ] + + + model_desc_path = os.path.join(self._save_path, 'model.yaml') + model_desc = { + 'inputs': [{"name": var.name, "shape": var.shape} for var in inputs], + 'outputs': [{"name": var.name, "shape": var.shape} for var in outputs], + 'labels': [{"name": var.name, "shape": var.shape} for var in labels], + 'loss': loss.name, + 'input_accessor': input_accessor + } + + with open(model_desc_path, 'w') as f: + yaml.safe_dump(model_desc, f, encoding='utf-8', allow_unicode=True, default_flow_style=None) -def inference_warpper(filename): - """Build inference network(without loss and optimizer) - Args: - filename: path of file which defined real inference function - Returns: - list: inputs - and - list: outputs - """ - - with open(filename, 'r') as f: - code = f.read() - compiled = compile(code, filename, 'exec') - - scope = dict() - exec(compiled, scope) - return scope['inference']() def main(argv): """Create programs Args: argv: arg list, length should be 2 """ - if len(argv) < 2 or not os.path.exists(argv[1]): + if len(argv) < 2: print_help(argv[0]) exit(1) - network_build_file = argv[1] + network_desc_path = argv[1] if len(argv) > 2: - model_dir = argv[2] + save_path = argv[2] else: - model_dir = './model' - - main_program = fluid.Program() - startup_program = fluid.Program() - with fluid.program_guard(main_program, startup_program): - inputs, outputs = inference_warpper(network_build_file) - - test_program = main_program.clone(for_test=True) - - labels = list() - losses = list() - for output in outputs: - label = fluid.layers.data(name='label_' + output.name, shape=output.shape, dtype='float32') - loss = fluid.layers.square_error_cost(input=output, label=label) - loss = fluid.layers.mean(loss, name='loss_' + output.name) - - labels.append(label) - losses.append(loss) - - loss_all = fluid.layers.sum(losses) - optimizer = fluid.optimizer.SGD(learning_rate=1.0) - params_grads = optimizer.backward(loss_all) - - if not os.path.exists(model_dir): - os.mkdir(model_dir) - - programs = { - 'startup_program': startup_program, - 'main_program': main_program, - 'test_program': test_program, - } - for save_path, program in programs.items(): - with open(os.path.join(model_dir, save_path), 'w') as f: - f.write(program.desc.serialize_to_string()) - - model_desc_path = os.path.join(model_dir, 'model.yaml') - model_desc = { - 'inputs': [{"name": var.name, "shape": var.shape} for var in inputs], - 'outputs': [{"name": var.name, "shape": var.shape, "label_name": label.name, "loss_name": loss.name} for var, label, loss in zip(outputs, labels, losses)], - 'loss_all': loss_all.name, - } - - with open(model_desc_path, 'w') as f: - yaml.safe_dump(model_desc, f, encoding='utf-8', allow_unicode=True) + save_path = None + builder = ModelBuilder() + if not builder.initialize(network_desc_path, save_path): + print_help(argv[0]) + exit(1) + builder.build_and_save() if __name__ == "__main__": main(sys.argv) diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/example.py b/paddle/fluid/train/custom_trainer/feed/scripts/example.py index 6773d036465fa2f6cac897f50bc35a21a90b88f8..4805e853b1daf9687c78814cab0fcfecbde738ba 100644 --- a/paddle/fluid/train/custom_trainer/feed/scripts/example.py +++ b/paddle/fluid/train/custom_trainer/feed/scripts/example.py @@ -18,16 +18,36 @@ def inference(): list: outputs """ # TODO: build network here - cvm_input = fluid.layers.data(name='cvm_input', shape=[4488], dtype='float32') + cvm_input = fluid.layers.data(name='cvm_input', shape=[4488], dtype='float32', stop_gradient=False) net = cvm_input - net = fluid.layers.fc(net, 512, act='relu') - net = fluid.layers.fc(net, 256, act='relu') - net = fluid.layers.fc(net, 256, act='relu') - net = fluid.layers.fc(net, 128, act='relu') - net = fluid.layers.fc(net, 128, act='relu') - net = fluid.layers.fc(net, 128, act='relu') - net = fluid.layers.fc(net, 128, act='relu') + net = fluid.layers.data_norm(input=net, name="bn6048", epsilon=1e-4, + param_attr={"batch_size":1e4, "batch_sum_default":0.0, "batch_square":1e4}) + net = fluid.layers.fc(net, 512, act='relu', name='fc_1') + net = fluid.layers.fc(net, 256, act='relu', name='fc_2') + net = fluid.layers.fc(net, 256, act='relu', name='fc_3') + net = fluid.layers.fc(net, 128, act='relu', name='fc_4') + net = fluid.layers.fc(net, 128, act='relu', name='fc_5') + net = fluid.layers.fc(net, 128, act='relu', name='fc_6') + net = fluid.layers.fc(net, 128, act='relu', name='fc_7') ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr') return [cvm_input], [ctr_output] + +def loss_function(ctr_output): + """ + Args: + *outputs: the second result of inference() + + Returns: + Variable: loss + and + list: labels + """ + # TODO: calc loss here + + label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32') + loss = fluid.layers.square_error_cost(input=ctr_output, label=label) + loss = fluid.layers.mean(loss, name='loss_ctr') + + return loss, [label] diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/join.py b/paddle/fluid/train/custom_trainer/feed/scripts/join.py new file mode 100644 index 0000000000000000000000000000000000000000..f96bdaa423ce01f48e8f956b01c9892efa8c4d52 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/scripts/join.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +""" +This is an example of network building +""" + +from __future__ import print_function, division +import paddle +from paddle import fluid + +def sparse_cvm_dim(sparse_info): + return sparse_info['slot_dim'] * len(sparse_info['slots']) + +def inference(): + """Build inference network(without loss and optimizer) + + Returns: + list: sparse_inputs + and + list: inputs + and + list: outputs + """ + sparse_cvm = { "name": "cvm_input", "slot_dim" : 11, "slots": [6048,6002,6145,6202,6201,6121,6738,6119,6146,6120,6147,6122,6123,6118,6142,6143,6008,6148,6151,6127,6144,6094,6083,6952,6739,6150,6109,6003,6099,6149,6129,6203,6153,6152,6128,6106,6251,7082,7515,6951,6949,7080,6066,7507,6186,6007,7514,6125,7506,10001,6006,7023,6085,10000,6098,6250,6110,6124,6090,6082,6067,6101,6004,6191,7075,6948,6157,6126,6188,7077,6070,6111,6087,6103,6107,6194,6156,6005,6247,6814,6158,7122,6058,6189,7058,6059,6115,7079,7081,6833,7024,6108,13342,13345,13412,13343,13350,13346,13409,6009,6011,6012,6013,6014,6015,6019,6023,6024,6027,6029,6031,6050,6060,6068,6069,6089,6095,6105,6112,6130,6131,6132,6134,6161,6162,6163,6166,6182,6183,6185,6190,6212,6213,6231,6233,6234,6236,6238,6239,6240,6241,6242,6243,6244,6245,6354,7002,7005,7008,7010,7012,7013,7015,7016,7017,7018,7019,7020,7045,7046,7048,7049,7052,7054,7056,7064,7066,7076,7078,7083,7084,7085,7086,7087,7088,7089,7090,7099,7100,7101,7102,7103,7104,7105,7109,7124,7126,7136,7142,7143,7144,7145,7146,7147,7148,7150,7151,7152,7153,7154,7155,7156,7157,7047,7050,6253,6254,6255,6256,6257,6259,6260,6261,7170,7185,7186,6751,6755,6757,6759,6760,6763,6764,6765,6766,6767,6768,6769,6770,7502,7503,7504,7505,7510,7511,7512,7513,6806,6807,6808,6809,6810,6811,6812,6813,6815,6816,6817,6819,6823,6828,6831,6840,6845,6875,6879,6881,6888,6889,6947,6950,6956,6957,6959,10006,10008,10009,10010,10011,10016,10017,10018,10019,10020,10021,10022,10023,10024,10029,10030,10031,10032,10033,10034,10035,10036,10037,10038,10039,10040,10041,10042,10044,10045,10046,10051,10052,10053,10054,10055,10056,10057,10060,10066,10069,6820,6821,6822,13333,13334,13335,13336,13337,13338,13339,13340,13341,13351,13352,13353,13359,13361,13362,13363,13366,13367,13368,13369,13370,13371,13375,13376,5700,5702,13400,13401,13402,13403,13404,13406,13407,13408,13410,13417,13418,13419,13420,13422,13425,13427,13428,13429,13430,13431,13433,13434,13436,13437,13326,13330,13331,5717,13442,13451,13452,13455,13456,13457,13458,13459,13460,13461,13462,13463,13464,13465,13466,13467,13468,1104,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,13812,13813,6740,1490,1491]} + + # TODO: build network here + cvm_input = fluid.layers.data(name='cvm_input', shape=[sparse_cvm_dim(sparse_cvm)], dtype='float32', stop_gradient=False) + + net = cvm_input + net = fluid.layers.data_norm(input=net, name="bn6048", epsilon=1e-4, + param_attr={"batch_size":1e4, "batch_sum_default":0.0, "batch_square":1e4}) + net = fluid.layers.fc(net, 511, act='relu', name='fc_1') + net = fluid.layers.fc(net, 255, act='relu', name='fc_2') + net = fluid.layers.fc(net, 255, act='relu', name='fc_3') + net = fluid.layers.fc(net, 127, act='relu', name='fc_4') + net = fluid.layers.fc(net, 127, act='relu', name='fc_5') + net = fluid.layers.fc(net, 127, act='relu', name='fc_6') + net = fluid.layers.fc(net, 127, act='relu', name='fc_7') + + ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr') + + accessors = [ + { "class": "AbacusSparseUpdateAccessor", "input": "sparses", "table_id": 0, "need_gradient": False}, + { "class": "DenseInputAccessor", "input": "vars", "table_id": 1, "need_gradient": True, "async_pull": True}, + { "class": "DenseInputAccessor", "input": "sums", "table_id": 2, "need_gradient": True, "async_pull": True}, + { "class": "LabelInputAccessor", "input": "labels"} + ] + return accessors, [sparse_cvm], [cvm_input], [ctr_output] + +def loss_function(ctr_output): + """ + Args: + *outputs: the second result of inference() + + Returns: + Variable: loss + and + list: labels + """ + # TODO: calc loss here + + label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32') + loss = fluid.layers.square_error_cost(input=ctr_output, label=label) + loss = fluid.layers.mean(loss, name='loss_ctr') + + return loss, [label] diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/example/main_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/main_program new file mode 100644 index 0000000000000000000000000000000000000000..cc20c06bd5bff5e10fae6efdcff8d78cffc2aa7c Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/main_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/example/model.yaml b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..849c39d23c12d4f005ffb4a7b703e1e784d52b2e --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/model.yaml @@ -0,0 +1,49 @@ +inputs: +- name: cvm_input + shape: [-1, 4488] +labels: +- name: label_ctr + shape: [-1, 1] +loss: loss_ctr +outputs: +- name: ctr.tmp_2 + shape: [-1, 1] +vars: +- name: bn6048.batch_size + shape: [4488] +- name: bn6048.batch_sum + shape: [4488] +- name: bn6048.batch_square_sum + shape: [4488] +- name: fc_1.w_0 + shape: [4488, 512] +- name: fc_1.b_0 + shape: [512] +- name: fc_2.w_0 + shape: [512, 256] +- name: fc_2.b_0 + shape: [256] +- name: fc_3.w_0 + shape: [256, 256] +- name: fc_3.b_0 + shape: [256] +- name: fc_4.w_0 + shape: [256, 128] +- name: fc_4.b_0 + shape: [128] +- name: fc_5.w_0 + shape: [128, 128] +- name: fc_5.b_0 + shape: [128] +- name: fc_6.w_0 + shape: [128, 128] +- name: fc_6.b_0 + shape: [128] +- name: fc_7.w_0 + shape: [128, 128] +- name: fc_7.b_0 + shape: [128] +- name: ctr.w_0 + shape: [128, 1] +- name: ctr.b_0 + shape: [1] diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/example/startup_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/startup_program new file mode 100644 index 0000000000000000000000000000000000000000..259839f93f4f390a55c4589d14f4a4cef3b07652 Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/startup_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/example/test_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/test_program new file mode 100644 index 0000000000000000000000000000000000000000..147833363f095e719093aa22b4da2ac31b847cfb Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/example/test_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/join/main_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/main_program new file mode 100644 index 0000000000000000000000000000000000000000..9d5a954648abd1895f32c5aab6ec502ec6767695 Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/main_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/join/model.yaml b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..284827fa35573c42f6fb4745eff4674b153d8dd4 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/model.yaml @@ -0,0 +1,103 @@ +input_accessor: +- class: AbacusSparseUpdateAccessor + input: + - name: cvm_input + slot_dim: 11 + slots: [6048, 6002, 6145, 6202, 6201, 6121, 6738, 6119, 6146, 6120, 6147, 6122, + 6123, 6118, 6142, 6143, 6008, 6148, 6151, 6127, 6144, 6094, 6083, 6952, 6739, + 6150, 6109, 6003, 6099, 6149, 6129, 6203, 6153, 6152, 6128, 6106, 6251, 7082, + 7515, 6951, 6949, 7080, 6066, 7507, 6186, 6007, 7514, 6125, 7506, 10001, 6006, + 7023, 6085, 10000, 6098, 6250, 6110, 6124, 6090, 6082, 6067, 6101, 6004, 6191, + 7075, 6948, 6157, 6126, 6188, 7077, 6070, 6111, 6087, 6103, 6107, 6194, 6156, + 6005, 6247, 6814, 6158, 7122, 6058, 6189, 7058, 6059, 6115, 7079, 7081, 6833, + 7024, 6108, 13342, 13345, 13412, 13343, 13350, 13346, 13409, 6009, 6011, 6012, + 6013, 6014, 6015, 6019, 6023, 6024, 6027, 6029, 6031, 6050, 6060, 6068, 6069, + 6089, 6095, 6105, 6112, 6130, 6131, 6132, 6134, 6161, 6162, 6163, 6166, 6182, + 6183, 6185, 6190, 6212, 6213, 6231, 6233, 6234, 6236, 6238, 6239, 6240, 6241, + 6242, 6243, 6244, 6245, 6354, 7002, 7005, 7008, 7010, 7012, 7013, 7015, 7016, + 7017, 7018, 7019, 7020, 7045, 7046, 7048, 7049, 7052, 7054, 7056, 7064, 7066, + 7076, 7078, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7099, 7100, 7101, + 7102, 7103, 7104, 7105, 7109, 7124, 7126, 7136, 7142, 7143, 7144, 7145, 7146, + 7147, 7148, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7047, 7050, 6253, + 6254, 6255, 6256, 6257, 6259, 6260, 6261, 7170, 7185, 7186, 6751, 6755, 6757, + 6759, 6760, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 7502, 7503, 7504, + 7505, 7510, 7511, 7512, 7513, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813, + 6815, 6816, 6817, 6819, 6823, 6828, 6831, 6840, 6845, 6875, 6879, 6881, 6888, + 6889, 6947, 6950, 6956, 6957, 6959, 10006, 10008, 10009, 10010, 10011, 10016, + 10017, 10018, 10019, 10020, 10021, 10022, 10023, 10024, 10029, 10030, 10031, + 10032, 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042, + 10044, 10045, 10046, 10051, 10052, 10053, 10054, 10055, 10056, 10057, 10060, + 10066, 10069, 6820, 6821, 6822, 13333, 13334, 13335, 13336, 13337, 13338, 13339, + 13340, 13341, 13351, 13352, 13353, 13359, 13361, 13362, 13363, 13366, 13367, + 13368, 13369, 13370, 13371, 13375, 13376, 5700, 5702, 13400, 13401, 13402, 13403, + 13404, 13406, 13407, 13408, 13410, 13417, 13418, 13419, 13420, 13422, 13425, + 13427, 13428, 13429, 13430, 13431, 13433, 13434, 13436, 13437, 13326, 13330, + 13331, 5717, 13442, 13451, 13452, 13455, 13456, 13457, 13458, 13459, 13460, + 13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 1104, 1106, 1107, 1108, + 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122, + 1123, 1124, 1125, 1126, 1127, 1128, 1129, 13812, 13813, 6740, 1490, 1491] + need_gradient: false + table_id: 0 +- async_pull: true + class: DenseInputAccessor + input: + - name: fc_1.w_0 + shape: [4488, 511] + - name: fc_1.b_0 + shape: [511] + - name: fc_2.w_0 + shape: [511, 255] + - name: fc_2.b_0 + shape: [255] + - name: fc_3.w_0 + shape: [255, 255] + - name: fc_3.b_0 + shape: [255] + - name: fc_4.w_0 + shape: [255, 127] + - name: fc_4.b_0 + shape: [127] + - name: fc_5.w_0 + shape: [127, 127] + - name: fc_5.b_0 + shape: [127] + - name: fc_6.w_0 + shape: [127, 127] + - name: fc_6.b_0 + shape: [127] + - name: fc_7.w_0 + shape: [127, 127] + - name: fc_7.b_0 + shape: [127] + - name: ctr.w_0 + shape: [127, 1] + - name: ctr.b_0 + shape: [1] + need_gradient: true + table_id: 1 +- async_pull: true + class: DenseInputAccessor + input: + - name: bn6048.batch_size + shape: [4488] + - name: bn6048.batch_sum + shape: [4488] + - name: bn6048.batch_square_sum + shape: [4488] + need_gradient: true + table_id: 2 +- class: LabelInputAccessor + input: + - label_name: label_ctr + output_name: ctr.tmp_2 + shape: [-1, 1] +inputs: +- name: cvm_input + shape: [-1, 4488] +labels: +- name: label_ctr + shape: [-1, 1] +loss: loss_ctr +outputs: +- name: ctr.tmp_2 + shape: [-1, 1] diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/join/startup_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/startup_program new file mode 100644 index 0000000000000000000000000000000000000000..1e0b9f82c10f926a5bf2a08abfcdc9b8cdd810cb Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/startup_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/join/test_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/test_program new file mode 100644 index 0000000000000000000000000000000000000000..f056e29f9a0bd56332391fd16aee5212ce5e3a20 Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/join/test_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/update/main_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/main_program new file mode 100644 index 0000000000000000000000000000000000000000..ab8290d6940dc12a2fb052df286f44a4abf8cb10 Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/main_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/update/model.yaml b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/model.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bf93e56714fcbd4d30363fa39fddd12b1147c32 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/model.yaml @@ -0,0 +1,84 @@ +input_accessor: +- class: AbacusSparseUpdateAccessor + input: + - name: cvm_input + slot_dim: 9 + slots: [6048, 6002, 6145, 6202, 6201, 6121, 6738, 6119, 6146, 6120, 6147, 6122, + 6123, 6118, 6142, 6143, 6008, 6148, 6151, 6127, 6144, 6094, 6083, 6952, 6739, + 6150, 6109, 6003, 6099, 6149, 6129, 6203, 6153, 6152, 6128, 6106, 6251, 7082, + 7515, 6951, 6949, 7080, 6066, 7507, 6186, 6007, 7514, 6125, 7506, 10001, 6006, + 7023, 6085, 10000, 6098, 6250, 6110, 6124, 6090, 6082, 6067, 6101, 6004, 6191, + 7075, 6948, 6157, 6126, 6188, 7077, 6070, 6111, 6087, 6103, 6107, 6194, 6156, + 6005, 6247, 6814, 6158, 7122, 6058, 6189, 7058, 6059, 6115, 7079, 7081, 6833, + 7024, 6108, 13342, 13345, 13412, 13343, 13350, 13346, 13409, 6009, 6011, 6012, + 6013, 6014, 6015, 6019, 6023, 6024, 6027, 6029, 6031, 6050, 6060, 6068, 6069, + 6089, 6095, 6105, 6112, 6130, 6131, 6132, 6134, 6161, 6162, 6163, 6166, 6182, + 6183, 6185, 6190, 6212, 6213, 6231, 6233, 6234, 6236, 6238, 6239, 6240, 6241, + 6242, 6243, 6244, 6245, 6354, 7002, 7005, 7008, 7010, 7012, 7013, 7015, 7016, + 7017, 7018, 7019, 7020, 7045, 7046, 7048, 7049, 7052, 7054, 7056, 7064, 7066, + 7076, 7078, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7099, 7100, 7101, + 7102, 7103, 7104, 7105, 7109, 7124, 7126, 7136, 7142, 7143, 7144, 7145, 7146, + 7147, 7148, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7047, 7050, 6253, + 6254, 6255, 6256, 6257, 6259, 6260, 6261, 7170, 7185, 7186, 6751, 6755, 6757, + 6759, 6760, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 7502, 7503, 7504, + 7505, 7510, 7511, 7512, 7513, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813, + 6815, 6816, 6817, 6819, 6823, 6828, 6831, 6840, 6845, 6875, 6879, 6881, 6888, + 6889, 6947, 6950, 6956, 6957, 6959, 10006, 10008, 10009, 10010, 10011, 10016, + 10017, 10018, 10019, 10020, 10021, 10022, 10023, 10024, 10029, 10030, 10031, + 10032, 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042, + 10044, 10045, 10046, 10051, 10052, 10053, 10054, 10055, 10056, 10057, 10060, + 10066, 10069, 6820, 6821, 6822, 13333, 13334, 13335, 13336, 13337, 13338, 13339, + 13340, 13341, 13351, 13352, 13353, 13359, 13361, 13362, 13363, 13366, 13367, + 13368, 13369, 13370, 13371, 13375, 13376, 5700, 5702, 13400, 13401, 13402, 13403, + 13404, 13406, 13407, 13408, 13410, 13417, 13418, 13419, 13420, 13422, 13425, + 13427, 13428, 13429, 13430, 13431, 13433, 13434, 13436, 13437, 13326, 13330, + 13331, 5717, 13442, 13451, 13452, 13455, 13456, 13457, 13458, 13459, 13460, + 13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 1104, 1106, 1107, 1108, + 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122, + 1123, 1124, 1125, 1126, 1127, 1128, 1129, 13812, 13813, 6740, 1490, 1491] + need_gradient: true + table_id: 0 +- async_pull: true + class: DenseInputAccessor + input: + - name: fc_1.w_0 + shape: [3672, 511] + - name: fc_1.b_0 + shape: [511] + - name: fc_2.w_0 + shape: [511, 255] + - name: fc_2.b_0 + shape: [255] + - name: fc_3.w_0 + shape: [255, 127] + - name: fc_3.b_0 + shape: [127] + - name: fc_4.w_0 + shape: [127, 127] + - name: fc_4.b_0 + shape: [127] + - name: fc_5.w_0 + shape: [127, 127] + - name: fc_5.b_0 + shape: [127] + - name: ctr.w_0 + shape: [127, 1] + - name: ctr.b_0 + shape: [1] + need_gradient: true + table_id: 3 +- class: LabelInputAccessor + input: + - label_name: label_ctr + output_name: ctr.tmp_2 + shape: [-1, 1] +inputs: +- name: cvm_input + shape: [-1, 3672] +labels: +- name: label_ctr + shape: [-1, 1] +loss: loss_ctr +outputs: +- name: ctr.tmp_2 + shape: [-1, 1] diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/update/startup_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/startup_program new file mode 100644 index 0000000000000000000000000000000000000000..ad56050377a0dce6412cfdab650483f88ab5f43a Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/startup_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/model/update/test_program b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/test_program new file mode 100644 index 0000000000000000000000000000000000000000..5e6ef75f3b3f7bb164aae179dd5951deecc3b04a Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/scripts/model/update/test_program differ diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/start_feed_trainer.sh b/paddle/fluid/train/custom_trainer/feed/scripts/start_feed_trainer.sh old mode 100644 new mode 100755 index a0fec52dfbb6d1fb7d864cb79018a2ddd9c3bd96..1d4c21148e05e2c9cba0c3a132d53b2871635963 --- a/paddle/fluid/train/custom_trainer/feed/scripts/start_feed_trainer.sh +++ b/paddle/fluid/train/custom_trainer/feed/scripts/start_feed_trainer.sh @@ -1,3 +1,3 @@ -#!bash +#!/bin/bash export LD_LIBRARY_PATH=LD_LIBRARY_PATH:./so -./bin/feed_trainer +./bin/feed_trainer "$@" diff --git a/paddle/fluid/train/custom_trainer/feed/scripts/update.py b/paddle/fluid/train/custom_trainer/feed/scripts/update.py new file mode 100644 index 0000000000000000000000000000000000000000..635253482f2c1e2e435354964d4953aa7c4b27ca --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/scripts/update.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +#-*- coding:utf-8 -*- + +""" +This is an example of network building +""" + +from __future__ import print_function, division +import paddle +from paddle import fluid + +def sparse_cvm_dim(sparse_info): + return sparse_info['slot_dim'] * len(sparse_info['slots']) + +def inference(): + """Build inference network(without loss and optimizer) + + Returns: + list: inputs + and + list: outputs + """ + sparse_cvm = { "name": "cvm_input", "slot_dim" : 9, "slots": [6048,6002,6145,6202,6201,6121,6738,6119,6146,6120,6147,6122,6123,6118,6142,6143,6008,6148,6151,6127,6144,6094,6083,6952,6739,6150,6109,6003,6099,6149,6129,6203,6153,6152,6128,6106,6251,7082,7515,6951,6949,7080,6066,7507,6186,6007,7514,6125,7506,10001,6006,7023,6085,10000,6098,6250,6110,6124,6090,6082,6067,6101,6004,6191,7075,6948,6157,6126,6188,7077,6070,6111,6087,6103,6107,6194,6156,6005,6247,6814,6158,7122,6058,6189,7058,6059,6115,7079,7081,6833,7024,6108,13342,13345,13412,13343,13350,13346,13409,6009,6011,6012,6013,6014,6015,6019,6023,6024,6027,6029,6031,6050,6060,6068,6069,6089,6095,6105,6112,6130,6131,6132,6134,6161,6162,6163,6166,6182,6183,6185,6190,6212,6213,6231,6233,6234,6236,6238,6239,6240,6241,6242,6243,6244,6245,6354,7002,7005,7008,7010,7012,7013,7015,7016,7017,7018,7019,7020,7045,7046,7048,7049,7052,7054,7056,7064,7066,7076,7078,7083,7084,7085,7086,7087,7088,7089,7090,7099,7100,7101,7102,7103,7104,7105,7109,7124,7126,7136,7142,7143,7144,7145,7146,7147,7148,7150,7151,7152,7153,7154,7155,7156,7157,7047,7050,6253,6254,6255,6256,6257,6259,6260,6261,7170,7185,7186,6751,6755,6757,6759,6760,6763,6764,6765,6766,6767,6768,6769,6770,7502,7503,7504,7505,7510,7511,7512,7513,6806,6807,6808,6809,6810,6811,6812,6813,6815,6816,6817,6819,6823,6828,6831,6840,6845,6875,6879,6881,6888,6889,6947,6950,6956,6957,6959,10006,10008,10009,10010,10011,10016,10017,10018,10019,10020,10021,10022,10023,10024,10029,10030,10031,10032,10033,10034,10035,10036,10037,10038,10039,10040,10041,10042,10044,10045,10046,10051,10052,10053,10054,10055,10056,10057,10060,10066,10069,6820,6821,6822,13333,13334,13335,13336,13337,13338,13339,13340,13341,13351,13352,13353,13359,13361,13362,13363,13366,13367,13368,13369,13370,13371,13375,13376,5700,5702,13400,13401,13402,13403,13404,13406,13407,13408,13410,13417,13418,13419,13420,13422,13425,13427,13428,13429,13430,13431,13433,13434,13436,13437,13326,13330,13331,5717,13442,13451,13452,13455,13456,13457,13458,13459,13460,13461,13462,13463,13464,13465,13466,13467,13468,1104,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,13812,13813,6740,1490,1491]} + # TODO: build network here + cvm_input = fluid.layers.data(name='cvm_input', shape=[sparse_cvm_dim(sparse_cvm)], dtype='float32', stop_gradient=False) + + net = cvm_input + net = fluid.layers.fc(net, 511, act='relu', name='fc_1') + net = fluid.layers.fc(net, 255, act='relu', name='fc_2') + net = fluid.layers.fc(net, 127, act='relu', name='fc_3') + net = fluid.layers.fc(net, 127, act='relu', name='fc_4') + net = fluid.layers.fc(net, 127, act='relu', name='fc_5') + + ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr') + + accessors = [ + { "class": "AbacusSparseUpdateAccessor", "input": "sparses", "table_id": 0, "need_gradient": True}, + { "class": "DenseInputAccessor", "input": "vars", "table_id": 3, "need_gradient": True, "async_pull": True}, + { "class": "LabelInputAccessor", "input": "labels"} + ] + + return accessors, [sparse_cvm], [cvm_input], [ctr_output] + +def loss_function(ctr_output): + """ + Args: + *outputs: the second result of inference() + + Returns: + Variable: loss + and + list: labels + """ + # TODO: calc loss here + + label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32') + loss = fluid.layers.square_error_cost(input=ctr_output, label=label) + loss = fluid.layers.mean(loss, name='loss_ctr') + + return loss, [label] diff --git a/paddle/fluid/train/custom_trainer/feed/tool/format_newcate_hotnews.awk b/paddle/fluid/train/custom_trainer/feed/tool/format_newcate_hotnews.awk new file mode 100755 index 0000000000000000000000000000000000000000..7820d4050110a1e1b59d739c126648d24681dd18 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/tool/format_newcate_hotnews.awk @@ -0,0 +1,21 @@ +#!/bin/awk -f +{ + if ($1 !~ /^([0-9a-zA-Z])+$/ || $2 !~ /^([0-9])+$/ || $3 !~ /^([0-9])+$/) { + next; + } + show = $2; + clk = $3; + if (clk > show) { + clk = show; + } + for (i = 0; i < clk; i++) { + $2 = "1"; + $3 = "1"; + print $0; + } + for (i = 0; i < show - clk; i++) { + $2 = "1"; + $3 = "0"; + print $0; + } +} diff --git a/paddle/fluid/train/custom_trainer/feed/tool/gdbinit b/paddle/fluid/train/custom_trainer/feed/tool/gdbinit new file mode 100644 index 0000000000000000000000000000000000000000..1979250bd0771d5a6ac3e17aeab4187a5605272f --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/tool/gdbinit @@ -0,0 +1,697 @@ +# +# STL GDB evaluators/views/utilities - 1.03 +# +# The new GDB commands: +# are entirely non instrumental +# do not depend on any "inline"(s) - e.g. size(), [], etc +# are extremely tolerant to debugger settings +# +# This file should be "included" in .gdbinit as following: +# source stl-views.gdb or just paste it into your .gdbinit file +# +# The following STL containers are currently supported: +# +# std::vector -- via pvector command +# std::list -- via plist or plist_member command +# std::map -- via pmap or pmap_member command +# std::multimap -- via pmap or pmap_member command +# std::set -- via pset command +# std::multiset -- via pset command +# std::deque -- via pdequeue command +# std::stack -- via pstack command +# std::queue -- via pqueue command +# std::priority_queue -- via ppqueue command +# std::bitset -- via pbitset command +# std::string -- via pstring command +# std::widestring -- via pwstring command +# +# The end of this file contains (optional) C++ beautifiers +# Make sure your debugger supports $argc +# +# Simple GDB Macros writen by Dan Marinescu (H-PhD) - License GPL +# Inspired by intial work of Tom Malnar, +# Tony Novac (PhD) / Cornell / Stanford, +# Gilad Mishne (PhD) and Many Many Others. +# Contact: dan_c_marinescu@yahoo.com (Subject: STL) +# +# Modified to work with g++ 4.3 by Anders Elton +# Also added _member functions, that instead of printing the entire class in map, prints a member. + + + +# +# std::vector<> +# + +define pvector + if $argc == 0 + help pvector + else + set $size = $arg0._M_impl._M_finish - $arg0._M_impl._M_start + set $capacity = $arg0._M_impl._M_end_of_storage - $arg0._M_impl._M_start + set $size_max = $size - 1 + end + if $argc == 1 + set $i = 0 + while $i < $size + printf "elem[%u]: ", $i + p *($arg0._M_impl._M_start + $i) + set $i++ + end + end + if $argc == 2 + set $idx = $arg1 + if $idx < 0 || $idx > $size_max + printf "idx1, idx2 are not in acceptable range: [0..%u].\n", $size_max + else + printf "elem[%u]: ", $idx + p *($arg0._M_impl._M_start + $idx) + end + end + if $argc == 3 + set $start_idx = $arg1 + set $stop_idx = $arg2 + if $start_idx > $stop_idx + set $tmp_idx = $start_idx + set $start_idx = $stop_idx + set $stop_idx = $tmp_idx + end + if $start_idx < 0 || $stop_idx < 0 || $start_idx > $size_max || $stop_idx > $size_max + printf "idx1, idx2 are not in acceptable range: [0..%u].\n", $size_max + else + set $i = $start_idx + while $i <= $stop_idx + printf "elem[%u]: ", $i + p *($arg0._M_impl._M_start + $i) + set $i++ + end + end + end + if $argc > 0 + printf "Vector size = %u\n", $size + printf "Vector capacity = %u\n", $capacity + printf "Element " + whatis $arg0._M_impl._M_start + end +end + +document pvector + Prints std::vector information. + Syntax: pvector + Note: idx, idx1 and idx2 must be in acceptable range [0...size()-1]. + Examples: + pvector v - Prints vector content, size, capacity and T typedef + pvector v 0 - Prints element[idx] from vector + pvector v 1 2 - Prints elements in range [idx1..idx2] from vector +end + +# +# std::list<> +# + +define plist + if $argc == 0 + help plist + else + set $head = &$arg0._M_impl._M_node + set $current = $arg0._M_impl._M_node._M_next + set $size = 0 + while $current != $head + if $argc == 2 + printf "elem[%u]: ", $size + p *($arg1*)($current + 1) + end + if $argc == 3 + if $size == $arg2 + printf "elem[%u]: ", $size + p *($arg1*)($current + 1) + end + end + set $current = $current._M_next + set $size++ + end + printf "List size = %u \n", $size + if $argc == 1 + printf "List " + whatis $arg0 + printf "Use plist to see the elements in the list.\n" + end + end +end + +document plist + Prints std::list information. + Syntax: plist : Prints list size, if T defined all elements or just element at idx + Examples: + plist l - prints list size and definition + plist l int - prints all elements and list size + plist l int 2 - prints the third element in the list (if exists) and list size +end + +define plist_member + if $argc == 0 + help plist_member + else + set $head = &$arg0._M_impl._M_node + set $current = $arg0._M_impl._M_node._M_next + set $size = 0 + while $current != $head + if $argc == 3 + printf "elem[%u]: ", $size + p (*($arg1*)($current + 1)).$arg2 + end + if $argc == 4 + if $size == $arg3 + printf "elem[%u]: ", $size + p (*($arg1*)($current + 1)).$arg2 + end + end + set $current = $current._M_next + set $size++ + end + printf "List size = %u \n", $size + if $argc == 1 + printf "List " + whatis $arg0 + printf "Use plist_member to see the elements in the list.\n" + end + end +end + +document plist_member + Prints std::list information. + Syntax: plist : Prints list size, if T defined all elements or just element at idx + Examples: + plist_member l int member - prints all elements and list size + plist_member l int member 2 - prints the third element in the list (if exists) and list size +end + + +# +# std::map and std::multimap +# + +define pmap + if $argc == 0 + help pmap + else + set $tree = $arg0 + set $i = 0 + set $node = $tree._M_t._M_impl._M_header._M_left + set $end = $tree._M_t._M_impl._M_header + set $tree_size = $tree._M_t._M_impl._M_node_count + if $argc == 1 + printf "Map " + whatis $tree + printf "Use pmap to see the elements in the map.\n" + end + if $argc == 3 + while $i < $tree_size + set $value = (void *)($node + 1) + printf "elem[%u].left: ", $i + p *($arg1*)$value + set $value = $value + sizeof($arg1) + printf "elem[%u].right: ", $i + p *($arg2*)$value + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + end + if $argc == 4 + set $idx = $arg3 + set $ElementsFound = 0 + while $i < $tree_size + set $value = (void *)($node + 1) + if *($arg1*)$value == $idx + printf "elem[%u].left: ", $i + p *($arg1*)$value + set $value = $value + sizeof($arg1) + printf "elem[%u].right: ", $i + p *($arg2*)$value + set $ElementsFound++ + end + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + printf "Number of elements found = %u\n", $ElementsFound + end + if $argc == 5 + set $idx1 = $arg3 + set $idx2 = $arg4 + set $ElementsFound = 0 + while $i < $tree_size + set $value = (void *)($node + 1) + set $valueLeft = *($arg1*)$value + set $valueRight = *($arg2*)($value + sizeof($arg1)) + if $valueLeft == $idx1 && $valueRight == $idx2 + printf "elem[%u].left: ", $i + p $valueLeft + printf "elem[%u].right: ", $i + p $valueRight + set $ElementsFound++ + end + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + printf "Number of elements found = %u\n", $ElementsFound + end + printf "Map size = %u\n", $tree_size + end +end + +document pmap + Prints std::map or std::multimap information. Works for std::multimap as well. + Syntax: pmap : Prints map size, if T defined all elements or just element(s) with val(s) + Examples: + pmap m - prints map size and definition + pmap m int int - prints all elements and map size + pmap m int int 20 - prints the element(s) with left-value = 20 (if any) and map size + pmap m int int 20 200 - prints the element(s) with left-value = 20 and right-value = 200 (if any) and map size +end + + +define pmap_member + if $argc == 0 + help pmap_member + else + set $tree = $arg0 + set $i = 0 + set $node = $tree._M_t._M_impl._M_header._M_left + set $end = $tree._M_t._M_impl._M_header + set $tree_size = $tree._M_t._M_impl._M_node_count + if $argc == 1 + printf "Map " + whatis $tree + printf "Use pmap to see the elements in the map.\n" + end + if $argc == 5 + while $i < $tree_size + set $value = (void *)($node + 1) + printf "elem[%u].left: ", $i + p (*($arg1*)$value).$arg2 + set $value = $value + sizeof($arg1) + printf "elem[%u].right: ", $i + p (*($arg3*)$value).$arg4 + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + end + if $argc == 6 + set $idx = $arg5 + set $ElementsFound = 0 + while $i < $tree_size + set $value = (void *)($node + 1) + if *($arg1*)$value == $idx + printf "elem[%u].left: ", $i + p (*($arg1*)$value).$arg2 + set $value = $value + sizeof($arg1) + printf "elem[%u].right: ", $i + p (*($arg3*)$value).$arg4 + set $ElementsFound++ + end + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + printf "Number of elements found = %u\n", $ElementsFound + end + printf "Map size = %u\n", $tree_size + end +end + +document pmap_member + Prints std::map or std::multimap information. Works for std::multimap as well. + Syntax: pmap : Prints map size, if T defined all elements or just element(s) with val(s) + Examples: + pmap_member m class1 member1 class2 member2 - prints class1.member1 : class2.member2 + pmap_member m class1 member1 class2 member2 lvalue - prints class1.member1 : class2.member2 where class1 == lvalue +end + + +# +# std::set and std::multiset +# + +define pset + if $argc == 0 + help pset + else + set $tree = $arg0 + set $i = 0 + set $node = $tree._M_t._M_impl._M_header._M_left + set $end = $tree._M_t._M_impl._M_header + set $tree_size = $tree._M_t._M_impl._M_node_count + if $argc == 1 + printf "Set " + whatis $tree + printf "Use pset to see the elements in the set.\n" + end + if $argc == 2 + while $i < $tree_size + set $value = (void *)($node + 1) + printf "elem[%u]: ", $i + p *($arg1*)$value + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + end + if $argc == 3 + set $idx = $arg2 + set $ElementsFound = 0 + while $i < $tree_size + set $value = (void *)($node + 1) + if *($arg1*)$value == $idx + printf "elem[%u]: ", $i + p *($arg1*)$value + set $ElementsFound++ + end + if $node._M_right != 0 + set $node = $node._M_right + while $node._M_left != 0 + set $node = $node._M_left + end + else + set $tmp_node = $node._M_parent + while $node == $tmp_node._M_right + set $node = $tmp_node + set $tmp_node = $tmp_node._M_parent + end + if $node._M_right != $tmp_node + set $node = $tmp_node + end + end + set $i++ + end + printf "Number of elements found = %u\n", $ElementsFound + end + printf "Set size = %u\n", $tree_size + end +end + +document pset + Prints std::set or std::multiset information. Works for std::multiset as well. + Syntax: pset : Prints set size, if T defined all elements or just element(s) having val + Examples: + pset s - prints set size and definition + pset s int - prints all elements and the size of s + pset s int 20 - prints the element(s) with value = 20 (if any) and the size of s +end + + + +# +# std::dequeue +# + +define pdequeue + if $argc == 0 + help pdequeue + else + set $size = 0 + set $start_cur = $arg0._M_impl._M_start._M_cur + set $start_last = $arg0._M_impl._M_start._M_last + set $start_stop = $start_last + while $start_cur != $start_stop + p *$start_cur + set $start_cur++ + set $size++ + end + set $finish_first = $arg0._M_impl._M_finish._M_first + set $finish_cur = $arg0._M_impl._M_finish._M_cur + set $finish_last = $arg0._M_impl._M_finish._M_last + if $finish_cur < $finish_last + set $finish_stop = $finish_cur + else + set $finish_stop = $finish_last + end + while $finish_first != $finish_stop + p *$finish_first + set $finish_first++ + set $size++ + end + printf "Dequeue size = %u\n", $size + end +end + +document pdequeue + Prints std::dequeue information. + Syntax: pdequeue : Prints dequeue size, if T defined all elements + Deque elements are listed "left to right" (left-most stands for front and right-most stands for back) + Example: + pdequeue d - prints all elements and size of d +end + + + +# +# std::stack +# + +define pstack + if $argc == 0 + help pstack + else + set $start_cur = $arg0.c._M_impl._M_start._M_cur + set $finish_cur = $arg0.c._M_impl._M_finish._M_cur + set $size = $finish_cur - $start_cur + set $i = $size - 1 + while $i >= 0 + p *($start_cur + $i) + set $i-- + end + printf "Stack size = %u\n", $size + end +end + +document pstack + Prints std::stack information. + Syntax: pstack : Prints all elements and size of the stack + Stack elements are listed "top to buttom" (top-most element is the first to come on pop) + Example: + pstack s - prints all elements and the size of s +end + + + +# +# std::queue +# + +define pqueue + if $argc == 0 + help pqueue + else + set $start_cur = $arg0.c._M_impl._M_start._M_cur + set $finish_cur = $arg0.c._M_impl._M_finish._M_cur + set $size = $finish_cur - $start_cur + set $i = 0 + while $i < $size + p *($start_cur + $i) + set $i++ + end + printf "Queue size = %u\n", $size + end +end + +document pqueue + Prints std::queue information. + Syntax: pqueue : Prints all elements and the size of the queue + Queue elements are listed "top to bottom" (top-most element is the first to come on pop) + Example: + pqueue q - prints all elements and the size of q +end + + + +# +# std::priority_queue +# + +define ppqueue + if $argc == 0 + help ppqueue + else + set $size = $arg0.c._M_impl._M_finish - $arg0.c._M_impl._M_start + set $capacity = $arg0.c._M_impl._M_end_of_storage - $arg0.c._M_impl._M_start + set $i = $size - 1 + while $i >= 0 + p *($arg0.c._M_impl._M_start + $i) + set $i-- + end + printf "Priority queue size = %u\n", $size + printf "Priority queue capacity = %u\n", $capacity + end +end + +document ppqueue + Prints std::priority_queue information. + Syntax: ppqueue : Prints all elements, size and capacity of the priority_queue + Priority_queue elements are listed "top to buttom" (top-most element is the first to come on pop) + Example: + ppqueue pq - prints all elements, size and capacity of pq +end + + + +# +# std::bitset +# + +define pbitset + if $argc == 0 + help pbitset + else + p /t $arg0._M_w + end +end + +document pbitset + Prints std::bitset information. + Syntax: pbitset : Prints all bits in bitset + Example: + pbitset b - prints all bits in b +end + + + +# +# std::string +# + +define pstring + if $argc == 0 + help pstring + else + printf "String \t\t\t= \"%s\"\n", $arg0._M_data() + printf "String size/length \t= %u\n", $arg0._M_rep()._M_length + printf "String capacity \t= %u\n", $arg0._M_rep()._M_capacity + printf "String ref-count \t= %d\n", $arg0._M_rep()._M_refcount + end +end + +document pstring + Prints std::string information. + Syntax: pstring + Example: + pstring s - Prints content, size/length, capacity and ref-count of string s +end + +# +# std::wstring +# + +define pwstring + if $argc == 0 + help pwstring + else + call printf("WString \t\t= \"%ls\"\n", $arg0._M_data()) + printf "WString size/length \t= %u\n", $arg0._M_rep()._M_length + printf "WString capacity \t= %u\n", $arg0._M_rep()._M_capacity + printf "WString ref-count \t= %d\n", $arg0._M_rep()._M_refcount + end +end + +document pwstring + Prints std::wstring information. + Syntax: pwstring + Example: + pwstring s - Prints content, size/length, capacity and ref-count of wstring s +end + +# +# C++ related beautifiers (optional) +# + +set print pretty on +set print object on +set print static-members on +set print vtbl on +set print demangle on +set demangle-style gnu-v3 +set print sevenbit-strings off + +set follow-fork-mode child +set detach-on-fork off diff --git a/paddle/fluid/train/custom_trainer/feed/tool/ins_weight.py b/paddle/fluid/train/custom_trainer/feed/tool/ins_weight.py new file mode 100755 index 0000000000000000000000000000000000000000..8b4d87c34300aaea048c07fd9e9c50aa70e3a07c --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/tool/ins_weight.py @@ -0,0 +1,122 @@ +#!/usr/bin/python +import sys +import re +import math + +del_text_slot = True +g_ratio = 1 +w_ratio = 0.01 +slots_str = "6048 6145 6202 6201 6121 6119 6146 6120 6147 6122 6123 6118 6142 6143 6008 6148 6151 6127 6144 6150 6109 6003 6096 6149 6129 6203 6153 6152 6128 6106 6251 7082 7515 7080 6066 7507 6186 6007 7514 6054 6125 7506 10001 6006 6080 7023 6085 10000 6250 6110 6124 6090 6082 6067 7516 6101 6004 6191 6188 6070 6194 6247 6814 7512 10007 6058 6189 6059 7517 10005 7510 7024 7502 7503 6183 7511 6060 6806 7504 6185 6810 6248 10004 6815 6182 10068 6069 6073 6196 6816 7513 6071 6809 6072 6817 6190 7505 6813 6192 6807 6808 6195 6826 6184 6197 6068 6812 7107 6811 6823 6824 6819 6818 6821 6822 6820 6094 6083 6952 6099 6951 6949 6098 7075 6948 6157 6126 7077 6111 6087 6103 6107 6156 6005 6158 7122 6155 7058 6115 7079 7081 6833 6108 6840 6837 7147 7129 6097 6231 6957 7145 6956 7143 6130 7149 7142 6212 6827 7144 6089 6161 7055 6233 6105 7057 6237 6828 6850 6163 7124 6354 6162 7146 6830 7123 6160 6235 7056 6081 6841 6132 6954 6131 6236 6831 6845 6832 6953 6839 6950 7125 7054 6138 6166 6076 6851 6353 7076 7148 6858 6842 6860 7126 6829 6835 7078 6866 6869 6871 7052 6134 6855 6947 6862 6215 6852 7128 6092 6112 6213 6232 6863 6113 6165 6214 6216 6873 6865 6870 6077 6234 6861 6164 6217 7127 6218 6962 7053 7051 6961 6002 6738 6739 10105 7064 6751 6770 7100 6014 6765 6755 10021 10022 6010 10056 6011 6756 10055 6768 10024 6023 10003 6769 10002 6767 6759 10018 6024 6064 6012 6050 10042 6168 6253 10010 10020 6015 6018 10033 10041 10039 10031 10016 6764 7083 7152 7066 6171 7150 7085 6255 10044 10008 7102 6167 6240 6238 6095 10017 10046 6019 6031 6763 6256 6169 6254 10034 7108 7186 6257 10019 6757 10040 6025 7019 7086 10029 10011 7104 6261 6013 6766 10106 7105 7153 7089 6057 7134 7151 7045 7005 7008 7101 6035 7137 10023 6036 6172 7099 7087 6239 7185 6170 10006 6243 6350 7103 7090 7157 6259 7171 6875 7084 7154 6242 6260 7155 7017 7048 7156 6959 7047 10053 7135 6244 7136 10030 7063 6760 7016 7065 7179 6881 7018 6876 10081 10052 10054 10038 6886 10069 7004 10051 7007 7109 10057 6029 6888 10009 6889 7021 10047 6245 6878 10067 6879 6884 7180 7182 10071 7002 6880 6890 6887 10061 6027 6877 6892 10060 6893 7050 10036 7049 10012 10025 7012 7183 10058 7181 10086 6891 6258 6894 6883 7046 6037 7106 10043 10048 10045 10087 6885 10013 10028 7187 10037 10035 10050 6895 7011 7170 7172 10026 10063 10095 10082 10084 6960 10092 10075 6038 7010 7015 10015 10027 10064 7184 10014 10059 7013 7020 10072 10066 10080 6896 10083 10090 6039 10049 7164 7165 10091 10099 6963 7166 10079 10103 7006 7009 7169 6034 7028 7029 7030 7034 7035 7036 7040 7041 7042 10032 6009 6241 7003 7014 7088 13326 13330 13331 13352 13353 6198" +slot_whitelist = slots_str.split(" ") + +def calc_ins_weight(params, label): + """calc ins weight""" + global g_ratio + global w_ratio + slots = [] + s_clk_num = 0 + s_show_num = 0 + active = 0 + attclk_num = 0 + attshow_num = 0 + attclk_avg = 0 + for items in params: + if len(items) != 2: + continue + slot_name = items[0] + slot_val = items[1] + if slot_name not in slots: + slots.append(slot_name) + if slot_name == "session_click_num": + s_clk_num = int(slot_val) + if slot_name == "session_show_num": + s_show_num = int(slot_val) + if slot_name == "activity": + active = float(slot_val) / 10000.0 + w = 1 + # for inactive user + if active >= 0 and active < 0.4 and s_show_num >=0 and s_show_num < 20: + w = math.log(w_ratio * (420 - (active * 50 + 1) * (s_show_num + 1)) + math.e) + if label == "0": + w = 1 + (w - 1) * g_ratio + return w + +def filter_whitelist_slot(tmp_line): + terms = tmp_line.split() + line = "%s %s %s" % (terms[0], terms[1], terms[2]) + for item in terms[3:]: + feasign = item.split(':') + if len(feasign) == 2 and \ + feasign[1] in slot_whitelist: + line = "%s %s" %(line, item) + return line + +def get_sample_type(line): + # vertical_type = 20 + # if line.find("13038012583501790:6738") > 0: + # return 30 + # vertical_type = 0/5/1/2/9/11/13/16/29/-1 + if (line.find("7408512894065610:6738") > 0) or \ + (line.find("8815887816424655:6738") > 0) or \ + (line.find("7689987878537419:6738") > 0) or \ + (line.find("7971462863009228:6738") > 0) or \ + (line.find("9941787754311891:6738") > 0) or \ + (line.find("10504737723255509:6738") > 0) or \ + (line.find("11067687692199127:6738") > 0) or \ + (line.find("11912112645614554:6738") > 0) or \ + (line.find("15571287443748071:6738") > 0) or \ + (line.find("7127025017546227:6738") > 0): + return 20 + return -1 + +def main(): + """ins adjust""" + global del_text_slot + for l in sys.stdin: + l = l.rstrip("\n") + items = l.split(" ") + if len(items) < 3: + continue + label = items[2] + lines = l.split("\t") + line = lines[0] + # streaming ins include all ins, sample_type only handle NEWS ins + sample_type = -1 + if 'NEWS' in l: + sample_type = get_sample_type(line) + #line = filter_whitelist_slot(tmp_line) + if len(lines) >= 4: + if 'VIDEO' in lines[3]: + continue + params = lines[2] + params = params.split(" ") + m = [tuple(i.split(":")) for i in params] + if m is None or len(m) == 0: + if sample_type > 0: + print "%s $%s *1" % (line, sample_type) + else: + print "%s *1" % line + sys.stdout.flush() + continue + weight = calc_ins_weight(m, label) + if sample_type > 0: + print "%s $%s *%s" % (line, sample_type, weight) + else: + print "%s *%s" % (line, weight) + sys.stdout.flush() + else: + if sample_type > 0: + print "%s $%s *1" % (line, sample_type) + else: + print "%s *1" % line + sys.stdout.flush() + +if __name__ == "__main__": + if len(sys.argv) > 1: + if sys.argv[1] == "0": + del_text_slot = False + if len(sys.argv) > 2: + g_ratio = float(sys.argv[2]) + if len(sys.argv) > 3: + w_ratio = float(sys.argv[3]) + main() diff --git a/paddle/fluid/train/custom_trainer/feed/tool/xbox_compressor_mf.py b/paddle/fluid/train/custom_trainer/feed/tool/xbox_compressor_mf.py new file mode 100755 index 0000000000000000000000000000000000000000..b306ddfeb183515c7652b2f0d08cbe98f95033b4 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/tool/xbox_compressor_mf.py @@ -0,0 +1,162 @@ +#!/usr/bin/python +""" +xbox model compressor +""" + +import sys +import math +import time +import re + +#WISE +#SHOW_COMPRESS_RATIO : 8192 +#CLICK_COMPRESS_RATIO : 8192 +#LR_COMPRESS_RATIO : 1048576 +#MIO_COMPRESS_RATIO:8192 + +#PC +#MIO_COMPRESS_RATIO : 1024 +#SHOW_COMPRESS_RATIO : 128 +#CLICK_COMPRESS_RATIO : 1024 +#LR_COMPRESS_RATIO : 8192 + +#STAMP_COL = 2 +SHOW_COL = 3 +CLICK_COL = 4 +LR_W_COL = 5 +LR_G2SUM_COL = 6 +FM_COL = 9 + +#DAY_SPAN = 300 + +#show clk lr = float +SHOW_RATIO = 1 +#SHOW_RATIO = 1024 +CLK_RATIO = 8 +#CLK_RATIO = 1024 +LR_RATIO = 1024 +MF_RATIO = 1024 + +base_update_threshold=0.965 +base_xbox_clk_cof=1 +base_xbox_nonclk_cof=0.2 + +def as_num(x): + y='{:.5f}'.format(x) + return(y) + +def compress_show(xx): + """ + compress show + """ + preci = SHOW_RATIO + + x = float(xx) + return str(int(math.floor(x * preci + 0.5))) + + +def compress_clk(xx): + """ + compress clk + """ + preci = CLK_RATIO + + x = float(xx) + clk = int(math.floor(x * preci + 0.5)) + if clk == 0: + return "" + return str(clk) + + +def compress_lr(xx): + """ + compress lr + """ + preci = LR_RATIO + + x = float(xx) + lr = int(math.floor(x * preci + 0.5)) + if lr == 0: + return "" + return str(lr) + +def compress_mf(xx): + """ + compress mf + """ + preci = MF_RATIO + + x = float(xx) + return int(math.floor(x * preci + 0.5)) + + +def show_clk_score(show, clk): + """ + calculate show_clk score + """ + return (show - clk) * 0.2 + clk + + +for l in sys.stdin: + cols = re.split(r'\s+', l.strip()) + key = cols[0].strip() + + #day = int(cols[STAMP_COL].strip()) + #cur_day = int(time.time()/3600/24) + #if (day + DAY_SPAN) <= cur_day: + # continue + + # cvm features + show = cols[SHOW_COL] + click = cols[CLICK_COL] + pred = "" + + f_show = float(show) + f_clk = float(click) + """ + if f_show != 0: + show_log = math.log(f_show) + else: + show_log = 0 + + if f_clk != 0: + click_log = math.log(f_clk) - show_log + else: + click_log = 0 + """ + show_log = f_show + click_log = f_clk + #print f_show, f_clk + #if show_clk_score(f_show, f_clk) < base_update_threshold: + # continue + + #show = compress_show(show) + show = compress_show(show_log) + #clk = compress_clk(click) + clk = compress_clk(click_log) + + # personal lr weight + lr_w = cols[LR_W_COL].strip() + lr_wei = compress_lr(lr_w) + + # fm weight + fm_wei = [] + fm_sum = 0 + if len(cols) > 7: + #fm_dim = int(cols[FM_COL].strip()) + #if fm_dim != 0: + for v in xrange(FM_COL, len(cols), 1): + mf_v = compress_mf(cols[v]) + #print mf_v + fm_wei.append(str(mf_v)) + fm_sum += (mf_v * mf_v) + + sys.stdout.write("%s\t%s\t%s\t%s" % (key, show, clk, pred)) + sys.stdout.write("\t") + sys.stdout.write("%s" % lr_wei) + if len(fm_wei) > 0 and fm_sum > 0: + sys.stdout.write("\t%s" % "\t".join(fm_wei)) + else: + sys.stdout.write("\t[\t]") + sys.stdout.write("\n") + diff --git a/paddle/fluid/train/custom_trainer/feed/tool/xbox_decompressor_mf.awk b/paddle/fluid/train/custom_trainer/feed/tool/xbox_decompressor_mf.awk new file mode 100755 index 0000000000000000000000000000000000000000..61b2f831cf8354dc5ee95487b3bfd0b280ceb998 --- /dev/null +++ b/paddle/fluid/train/custom_trainer/feed/tool/xbox_decompressor_mf.awk @@ -0,0 +1,52 @@ +#!/bin/awk -f +{ + OFS="\t"; + SHOW_RATIO = 1; + CLK_RATIO = 8; + LR_RATIO = 1024; + MF_RATIO = 1024; +} + +function decompress_show(x) { + x = x * 1.0 / SHOW_RATIO; + return x; +} + +function decompress_clk(x) { + if (x == "") { + x = 0; + } + x = x * 1.0 / CLK_RATIO; + return x; +} + +function decompress_lr(x) { + return x * 1.0 / LR_RATIO; +} + +function decompress_mf(x) { + return x * 1.0 / MF_RATIO; +} + +function show_clk_sore(show, clk, nonclk_coeff, clk_coeff) { + return (show - clk) * nonclk_coeff + clk * clk_coeff; +} + +#key, show, clk, pred, lr_w, mf_w or [\t] +{ + l=split($0, a, "\t"); + + show = decompress_show(a[2]); + click = decompress_clk(a[3]); + lr = decompress_lr(a[5]); + printf("%s\t0\t0\t%s\t%s\t%s\t0\t", a[1], show, click, lr); + if (l == 7) { + printf("0\n"); + } else { + printf("%d", l-5) + for(i = 6; i <= l; i++) { + printf("\t%s", decompress_mf(a[i])); + } + printf("\t0\n"); + } +} diff --git a/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_converter b/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_converter new file mode 100755 index 0000000000000000000000000000000000000000..04d925a88cd58ee3719a34b3e51fcacd3c8757da Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_converter differ diff --git a/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_deconverter b/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_deconverter new file mode 100755 index 0000000000000000000000000000000000000000..e0c1a18c4ddce4badfa14f1c68e7055a74aeb158 Binary files /dev/null and b/paddle/fluid/train/custom_trainer/feed/tool/xbox_pb_deconverter differ diff --git a/paddle/fluid/train/custom_trainer/feed/trainer_context.h b/paddle/fluid/train/custom_trainer/feed/trainer_context.h old mode 100644 new mode 100755 index 212f32d370a5399eb3566b9ef1e741854c8de03d..f9c86a45a307791d6700e8bac9759e2184022368 --- a/paddle/fluid/train/custom_trainer/feed/trainer_context.h +++ b/paddle/fluid/train/custom_trainer/feed/trainer_context.h @@ -2,8 +2,8 @@ #include #include #include -#include #include "paddle/fluid/platform/place.h" +#include "paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h" #include "paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h" #include "paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h" @@ -17,35 +17,43 @@ class Dataset; class FileSystem; class EpochAccessor; +const uint32_t SecondsPerMin = 60; +const uint32_t SecondsPerHour = 3600; +const uint32_t SecondsPerDay = 24 * 3600; + enum class ModelSaveWay { ModelSaveTrainCheckpoint = 0, ModelSaveInferenceDelta = 1, ModelSaveInferenceBase = 2 }; -class TableMeta { +class SignCacheDict { public: - TableMeta() {} - ~TableMeta() {} - int table_id() { - return _id; + int32_t sign2index(uint64_t sign) { + return -1; + } + + uint64_t index2sign(int32_t index) { + return 0; } -private: - int _id; }; class TrainerContext { public: -YAML::Node trainer_config; -paddle::platform::CPUPlace cpu_place; - -std::shared_ptr pslib; -std::shared_ptr dataset; //训练样本 -std::shared_ptr file_system; //文件操作辅助类 -std::vector params_table_list; //参数表 -std::shared_ptr epoch_accessor; //训练轮次控制 -std::shared_ptr environment; //运行环境 -std::vector> process_list; //训练流程 + inline paddle::ps::PSClient* ps_client() { + return pslib->ps_client(); + } + + YAML::Node trainer_config; + paddle::platform::CPUPlace cpu_place; + + std::shared_ptr pslib; + std::shared_ptr dataset; //训练样本 + std::shared_ptr file_system; //文件操作辅助类 + std::shared_ptr epoch_accessor; //训练轮次控制 + std::shared_ptr environment; //运行环境 + std::vector> process_list; //训练流程 + std::shared_ptr cache_dict; //大模型cache词典 }; } // namespace feed diff --git a/paddle/fluid/train/custom_trainer/feed/unit_test/test_create_programs.cc b/paddle/fluid/train/custom_trainer/feed/unit_test/test_create_programs.cc index d04e2f5cda143c9bf95b7ed08e01cdd3760027d0..bff27b2ae2e725f1ca20428c244c1f064b16a7d5 100644 --- a/paddle/fluid/train/custom_trainer/feed/unit_test/test_create_programs.cc +++ b/paddle/fluid/train/custom_trainer/feed/unit_test/test_create_programs.cc @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include "paddle/fluid/train/custom_trainer/feed/executor/executor.h" #include "paddle/fluid/framework/tensor_util.h" @@ -51,6 +53,11 @@ public: context_ptr = nullptr; } + float random(float min_x = 0.0, float max_x = 1.0) { + float r = static_cast(rand()) / RAND_MAX; + return min_x + (max_x - min_x) * r; + } + std::shared_ptr context_ptr; }; @@ -61,32 +68,43 @@ TEST_F(CreateProgramsTest, example_network) { auto config = YAML::Load(string::format_string("{thread_num: 2, startup_program: %s, main_program: %s}", startup_program_path, main_program_path)); auto model_desc = YAML::LoadFile(model_desc_path); ASSERT_EQ(0, executor->initialize(config, context_ptr)); - + std::string input_name = "cvm_input"; + std::string loss_name = "loss_ctr"; + std::string label_name = "label_ctr"; + + // loss + ASSERT_TRUE(model_desc["loss"]); + ASSERT_EQ(loss_name, model_desc["loss"].as()); + + // input ASSERT_TRUE(model_desc["inputs"]); ASSERT_EQ(1, model_desc["inputs"].size()); ASSERT_TRUE(model_desc["inputs"][0]["name"]); ASSERT_TRUE(model_desc["inputs"][0]["shape"]); ASSERT_EQ(input_name, model_desc["inputs"][0]["name"].as()); - std::vector input_shape = model_desc["inputs"][0]["shape"].as>(std::vector()); + auto input_shape = model_desc["inputs"][0]["shape"].as>(std::vector()); ASSERT_EQ(2, input_shape.size()); ASSERT_EQ(-1, input_shape[0]); ASSERT_EQ(4488, input_shape[1]); - ASSERT_TRUE(model_desc["loss_all"]); - auto loss_all_name = model_desc["loss_all"].as(); + // label + ASSERT_TRUE(model_desc["labels"]); + ASSERT_EQ(1, model_desc["labels"].size()); + ASSERT_TRUE(model_desc["labels"][0]["name"]); + ASSERT_TRUE(model_desc["labels"][0]["shape"]); + ASSERT_EQ(label_name, model_desc["labels"][0]["name"].as()); + auto label_shape = model_desc["labels"][0]["shape"].as>(std::vector()); + ASSERT_EQ(2, label_shape.size()); + ASSERT_EQ(-1, label_shape[0]); + ASSERT_EQ(1, label_shape[1]); ASSERT_TRUE(model_desc["outputs"]); ASSERT_EQ(1, model_desc["outputs"].size()); ASSERT_TRUE(model_desc["outputs"][0]["name"]); ASSERT_TRUE(model_desc["outputs"][0]["shape"]); - ASSERT_TRUE(model_desc["outputs"][0]["label_name"]); - ASSERT_TRUE(model_desc["outputs"][0]["loss_name"]); - - auto ctr_output_label_name = model_desc["outputs"][0]["label_name"].as(); - auto ctr_output_loss_name = model_desc["outputs"][0]["loss_name"].as(); - auto ctr_output_name = model_desc["outputs"][0]["name"].as(); - std::vector output_shape = model_desc["outputs"][0]["shape"].as>(std::vector()); + auto output_name = model_desc["outputs"][0]["name"].as(); + auto output_shape = model_desc["outputs"][0]["shape"].as>(std::vector()); ASSERT_EQ(2, output_shape.size()); ASSERT_EQ(-1, output_shape[0]); ASSERT_EQ(1, output_shape[1]); @@ -94,7 +112,7 @@ TEST_F(CreateProgramsTest, example_network) { paddle::framework::Scope scope; executor->initialize_scope(&scope); auto input_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, input_name); - auto label_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, ctr_output_label_name); + auto label_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, label_name); ASSERT_NE(nullptr, input_var); ASSERT_NE(nullptr, label_var); @@ -102,28 +120,26 @@ TEST_F(CreateProgramsTest, example_network) { auto input_data = input_var->mutable_data(context_ptr->cpu_place); ASSERT_NE(nullptr, input_data); for (int i = 0; i < input_shape[1]; ++i) { - input_data[i] = 0.1; + input_data[i] = random(); } label_var->Resize({1, 1}); auto label_data = label_var->mutable_data(context_ptr->cpu_place); ASSERT_NE(nullptr, label_data); - label_data[0] = 0.5; + label_data[0] = random(); ASSERT_EQ(0, executor->run(&scope)); - auto loss_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, ctr_output_loss_name); + auto loss_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, loss_name); auto loss = loss_var.data()[0]; - auto loss_all_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, loss_all_name); - auto loss_all = loss_all_var.data()[0]; - - auto ctr_output_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, ctr_output_name); - auto ctr_output = ctr_output_var.data()[0]; + auto output_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, output_name); + auto output = output_var.data()[0]; - std::cout << "loss: " << loss << std::endl; - std::cout << "ctr_output: " << ctr_output << std::endl; - ASSERT_NEAR(loss, loss_all, 1e-9); + LOG(INFO) << "loss: " << loss << std::endl; + LOG(INFO) << "label: " << label_data[0] << std::endl; + LOG(INFO) << "output: " << output << std::endl; + ASSERT_NEAR(loss, pow(output - label_data[0], 2), 1e-8); } } // namespace feed diff --git a/paddle/fluid/train/custom_trainer/feed/unit_test/test_datareader.cc b/paddle/fluid/train/custom_trainer/feed/unit_test/test_datareader.cc index 03b47a1e950bb8912b9a307fcbcde0872302a0e1..9e941326ae36251ea2cbf2a05957ef21b71d3b99 100644 --- a/paddle/fluid/train/custom_trainer/feed/unit_test/test_datareader.cc +++ b/paddle/fluid/train/custom_trainer/feed/unit_test/test_datareader.cc @@ -194,14 +194,14 @@ TEST_F(DataReaderTest, LineDataReader_FileSystem) { "file_system:\n" " class: AutoFileSystem\n" " file_systems:\n" - " 'afs:': &HDFS \n" + " 'afs': &HDFS \n" " class: HadoopFileSystem\n" " hdfs_command: 'hadoop fs'\n" " ugis:\n" " 'default': 'feed_video,D3a0z8'\n" " 'xingtian.afs.baidu.com:9902': 'feed_video,D3a0z8'\n" " \n" - " 'hdfs:': *HDFS\n"); + " 'hdfs': *HDFS\n"); ASSERT_EQ(0, data_reader->initialize(config, context_ptr)); { auto data_file_list = data_reader->data_file_list(test_data_dir);