trainflow v0.1

BCLOUD

 WORKROOT('../../../')
 COMPILER('gcc482')
 CPPFLAGS('-D_GNU_SOURCE -DNDEBUG')
-GLOBAL_CFLAGS_STR = '-g -O0 -pipe -fopenmp '
+GLOBAL_CFLAGS_STR = '-g -O3 -pipe -fopenmp '
 CFLAGS(GLOBAL_CFLAGS_STR)
 GLOBAL_CXXFLAGS_STR = GLOBAL_CFLAGS_STR + ' -std=c++11 '
 CXXFLAGS(GLOBAL_CXXFLAGS_STR)
-
 INCPATHS('./')
 INCPATHS('$OUT/../')
 INCPATHS('../../third-party')
@@ -36,9 +35,8 @@ CONFIGS('baidu/third-party/pybind11@v2.2.4@git_branch')
 CONFIGS('baidu/third-party/python@gcc482output@git_branch')
 CONFIGS('baidu/third-party/yaml-cpp@yaml-cpp_0-6-2-0_GEN_PD_BL@git_tag')
 CONFIGS('baidu/third-party/openmpi@openmpi_1-4-5-0-feed_mlarch@git_branch')
-CONFIGS('baidu/paddlepaddle/pslib@master@git_branch')
+CONFIGS('baidu/paddlepaddle/pslib@no_abacus_in_proto@git_branch')
 CONFIGS('third-64/gtest@base')
-
 HEADERS('paddle/fluid/memory/*.h', '$INC/paddle/fluid/memory/')
 HEADERS('paddle/fluid/memory/detail/*.h', '$INC/paddle/fluid/memory/detail/')
 HEADERS('paddle/fluid/memory/allocation/*.h', '$INC/paddle/fluid/memory/allocation/')
@@ -59,12 +57,9 @@ HEADERS('paddle/fluid/pybind/pybind.h', '$INC/paddle/fluid/pybind')
 HEADERS('paddle/fluid/inference/api/*.h', '$INC/paddle/fluid/inference/api/')
 HEADERS(GLOB_GEN_SRCS('paddle/fluid/framework/*pb.h'), '$INC/paddle/fluid/framework')
 HEADERS(GLOB_GEN_SRCS('paddle/fluid/platform/*pb.h'), '$INC/paddle/fluid/platform')
-
 PROTOC('../../third-party/protobuf/bin/protoc')
-
 #proto
 StaticLibrary("fake_paddle_proto", Sources(GLOB("paddle/fluid/framework/*.proto"), GLOB("paddle/fluid/platform/*.proto")))
-
 #feed
 HEADERS('paddle/fluid/train/custom_trainer/feed/*.h', '$INC/paddle/fluid/train/custom_trainer/feed/')
 HEADERS('paddle/fluid/train/custom_trainer/feed/common/*.h', '$INC/paddle/fluid/train/custom_trainer/feed/common/')
@@ -80,16 +75,15 @@ NEED_OUTPUT("baidu/third-party/openmpi")
 OUTPUT('paddle/fluid/train/custom_trainer/feed/conf', '$OUT')
 OUTPUT('paddle/fluid/train/custom_trainer/feed/scripts', '$OUT')
 
+
 def UT_FILE(filename):
     UT_DIR = 'paddle/fluid/train/custom_trainer/feed/unit_test'
     import os
     return os.path.join(UT_DIR, filename)
-
 custom_trainer_src = GLOB('paddle/fluid/train/custom_trainer/feed/*/*.cc', Exclude(UT_FILE('*')))
-
 CPPFLAGS_STR = '-DHPPL_STUB_FUNC -DLAPACK_FOUND -DPADDLE_DISABLE_PROFILER -DPADDLE_NO_PYTHON -DCUSTOM_TRAINER -DPADDLE_ON_INFERENCE -DPADDLE_USE_DSO -DPADDLE_USE_PTHREAD_BARRIER -DPADDLE_USE_PTHREAD_SPINLOCK -DPADDLE_VERSION=0.0.0 -DPADDLE_WITH_AVX -DPADDLE_WITH_MKLML -DPADDLE_WITH_XBYAK -DXBYAK64 -DXBYAK_NO_OP_NAMES -D_GNU_SOURCE -D__STDC_LIMIT_MACROS -DPYBIND_AVX_MKLML' + r" -DPADDLE_REVISION=\"%s@%s@%s\"" % (REPO_URL(), REPO_BRANCH(), REPO_REVISION())
 
-CFLAGS_STR = '-m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -fopenmp -mavx -O0 -DNDEBUG '
+CFLAGS_STR = '-m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -Wno-narrowing -Wnarrowing -fopenmp -mavx -O3 -DNDEBUG '
 CXXFLAGS_STR = '-std=c++11 ' + CFLAGS_STR
 
 SharedLibrary("paddle_fluid_avx_mklml", PreBuilt(True))

paddle/fluid/framework/archive.h

@@ -518,11 +518,11 @@ Archive<AR>& operator>>(Archive<AR>& ar, std::tuple<T...>& x) {
   }                                                                            \
   template <class AR, class KEY, class VALUE, class... ARGS>                   \
   Archive<AR>& operator>>(Archive<AR>& ar, MAP_TYPE<KEY, VALUE, ARGS...>& p) { \
-    size_t size = ar.template get<size_t>();                                   \
+    size_t size = ar.template Get<size_t>();                                   \
     p.clear();                                                                 \
     RESERVE_STATEMENT;                                                         \
     for (size_t i = 0; i < size; i++) {                                        \
-      p.insert(ar.template get<std::pair<KEY, VALUE>>());                      \
+      p.insert(ar.template Get<std::pair<KEY, VALUE>>());                      \
     }                                                                          \
     return ar;                                                                 \
   }
@@ -539,11 +539,11 @@ Archive<AR>& operator>>(Archive<AR>& ar, std::tuple<T...>& x) {
   }                                                                            \
   template <class AR, class KEY, class VALUE, class... ARGS>                   \
   Archive<AR>& operator>>(Archive<AR>& ar, MAP_TYPE<KEY, VALUE, ARGS...>& p) { \
-    size_t size = ar.template get<uint64_t>();                                 \
+    size_t size = ar.template Get<uint64_t>();                                 \
     p.clear();                                                                 \
     RESERVE_STATEMENT;                                                         \
     for (size_t i = 0; i < size; i++) {                                        \
-      p.insert(ar.template get<std::pair<KEY, VALUE>>());                      \
+      p.insert(ar.template Get<std::pair<KEY, VALUE>>());                      \
     }                                                                          \
     return ar;                                                                 \
   }
@@ -568,11 +568,11 @@ ARCHIVE_REPEAT(std::unordered_multimap, p.reserve(size))
   }                                                                           \
   template <class AR, class KEY, class... ARGS>                               \
   Archive<AR>& operator>>(Archive<AR>& ar, SET_TYPE<KEY, ARGS...>& p) {       \
-    size_t size = ar.template get<size_t>();                                  \
+    size_t size = ar.template Get<size_t>();                                  \
     p.clear();                                                                \
     RESERVE_STATEMENT;                                                        \
     for (size_t i = 0; i < size; i++) {                                       \
-      p.insert(ar.template get<KEY>());                                       \
+      p.insert(ar.template Get<KEY>());                                       \
     }                                                                         \
     return ar;                                                                \
   }
@@ -588,11 +588,11 @@ ARCHIVE_REPEAT(std::unordered_multimap, p.reserve(size))
   }                                                                           \
   template <class AR, class KEY, class... ARGS>                               \
   Archive<AR>& operator>>(Archive<AR>& ar, SET_TYPE<KEY, ARGS...>& p) {       \
-    size_t size = ar.template get<uint64_t>();                                \
+    size_t size = ar.template Get<uint64_t>();                                \
     p.clear();                                                                \
     RESERVE_STATEMENT;                                                        \
     for (size_t i = 0; i < size; i++) {                                       \
-      p.insert(ar.template get<KEY>());                                       \
+      p.insert(ar.template Get<KEY>());                                       \
     }                                                                         \
     return ar;                                                                \
   }

paddle/fluid/string/string_helper.cc

@@ -24,26 +24,6 @@
 namespace paddle {
 namespace string {
 
-inline size_t count_spaces(const char* s) {
-  size_t count = 0;
-
-  while (*s != 0 && isspace(*s++)) {
-    count++;
-  }
-
-  return count;
-}
-
-inline size_t count_nonspaces(const char* s) {
-  size_t count = 0;
-
-  while (*s != 0 && !isspace(*s++)) {
-    count++;
-  }
-
-  return count;
-}
-
 // remove leading and tailing spaces
 std::string trim_spaces(const std::string& str) {
   const char* p = str.c_str();

paddle/fluid/string/string_helper.h

@@ -26,9 +26,25 @@
 namespace paddle {
 namespace string {
 
-inline size_t count_spaces(const char* s);
+inline size_t count_spaces(const char* s) {
+  size_t count = 0;
 
-inline size_t count_nonspaces(const char* s);
+  while (*s != 0 && isspace(*s++)) {
+    count++;
+  }
+
+  return count;
+}
+
+inline size_t count_nonspaces(const char* s) {
+  size_t count = 0;
+
+  while (*s != 0 && !isspace(*s++)) {
+    count++;
+  }
+
+  return count;
+}
 
 template <class... ARGS>
 void format_string_append(std::string& str, const char* fmt,  // NOLINT

paddle/fluid/string/to_string.h

@@ -31,6 +31,15 @@ inline std::string to_string(T v) {
   return sout.str();
 }
 
+template <typename T>
+inline std::string to_string(const std::vector<T>& v_list) {
+    std::ostringstream sout;
+    for (const auto& v : v_list) {
+        sout << v << " ";
+    }
+    return sout.str();
+}
+
 template <>
 inline std::string to_string(std::type_index t) {
   return t.name();

paddle/fluid/train/custom_trainer/feed/accessor/dense_input_accessor.cc

@@ -7,14 +7,13 @@ namespace feed {
 int DenseInputAccessor::initialize(YAML::Node config,
         std::shared_ptr<TrainerContext> context_ptr) {
     CHECK(DataInputAccessor::initialize(config, context_ptr) == 0);
-    CHECK(config["input"] && config["input"].Type() == YAML::NodeType::Map);
     _total_dim = 0;
     _pull_request_num.store(0);
     for (const auto& input : config["input"]) {
         DenseInputVariable variable;
-        variable.name = input.first.as<std::string>();
+        variable.name = input["name"].as<std::string>();
         variable.gradient_name = paddle::framework::GradVarName(variable.name);
-        variable.shape = input.second["shape"].as<std::vector<int>>();
+        variable.shape = input["shape"].as<std::vector<int>>();
         variable.dim = 1;
         for (int i = 0; i < variable.shape.size(); ++i) {
             if (variable.shape[i] <= 0) {
@@ -23,6 +22,7 @@ int DenseInputAccessor::initialize(YAML::Node config,
             variable.dim *= variable.shape[i];    
         }
         _total_dim += variable.dim;
+        _x_variables.emplace_back(variable);
     }
     if (config["async_pull"] && config["async_pull"].as<bool>()) {
         _need_async_pull = true;
@@ -30,14 +30,28 @@ int DenseInputAccessor::initialize(YAML::Node config,
     return 0;
 }
 
+int32_t DenseInputAccessor::create(::paddle::framework::Scope* scope) {
+    size_t data_buffer_idx = 0;
+    std::vector<paddle::ps::Region> regions;
+    for (auto& variable : _x_variables) {
+        auto* tensor = scope->Var(variable.name)->
+            GetMutable<paddle::framework::LoDTensor>(); 
+        auto* data = tensor->data<float>();
+        regions.emplace_back(data, variable.dim);
+    }
+    auto* ps_client = _trainer_context->pslib->ps_client();
+    auto push_status = ps_client->push_dense_param(regions.data(), regions.size(), _table_id);
+    return push_status.get();
+}
+
 // rpc拉取数据，需保证单线程运行
 int32_t DenseInputAccessor::pull_dense(size_t table_id) {
     float* data_buffer = NULL;
-    if (_data_buffer != nullptr) {
-        data_buffer = _data_buffer;
-    } else {
-        data_buffer = new float[_total_dim];
+    if (_data_buffer == nullptr) {
+        _data_buffer = new float[_total_dim];
     }
+    // TODO 使用双buffer DataBuffer,避免训练期改写，当前异步SGD下，问题不大
+    data_buffer = _data_buffer;
     
     size_t data_buffer_idx = 0;
     std::vector<paddle::ps::Region> regions;
@@ -77,6 +91,8 @@ int32_t DenseInputAccessor::forward(SampleInstance* samples, size_t num,
         auto* shape_ptr = &(variable.shape[0]);
         paddle::framework::DDim ddim(shape_ptr, variable.shape.size());
         auto* tensor = ScopeHelper::resize_lod_tensor(scope, variable.name, ddim);  
+        auto* grad_tensor = ScopeHelper::resize_lod_tensor(scope, variable.gradient_name, ddim);
+        VLOG(5) << "fill scope variable:" << variable.name << ", " << variable.gradient_name;
         auto* var_data = tensor->mutable_data<float>(_trainer_context->cpu_place);
         memcpy(var_data, _data_buffer + data_buffer_idx, variable.dim * sizeof(float));
         data_buffer_idx += variable.dim;
@@ -102,7 +118,8 @@ int32_t DenseInputAccessor::backward(SampleInstance* samples, size_t num,
     }
     auto* ps_client = _trainer_context->pslib->ps_client();
     auto push_status = ps_client->push_dense(regions.data(), regions.size(), _table_id);
-    return push_status.get();
+    //return push_status.get();
+    return 0;
 }
 
 int32_t EbdVariableInputAccessor::forward(SampleInstance* samples, size_t num,

paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.cc

@@ -7,88 +7,149 @@ namespace custom_trainer {
 namespace feed {
     int EpochAccessor::initialize(YAML::Node config,
         std::shared_ptr<TrainerContext> context_ptr) {
-        _model_root_path =  config["model_root_path"].as<std::string>() + "/";
-        
-        _done_file_path = _model_root_path;
+        _model_root_path =  config["model_root_path"].as<std::string>();
+
+        _trainer_context = context_ptr.get();
+        if (context_ptr->file_system == nullptr) {
+            VLOG(0) << "file_system is not initialized";
+            return -1;
+        }
+        auto fs = _trainer_context->file_system.get();
         if (config["donefile"]) {
-            _done_file_path.append(config["donefile"].as<std::string>());
+            _done_file_path = fs->path_join(_model_root_path, config["donefile"].as<std::string>());
         } else {
-            _done_file_path.append("epoch_donefile.txt");
+            _done_file_path = fs->path_join(_model_root_path, "epoch_donefile.txt");
         }
         
-        if (!context_ptr->file_system->exists(_done_file_path)) {
+        if (!fs->exists(_done_file_path)) {
             VLOG(0) << "missing done file, path:" << _done_file_path;
+            return -1;
         }
 
-        std::string done_text = context_ptr->file_system->tail(_done_file_path);
+        std::string done_text = fs->tail(_done_file_path);
         _done_status = paddle::string::split_string(done_text, std::string("\t"));
         _current_epoch_id = get_status<uint64_t>(EpochStatusFiled::EpochIdField);
         _last_checkpoint_epoch_id = get_status<uint64_t>(EpochStatusFiled::CheckpointIdField);
         _last_checkpoint_path = get_status<std::string>(EpochStatusFiled::CheckpointPathField);
         return 0;
     }
+    
+    int32_t EpochAccessor::epoch_done(uint64_t epoch_id) {
+        struct timeval now; 
+        gettimeofday(&now, NULL); 
+        if (need_save_model(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint)) {
+            _last_checkpoint_epoch_id = epoch_id;
+            _last_checkpoint_path = model_save_path(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint);
+        }
+        set_status(EpochStatusFiled::EpochIdField, epoch_id);
+        set_status(EpochStatusFiled::TimestampField, now.tv_sec);
+        set_status(EpochStatusFiled::CheckpointIdField, _last_checkpoint_epoch_id);
+        set_status(EpochStatusFiled::CheckpointPathField, _last_checkpoint_path);
+        set_status(EpochStatusFiled::DateField, format_timestamp(epoch_id, "%Y%m%d"));
 
-    int HourlyEpochAccessor::initialize(YAML::Node config,
-        std::shared_ptr<TrainerContext> context_ptr) {
-        EpochAccessor::initialize(config, context_ptr); 
+        // 非主节点不做状态持久化
+        if (!_trainer_context->environment->is_master_node(EnvironmentRole::WORKER)) {
+            return 0;
+        }
+        auto fs = _trainer_context->file_system.get();
+        std::string done_str = paddle::string::join_strings(_done_status, '\t');
+        // 保留末尾1000数据
+        std::string tail_done_info = paddle::string::trim_spaces(fs->tail(_done_file_path, 1000)); 
+        if (tail_done_info.size() > 0) {
+            tail_done_info = tail_done_info + "\n" + done_str;
+        } else {
+            tail_done_info = done_str;
+        }
+        VLOG(2) << "Write epoch donefile to " << _done_file_path << ", str:" << done_str;
+        bool write_success = false;
+        while (true) {
+            fs->remove(_done_file_path);
+            auto fp = fs->open_write(_done_file_path, "");
+            if (fwrite(tail_done_info.c_str(), tail_done_info.length(), 1, &*fp) == 1) {
+                break;
+            }     
+            sleep(10);   
+        }
+        VLOG(2) << "Write epoch donefile success";
         return 0;
     }
 
-    void HourlyEpochAccessor::next_epoch() {
+    int TimelyEpochAccessor::initialize(YAML::Node config,
+        std::shared_ptr<TrainerContext> context_ptr) {
+        _time_zone_seconds = config["time_zone_seconds"].as<int>();
+        _train_time_interval = config["train_time_interval"].as<int>();
+        CHECK(_train_time_interval > 0 && (_train_time_interval % SecondsPerMin) == 0);
+        _train_num_per_day = SecondsPerDay / _train_time_interval;
+        return EpochAccessor::initialize(config, context_ptr); 
+    }
+
+    void TimelyEpochAccessor::next_epoch() {
         _current_epoch_id = next_epoch_id(_current_epoch_id);
     }
 
-    std::string HourlyEpochAccessor::text(uint64_t epoch_id) {
-        return format_timestamp(epoch_id, "%Y%m%d delta-%H");
+    std::string TimelyEpochAccessor::text(uint64_t epoch_id) {
+        auto delta = delta_id(epoch_id);
+        std::string date = format_timestamp(epoch_id, "%Y%m%d%H%M");
+        return string::format_string("%s delta-%d", date.c_str(), delta);
     }
 
-    uint64_t HourlyEpochAccessor::next_epoch_id(uint64_t epoch_id) {
+    uint64_t TimelyEpochAccessor::next_epoch_id(uint64_t epoch_id) {
         if (epoch_id == 0) {
             struct timeval now; 
             gettimeofday(&now, NULL); 
-            return now.tv_sec / (24 * 3600) * (24 * 3600);
+            // 归整到零点
+            return now.tv_sec / SecondsPerDay * SecondsPerDay;
         } 
-        return epoch_id + 3600;
+        return epoch_id + _train_time_interval;
     }
 
-    bool HourlyEpochAccessor::is_last_epoch(uint64_t epoch_id) {
-        return ((epoch_id / 3600) % 24) == 23;
+    bool TimelyEpochAccessor::is_last_epoch(uint64_t epoch_id) {
+        auto delta = delta_id(epoch_id);
+        return delta == _train_num_per_day;
     }
  
-    uint64_t HourlyEpochAccessor::epoch_time_interval() {
-        return 3600;
+    uint64_t TimelyEpochAccessor::epoch_time_interval() {
+        return _train_time_interval;
     }
 
-    uint64_t HourlyEpochAccessor::epoch_timestamp(uint64_t epoch_id) {
+    uint64_t TimelyEpochAccessor::epoch_timestamp(uint64_t epoch_id) {
         return epoch_id;
     }
  
-    bool HourlyEpochAccessor::need_save_model(uint64_t epoch_id, ModelSaveWay save_way) {
+    bool TimelyEpochAccessor::need_save_model(uint64_t epoch_id, ModelSaveWay save_way) {
         if (epoch_id == 0) {
             return false;
         }
-        if (save_way == ModelSaveWay::ModelSaveInferenceDelta) {
-            return true;
-        } else if (save_way == ModelSaveWay::ModelSaveInferenceBase) {
-            return is_last_epoch(epoch_id);
-        } else if (save_way == ModelSaveWay::ModelSaveTrainCheckpoint) {
-            return ((epoch_id / 3600) % 8) == 0;
+        switch (save_way) {
+            case ModelSaveWay::ModelSaveInferenceDelta:
+                return true;
+            case ModelSaveWay::ModelSaveInferenceBase:
+                return is_last_epoch(epoch_id);
+            case ModelSaveWay::ModelSaveTrainCheckpoint:
+                return ((epoch_id / SecondsPerHour) % 8) == 0;
         }
         return false;
     }
 
-    std::string HourlyEpochAccessor::model_save_path(uint64_t epoch_id, ModelSaveWay save_way) {
-        if (save_way == ModelSaveWay::ModelSaveInferenceDelta) {
-            return _model_root_path + "/xbox/delta-" + std::to_string(epoch_id);
-        } else if (save_way == ModelSaveWay::ModelSaveInferenceBase) {
-            return _model_root_path + "/xbox/base";
-        } else if (save_way == ModelSaveWay::ModelSaveTrainCheckpoint) {
-            return _model_root_path + "/xbox/checkpoint";
+    std::string TimelyEpochAccessor::model_save_path(uint64_t epoch_id, ModelSaveWay save_way) {
+        int32_t delta = delta_id(epoch_id);
+        std::string date = format_timestamp(epoch_id, "%Y%m%d");
+        std::string date_with_hour = format_timestamp(epoch_id, "%Y%m%d%H");
+        switch (save_way) {
+            case ModelSaveWay::ModelSaveInferenceDelta:
+                return _trainer_context->file_system->path_join(_model_root_path, 
+                    string::format_string("xbox/%s/delta-%d", date.c_str(), delta));
+            case ModelSaveWay::ModelSaveInferenceBase:
+                return _trainer_context->file_system->path_join(_model_root_path, 
+                    string::format_string("xbox/%s/base", date.c_str()));
+            case ModelSaveWay::ModelSaveTrainCheckpoint:
+                return _trainer_context->file_system->path_join(_model_root_path, 
+                    string::format_string("batch_model/%s", date_with_hour.c_str()));
         }
         return "";
     }
 
-    REGIST_CLASS(EpochAccessor, HourlyEpochAccessor);
+    REGIST_CLASS(EpochAccessor, TimelyEpochAccessor);
 
 }  // namespace feed
 }  // namespace custom_trainer

paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h

@@ -31,27 +31,39 @@ public:
     virtual const std::string& checkpoint_path() {
         return _last_checkpoint_path;
     }
-    
+
+    virtual int32_t epoch_done(uint64_t epoch_id);
+
     template <class T>
     T get_status(EpochStatusFiled field) {
         auto status = paddle::string::trim_spaces(_done_status[static_cast<int>(field)]);
         return boost::lexical_cast<T>(status.c_str());
     }
-    
-    virtual void next_epoch()                     = 0;
+    template <class T>
+    void set_status(EpochStatusFiled field, const T& status) {
+        auto str_status = paddle::string::to_string(status);
+        _done_status[static_cast<int>(field)] = str_status;
+        return;
+    }
     virtual std::string model_root_path() {
         return _model_root_path;
     }
+
+    virtual void next_epoch()                     = 0;
+    
     virtual std::string text(uint64_t epoch_id)   = 0;
     virtual uint64_t next_epoch_id(uint64_t epoch_id)  = 0;
     virtual bool is_last_epoch(uint64_t epoch_id) = 0; 
+
     //epoch间的数据时间间隔（秒）
     virtual uint64_t epoch_time_interval() = 0;
     //获取epoch的样本数据时间
     virtual uint64_t epoch_timestamp(uint64_t epoch_id) = 0; 
+
     virtual bool need_save_model(uint64_t epoch_id, ModelSaveWay save_way) = 0;
     virtual std::string model_save_path(uint64_t epoch_id, ModelSaveWay save_way) = 0;
 protected:
+    TrainerContext* _trainer_context;
     std::string _done_file_path;
     std::string _model_root_path;
     uint64_t _current_epoch_id = 0;
@@ -62,10 +74,10 @@ protected:
 };
 REGIST_REGISTERER(EpochAccessor);
 
-class HourlyEpochAccessor : public EpochAccessor {
+class TimelyEpochAccessor : public EpochAccessor {
 public:
-    HourlyEpochAccessor() {}
-    virtual ~HourlyEpochAccessor() {}
+    TimelyEpochAccessor() {}
+    virtual ~TimelyEpochAccessor() {}
     virtual int initialize(YAML::Node config,
         std::shared_ptr<TrainerContext> context_ptr);
     virtual void next_epoch();
@@ -76,6 +88,14 @@ public:
     virtual uint64_t epoch_timestamp(uint64_t epoch_id); 
     virtual bool need_save_model(uint64_t epoch_id, ModelSaveWay save_way);
     virtual std::string model_save_path(uint64_t epoch_id, ModelSaveWay save_way);
+
+private:
+    inline size_t delta_id(uint64_t epoch_id) {
+        return ((epoch_id + _time_zone_seconds) % SecondsPerDay) / _train_time_interval; 
+    } 
+    uint32_t _time_zone_seconds;   // 相对UTC时差(秒)
+    uint32_t _train_time_interval; // 训练时间间隔(秒)
+    uint32_t _train_num_per_day;   // 天级训练总轮数
 };
 
 }  // namespace feed

paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h

@@ -22,6 +22,11 @@ public:
         return 0;
     }
 
+    // 创建，一般用于模型冷启的随机初始化
+    virtual int32_t create(::paddle::framework::Scope* scope) {
+        return 0;
+    }
+
     // 前向， 一般用于填充输入，在训练网络执行前调用
     virtual int32_t forward(SampleInstance* samples, size_t num,
         ::paddle::framework::Scope* scope) = 0;
@@ -30,12 +35,34 @@ public:
     virtual int32_t backward(SampleInstance* samples, size_t num,
         ::paddle::framework::Scope* scope) = 0;
 protected:
-    size_t _table_id;
+    size_t _table_id = 0;
     bool _need_gradient = false;
     TrainerContext* _trainer_context = nullptr;
 };
 REGIST_REGISTERER(DataInputAccessor);
 
+struct LabelInputVariable {
+    std::string label_name;
+    std::string output_name;
+    size_t label_dim = 0;
+};
+class LabelInputAccessor : public DataInputAccessor {
+public:
+    LabelInputAccessor() {}
+    virtual ~LabelInputAccessor() {}
+    
+    virtual int initialize(YAML::Node config,
+         std::shared_ptr<TrainerContext> context_ptr);
+
+    virtual int32_t forward(SampleInstance* samples, size_t num,
+        ::paddle::framework::Scope* scope);
+
+    virtual int32_t backward(SampleInstance* samples, size_t num,
+        ::paddle::framework::Scope* scope);
+protected:
+    size_t _label_total_dim = 0; 
+    std::vector<LabelInputVariable> _labels;
+};
 
 struct SparseInputVariable {
     size_t slot_dim;
@@ -104,6 +131,8 @@ public:
     
     virtual int initialize(YAML::Node config,
         std::shared_ptr<TrainerContext> context_ptr);
+    
+    virtual int32_t create(::paddle::framework::Scope* scope);
 
     virtual int32_t forward(SampleInstance* samples, size_t num,
         paddle::framework::Scope* scope);

paddle/fluid/train/custom_trainer/feed/accessor/label_input_accessor.cc

+#include "paddle/fluid/train/custom_trainer/feed/accessor/input_data_accessor.h"
+
+namespace paddle {
+namespace custom_trainer {
+namespace feed {
+    
+int LabelInputAccessor::initialize(YAML::Node config,
+        std::shared_ptr<TrainerContext> context_ptr) {
+    _trainer_context = context_ptr.get();
+    _label_total_dim = 0;
+    for (const auto& input : config["input"]) {
+        LabelInputVariable variable;
+        variable.label_name = input["label_name"].as<std::string>();
+        variable.output_name = input["output_name"].as<std::string>();
+        auto shape = input["shape"].as<std::vector<int>>();
+        variable.label_dim = 0;
+        for (auto dim : shape) {
+            variable.label_dim += (dim > 0 ? dim : 0);
+        }
+        _label_total_dim += variable.label_dim;
+        _labels.emplace_back(variable);
+    }
+    return 0;
+}
+
+int32_t LabelInputAccessor::forward(SampleInstance* samples, size_t num,
+    paddle::framework::Scope* scope) {
+    if (num < 1) {
+        return 0;
+    }
+    size_t sample_label_data_idx = 0;
+    for (auto& label : _labels) {
+        auto* tensor = ScopeHelper::resize_lod_tensor(scope, label.label_name, {num, label.label_dim}); 
+        auto* res_tens = ScopeHelper::resize_lod_tensor(scope, label.output_name, {num, label.label_dim}); 
+        auto* var_data = tensor->mutable_data<float>(_trainer_context->cpu_place);        
+        for (size_t i = 0; i < num; ++i) {
+            auto& sample = samples[i];
+            CHECK(sample.labels.size() > sample_label_data_idx);
+            float* sample_label_buffer = sample.labels.data();
+            memcpy(var_data + i * label.label_dim, 
+                sample_label_buffer + sample_label_data_idx, label.label_dim * sizeof(float));
+        }
+        sample_label_data_idx += label.label_dim;  
+    }
+    return 0;
+}
+
+int32_t LabelInputAccessor::backward(SampleInstance* samples, size_t num,
+        paddle::framework::Scope* scope) {
+    if (num < 1) {
+        return 0;
+    }
+    for (size_t i = 0; i < num; ++i) {
+        auto& sample = samples[i];
+        sample.predicts.resize(_label_total_dim);
+        size_t sample_predict_data_idx = 0;
+        float* sample_predict_buffer = sample.predicts.data();
+        for (auto& label : _labels) {
+            auto* tensor = scope->Var(label.output_name)->
+                GetMutable<paddle::framework::LoDTensor>(); 
+            auto* var_data = tensor->mutable_data<float>(_trainer_context->cpu_place);        
+            memcpy(sample_predict_buffer + sample_predict_data_idx, 
+                var_data + i * label.label_dim, label.label_dim * sizeof(float));
+            sample_predict_data_idx += label.label_dim;  
+        }
+    }
+    return 0;
+}
+
+REGIST_CLASS(DataInputAccessor, LabelInputAccessor);
+
+}  // namespace feed
+}  // namespace custom_trainer
+}  // namespace paddle

paddle/fluid/train/custom_trainer/feed/accessor/sparse_input_accessor.cc

@@ -12,20 +12,18 @@ namespace feed {
 int BaseSparseInputAccessor::initialize(YAML::Node config,
     std::shared_ptr<TrainerContext> context_ptr) {
     CHECK(DataInputAccessor::initialize(config, context_ptr) == 0);
-    CHECK(config["input"] && config["input"].Type() == YAML::NodeType::Map);
     for (const auto& input : config["input"]) {
         SparseInputVariable variable;
-        variable.name = input.first.as<std::string>();
+        variable.name = input["name"].as<std::string>();
         variable.gradient_name = paddle::framework::GradVarName(variable.name);
-        std::string slots_str = input.second["slots"].as<std::string>();
-        std::vector<std::string> slots = paddle::string::split_string(slots_str, ","); 
+        auto slots = input["slots"].as<std::vector<int>>();
         variable.slot_idx.resize(UINT16_MAX, -1);
         for (int i = 0; i < slots.size(); ++i) {
-            auto slot = (uint16_t)atoi(slots[i].c_str());
+            uint16_t slot = (uint16_t)slots[i];
             variable.slot_idx[slot] = i;
             variable.slot_list.push_back(slot);
         }
-        variable.slot_dim = input.second["slot_dim"].as<int>();
+        variable.slot_dim = input["slot_dim"].as<int>();
         variable.total_dim = variable.slot_list.size() * variable.slot_dim; 
         _x_variables.push_back(variable);
     }
@@ -52,7 +50,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples,
         auto& features = samples[i].features;
         for (auto& feature_item : features) {
             feature_item.weights.resize(pull_value_dim, 0.0);
-            keys[key_idx] = feature_item.feature_sign;
+            keys[key_idx] = feature_item.sign();
             pull_values[key_idx++] = &(feature_item.weights[0]);
         }
     }
@@ -60,6 +58,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples,
     auto ret = pull_status.get();
     delete[] pull_values;
     if (ret != 0) {
+        VLOG(0) << "pull sparse failed, table_id:" << _table_id << ", key_num:" << key_num << ", ret:" << ret;
         return ret;
     }
 
@@ -78,6 +77,7 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples,
             scope, variable.name, {num, variable.total_dim});
         auto* grad_tensor = ScopeHelper::resize_lod_tensor(
             scope, variable.gradient_name, {num, variable.total_dim});
+        VLOG(5) << "fill scope variable:" << variable.name << ", " << variable.gradient_name;
         var_runtime_data[i].variable_data = tensor->mutable_data<float>(_trainer_context->cpu_place);
         var_runtime_data[i].gradient_data = grad_tensor->mutable_data<float>(_trainer_context->cpu_place);
         memset((void*) var_runtime_data[i].variable_data, 0, var_runtime_data[i].total_size * sizeof(float)); 
@@ -89,12 +89,12 @@ int32_t BaseSparseInputAccessor::forward(SampleInstance* samples,
         for (auto& feature_item : features) {
             for (size_t i = 0; i < _x_variables.size(); ++i) {
                 auto& variable = _x_variables[i];
-                auto slot_idx = variable.slot_idx[feature_item.slot_id]; 
+                auto slot_idx = variable.slot_idx[feature_item.slot()]; 
                 if (slot_idx < 0) {
                     continue;
                 }
                 float* item_data =  var_runtime_data[i].variable_data +  
-                samp_idx * var_runtime_data[i].total_size + variable.slot_dim * slot_idx; 
+                samp_idx * variable.total_dim + variable.slot_dim * slot_idx; 
                 fill_input(item_data, &(feature_item.weights[0]), *value_accessor, variable, samples[samp_idx]);
             }
         }
@@ -138,24 +138,24 @@ int32_t BaseSparseInputAccessor::backward(SampleInstance* samples,
             feature_item.gradients.resize(push_value_dim, 0.0);
             for (size_t i = 0; i < _x_variables.size(); ++i) {
                 auto& variable = _x_variables[i];
-                auto slot_idx = variable.slot_idx[feature_item.slot_id]; 
+                auto slot_idx = variable.slot_idx[feature_item.slot()]; 
                 if (slot_idx < 0) {
                     continue;
                 }
                 const float* grad_data = var_runtime_data[i].gradient_data +  
-                    samp_idx * var_runtime_data[i].total_size + variable.slot_dim * slot_idx; 
+                    samp_idx * variable.total_dim + variable.slot_dim * slot_idx; 
                 fill_gradient(&(feature_item.gradients[0]), grad_data, 
                     *value_accessor, variable, samples[samp_idx]);
-                keys[key_idx] = feature_item.feature_sign;
+                keys[key_idx] = feature_item.sign();
                 push_values[key_idx++] = &(feature_item.gradients[0]);
             }
         }
     }
     auto push_status = ps_client->push_sparse(_table_id, 
-        keys.data(), (const float**)push_values, key_num);
-    auto ret = push_status.get();
+        keys.data(), (const float**)push_values, key_idx);
+    //auto ret = push_status.get();
     delete[] push_values;
-    return ret;
+    return 0;
 } 
 
 class AbacusSparseJoinAccessor : public BaseSparseInputAccessor {

paddle/fluid/train/custom_trainer/feed/common/pipeline.h

@@ -59,6 +59,10 @@ public:
         _input_data_buffer.resize(input_batch_size * options.buffer_batch_count);
         _output_data_buffer.resize(options.batch_size * options.buffer_batch_count);
         _output_channel->SetCapacity(_output_data_buffer.size());
+        if (_options.need_hold_input_data) {
+            _input_channel_backup = ::paddle::framework::MakeChannel<TypeIn>();
+            _input_channel_backup->SetBlockSize(input_batch_size);
+        }
         CHECK(_input_channel != nullptr) << " Input Channel is null";
         _convert_thread = std::make_shared<std::thread>([this](){
             async_convert_data();
@@ -114,12 +118,16 @@ private:
                 CHECK(_converter(&_input_data_buffer[0], read_size,
                     &_output_data_buffer[0], &write_size, 0) == 0) << "Data Converter Do Failed";
                 _output_channel->WriteMove(write_size, &_output_data_buffer[0]);
-                if (_options.need_hold_input_data) {
+                if (_input_channel_backup) {
                     _input_channel_backup->WriteMove(read_size, &_input_data_buffer[0]);
                 }
             }  
             sleep(1);
         }
+        _output_channel->Close();
+        if (_input_channel_backup) {
+            _input_channel_backup->Close();
+        }
     }    
    
     

paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.cc

@@ -12,7 +12,8 @@ namespace custom_trainer {
 namespace feed {
 
 int PSlib::initialize(const std::string& conf_path, 
-    RuntimeEnvironment* environment, EnvironmentRole role) {
+    RuntimeEnvironment* environment) {
+    _environment = environment;
     init_gflag();    
     int file_descriptor = open(conf_path.c_str(), O_RDONLY);
     if (file_descriptor == -1){
@@ -25,26 +26,26 @@ int PSlib::initialize(const std::string& conf_path,
         return -1;
     }
     close(file_descriptor); 
-    init_server(role);
-    init_client(EnvironmentRole::ALL);
+    init_server();
+    init_client();
     return 0;
 }
         
-int PSlib::init_server(EnvironmentRole role) {
-    if (role == EnvironmentRole::PSERVER) {
+int PSlib::init_server() {
+    if (_environment->is_role(EnvironmentRole::PSERVER)) {
         _server_ptr.reset(paddle::ps::PSServerFactory::create(_ps_param));
         _server_ptr->configure(_ps_param, *(_environment->ps_environment()), 
-            _environment->rank_id(role));
+            _environment->rank_id(EnvironmentRole::PSERVER));
         _server_ptr->start(); 
     }
     _environment->ps_environment()->gather_ps_servers();
     return 0;
 }
 
-int PSlib::init_client(EnvironmentRole role) {
+int PSlib::init_client() {
     _client_ptr.reset(paddle::ps::PSClientFactory::create(_ps_param));
     _client_ptr->configure(_ps_param, *(_environment->ps_environment()), 
-        _environment->rank_id(role));
+        _environment->rank_id(EnvironmentRole::ALL));
     return 0;
 }
 

paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h

@@ -14,6 +14,9 @@ limitations under the License. */
 
 #pragma once
 
+// Hide BLOG
+#define BUTIL_LOGGING_H_
+#define COMPACT_GOOGLE_LOG_NOTICE COMPACT_GOOGLE_LOG_INFO
 #include "communicate/ps_server.h"
 #include "communicate/ps_client.h"
 
@@ -22,21 +25,20 @@ namespace custom_trainer {
 namespace feed {
     
 class RuntimeEnvironment;
-enum class EnvironmentRole;
 class PSlib {
 public:
     PSlib() {}
     virtual ~PSlib() {}
     int initialize(const std::string& conf_path, 
-        RuntimeEnvironment* environment, EnvironmentRole role);
+        RuntimeEnvironment* environment);
         
     virtual paddle::ps::PSServer* ps_server();
     virtual paddle::ps::PSClient* ps_client();
     virtual paddle::PSParameter* get_param();
 private:
     void init_gflag();
-    virtual int init_server(EnvironmentRole role);
-    virtual int init_client(EnvironmentRole role);
+    virtual int init_server();
+    virtual int init_client();
 
     paddle::PSParameter _ps_param;
     RuntimeEnvironment* _environment;

paddle/fluid/train/custom_trainer/feed/common/runtime_environment.cc

@@ -41,7 +41,7 @@ public:
             return -1;
         }
         _roles_node_info.resize(static_cast<int>(EnvironmentRole::ALL) + 1);
-        set_role(EnvironmentRole::ALL);
+        add_role(EnvironmentRole::ALL);
         return 0;
     }
     
@@ -56,7 +56,7 @@ public:
     virtual uint32_t node_num(EnvironmentRole role) {
         return mpi_node_info(role).node_num;
     }
-    virtual int set_role(EnvironmentRole role) {
+    virtual int add_role(EnvironmentRole role) {
         auto& node_info = mpi_node_info(role);
         if (node_info.rank_id < 0) {
             if (role == EnvironmentRole::ALL) {
@@ -68,8 +68,12 @@ public:
             MPI_Comm_rank(node_info.mpi_comm, &(node_info.rank_id));
             MPI_Comm_size(node_info.mpi_comm, &(node_info.node_num));
         }
+        _role_set.insert(role);
         return 0;
     }
+    virtual bool is_role(EnvironmentRole role) {
+        return _role_set.count(role) > 0;
+    }
 
     virtual void barrier(EnvironmentRole role) {
         MPI_Barrier(mpi_node_info(role).mpi_comm);
@@ -98,6 +102,7 @@ protected:
     }
 
 private:
+    std::set<EnvironmentRole> _role_set;
     std::vector<MpiNodeInfo> _roles_node_info;
 };
 REGIST_CLASS(RuntimeEnvironment, MPIRuntimeEnvironment);
@@ -123,9 +128,12 @@ public:
     virtual uint32_t node_num(EnvironmentRole role) {
         return 1;
     }
-    virtual int set_role(EnvironmentRole role) {
+    virtual int add_role(EnvironmentRole role) {
         return 0;
     }
+    virtual bool is_role(EnvironmentRole role) {
+        return true;
+    }
     virtual void barrier(EnvironmentRole role) {
         return;
     }

paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h

@@ -42,7 +42,9 @@ public:
     // 配置初始化
     virtual int initialize(YAML::Node config) = 0;
     // 设置role
-    virtual int set_role(EnvironmentRole role) = 0;
+    virtual int add_role(EnvironmentRole role) = 0;
+    // 判断role
+    virtual bool is_role(EnvironmentRole role) = 0;
     // 环境初始化，会在所有依赖模块initialize后调用
     virtual int wireup() = 0;
     

paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h

+#pragma once
+#include <glog/logging.h>
+#include <yaml-cpp/yaml.h>
+
+namespace paddle {
+namespace custom_trainer {
+namespace feed {
+    
+class YamlHelper {
+public:
+    // 直接使用node["key"]判断，会导致node数据被加入key键
+    static bool has_key(const YAML::Node& node, const std::string& key) {
+        CHECK(node.Type() == YAML::NodeType::Map);
+        for (const auto& itr : node) {
+            if (key == itr.first.as<std::string>()) {
+                return true;
+            }
+        } 
+        return false;
+    }
+    template <class T>
+    static T get_with_default(YAML::Node node, const std::string& key, const T& default_v) {
+        if (has_key(node, key)) {
+            return node[key].as<T>();
+        }
+        return default_v;
+    }
+};
+
+}  // namespace feed
+}  // namespace custom_trainer
+}  // namespace paddle

paddle/fluid/train/custom_trainer/feed/conf/gflags.conf

--v=10
+-log_dir=log
+-v=4
+-logbufsecs=0
+-pslib_push_dense_merge_limit=1
+-pslib_push_sparse_merge_limit=1

paddle/fluid/train/custom_trainer/feed/conf/ps_table_config

+server_param {
+  downpour_server_param {
+    downpour_table_param {
+      table_id: 0
+      table_class: "DownpourSparseTable"
+      shard_num: 1950
+      accessor {
+        accessor_class: "DownpourCtrAccessor"
+        sparse_sgd_param {
+          learning_rate: 0.05
+          initial_g2sum: 3.0
+          initial_range: 0.0001
+          weight_bounds: -10.0
+          weight_bounds: 10.0
+        }
+        fea_dim: 11
+        embedx_dim: 8
+        embedx_threshold: 10
+        downpour_accessor_param {
+          nonclk_coeff: 0.1
+          click_coeff: 1
+          base_threshold: 1.5
+          delta_threshold: 0.25
+          delta_keep_days: 16
+          delete_after_unseen_days: 30
+          show_click_decay_rate: 0.98
+          delete_threshold: 0.8
+        }
+        table_accessor_save_param {
+          param: 1
+          converter: "(tool/xbox_compressor_mf.py | tool/xbox_pb_converter)"
+          deconverter:  "(tool/xbox_pb_deconverter | tool/xbox_decompressor_mf.awk)"
+        }   
+        table_accessor_save_param {
+          param: 2
+          converter: "(tool/xbox_compressor_mf.py | tool/xbox_pb_converter)"
+          deconverter:  "(tool/xbox_pb_deconverter | tool/xbox_decompressor_mf.awk)"
+        }
+      }
+      type: PS_SPARSE_TABLE
+      compress_in_save: true
+    }
+    downpour_table_param {
+      table_id: 1
+      table_class: "DownpourDenseTable"
+      accessor {
+        accessor_class: "DownpourDenseValueAccessor"
+        dense_sgd_param {
+          name: "adam"
+          adam {
+            learning_rate: 5e-06
+            avg_decay_rate: 0.999993
+            ada_decay_rate: 0.9999
+            ada_epsilon: 1e-08
+            mom_decay_rate: 0.99
+          }
+          naive {
+            learning_rate: 0.0002
+          }
+        }
+        fea_dim: 2571127
+      }
+      type: PS_DENSE_TABLE
+      compress_in_save: true
+    }
+    downpour_table_param {
+      table_id: 2
+      table_class: "DownpourDenseDoubleTable"
+      accessor {
+        accessor_class: "DownpourDenseValueDoubleAccessor"
+        dense_sgd_param {
+          name: "summarydouble"
+          summary {
+            summary_decay_rate: 0.999999
+          }
+        }
+        fea_dim: 13464
+      }
+      type: PS_DENSE_TABLE
+      compress_in_save: true
+    }
+    downpour_table_param {
+      table_id: 3
+      table_class: "DownpourDenseTable"
+      accessor {
+        accessor_class: "DownpourDenseValueAccessor"
+        dense_sgd_param {
+          name: "adam"
+          adam {
+            learning_rate: 5e-06
+            avg_decay_rate: 0.999993
+            ada_decay_rate: 0.9999
+            ada_epsilon: 1e-08
+            mom_decay_rate: 0.99
+          }
+          naive {
+            learning_rate: 0.0002
+          }
+        }
+        fea_dim: 2072615
+      }
+      type: PS_DENSE_TABLE
+      compress_in_save: true
+    }
+    service_param {
+      server_class: "DownpourBrpcPsServer"
+      client_class: "DownpourBrpcPsClient"
+      service_class: "DownpourPsService"
+      start_server_port: 0
+      server_thread_num: 12
+    }
+  }
+}
+
+fs_client_param {
+  uri: "afs://xingtian.afs.baidu.com:9902"
+  user: "mlarch"
+  passwd: "Fv1M87"
+  hadoop_bin: "$HADOOP_HOME/bin/hadoop"
+}

paddle/fluid/train/custom_trainer/feed/conf/trainer.yaml

-train_thread_num : 10
+train_thread_num: 10
 
-environment :
-    environment_class : LocalRuntimeEnvironment
+environment:
+    environment_class: LocalRuntimeEnvironment
+    ps: ./conf/ps_table_config
+        
+    
 
-io :
-    file_systems :
-        afs :
-            class : HadoopFileSystem
-            buffer_size : 1024000
-            ugis : 
+io:
+    file_systems:
+        afs:
+            class: HadoopFileSystem
+            buffer_size: 1024000
+            ugis: 
                 'default': 'feed_video,D3a0z8'
                 'xingtian.afs.baidu.com:9902': 'feed_video,D3a0z8'
-        local :
-            class : LocalFileSystem
-            buffer_size : 1024000
-dataset :
-    data_list :
-        train_sample :
-            prefetch_num : 2
-            root_path : ./sample
-            data_spit_interval : 300
-            data_path_formater : '%Y%m%d/%H%M' 
-            data_reader : LineDataReader 
-            done_file : to.hadoop.done 
-            filename_prefix : part
-            pipeline_cmd : cat
-            parser :
-                class : LineDataParser 
+        default:
+            class: LocalFileSystem
+            buffer_size: 1024000
+dataset:
+    data_list:
+        train_sample:
+            prefetch_num: 2
+            root_path : [./sample]
+            data_spit_interval: 300
+            data_path_formater: '%Y%m%d/%H%M' 
+            data_reader: LineDataReader 
+            done_file: to.hadoop.done 
+            filename_prefix: part
+            pipeline_cmd: './tool/ins_weight.py | awk -f ./tool/format_newcate_hotnews.awk'
+            parser:
+                class: AbacusTextDataParser 
     
 epoch:
-    epoch_class : HourlyEpochAccessor 
-    model_root_path : ./model/
-    
+    epoch_class: TimelyEpochAccessor 
+    model_root_path: ./model/
+    train_time_interval: 600
+    time_zone_seconds: 28800
+
+executor:
+- name: join
+  class: SimpleExecutor
+  train_data_name: train_sample
+  train_batch_size: 32
+  input_parse_thread_num: 10
+  push_gradient_thread_num: 16
+  train_thread_num: 16
+  need_dump_all_model: true
+- name: update
+  class: SimpleExecutor
+  train_data_name: train_sample
+  train_batch_size: 32
+  input_parse_thread_num: 10
+  push_gradient_thread_num: 16
+  train_thread_num: 16
+  need_dump_all_model: false 

paddle/fluid/train/custom_trainer/feed/dataset/abacus_data_reader.cc

+#include "paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h"
+
+#include <cstdio>
+#include <atomic>
+
+#include <glog/logging.h>
+#include <omp.h>
+
+#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h"
+
+namespace paddle {
+namespace custom_trainer {
+namespace feed {
+
+/*解析Abacus格式明文Feasign
+ */
+class AbacusTextDataParser : public LineDataParser {
+public:
+    AbacusTextDataParser() {}
+    virtual ~AbacusTextDataParser() {}
+
+    virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const {
+        instance.id = data.id;
+        instance.labels.resize(1);
+        size_t len = data.data.size();
+        const char* str = data.data.c_str();
+        const char* line_end = str + len;
+
+        char* cursor = NULL;
+        int show = (int)strtol(str, &cursor, 10);
+        str = cursor;
+        instance.labels[0] = (float)strtol(str, &cursor, 10);// click
+        str = cursor;
+
+        while (*(str += paddle::string::count_nonspaces(str)) != 0) {
+            if (*str == '*') {
+                str++;
+                size_t len = paddle::string::count_nonspaces(str);
+                str += len;
+            } else if (*str == '$') {
+                str++;
+                CHECK(((int)strtol(str, &cursor, 10), cursor != str))<<" sample type parse err:" << str;
+                str = cursor;
+            } else if (*str == '#') {
+                str++;
+                break;
+            } else if (*str == '@') {
+                str++;
+                size_t len = paddle::string::count_nonspaces(str);
+                std::string all_str(str, str + len);
+                str += len;
+            } else {
+                FeatureItem feature_item;
+                feature_item.sign() = (uint64_t)strtoull(str, &cursor, 10);
+                if (cursor == str) { //FIXME abacus没有这种情况
+                    str++;
+                    continue;
+                }
+                str = cursor;
+                CHECK(*str++ == ':');
+                CHECK(!isspace(*str));
+                CHECK((feature_item.slot() = (int) strtol(str, &cursor, 10), cursor != str)) << " format error: " << str;
+                str = cursor;
+                instance.features.emplace_back(feature_item);
+            }
+        }
+        VLOG(5) << "parse sample success, id:" << instance.id << ", fea_sum:" 
+            << instance.features.size() << ", label:" << instance.labels[0];
+        return 0;
+    }
+};
+REGIST_CLASS(DataParser, AbacusTextDataParser);
+
+}  // namespace feed
+}  // namespace custom_trainer
+}  // namespace paddle

paddle/fluid/train/custom_trainer/feed/dataset/data_reader.cc

@@ -12,51 +12,389 @@ namespace paddle {
 namespace custom_trainer {
 namespace feed {
 
-class LineDataParser : public DataParser {
+int LineDataParser::parse(const char* str, size_t len, DataItem& data) const {
+    size_t pos = 0;
+    while (pos < len && str[pos] != ' ') {
+        ++pos;
+    }
+    if (pos >= len) {
+        VLOG(2) << "fail to parse line: " << std::string(str, len) << ", strlen: " << len;
+        return -1;
+    }
+    VLOG(5) << "getline: " << str << " , pos: " << pos << ", len: " << len;
+    data.id.assign(str, pos);
+    data.data.assign(str + pos + 1, len - pos - 1);
+    return 0;
+}
+REGIST_CLASS(DataParser, LineDataParser);
+
+/********************************
+ * feasign压缩格式
+ * 情形1：slot:hot
+ * |4b|4b|4b|4b|4b| 28b |
+ * |slot       |0 |sign |
+ * 情形2：slot:hot*n
+ * |4b|4b|4b|4b|4b|4b|4b|4b|32b*n|
+ * |slot       |1 |0 |len  |sign |
+ * 情形3：slot:cold
+ * |4b|4b|4b|4b|4b|4b| 64b |
+ * |slot       |2 |0 |sign |
+ * 情形4：slot:cold*n
+ * |4b|4b|4b|4b|4b|4b|4b|4b|64b*n|
+ * |slot       |3 |0 |len  |sign |
+ ********************************/
+class ArchiveDataParse : public DataParser {
+public:
+    static const uint8_t HOT_SIGN_SIZE = 4;
+    static const uint8_t COLD_SIGN_SIZE = 8;
+
 public:
-    LineDataParser() {}
+    ArchiveDataParse() {}
+    virtual ~ArchiveDataParse() {}
+
+    struct Record {
+        int show, clk;
+        std::string tags;
+        std::map<std::string, std::vector<float>> vec_feas;
+        int sample_type;
+        std::map<std::string, std::vector<std::string>> auc_category_info_map; //为细维度计算auc准备的数据
+        std::vector<FeatureItem> hot_feas, cold_feas; //冷(int32_t)热(uint64_t)feasign
+
+        void clear() {
+            show = 0;
+            clk = 0;
+            tags.clear();
+            vec_feas.clear();
+            sample_type = 0;
+            auc_category_info_map.clear();
+            hot_feas.clear();
+            cold_feas.clear();
+        }
+
+        uint32_t calc_compress_feas_lens() const {
+            uint32_t hot_len = hot_feas.size();
+            uint32_t cold_len = cold_feas.size();
+            uint32_t cursor = 0;
+            int32_t pre_slot = -1;
+            uint32_t k = 0;
+            //热编码
+            if (hot_len > 0) {
+                pre_slot = hot_feas[0].slot();
 
-    virtual ~LineDataParser() {}
+                for (uint32_t i = 0; i < hot_len + 1; ++i) {
+                    if (i == hot_len || pre_slot != hot_feas[i].slot()) {
+                        cursor += 2;
+                        //情形2
+                        if (i - k > 1) {
+                            cursor += 2;
+                        }
+                        //情形1/2
+                        cursor += (HOT_SIGN_SIZE * (i - k));
+                        k = i;
+                    }
+                    pre_slot = hot_feas[i].slot();
+                }
+            }
+            //冷编码
+            if (cold_len > 0) {
+                pre_slot = cold_feas[0].slot();
+                k = 0;
+
+                for (uint32_t i = 0; i < cold_len + 1; ++i) {
+                    if (i == cold_len || pre_slot != cold_feas[i].slot()) {
+                        cursor += 2;
+                        //情形4
+                        if (i - k > 1) {
+                            cursor += 2;
+                        } else { //情形3
+                            cursor++;
+                        }
+                        //情形3/4
+                        cursor += (COLD_SIGN_SIZE * (i - k));
+                        k = i;
+                    }
+                    pre_slot = cold_feas[i].slot();
+                }
+            }
+            return cursor;
+        }
+
+        void serialize_to_compress_feas(char* buffer) const {
+            if (buffer == nullptr) {
+                return ;
+            }
+            uint32_t cursor = 0;
+            uint32_t hot_len = hot_feas.size();
+            uint32_t cold_len = cold_feas.size();
+            int32_t pre_slot = -1;
+            int32_t hot_sign;
+            uint16_t slot;
+            uint8_t flag = 0, len = 0;
+            uint32_t k = 0;
+            //热编码
+            if (hot_len > 0) {
+                pre_slot = hot_feas[0].slot();
+
+                for (uint32_t i = 0; i < hot_len + 1; ++i) {
+                    if (i == hot_len || pre_slot != hot_feas[i].slot()) {
+                        memcpy(buffer + cursor, &pre_slot, 2);
+                        cursor += 2;
+                        //情形2
+                        if (i - k > 1) {
+                            flag = 0x10;
+                            memcpy(buffer + cursor, &flag, 1);
+                            cursor++;
+                            len = i - k;
+                            memcpy(buffer + cursor, &len, 1);
+                            cursor++;
+                        }
+                        //情形1/2
+                        for (uint32_t j = k; j < i; ++j) {
+                            hot_sign = (int32_t) hot_feas[j].sign();
+                            for (uint8_t b = 0; b < HOT_SIGN_SIZE; ++b) {
+                                flag = (hot_sign >> ((HOT_SIGN_SIZE - b - 1) * 8)) & 0xFF;
+                                memcpy(buffer + cursor, &flag, 1);
+                                cursor++;
+                            }
+                        }
+                        k = i;
+                    }
+                    pre_slot = hot_feas[i].slot();
+                }
+            }
+            //冷编码
+            if (cold_len > 0) {
+                pre_slot = cold_feas[0].slot();
+                k = 0;
+
+                for (uint32_t i = 0; i < cold_len + 1; ++i) {
+                    if (i == cold_len || pre_slot != cold_feas[i].slot()) {
+                        memcpy(buffer + cursor, &pre_slot, 2);
+                        cursor += 2;
+                        //情形4
+                        if (i - k > 1) {
+                            flag = 0x30;
+                            memcpy(buffer + cursor, &flag, 1);
+                            cursor++;
+                            len = i - k;
+                            memcpy(buffer + cursor, &len, 1);
+                            cursor++;
+                        }
+                        //情形3/4
+                        for (uint32_t j = k; j < i; ++j) {
+                            if (i - k == 1) {
+                                flag = 0x20;
+                                memcpy(buffer + cursor, &flag, 1);
+                                cursor++;
+                            }
+                            memcpy(buffer + cursor, &cold_feas[j].sign(), COLD_SIGN_SIZE);
+                            cursor += COLD_SIGN_SIZE;
+                        }
+                        k = i;
+                    }
+                    pre_slot = cold_feas[i].slot();
+                }
+            }
+        }
+    };
 
     virtual int initialize(const YAML::Node& config, std::shared_ptr<TrainerContext> context) {
+        _index = context->cache_dict;
+
         return 0;
     }
 
     virtual int parse(const char* str, size_t len, DataItem& data) const {
-        size_t pos = 0;
-        while (pos < len && str[pos] != ' ') {
-            ++pos;
-        }
+        size_t pos = paddle::string::count_nonspaces(str);
         if (pos >= len) {
             VLOG(2) << "fail to parse line: " << std::string(str, len) << ", strlen: " << len;
             return -1;
         }
         VLOG(5) << "getline: " << str << " , pos: " << pos << ", len: " << len;
         data.id.assign(str, pos);
-        data.data.assign(str + pos + 1, len - pos - 1);
+        str += pos;
+
+        static thread_local std::vector<float> vec_feas;
+        static thread_local Record rec;
+        rec.clear();
+
+        const char* line_end = str + len;
+        char* cursor = NULL;
+        CHECK((rec.show = (int)strtol(str, &cursor, 10), cursor != str));
+        str = cursor;
+        CHECK((rec.clk = (int)strtol(str, &cursor, 10), cursor != str));
+        str = cursor;
+        CHECK(rec.show >= 1 && rec.clk >= 0 && rec.clk <= rec.show);
+
+        while (*(str += paddle::string::count_nonspaces(str)) != 0) {
+            if (*str == '*') {
+                str++;
+                size_t len = paddle::string::count_nonspaces(str);
+                std::string tag(str, str + len);
+                rec.tags = tag;
+                str += len;
+            } else if (*str == '$') {
+                str++;
+                CHECK((rec.sample_type = (int)strtol(str, &cursor, 10), cursor != str))<<" sample type parse err:" << str;
+                str = cursor;
+            } else if (*str == '#') {
+                str++;
+                size_t len = std::find_if_not(str, line_end,
+                                              [](char c) { return std::isalnum(c) != 0 || c == '_';}) - str;
+                CHECK(len > 0 && *(str + len) == ':');
+                std::string name(str, len);
+                str += len;
+                vec_feas.clear();
+                while (*str == ':') {
+                    float val = 0;
+                    CHECK((val = strtof(str + 1, &cursor), cursor > str));
+                    vec_feas.push_back(val);
+                    str = cursor;
+                }
+                CHECK(rec.vec_feas.insert({name, vec_feas}).second);
+            } else if (*str == '@') {
+                str++;
+                size_t len = paddle::string::count_nonspaces(str);
+                std::string all_str(str, str + len);
+                str += len;
+                //category_name1=value1,value2,value3|category_name2=value1,value2|....
+                std::vector<std::string> all_category_vec = paddle::string::split_string(all_str, "|");
+                for (size_t i = 0; i < all_category_vec.size(); ++i) {
+                    std::string& single_category_str = all_category_vec[i];
+                    std::vector<std::string> str_vec = paddle::string::split_string(single_category_str, "=");
+                    CHECK(str_vec.size() == 2);
+                    std::string category_name = str_vec[0];
+                    std::vector<std::string> category_info_vec = paddle::string::split_string<std::string>(str_vec[1], ",");
+                    CHECK(category_info_vec.size() > 0);
+
+                    CHECK(rec.auc_category_info_map.insert({category_name, category_info_vec}).second);
+                }
+            } else {
+                uint64_t sign = 0;
+                int slot = -1;
+                sign = (uint64_t)strtoull(str, &cursor, 10);
+                if (cursor == str) { //FIXME abacus没有这种情况
+                    str++;
+                    continue;
+                }
+                //CHECK((sign = (uint64_t)strtoull(str, &cursor, 10), cursor != str));
+                str = cursor;
+                CHECK(*str++ == ':');
+                CHECK(!isspace(*str));
+                CHECK((slot = (int) strtol(str, &cursor, 10), cursor != str)) << " format error: " << str;
+                CHECK((uint16_t) slot == slot);
+                str = cursor;
+
+                int32_t compress_sign = _index->sign2index(sign);
+                if (compress_sign < 0) {
+                    rec.cold_feas.emplace_back(sign, (uint16_t)slot);
+                } else {
+                    rec.hot_feas.emplace_back(compress_sign, (uint16_t)slot);
+                }
+            }
+        }
+
+        paddle::framework::BinaryArchive bar;
+        bar << rec.show << rec.clk << rec.tags << rec.vec_feas << rec.sample_type << rec.auc_category_info_map;
+        uint32_t feas_len = rec.calc_compress_feas_lens(); //事先计算好压缩后feasign的空间
+        bar << feas_len;
+        bar.Resize(bar.Length() + feas_len);
+        rec.serialize_to_compress_feas(bar.Finish() - feas_len); //直接在archive内部buffer进行压缩，避免不必要的拷贝
+        data.data.assign(bar.Buffer(), bar.Length()); //TODO 这一步拷贝是否也能避免
+
         return 0;
     }
 
-    virtual int parse(const char* str, DataItem& data) const {
-        size_t pos = 0;
-        while (str[pos] != '\0' && str[pos] != ' ') {
-            ++pos;
-        }
-        if (str[pos] == '\0') {
-            VLOG(2) << "fail to parse line: " << str << ", get '\\0' at pos: " << pos;
+    virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const {
+        instance.id = data.id;
+        if (data.data.empty()) {
             return -1;
         }
-        VLOG(5) << "getline: " << str << " , pos: " << pos;
-        data.id.assign(str, pos);
-        data.data.assign(str + pos + 1);
+
+        //FIXME temp
+        int show = 0, clk = 0;
+        std::string tags;
+        std::map<std::string, std::vector<float>> vec_feas;
+        int sample_type = 0;
+        std::map<std::string, std::vector<int>> auc_category_info_map;
+        uint32_t feas_len = 0;
+
+        paddle::framework::BinaryArchive bar;
+        bar.SetReadBuffer(const_cast<char*>(&data.data[0]), data.data.size(), nullptr);
+        bar >> show;
+        bar >> clk;
+        bar >> tags;
+        bar >> vec_feas;
+        bar >> sample_type;
+        bar >> auc_category_info_map;
+        bar >> feas_len;
+        CHECK((bar.Finish() - bar.Cursor()) == feas_len);
+        deserialize_feas_to_ins(bar.Cursor(), feas_len, instance.features);
+
         return 0;
     }
 
-    virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const {
-        return 0;
+private:
+    void deserialize_feas_to_ins(char* buffer, uint32_t len, std::vector<FeatureItem>& ins) const {
+        if (buffer == nullptr) {
+            return ;
+        }
+
+        uint32_t cursor = 0;
+        uint16_t slot;
+        uint8_t flag;
+        while (cursor < len) {
+            memcpy(&slot, buffer + cursor, 2);
+            cursor += 2;
+
+            memcpy(&flag, buffer + cursor, 1);
+            flag &= 0xF0;
+
+            CHECK(flag == 0x00 || flag == 0x10|| flag == 0x20 || flag == 0x30);
+
+            if (flag == 0x00 || flag == 0x10) {
+                uint8_t len = 1;
+                if (flag == 0x10) {
+                    cursor++;
+                    memcpy(&len, buffer + cursor, 1);
+                    cursor++;
+                }
+                for (uint8_t i = 0; i < len; ++i) {
+                    int32_t sign;
+                    for (uint8_t j = 0; j < HOT_SIGN_SIZE; ++j) {
+                        memcpy((char*)&sign + HOT_SIGN_SIZE-j-1, buffer + cursor, 1);
+                        cursor++;
+                    }
+
+                    uint64_t sign64 = sign & 0x0FFFFFFF;
+                    sign64 = _index->index2sign((int32_t)sign64);
+                    ins.emplace_back(sign64, slot);
+                }
+            }
+
+            if (flag == 0x20 || flag == 0x30) {
+                uint8_t len = 1;
+                cursor++;
+                if (flag == 0x30) {
+                    memcpy(&len, buffer + cursor, 1);
+                    cursor++;
+                }
+
+                for (uint8_t i = 0; i < len; ++i) {
+                    uint64_t sign64;
+                    memcpy(&sign64, buffer + cursor, COLD_SIGN_SIZE);
+                    cursor += COLD_SIGN_SIZE;
+                    ins.emplace_back(sign64, slot);
+                }
+            }
+        }
     }
+
+private:
+    std::shared_ptr<SignCacheDict> _index;
+
 };
-REGIST_CLASS(DataParser, LineDataParser);
+REGIST_CLASS(DataParser, ArchiveDataParse);
 
 int DataReader::initialize(const YAML::Node& config, std::shared_ptr<TrainerContext> context) {
     _parser.reset(CREATE_INSTANCE(DataParser, config["parser"]["class"].as<std::string>()));
@@ -162,7 +500,7 @@ public:
             size_t buffer_size = 0;
             ssize_t line_len = 0;
             while ((line_len = getline(&buffer, &buffer_size, fin.get())) != -1) {
-                // 去掉行位回车
+                // 去掉行尾回车
                 if (line_len > 0 && buffer[line_len - 1] == '\n') {
                     buffer[--line_len] = '\0';
                 }
@@ -175,7 +513,8 @@ public:
                     VLOG(5) << "parse data: " << data_item.id << " " << data_item.data << ", filename: " << filepath << ", thread_num: " << thread_num << ", max_threads: " << max_threads;
                     if (writer == nullptr) {
                         if (!data_channel->Put(std::move(data_item))) {
-                            VLOG(2) << "fail to put data, thread_num: " << thread_num;
+                            LOG(WARNING) << "fail to put data, thread_num: " << thread_num;
+                            is_failed = true;
                         }
                     } else {
                         (*writer) << std::move(data_item);
@@ -192,11 +531,6 @@ public:
                 is_failed = true;
                 continue;
             }
-            if (_file_system->err_no() != 0) {
-                _file_system->reset_err_no();
-                is_failed = true;
-                continue;
-            }
         }
         // omp end
 

paddle/fluid/train/custom_trainer/feed/dataset/data_reader.h

@@ -9,6 +9,8 @@
 #include <yaml-cpp/yaml.h>
 #include "paddle/fluid/framework/channel.h"
 #include "paddle/fluid/train/custom_trainer/feed/common/pipeline.h"
+#include "paddle/fluid/framework/archive.h"
+#include "paddle/fluid/string/string_helper.h"
 #include "paddle/fluid/train/custom_trainer/feed/common/registerer.h"
 
 namespace paddle {
@@ -18,14 +20,40 @@ namespace feed {
 class TrainerContext;
 
 struct FeatureItem {
-    uint64_t feature_sign;
-    uint16_t slot_id;
     std::vector<float> weights;
     std::vector<float> gradients;
+public:
+    FeatureItem() {
+    }
+    FeatureItem(uint64_t sign_, uint16_t slot_) {
+        sign() = sign_;
+        slot() = slot_;
+    }
+    uint64_t& sign() {
+        return *(uint64_t*)sign_buffer();
+    }
+    const uint64_t& sign() const {
+        return *(const uint64_t*)sign_buffer();
+    }
+    uint16_t& slot() {
+        return _slot;
+    }
+    const uint16_t& slot() const {
+        return _slot;
+    }
+
+private:
+    char _sign[sizeof(uint64_t)];
+    uint16_t _slot;
+
+    char* sign_buffer() const {
+        return (char*)_sign;
+    }
 };
 
 struct SampleInstance {
     std::string id;
+    std::vector<float> predicts;
     std::vector<float> labels;
     std::vector<FeatureItem> features;
     std::vector<float> embedx;
@@ -50,10 +78,9 @@ public:
     virtual ~DataParser() {}
     virtual int initialize(const YAML::Node& config, std::shared_ptr<TrainerContext> context) = 0;
     virtual int parse(const std::string& str, DataItem& data) const {
-        return parse(str.c_str(), data);
+        return parse(str.c_str(), str.size(), data);
     }
     virtual int parse(const char* str, size_t len, DataItem& data) const = 0;
-    virtual int parse(const char* str, DataItem& data) const = 0;
     virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const = 0;  
 };
 REGIST_REGISTERER(DataParser);
@@ -80,6 +107,23 @@ protected:
 };
 REGIST_REGISTERER(DataReader);
 
+class LineDataParser : public DataParser {
+public:
+    LineDataParser() {}
+
+    virtual ~LineDataParser() {}
+
+    virtual int initialize(const YAML::Node& config, std::shared_ptr<TrainerContext> context) {
+        return 0;
+    }
+
+    virtual int parse(const char* str, size_t len, DataItem& data) const;
+
+    virtual int parse_to_sample(const DataItem& data, SampleInstance& instance) const {
+        return 0;
+    }
+};
+
 }//namespace feed
 }//namespace custom_trainer
 }//namespace paddle

paddle/fluid/train/custom_trainer/feed/dataset/dataset.cc

@@ -7,14 +7,14 @@ namespace feed {
 int Dataset::initialize(
     const YAML::Node& config, std::shared_ptr<TrainerContext> context) {
     if (config["data_list"].Type() != YAML::NodeType::Map) {
-        VLOG(0) << "miss data_list config in dataset, or type error please check";
+        LOG(FATAL) << "miss data_list config in dataset, or type error please check";
         return -1;
     }
     for (auto& data_config : config["data_list"]) {
         std::string name = data_config.first.as<std::string>();
         auto data_ptr = std::make_shared<DatasetContainer>();
         if (data_ptr->initialize(data_config.second, context) != 0) {
-            VLOG(0) << "dataset initialize failed, name:" << name;
+            LOG(FATAL) << "dataset initialize failed, name:" << name;
             return -1;
         }
         _data_containers[name] = data_ptr;
@@ -48,6 +48,22 @@ inline DatasetStatus Dataset::epoch_data_status(
     return _data_containers[data_name]->epoch_data_status(epoch_id);
 }
 
+inline std::vector<std::string> Dataset::epoch_data_path(
+    const std::string& data_name, uint64_t epoch_id) {
+    return _data_containers[data_name]->epoch_data_path(epoch_id);
+}
+
+inline std::vector<std::string> Dataset::epoch_data_path(uint64_t epoch_id) {
+    std::vector<std::string> results;
+    for (auto it = _data_containers.begin(); it != _data_containers.end(); ++it) {
+        auto items = std::move(it->second->epoch_data_path(epoch_id));
+        for (auto& item : items) {
+            results.emplace_back(item);
+        }
+    }
+    return results;
+}
+
 inline ::paddle::framework::Channel<DataItem> Dataset::fetch_data(
     const std::string& data_name, uint64_t epoch_id) {
     return _data_containers[data_name]->fetch(epoch_id);

paddle/fluid/train/custom_trainer/feed/dataset/dataset.h

@@ -29,6 +29,10 @@ public:
     virtual DatasetStatus epoch_data_status(uint64_t epoch_id);
     virtual DatasetStatus epoch_data_status(const std::string& data_name, uint64_t epoch_id);
 
+    //获取数据路径
+    virtual std::vector<std::string> epoch_data_path(uint64_t epoch_id);
+    virtual std::vector<std::string> epoch_data_path(const std::string& data_name, uint64_t epoch_id);
+
     //返回各DataContainer内的原始数据(maybe 压缩格式)
     virtual ::paddle::framework::Channel<DataItem> fetch_data(
             const std::string& data_name, uint64_t epoch_id);

paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.cc

@@ -6,10 +6,10 @@
 #include <vector>
 #include <memory>
 #include <yaml-cpp/yaml.h>
-#include "paddle/fluid/framework/io/shell.h"
 #include "paddle/fluid/string/string_helper.h"
 #include "paddle/fluid/train/custom_trainer/feed/trainer_context.h"
 #include "paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h"
+#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h"
 #include "paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h"
 
 namespace paddle {
@@ -27,8 +27,7 @@ int DatasetContainer::initialize(
         _dataset_list[i].reset(new DatasetInfo);
     }
 
-    _data_root_paths = paddle::string::split_string(
-        config["root_path"].as<std::string>(), " ");
+    _data_root_paths = config["root_path"].as<std::vector<std::string>>();
     _data_split_interval = config["data_spit_interval"].as<int>();
     _data_path_formater = config["data_path_formater"].as<std::string>();
     std::string data_reader_class = config["data_reader"].as<std::string>();
@@ -42,6 +41,21 @@ std::shared_ptr<DatasetInfo> DatasetContainer::dataset(uint64_t timestamp) {
     auto data_idx = timestamp / epoch_accessor->epoch_time_interval();
     return _dataset_list[data_idx % _prefetch_num];
 }
+std::vector<std::string> DatasetContainer::epoch_data_path(uint64_t epoch_id) {
+    std::vector<std::string> results;
+    auto* epoch_accessor = _trainer_context->epoch_accessor.get();
+    time_t timestamp = epoch_accessor->epoch_timestamp(epoch_id);
+    size_t data_num = data_num_for_train(timestamp, epoch_accessor->epoch_time_interval(), _data_split_interval);
+    uint64_t data_timestamp = timestamp % _data_split_interval == 0 ? timestamp : (timestamp / _data_split_interval + 1) * _data_split_interval;
+    for (int i = 0; i < _data_root_paths.size(); ++i) {
+        for (int j = 0; j < data_num; ++j) {
+            std::string path_suffix = format_timestamp(data_timestamp + j * _data_split_interval, _data_path_formater);
+            std::string data_dir = _trainer_context->file_system->path_join(_data_root_paths[i], path_suffix);
+            results.emplace_back(data_dir);
+        }
+    }
+    return results;
+}
 
 void DatasetContainer::pre_detect_data(uint64_t epoch_id) {
     int status = 0;
@@ -56,7 +70,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) {
             async_download_data(timestamp);
         }));
     }
-    for (int detect_idx = 0 ; detect_idx < _prefetch_num; ++detect_idx) {
+    for (int detect_idx = 0 ; detect_idx < _prefetch_num; ++detect_idx, ++epoch_id) {
         if (DatasetStatus::Empty != data_status(timestamp)) {
             continue;
         }
@@ -66,7 +80,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) {
         for (int i = 0; i < _data_root_paths.size() && status == 0; ++i) {
             for (int j = 0; j < data_num && status == 0; ++j) {
                 std::string path_suffix = format_timestamp(data_timestamp + j * _data_split_interval, _data_path_formater);
-                std::string data_dir = _data_root_paths[i] + "/" + path_suffix;
+                std::string data_dir = _trainer_context->file_system->path_join(_data_root_paths[i], path_suffix);
                 status = read_data_list(data_dir, data_path_list);
             }
         }
@@ -75,6 +89,7 @@ void DatasetContainer::pre_detect_data(uint64_t epoch_id) {
             dataset_info->timestamp = timestamp;
             dataset_info->file_path_list = std::move(data_path_list);
             dataset_info->status = DatasetStatus::Detected;
+            VLOG(2) << epoch_accessor->text(epoch_id) << ", data is detected";
         }
         timestamp += epoch_accessor->epoch_time_interval();
     }
@@ -150,16 +165,25 @@ void DatasetContainer::async_download_data(uint64_t start_timestamp) {
     }
     while (!_stop_download) {
         auto dataset_info = dataset(start_timestamp);
-        while (data_status(start_timestamp) != DatasetStatus::Detected) {
+        while (data_status(start_timestamp) == DatasetStatus::Empty) {
             sleep(30);
         }
+        dataset_info->status = DatasetStatus::Downloding;
+
+        VLOG(2) << "Start download data, data_timestap:" << start_timestamp
+            << ", for epoch:" << epoch_accessor->text(start_timestamp);
         const auto& file_list = dataset_info->file_path_list;
         dataset_info->data_channel->Clear();
         while (_data_reader->read_all(file_list, dataset_info->data_channel) != 0) {
             dataset_info->data_channel->Clear();
-            VLOG(0) << "timestamp:" << start_timestamp << " data read failed, retry";
+            VLOG(0) << "Failed download data, data_timestap:" << start_timestamp
+                << ", for epoch:" << epoch_accessor->text(start_timestamp) << ", Retry it";
             sleep(30); 
         }
+        VLOG(2) << "End download data num:" << dataset_info->data_channel->Size()
+            << ", data_timestap:" << start_timestamp
+            << ", for epoch:" << epoch_accessor->text(start_timestamp);
+        dataset_info->status = DatasetStatus::Ready;
         start_timestamp += epoch_accessor->epoch_time_interval();
     }
 }

paddle/fluid/train/custom_trainer/feed/dataset/dataset_container.h

@@ -49,20 +49,22 @@ public:
     }
     virtual int initialize(
         const YAML::Node& config, std::shared_ptr<TrainerContext> context);
-    //触发可预取的数据判断
+    // 触发可预取的数据判断
     virtual void pre_detect_data(uint64_t epoch_id);
-    //获取数据状态
+    // 获取epoch对应的样本数据目录
+    std::vector<std::string> epoch_data_path(uint64_t epoch_id);
+    // 获取数据状态
     virtual DatasetStatus epoch_data_status(uint64_t epoch_id);
-    //获取特定epoch_i样本，如果数据未ready，Channel内为空指针
+    // 获取特定epoch_i样本，如果数据未ready，Channel内为空指针
     virtual ::paddle::framework::Channel<DataItem> fetch(uint64_t epoch_id);
-    //获取DataItem解析器
+    // 获取DataItem解析器
     virtual const DataParser* data_parser() {
         return _data_reader->get_parser();
     }
 protected:
     virtual DatasetStatus data_status(uint64_t timestamp);
     virtual int read_data_list(const std::string& data_dir, std::vector<std::string>& data_list);
-    //异步样本download
+    // 异步样本download
     virtual void async_download_data(uint64_t start_timestamp);
     virtual std::shared_ptr<DatasetInfo> dataset(uint64_t timestamp);
    

paddle/fluid/train/custom_trainer/feed/executor/executor.cc

+#include <sstream>
 #include "paddle/fluid/train/custom_trainer/feed/executor/executor.h"
 
 #include "paddle/fluid/framework/program_desc.h"
@@ -17,7 +18,7 @@ namespace {
 int ReadBinaryFile(const std::string& filename, std::string* contents) {
     std::ifstream fin(filename, std::ios::in | std::ios::binary);
     if (!fin) {
-        VLOG(2) << "Cannot open file " << filename;
+        LOG(FATAL) << "Cannot open file " << filename;
         return -1;
     }
     fin.seekg(0, std::ios::end);
@@ -31,7 +32,7 @@ int ReadBinaryFile(const std::string& filename, std::string* contents) {
 
 std::unique_ptr<paddle::framework::ProgramDesc> Load(
         paddle::framework::Executor* /*executor*/, const std::string& model_filename) {
-    VLOG(3) << "loading model from " << model_filename;
+    LOG(INFO) << "loading model from " << model_filename;
     std::string program_desc_str;
     if (ReadBinaryFile(model_filename, &program_desc_str) != 0) {
         return nullptr;
@@ -50,38 +51,34 @@ public:
     virtual ~SimpleExecutor() {};
     virtual int initialize(YAML::Node exe_config,
         std::shared_ptr<TrainerContext> context_ptr) {
-        
         paddle::framework::InitDevices(false);
-        if (exe_config["num_threads"]) {
-            paddle::platform::SetNumThreads(exe_config["num_threads"].as<int>());
-        }
-
-        if (!exe_config["startup_program"] || 
-            !exe_config["main_program"]) {
-            VLOG(2) << "fail to load config";
-            return -1;
-        }
-
+        //if (exe_config["num_threads"]) {
+            
+        //}
+        paddle::platform::SetNumThreads(1);
+        std::string name = exe_config["name"].as<std::string>();
+        std::string main_program = YamlHelper::get_with_default(exe_config, "main_program",
+            string::format_string("./model/%s/main_program", name.c_str()));
+        std::string startup_program = YamlHelper::get_with_default(exe_config, "startup_program",
+            string::format_string("./model/%s/startup_program", name.c_str()));
         try {
             _context.reset(new SimpleExecutor::Context(context_ptr->cpu_place));
-            _context->startup_program = Load(&_context->executor, exe_config["startup_program"].as<std::string>());
+            _context->startup_program = Load(&_context->executor, startup_program);
             if (_context->startup_program == nullptr) {
-                VLOG(2) << "fail to load startup_program: " << exe_config["startup_program"].as<std::string>();
+                VLOG(0) << "fail to load startup_program: " << startup_program;
                 return -1;
             }
-
-            _context->main_program = Load(&_context->executor, exe_config["main_program"].as<std::string>());
+            _context->main_program = Load(&_context->executor, main_program);
             if (_context->main_program == nullptr) {
-                VLOG(2) << "fail to load main_program: " << exe_config["main_program"].as<std::string>();
+                VLOG(0) << "fail to load main_program: " << main_program;
                 return -1;
             }
             _context->prepare_context = _context->executor.Prepare(*_context->main_program, 0);
         } catch (::paddle::platform::EnforceNotMet& err) {
-            VLOG(2) << err.what();
+            VLOG(0) << err.what();
             _context.reset(nullptr);
             return -1;
         }
-
         return 0;
     }
     virtual int initialize_scope(::paddle::framework::Scope* scope) {
@@ -96,8 +93,8 @@ public:
         }
         try {
             _context->executor.RunPreparedContext(_context->prepare_context.get(), scope,
-                                    false, /* don't create local scope each time*/
-                                    false /* don't create variable each time */);
+                false, /* don't create local scope each time*/
+                false /* don't create variable each time */);
 
             // For some other vector like containers not cleaned after each batch.
             _context->tensor_array_batch_cleaner.CollectNoTensorVars(scope);

paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.cc

@@ -12,7 +12,7 @@ int MultiThreadExecutor::initialize(YAML::Node exe_config,
     _train_data_name = exe_config["train_data_name"].as<std::string>();
     _train_batch_size = exe_config["train_batch_size"].as<int>();
     _input_parse_thread_num = exe_config["input_parse_thread_num"].as<int>();
-    _push_gradient_thread_num = exe_config["push_gradient_thread_num "].as<int>();
+    _push_gradient_thread_num = exe_config["push_gradient_thread_num"].as<int>();
     _train_thread_num = exe_config["train_thread_num"].as<int>();
     _need_dump_all_model = exe_config["need_dump_all_model"].as<bool>();
     CHECK(_train_thread_num > 0 && _train_batch_size > 0);
@@ -26,28 +26,41 @@ int MultiThreadExecutor::initialize(YAML::Node exe_config,
         auto* e_ptr = CREATE_INSTANCE(Executor, e_class);
         _thread_executors[i].reset(e_ptr);
         if (e_ptr->initialize(exe_config, context_ptr) != 0) {
+            VLOG(0) << "executor initialize failed, name:" << _train_exe_name
+                << " class:" << e_class;
             ret = -1;
         }
     }
     CHECK(ret == 0);
-    _scope_obj_pool.config(
+
+    // buffer
+    _scope_obj_pool.reset(new paddle::ps::ObjectPool<::paddle::framework::Scope>(
         [this]() -> ::paddle::framework::Scope* {
             auto* scope = new ::paddle::framework::Scope();
             _thread_executors[0]->initialize_scope(scope);
             return scope;
-        }, _train_thread_num * 8);
+        }, _train_thread_num * 8, 0, _train_thread_num * 8));
 
+    // 模型网络加载
     std::string model_config_path = _trainer_context->file_system->path_join(
-        "./model", string::format_string("%s.yaml", _train_exe_name.c_str()));
+        "./model", string::format_string("%s/model.yaml", _train_exe_name.c_str()));
     CHECK(_trainer_context->file_system->exists(model_config_path)) 
         << "miss model config file:" << model_config_path;
     _model_config = YAML::LoadFile(model_config_path);
-    _input_accessors.resize(_model_config["input_accessor"].size());
     for (const auto& accessor_config : _model_config["input_accessor"]) {
         auto accessor_class = accessor_config["class"].as<std::string>();
-        _input_accessors.emplace_back(CREATE_INSTANCE(DataInputAccessor, accessor_class));
-        CHECK(_input_accessors.back()->initialize(accessor_config, context_ptr) == 0)
+        auto* accessor_ptr = CREATE_INSTANCE(DataInputAccessor, accessor_class);
+        _input_accessors.emplace_back(accessor_ptr);
+        CHECK(accessor_ptr->initialize(accessor_config, context_ptr) == 0)
             << "InputAccessor init Failed, class:" << accessor_class;
+        if (accessor_config["table_id"]) {
+            auto table_id = accessor_config["table_id"].as<int>();
+            if (_table_to_accessors.count(table_id) > 0) {
+                _table_to_accessors[table_id].push_back(accessor_ptr);
+            } else {
+                _table_to_accessors[table_id] = {accessor_ptr};
+            }
+        }
     } 
 
     return ret;
@@ -57,7 +70,7 @@ paddle::framework::Channel<DataItem> MultiThreadExecutor::run(
     paddle::framework::Channel<DataItem> input, const DataParser* parser) {
     PipelineOptions input_pipe_option;
     input_pipe_option.need_hold_input_data = true;
-    input_pipe_option.batch_size = _train_batch_size;
+    input_pipe_option.batch_size = 1;
     input_pipe_option.thread_num = _input_parse_thread_num;
     input_pipe_option.input_output_rate = _train_batch_size;
     input_pipe_option.buffer_batch_count = _train_thread_num;
@@ -66,7 +79,7 @@ paddle::framework::Channel<DataItem> MultiThreadExecutor::run(
         [this, parser](DataItem* item, size_t item_num, 
             ScopePoolObj* scope, size_t* scope_num, size_t thread_idx) -> int {
             *scope_num = 1;
-            auto scope_obj = _scope_obj_pool.get();   
+            auto scope_obj = _scope_obj_pool->get();   
             auto* samples = new SampleInstance[item_num];
             for (size_t i = 0; i <item_num; ++i) {
                 CHECK(parser->parse_to_sample(item[i], samples[i]) == 0);
@@ -95,7 +108,7 @@ paddle::framework::Channel<DataItem> MultiThreadExecutor::run(
             auto* executor = _thread_executors[thread_idx].get();
             size_t& out_idx = *out_num;
             for (out_idx = 0; out_idx < in_num; ++out_idx) {
-                executor->run(in_items[out_idx].get());
+                //CHECK(executor->run(in_items[out_idx].get()) == 0);
                 out_items[out_idx] = std::move(in_items[out_idx]);
             }
             return 0;
@@ -123,7 +136,6 @@ paddle::framework::Channel<DataItem> MultiThreadExecutor::run(
                 }
                 delete[] samples; // 所有pipe完成后，再回收sample
             }
-
             return 0;
         });
 

paddle/fluid/train/custom_trainer/feed/executor/multi_thread_executor.h

@@ -8,6 +8,8 @@ namespace paddle {
 namespace custom_trainer {
 namespace feed {
 
+typedef paddle::ps::ObjectPool<::paddle::framework::Scope>::PooledObject ScopePoolObj;
+
 class MultiThreadExecutor {
 public:
     MultiThreadExecutor() {}
@@ -24,10 +26,19 @@ public:
     virtual bool is_dump_all_model() {
         return _need_dump_all_model;
     }
-
+    virtual const std::string& train_exe_name() {
+        return _train_exe_name;
+    }
     virtual const std::string& train_data_name() {
         return _train_data_name;
     }
+    virtual const std::map<uint32_t, std::vector<DataInputAccessor*>>& table_accessors() {
+        return _table_to_accessors;
+    }
+    virtual ScopePoolObj fetch_scope() {
+        ScopePoolObj scope_obj(_scope_obj_pool->get());
+        return scope_obj;
+    }
 protected:
     std::string _train_data_name;
     size_t _train_batch_size = 32;
@@ -41,8 +52,8 @@ protected:
     TrainerContext* _trainer_context = nullptr;
     std::vector<std::shared_ptr<Executor>> _thread_executors;
     std::vector<std::shared_ptr<DataInputAccessor>> _input_accessors;
-    paddle::ps::ObjectPool<::paddle::framework::Scope> _scope_obj_pool;
-    typedef paddle::ps::ObjectPool<::paddle::framework::Scope>::PooledObject ScopePoolObj;
+    std::map<uint32_t, std::vector<DataInputAccessor*>> _table_to_accessors;
+    std::shared_ptr<paddle::ps::ObjectPool<::paddle::framework::Scope>> _scope_obj_pool;
 };
 
 }  // namespace feed

paddle/fluid/train/custom_trainer/feed/io/auto_file_system.cc

@@ -19,17 +19,18 @@ public:
             for (auto& prefix_fs: config["file_systems"]) {
                 std::unique_ptr<FileSystem> fs(CREATE_INSTANCE(FileSystem, prefix_fs.second["class"].as<std::string>("")));
                 if (fs == nullptr) {
-                    VLOG(2) << "fail to create class: " << prefix_fs.second["class"].as<std::string>("");
+                    LOG(FATAL)  << "fail to create class: " << prefix_fs.second["class"].as<std::string>("");
                     return -1;
                 }
                 if (fs->initialize(prefix_fs.second, context) != 0) {
-                    VLOG(2) << "fail to initialize class: " << prefix_fs.second["class"].as<std::string>("");
-                    return 0;
+                    LOG(FATAL)  << "fail to initialize class: " << prefix_fs.second["class"].as<std::string>("");
+                    return -1;
                 }
                 _file_system.emplace(prefix_fs.first.as<std::string>(""), std::move(fs));
             }
         }
         if (_file_system.find("default") == _file_system.end()) {
+            LOG(WARNING) << "miss default file_system, use LocalFileSystem as default";
             std::unique_ptr<FileSystem> fs(CREATE_INSTANCE(FileSystem, "LocalFileSystem"));
             if (fs == nullptr || fs->initialize(YAML::Load(""), context) != 0) {
                 return -1;
@@ -61,8 +62,8 @@ public:
         return get_file_system(path)->list(path);
     }
 
-    std::string tail(const std::string& path) override {
-        return get_file_system(path)->tail(path);
+    std::string tail(const std::string& path,  size_t tail_num = 1) override {
+        return get_file_system(path)->tail(path, tail_num);
     }
 
     bool exists(const std::string& path) override {
@@ -76,35 +77,15 @@ public:
     FileSystem* get_file_system(const std::string& path) {
         auto pos = path.find_first_of(":");
         if (pos != std::string::npos) {
-            auto substr = path.substr(0, pos + 1);
+            auto substr = path.substr(0, pos); // example: afs:/xxx -> afs
             auto fs_it = _file_system.find(substr);
             if (fs_it != _file_system.end()) {
                 return fs_it->second.get();
             }
         }
-        VLOG(5) << "path: " << path << ", select default file system";
         return _file_system["default"].get();
     }
 
-    int err_no() const override {
-        if (_err_no == 0) {
-            for (const auto& file_system : _file_system) {
-                if (file_system.second->err_no() != 0) {
-                    const_cast<int&>(_err_no) = -1;
-                    break;
-                }
-            }
-        }
-        return FileSystem::err_no();
-    }
-
-    void reset_err_no() override {
-        _err_no = 0;
-        for (auto& file_system : _file_system) {
-            file_system.second->reset_err_no();
-        }
-    }
-
 private:
     std::unordered_map<std::string, std::unique_ptr<FileSystem>> _file_system;
 };

paddle/fluid/train/custom_trainer/feed/io/file_system.h

@@ -21,22 +21,12 @@ public:
     virtual int64_t file_size(const std::string& path) = 0;
     virtual void remove(const std::string& path) = 0;
     virtual std::vector<std::string> list(const std::string& path) = 0;
-    virtual std::string tail(const std::string& path) = 0;
+    virtual std::string tail(const std::string& path, size_t tail_num = 1) = 0;
     virtual bool exists(const std::string& path) = 0;
     virtual void mkdir(const std::string& path) = 0;
     virtual std::string path_join(const std::string& dir, const std::string& path);
     virtual std::pair<std::string, std::string> path_split(const std::string& path);
-    virtual int err_no() const {
-        return _err_no;
-    }
-    inline operator bool() {
-        return err_no() == 0;
-    }
-    virtual void reset_err_no() {
-        _err_no = 0;
-    }
 protected:
-    int _err_no = 0;
 };
 REGIST_REGISTERER(FileSystem);
 

paddle/fluid/train/custom_trainer/feed/io/hadoop_file_system.cc

@@ -25,7 +25,7 @@ public:
             }
         }
         if (_ugi.find("default") == _ugi.end()) {
-            VLOG(2) << "fail to load default ugi";
+            LOG(FATAL) << "fail to load default ugi";
             return -1;
         }
         return 0;
@@ -33,6 +33,7 @@ public:
 
     std::shared_ptr<FILE> open_read(const std::string& path, const std::string& converter)
             override {
+        int err_no = 0;
         std::string cmd;
         if (string::end_with(path, ".gz")) {
             cmd = string::format_string(
@@ -43,11 +44,12 @@ public:
 
         bool is_pipe = true;
         shell_add_read_converter(cmd, is_pipe, converter);
-        return shell_open(cmd, is_pipe, "r", _buffer_size, &_err_no);
+        return shell_open(cmd, is_pipe, "r", _buffer_size, &err_no);
     }
 
     std::shared_ptr<FILE> open_write(const std::string& path, const std::string& converter)
             override {
+        int err_no = 0;
         std::string cmd =
                 string::format_string("%s -put - \"%s\"", hdfs_command(path).c_str(), path.c_str());
         bool is_pipe = true;
@@ -57,12 +59,11 @@ public:
         }
 
         shell_add_write_converter(cmd, is_pipe, converter);
-        return shell_open(cmd, is_pipe, "w", _buffer_size, &_err_no);
+        return shell_open(cmd, is_pipe, "w", _buffer_size, &err_no);
     }
 
     int64_t file_size(const std::string& path) override {
-        _err_no = -1;
-        VLOG(2) << "not support";
+        LOG(FATAL) << "not support";
         return 0;
     }
 
@@ -107,13 +108,13 @@ public:
         return list;
     }
 
-    std::string tail(const std::string& path) override {
+    std::string tail(const std::string& path, size_t tail_num = 1) override {
         if (path == "") {
             return "";
         }
 
         return shell_get_command_output(string::format_string(
-                "%s -text %s | tail -1 ", hdfs_command(path).c_str(), path.c_str()));
+                "%s -text %s | tail -%u", hdfs_command(path).c_str(), path.c_str(), tail_num));
     }
 
     bool exists(const std::string& path) override {

paddle/fluid/train/custom_trainer/feed/io/local_file_system.cc

@@ -64,10 +64,10 @@ public:
         if (path == "") {
             return {};
         }
-
+        int err_no;
         std::shared_ptr<FILE> pipe;
         pipe = shell_popen(
-                string::format_string("find %s -maxdepth 1 -type f", path.c_str()), "r", &_err_no);
+                string::format_string("find %s -maxdepth 1 -type f", path.c_str()), "r", &err_no);
         string::LineFileReader reader;
         std::vector<std::string> list;
 
@@ -78,12 +78,12 @@ public:
         return list;
     }
 
-    std::string tail(const std::string& path) override {
+    std::string tail(const std::string& path,  size_t tail_num = 1) override {
         if (path == "") {
             return "";
         }
 
-        return shell_get_command_output(string::format_string("tail -1 %s ", path.c_str()));
+        return shell_get_command_output(string::format_string("tail -%u %s ", tail_num, path.c_str()));
     }
 
     bool exists(const std::string& path) override {

paddle/fluid/train/custom_trainer/feed/main.cc

@@ -13,6 +13,7 @@ using namespace paddle::custom_trainer::feed;
 DEFINE_string(feed_trainer_conf_path, "./conf/trainer.yaml", "path of trainer conf");
 
 int main(int argc, char* argv[]) {
+    google::InitGoogleLogging(argv[0]);
     //gflags
     google::ParseCommandLineFlags(&argc, &argv, true);
     std::string gflag_conf = "./conf/gflags.conf";
@@ -29,26 +30,27 @@ int main(int argc, char* argv[]) {
     if (trainer_context_ptr->environment->initialize(config["environment"]) != 0) {
         return -1;
     }
-    EnvironmentRole role;
     auto* environment = trainer_context_ptr->environment.get();
     environment->wireup();
-    if (environment->rank_id(EnvironmentRole::ALL) % 2 == 0) {
-        role = EnvironmentRole::WORKER;
+    if (environment->node_num(EnvironmentRole::ALL) == 1) {
+        environment->add_role(EnvironmentRole::WORKER);
+        environment->add_role(EnvironmentRole::PSERVER);
+    } else if (environment->rank_id(EnvironmentRole::ALL) % 2 == 0) {
+        environment->add_role(EnvironmentRole::WORKER);
     } else {
-        role = EnvironmentRole::PSERVER;
+        environment->add_role(EnvironmentRole::PSERVER);
     } 
-    environment->set_role(role);
     trainer_context_ptr->pslib.reset(new PSlib());
     std::string ps_config = config["environment"]["ps"].as<std::string>();
-    trainer_context_ptr->pslib->initialize(ps_config, environment, role);
+    trainer_context_ptr->pslib->initialize(ps_config, environment);
     //VLOG(3) << "Node Start With Role:" << role;    
      
-    std::vector<std::string> process_name_list = {
-        "InitEnvProcess",
-        "LearnerProcess"
-    };
-    switch (role) {
-    case EnvironmentRole::WORKER:
+    
+    if (environment->is_role(EnvironmentRole::WORKER)) {
+        std::vector<std::string> process_name_list = {
+            "InitEnvProcess",
+            "LearnerProcess"
+        };
         for (const auto& process_name : process_name_list) {
             Process* process = CREATE_INSTANCE(Process, process_name);
             if (process == NULL) {
@@ -64,14 +66,13 @@ int main(int argc, char* argv[]) {
         for (auto& process : trainer_context_ptr->process_list) {
             process->run();
         }
-        break;
-    case EnvironmentRole::PSERVER:
-        //wait server done
-        while (true) {
-            sleep(10000);
-        }
-        break;
+     
+    }
+    
+    //TODO exit control
+    bool running = true;
+    while (running) {
+        sleep(10000);
     }
-
     return 0;
 }

paddle/fluid/train/custom_trainer/feed/process/init_env_process.cc

@@ -40,7 +40,7 @@ int InitEnvProcess::initialize(std::shared_ptr<TrainerContext> context_ptr) {
         return -1;
     }
     
-    VLOG(3) << "Env initialize success"; 
+    VLOG(3) << "Env initialize success";
     return 0;
 }
 
@@ -49,14 +49,6 @@ int InitEnvProcess::run() {
     VLOG(3) << "Trainer Resume From epoch:" << epoch_accessor->current_epoch_id();
     auto next_epoch_id = epoch_accessor->next_epoch_id(epoch_accessor->current_epoch_id());
     _context_ptr->dataset->pre_detect_data(next_epoch_id);
-
-    if (epoch_accessor->checkpoint_path().size() > 0) {
-        //Load Model
-    } else {
-        //Random Init Model
-    }
-    //context_ptr->pslib_client()->load_model();
-    VLOG(3) << "Psserver Load Model Success";
     return 0;
 }
 

paddle/fluid/train/custom_trainer/feed/process/learner_process.cc

@@ -3,6 +3,7 @@
  *Train样本
  */
 #include <omp.h>
+#include "paddle/fluid/train/custom_trainer/feed/io/file_system.h"
 #include "paddle/fluid/train/custom_trainer/feed/dataset/dataset.h"
 #include "paddle/fluid/train/custom_trainer/feed/accessor/epoch_accessor.h"
 #include "paddle/fluid/train/custom_trainer/feed/process/learner_process.h"
@@ -27,8 +28,12 @@ int LearnerProcess::initialize(std::shared_ptr<TrainerContext> context_ptr) {
 std::future<int> LearnerProcess::save_model(uint64_t epoch_id, int table_id, ModelSaveWay way) {
     std::promise<int> p;
     auto ret = p.get_future();
-    if (_context_ptr->epoch_accessor->need_save_model(epoch_id, way)) {
-        LOG(NOTICE) << "save table, table_id:" << table_id;
+    auto* ps_client = _context_ptr->pslib->ps_client();
+    auto* epoch_accessor = _context_ptr->epoch_accessor.get();
+    if (epoch_accessor->need_save_model(epoch_id, way)) {
+        VLOG(2) << "Start save model, table_id:" << table_id;
+        auto model_dir = epoch_accessor->model_save_path(epoch_id, way);
+        return ps_client->save(table_id, model_dir, std::to_string((int)way));
     } else {
         p.set_value(0);
     }
@@ -40,14 +45,19 @@ int LearnerProcess::wait_save_model(uint64_t epoch_id, ModelSaveWay way) {
     if (!environment->is_master_node(EnvironmentRole::WORKER)) {
         return 0;
     }
+    std::set<uint32_t> table_set;
+    for (auto& executor : _executors) {
+        const auto& table_accessors = executor->table_accessors();
+        for (auto& itr : table_accessors) {
+            table_set.insert(itr.first);
+        }
+    }
     int ret_size = 0;
-    auto table_num = _context_ptr->params_table_list.size();
+    auto table_num = table_set.size();
     std::future<int> rets[table_num];
-    for (int i = 0; i < table_num; ++i) {
-        auto table_id = _context_ptr->params_table_list[i].table_id();
+    for (auto table_id : table_set) {
         rets[ret_size++] = save_model(epoch_id, table_id, way); 
     }
-
     int all_ret = 0;
     for (int i = 0; i < ret_size; ++i) {
         rets[i].wait();
@@ -56,6 +66,36 @@ int LearnerProcess::wait_save_model(uint64_t epoch_id, ModelSaveWay way) {
     return all_ret;
 }
 
+int LearnerProcess::load_model(uint64_t epoch_id) {
+    auto* environment = _context_ptr->environment.get();
+    if (!environment->is_master_node(EnvironmentRole::WORKER)) {
+        return 0;
+    }
+    std::set<uint32_t> loaded_table_set;
+    auto model_dir = _context_ptr->epoch_accessor->checkpoint_path();
+    for (auto& executor : _executors) {
+        const auto& table_accessors = executor->table_accessors();
+        for (auto& itr : table_accessors) {
+            if (loaded_table_set.count(itr.first)) {
+                continue;
+            }
+            auto table_model_path = _context_ptr->file_system->path_join(
+                model_dir, string::format_string("%03d", itr.first));
+            if (_context_ptr->file_system->list(table_model_path).size() == 0) {
+                VLOG(2) << "miss table_model:" << table_model_path << ", initialize by default";
+                auto scope = std::move(executor->fetch_scope());
+                CHECK(itr.second[0]->create(scope.get()) == 0);
+            } else {
+                auto status = _context_ptr->ps_client()->load(itr.first, 
+                    model_dir, std::to_string((int)ModelSaveWay::ModelSaveTrainCheckpoint));
+                CHECK(status.get() == 0) << "table load failed, id:" << itr.first;
+            }
+            loaded_table_set.insert(itr.first);
+        }
+    }
+    return 0;
+}
+
 int LearnerProcess::run() {
     auto* dataset = _context_ptr->dataset.get();
     auto* environment = _context_ptr->environment.get();
@@ -63,8 +103,12 @@ int LearnerProcess::run() {
     uint64_t epoch_id = epoch_accessor->current_epoch_id();
 
     environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, 
-        "Resume traine with epoch_id:%d label:%s", epoch_id, _context_ptr->epoch_accessor->text(epoch_id).c_str());
+        "Resume train with epoch_id:%d %s", epoch_id, _context_ptr->epoch_accessor->text(epoch_id).c_str());
     
+    //尝试加载模型 or 初始化
+    CHECK(load_model(epoch_id) == 0);
+    environment->barrier(EnvironmentRole::WORKER); 
+
     //判断是否先dump出base
     wait_save_model(epoch_id, ModelSaveWay::ModelSaveInferenceBase);
     environment->barrier(EnvironmentRole::WORKER); 
@@ -73,21 +117,22 @@ int LearnerProcess::run() {
         epoch_accessor->next_epoch();
         bool already_dump_inference_model = false;
         epoch_id = epoch_accessor->current_epoch_id();
-        std::string epoch_log_title= paddle::string::format_string(
+        std::string epoch_log_title = paddle::string::format_string(
             "train epoch_id:%d label:%s", epoch_id, epoch_accessor->text(epoch_id).c_str());
+        std::string data_path = paddle::string::to_string<std::string>(dataset->epoch_data_path(epoch_id));
         
         //Step1. 等待样本ready
         {
             environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, 
-                "Start %s, wait data ready", epoch_log_title.c_str());
+                "%s, wait data ready:%s", epoch_log_title.c_str(), data_path.c_str());
             while (dataset->epoch_data_status(epoch_id) != DatasetStatus::Ready) {
                 sleep(30);  
                 dataset->pre_detect_data(epoch_id);
                 environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, 
-                    "%s, data not ready, wait 30s", epoch_log_title.c_str());
+                "data not ready, wait 30s");
             } 
             environment->log(EnvironmentRole::WORKER, EnvironmentLogType::MASTER_LOG, EnvironmentLogLevel::NOTICE, 
-                "%s, data is ready, start traning", epoch_log_title.c_str());
+                "Start %s, data is ready", epoch_log_title.c_str());
             environment->barrier(EnvironmentRole::WORKER); 
         }
     
@@ -96,6 +141,7 @@ int LearnerProcess::run() {
             std::map<std::string, paddle::framework::Channel<DataItem>> backup_input_map;
             for (auto& executor : _executors) {
                 environment->barrier(EnvironmentRole::WORKER); 
+                VLOG(2) << "Start executor:" << executor->train_exe_name();
                 auto data_name = executor->train_data_name();
                 paddle::framework::Channel<DataItem> input_channel;
                 if (backup_input_map.count(data_name)) {
@@ -104,6 +150,12 @@ int LearnerProcess::run() {
                     input_channel = dataset->fetch_data(data_name, epoch_id);
                 }
                 input_channel = executor->run(input_channel, dataset->data_parser(data_name));
+                VLOG(2) << "End executor:" << executor->train_exe_name();
+
+                // 等待异步梯度完成
+                _context_ptr->ps_client()->flush();
+                environment->barrier(EnvironmentRole::WORKER); 
+
                 if (executor->is_dump_all_model()) {
                     already_dump_inference_model = true;
                     wait_save_model(epoch_id, ModelSaveWay::ModelSaveInferenceDelta);
@@ -121,6 +173,10 @@ int LearnerProcess::run() {
             } 
             wait_save_model(epoch_id, ModelSaveWay::ModelSaveTrainCheckpoint);
             environment->barrier(EnvironmentRole::WORKER); 
+
+            epoch_accessor->epoch_done(epoch_id);
+            environment->barrier(EnvironmentRole::WORKER); 
+
         }
     
         //Step4. Output Monitor && RunStatus

paddle/fluid/train/custom_trainer/feed/process/learner_process.h

@@ -18,6 +18,8 @@ public:
     virtual int initialize(std::shared_ptr<TrainerContext> context_ptr);
 
 protected:
+// 加载所有模型
+virtual int load_model(uint64_t epoch_id);
 // 同步保存所有模型
 virtual int wait_save_model(uint64_t epoch_id, ModelSaveWay way);
 // 异步保存指定模型

paddle/fluid/train/custom_trainer/feed/scripts/create_programs.py

@@ -12,87 +12,163 @@ def print_help(this_name):
     """Print help
     """
     dirname = os.path.dirname(this_name)
-    print("Usage: {} <network building filename> [model_dir]\n".format(this_name))
-    print("    example: {} {}".format(this_name, os.path.join(dirname, 'example.py')))
+    print('Usage: {} <network building filename> [model_dir]\n'.format(this_name))
+    print('    example: {} {}'.format(this_name, os.path.join(dirname, 'example.py')))
 
+class ModelBuilder:
+    """
+    Attributes:
+        _save_path: Save path of programs
+
+        def _inference():
+            Build inference network(without loss and optimizer)
+            **This function is declared in the network_desc_path file, and will be set in initialize()**
+
+            Returns:
+                list<Variable>: inputs
+                and
+                list<Variable>: outputs
+            pass
+
+        def _loss_function(*outputs):
+            **This function is declared in the network_desc_path file, and will be set in initialize()**
+            Args:
+                *outputs: the second result of inference()
+
+            Returns:
+                Variable: loss
+                and
+                list<Variable>: labels
+            pass
+    """
 
+    def initialize(self, network_desc_path, save_path=None):
+        """compile the network description module
+        Args:
+            network_desc_path: path
+            save_path: model save path, default is ./model/<network_desc_path without .py>/
+
+        Returns:
+            bool: True if succeed else False
+        """
+        if not isinstance(network_desc_path, str):
+            print('network_desc_path must be str')
+            return False
+
+        if not network_desc_path.endswith('.py'):
+            print('network_desc_path must be end with .py')
+            return False
+
+        if not os.path.exists(network_desc_path):
+            print('file not exists:', network_desc_path)
+            return False
+
+        scope = dict()
+        with open(network_desc_path, 'r') as f:
+            code = f.read()
+            compiled = compile(code, network_desc_path, 'exec')
+            exec(compiled, scope)
+
+        if not 'inference' in scope:
+            print('inference not defined')
+            return False
+
+        if not 'loss_function' in scope:
+            print('loss_function not defined')
+            return False
+
+        if save_path is None:
+            # example /a/b/c.d -> ./model/c
+            save_path = os.path.join('./model', os.path.splitext(os.path.split(network_desc_path)[1])[0])
+            print('save in the default path:', save_path)
+
+        self._save_path = save_path
+
+        self._inference = scope['inference']
+        self._loss_function = scope['loss_function']
+
+        return True
+
+    def build_and_save(self):
+        """Build programs and save to _save_path
+        """
+        main_program = fluid.Program()
+        startup_program = fluid.Program()
+        with fluid.program_guard(main_program, startup_program):
+            input_accessor, sparses, inputs, outputs = self._inference()
+            test_program = main_program.clone(for_test=True)
+            loss, labels = self._loss_function(*outputs)
+
+            optimizer = fluid.optimizer.SGD(learning_rate=1.0)
+            params_grads = optimizer.backward(loss)
+
+        if not os.path.exists(self._save_path):
+            os.makedirs(self._save_path)
+
+        programs = {
+            'startup_program': startup_program,
+            'main_program': main_program,
+            'test_program': test_program,
+        }
+        for name, program in programs.items():
+            with open(os.path.join(self._save_path, name), 'w') as f:
+                f.write(program.desc.serialize_to_string())
+
+        params = filter(fluid.io.is_parameter, main_program.list_vars())
+        vars = []
+        sums=[]
+        for param in params:
+            if param.name.find("bn") == 0:
+                sums.append({"name": param.name, "shape": param.shape});
+            else:
+                vars.append({"name": param.name, "shape": param.shape});
+
+        for accessor in input_accessor:
+            if (accessor["input"] == "sparses"):
+                accessor["input"] = sparses
+            if (accessor["input"] == "vars"):
+                accessor["input"] = vars
+            if (accessor["input"] == "sums"):
+                accessor["input"] = sums
+            if (accessor["input"] == "labels"):
+                accessor["input"] = [
+                    {"label_name": label.name, "shape": label.shape, "output_name": output.name } 
+                    for (label, output) in zip(labels, outputs) ]
+            
+
+        model_desc_path = os.path.join(self._save_path, 'model.yaml')
+        model_desc = {
+            'inputs': [{"name": var.name, "shape": var.shape} for var in inputs],
+            'outputs': [{"name": var.name, "shape": var.shape} for var in outputs],
+            'labels': [{"name": var.name, "shape": var.shape} for var in labels],
+            'loss': loss.name,
+            'input_accessor': input_accessor
+        }
+
+        with open(model_desc_path, 'w') as f:
+            yaml.safe_dump(model_desc, f, encoding='utf-8', allow_unicode=True, default_flow_style=None)
 
-def inference_warpper(filename):
-    """Build inference network(without loss and optimizer)
-    Args:
-        filename: path of file which defined real inference function
-    Returns:
-        list<Variable>: inputs
-        and
-        list<Variable>: outputs
-    """
-    
-    with open(filename, 'r') as f:
-        code = f.read()
-    compiled = compile(code, filename, 'exec')
-    
-    scope = dict()
-    exec(compiled, scope)
-    return scope['inference']()
 
 def main(argv):
     """Create programs
     Args:
         argv: arg list, length should be 2
     """
-    if len(argv) < 2 or not os.path.exists(argv[1]):
+    if len(argv) < 2:
         print_help(argv[0])
         exit(1)
-    network_build_file = argv[1]
+    network_desc_path = argv[1]
 
     if len(argv) > 2:
-        model_dir = argv[2]
+        save_path = argv[2]
     else:
-        model_dir = './model'
-
-    main_program = fluid.Program()
-    startup_program = fluid.Program()
-    with fluid.program_guard(main_program, startup_program):
-        inputs, outputs = inference_warpper(network_build_file)
-
-        test_program = main_program.clone(for_test=True)
-
-        labels = list()
-        losses = list()
-        for output in outputs:
-            label = fluid.layers.data(name='label_' + output.name, shape=output.shape, dtype='float32')
-            loss = fluid.layers.square_error_cost(input=output, label=label)
-            loss = fluid.layers.mean(loss, name='loss_' + output.name)
-
-            labels.append(label)
-            losses.append(loss)
-
-        loss_all = fluid.layers.sum(losses)
-        optimizer = fluid.optimizer.SGD(learning_rate=1.0)
-        params_grads = optimizer.backward(loss_all)
-
-    if not os.path.exists(model_dir):
-        os.mkdir(model_dir)
-
-    programs = {
-        'startup_program': startup_program,
-        'main_program': main_program,
-        'test_program': test_program,
-    }
-    for save_path, program in programs.items():
-        with open(os.path.join(model_dir, save_path), 'w') as f:
-            f.write(program.desc.serialize_to_string())
-
-    model_desc_path = os.path.join(model_dir, 'model.yaml')
-    model_desc = {
-        'inputs': [{"name": var.name, "shape": var.shape} for var in inputs],
-        'outputs': [{"name": var.name, "shape": var.shape, "label_name": label.name, "loss_name": loss.name} for var, label, loss in zip(outputs, labels, losses)],
-        'loss_all': loss_all.name,
-    }
-    
-    with open(model_desc_path, 'w') as f:
-        yaml.safe_dump(model_desc, f, encoding='utf-8', allow_unicode=True)
+        save_path = None
 
+    builder = ModelBuilder()
+    if not builder.initialize(network_desc_path, save_path):
+        print_help(argv[0])
+        exit(1)
+    builder.build_and_save()
 
 if __name__ == "__main__":
     main(sys.argv)

paddle/fluid/train/custom_trainer/feed/scripts/example.py

@@ -18,16 +18,36 @@ def inference():
         list<Variable>: outputs
     """
     # TODO: build network here
-    cvm_input = fluid.layers.data(name='cvm_input', shape=[4488], dtype='float32')
+    cvm_input = fluid.layers.data(name='cvm_input', shape=[4488], dtype='float32', stop_gradient=False)
 
     net = cvm_input
-    net = fluid.layers.fc(net, 512, act='relu')
-    net = fluid.layers.fc(net, 256, act='relu')
-    net = fluid.layers.fc(net, 256, act='relu')
-    net = fluid.layers.fc(net, 128, act='relu')
-    net = fluid.layers.fc(net, 128, act='relu')
-    net = fluid.layers.fc(net, 128, act='relu')
-    net = fluid.layers.fc(net, 128, act='relu')
+    net = fluid.layers.data_norm(input=net, name="bn6048", epsilon=1e-4,
+        param_attr={"batch_size":1e4, "batch_sum_default":0.0, "batch_square":1e4})
+    net = fluid.layers.fc(net, 512, act='relu', name='fc_1')
+    net = fluid.layers.fc(net, 256, act='relu', name='fc_2')
+    net = fluid.layers.fc(net, 256, act='relu', name='fc_3')
+    net = fluid.layers.fc(net, 128, act='relu', name='fc_4')
+    net = fluid.layers.fc(net, 128, act='relu', name='fc_5')
+    net = fluid.layers.fc(net, 128, act='relu', name='fc_6')
+    net = fluid.layers.fc(net, 128, act='relu', name='fc_7')
 
     ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr')
     return [cvm_input], [ctr_output]
+
+def loss_function(ctr_output):
+    """
+    Args:
+        *outputs: the second result of inference()
+
+    Returns:
+        Variable: loss
+        and
+        list<Variable>: labels
+    """
+    # TODO: calc loss here
+
+    label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32')
+    loss = fluid.layers.square_error_cost(input=ctr_output, label=label)
+    loss = fluid.layers.mean(loss, name='loss_ctr')
+
+    return loss, [label]

paddle/fluid/train/custom_trainer/feed/scripts/join.py

+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+"""
+This is an example of network building
+"""
+
+from __future__ import print_function, division
+import paddle
+from paddle import fluid
+
+def sparse_cvm_dim(sparse_info):
+    return sparse_info['slot_dim'] * len(sparse_info['slots'])
+
+def inference():
+    """Build inference network(without loss and optimizer)
+
+    Returns:
+        list<Dict>: sparse_inputs
+        and
+        list<Variable>: inputs
+        and
+        list<Variable>: outputs
+    """
+    sparse_cvm = { "name": "cvm_input", "slot_dim" : 11, "slots": [6048,6002,6145,6202,6201,6121,6738,6119,6146,6120,6147,6122,6123,6118,6142,6143,6008,6148,6151,6127,6144,6094,6083,6952,6739,6150,6109,6003,6099,6149,6129,6203,6153,6152,6128,6106,6251,7082,7515,6951,6949,7080,6066,7507,6186,6007,7514,6125,7506,10001,6006,7023,6085,10000,6098,6250,6110,6124,6090,6082,6067,6101,6004,6191,7075,6948,6157,6126,6188,7077,6070,6111,6087,6103,6107,6194,6156,6005,6247,6814,6158,7122,6058,6189,7058,6059,6115,7079,7081,6833,7024,6108,13342,13345,13412,13343,13350,13346,13409,6009,6011,6012,6013,6014,6015,6019,6023,6024,6027,6029,6031,6050,6060,6068,6069,6089,6095,6105,6112,6130,6131,6132,6134,6161,6162,6163,6166,6182,6183,6185,6190,6212,6213,6231,6233,6234,6236,6238,6239,6240,6241,6242,6243,6244,6245,6354,7002,7005,7008,7010,7012,7013,7015,7016,7017,7018,7019,7020,7045,7046,7048,7049,7052,7054,7056,7064,7066,7076,7078,7083,7084,7085,7086,7087,7088,7089,7090,7099,7100,7101,7102,7103,7104,7105,7109,7124,7126,7136,7142,7143,7144,7145,7146,7147,7148,7150,7151,7152,7153,7154,7155,7156,7157,7047,7050,6253,6254,6255,6256,6257,6259,6260,6261,7170,7185,7186,6751,6755,6757,6759,6760,6763,6764,6765,6766,6767,6768,6769,6770,7502,7503,7504,7505,7510,7511,7512,7513,6806,6807,6808,6809,6810,6811,6812,6813,6815,6816,6817,6819,6823,6828,6831,6840,6845,6875,6879,6881,6888,6889,6947,6950,6956,6957,6959,10006,10008,10009,10010,10011,10016,10017,10018,10019,10020,10021,10022,10023,10024,10029,10030,10031,10032,10033,10034,10035,10036,10037,10038,10039,10040,10041,10042,10044,10045,10046,10051,10052,10053,10054,10055,10056,10057,10060,10066,10069,6820,6821,6822,13333,13334,13335,13336,13337,13338,13339,13340,13341,13351,13352,13353,13359,13361,13362,13363,13366,13367,13368,13369,13370,13371,13375,13376,5700,5702,13400,13401,13402,13403,13404,13406,13407,13408,13410,13417,13418,13419,13420,13422,13425,13427,13428,13429,13430,13431,13433,13434,13436,13437,13326,13330,13331,5717,13442,13451,13452,13455,13456,13457,13458,13459,13460,13461,13462,13463,13464,13465,13466,13467,13468,1104,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,13812,13813,6740,1490,1491]} 
+
+    # TODO: build network here
+    cvm_input = fluid.layers.data(name='cvm_input', shape=[sparse_cvm_dim(sparse_cvm)], dtype='float32', stop_gradient=False)
+
+    net = cvm_input
+    net = fluid.layers.data_norm(input=net, name="bn6048", epsilon=1e-4,
+        param_attr={"batch_size":1e4, "batch_sum_default":0.0, "batch_square":1e4})
+    net = fluid.layers.fc(net, 511, act='relu', name='fc_1')
+    net = fluid.layers.fc(net, 255, act='relu', name='fc_2')
+    net = fluid.layers.fc(net, 255, act='relu', name='fc_3')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_4')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_5')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_6')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_7')
+
+    ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr')
+    
+    accessors = [
+        { "class": "AbacusSparseUpdateAccessor", "input": "sparses", "table_id": 0, "need_gradient": False},
+        { "class": "DenseInputAccessor", "input": "vars", "table_id": 1, "need_gradient": True, "async_pull": True},
+        { "class": "DenseInputAccessor", "input": "sums", "table_id": 2, "need_gradient": True, "async_pull": True},
+        { "class": "LabelInputAccessor", "input": "labels"}
+    ]
+    return accessors, [sparse_cvm], [cvm_input], [ctr_output]
+
+def loss_function(ctr_output):
+    """
+    Args:
+        *outputs: the second result of inference()
+
+    Returns:
+        Variable: loss
+        and
+        list<Variable>: labels
+    """
+    # TODO: calc loss here
+
+    label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32')
+    loss = fluid.layers.square_error_cost(input=ctr_output, label=label)
+    loss = fluid.layers.mean(loss, name='loss_ctr')
+
+    return loss, [label]

paddle/fluid/train/custom_trainer/feed/scripts/model/example/model.yaml

+inputs:
+- name: cvm_input
+  shape: [-1, 4488]
+labels:
+- name: label_ctr
+  shape: [-1, 1]
+loss: loss_ctr
+outputs:
+- name: ctr.tmp_2
+  shape: [-1, 1]
+vars:
+- name: bn6048.batch_size
+  shape: [4488]
+- name: bn6048.batch_sum
+  shape: [4488]
+- name: bn6048.batch_square_sum
+  shape: [4488]
+- name: fc_1.w_0
+  shape: [4488, 512]
+- name: fc_1.b_0
+  shape: [512]
+- name: fc_2.w_0
+  shape: [512, 256]
+- name: fc_2.b_0
+  shape: [256]
+- name: fc_3.w_0
+  shape: [256, 256]
+- name: fc_3.b_0
+  shape: [256]
+- name: fc_4.w_0
+  shape: [256, 128]
+- name: fc_4.b_0
+  shape: [128]
+- name: fc_5.w_0
+  shape: [128, 128]
+- name: fc_5.b_0
+  shape: [128]
+- name: fc_6.w_0
+  shape: [128, 128]
+- name: fc_6.b_0
+  shape: [128]
+- name: fc_7.w_0
+  shape: [128, 128]
+- name: fc_7.b_0
+  shape: [128]
+- name: ctr.w_0
+  shape: [128, 1]
+- name: ctr.b_0
+  shape: [1]

paddle/fluid/train/custom_trainer/feed/scripts/model/join/model.yaml

+input_accessor:
+- class: AbacusSparseUpdateAccessor
+  input:
+  - name: cvm_input
+    slot_dim: 11
+    slots: [6048, 6002, 6145, 6202, 6201, 6121, 6738, 6119, 6146, 6120, 6147, 6122,
+      6123, 6118, 6142, 6143, 6008, 6148, 6151, 6127, 6144, 6094, 6083, 6952, 6739,
+      6150, 6109, 6003, 6099, 6149, 6129, 6203, 6153, 6152, 6128, 6106, 6251, 7082,
+      7515, 6951, 6949, 7080, 6066, 7507, 6186, 6007, 7514, 6125, 7506, 10001, 6006,
+      7023, 6085, 10000, 6098, 6250, 6110, 6124, 6090, 6082, 6067, 6101, 6004, 6191,
+      7075, 6948, 6157, 6126, 6188, 7077, 6070, 6111, 6087, 6103, 6107, 6194, 6156,
+      6005, 6247, 6814, 6158, 7122, 6058, 6189, 7058, 6059, 6115, 7079, 7081, 6833,
+      7024, 6108, 13342, 13345, 13412, 13343, 13350, 13346, 13409, 6009, 6011, 6012,
+      6013, 6014, 6015, 6019, 6023, 6024, 6027, 6029, 6031, 6050, 6060, 6068, 6069,
+      6089, 6095, 6105, 6112, 6130, 6131, 6132, 6134, 6161, 6162, 6163, 6166, 6182,
+      6183, 6185, 6190, 6212, 6213, 6231, 6233, 6234, 6236, 6238, 6239, 6240, 6241,
+      6242, 6243, 6244, 6245, 6354, 7002, 7005, 7008, 7010, 7012, 7013, 7015, 7016,
+      7017, 7018, 7019, 7020, 7045, 7046, 7048, 7049, 7052, 7054, 7056, 7064, 7066,
+      7076, 7078, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7099, 7100, 7101,
+      7102, 7103, 7104, 7105, 7109, 7124, 7126, 7136, 7142, 7143, 7144, 7145, 7146,
+      7147, 7148, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7047, 7050, 6253,
+      6254, 6255, 6256, 6257, 6259, 6260, 6261, 7170, 7185, 7186, 6751, 6755, 6757,
+      6759, 6760, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 7502, 7503, 7504,
+      7505, 7510, 7511, 7512, 7513, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813,
+      6815, 6816, 6817, 6819, 6823, 6828, 6831, 6840, 6845, 6875, 6879, 6881, 6888,
+      6889, 6947, 6950, 6956, 6957, 6959, 10006, 10008, 10009, 10010, 10011, 10016,
+      10017, 10018, 10019, 10020, 10021, 10022, 10023, 10024, 10029, 10030, 10031,
+      10032, 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042,
+      10044, 10045, 10046, 10051, 10052, 10053, 10054, 10055, 10056, 10057, 10060,
+      10066, 10069, 6820, 6821, 6822, 13333, 13334, 13335, 13336, 13337, 13338, 13339,
+      13340, 13341, 13351, 13352, 13353, 13359, 13361, 13362, 13363, 13366, 13367,
+      13368, 13369, 13370, 13371, 13375, 13376, 5700, 5702, 13400, 13401, 13402, 13403,
+      13404, 13406, 13407, 13408, 13410, 13417, 13418, 13419, 13420, 13422, 13425,
+      13427, 13428, 13429, 13430, 13431, 13433, 13434, 13436, 13437, 13326, 13330,
+      13331, 5717, 13442, 13451, 13452, 13455, 13456, 13457, 13458, 13459, 13460,
+      13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 1104, 1106, 1107, 1108,
+      1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122,
+      1123, 1124, 1125, 1126, 1127, 1128, 1129, 13812, 13813, 6740, 1490, 1491]
+  need_gradient: false
+  table_id: 0
+- async_pull: true
+  class: DenseInputAccessor
+  input:
+  - name: fc_1.w_0
+    shape: [4488, 511]
+  - name: fc_1.b_0
+    shape: [511]
+  - name: fc_2.w_0
+    shape: [511, 255]
+  - name: fc_2.b_0
+    shape: [255]
+  - name: fc_3.w_0
+    shape: [255, 255]
+  - name: fc_3.b_0
+    shape: [255]
+  - name: fc_4.w_0
+    shape: [255, 127]
+  - name: fc_4.b_0
+    shape: [127]
+  - name: fc_5.w_0
+    shape: [127, 127]
+  - name: fc_5.b_0
+    shape: [127]
+  - name: fc_6.w_0
+    shape: [127, 127]
+  - name: fc_6.b_0
+    shape: [127]
+  - name: fc_7.w_0
+    shape: [127, 127]
+  - name: fc_7.b_0
+    shape: [127]
+  - name: ctr.w_0
+    shape: [127, 1]
+  - name: ctr.b_0
+    shape: [1]
+  need_gradient: true
+  table_id: 1
+- async_pull: true
+  class: DenseInputAccessor
+  input:
+  - name: bn6048.batch_size
+    shape: [4488]
+  - name: bn6048.batch_sum
+    shape: [4488]
+  - name: bn6048.batch_square_sum
+    shape: [4488]
+  need_gradient: true
+  table_id: 2
+- class: LabelInputAccessor
+  input:
+  - label_name: label_ctr
+    output_name: ctr.tmp_2
+    shape: [-1, 1]
+inputs:
+- name: cvm_input
+  shape: [-1, 4488]
+labels:
+- name: label_ctr
+  shape: [-1, 1]
+loss: loss_ctr
+outputs:
+- name: ctr.tmp_2
+  shape: [-1, 1]

paddle/fluid/train/custom_trainer/feed/scripts/model/update/model.yaml

+input_accessor:
+- class: AbacusSparseUpdateAccessor
+  input:
+  - name: cvm_input
+    slot_dim: 9
+    slots: [6048, 6002, 6145, 6202, 6201, 6121, 6738, 6119, 6146, 6120, 6147, 6122,
+      6123, 6118, 6142, 6143, 6008, 6148, 6151, 6127, 6144, 6094, 6083, 6952, 6739,
+      6150, 6109, 6003, 6099, 6149, 6129, 6203, 6153, 6152, 6128, 6106, 6251, 7082,
+      7515, 6951, 6949, 7080, 6066, 7507, 6186, 6007, 7514, 6125, 7506, 10001, 6006,
+      7023, 6085, 10000, 6098, 6250, 6110, 6124, 6090, 6082, 6067, 6101, 6004, 6191,
+      7075, 6948, 6157, 6126, 6188, 7077, 6070, 6111, 6087, 6103, 6107, 6194, 6156,
+      6005, 6247, 6814, 6158, 7122, 6058, 6189, 7058, 6059, 6115, 7079, 7081, 6833,
+      7024, 6108, 13342, 13345, 13412, 13343, 13350, 13346, 13409, 6009, 6011, 6012,
+      6013, 6014, 6015, 6019, 6023, 6024, 6027, 6029, 6031, 6050, 6060, 6068, 6069,
+      6089, 6095, 6105, 6112, 6130, 6131, 6132, 6134, 6161, 6162, 6163, 6166, 6182,
+      6183, 6185, 6190, 6212, 6213, 6231, 6233, 6234, 6236, 6238, 6239, 6240, 6241,
+      6242, 6243, 6244, 6245, 6354, 7002, 7005, 7008, 7010, 7012, 7013, 7015, 7016,
+      7017, 7018, 7019, 7020, 7045, 7046, 7048, 7049, 7052, 7054, 7056, 7064, 7066,
+      7076, 7078, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7099, 7100, 7101,
+      7102, 7103, 7104, 7105, 7109, 7124, 7126, 7136, 7142, 7143, 7144, 7145, 7146,
+      7147, 7148, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7047, 7050, 6253,
+      6254, 6255, 6256, 6257, 6259, 6260, 6261, 7170, 7185, 7186, 6751, 6755, 6757,
+      6759, 6760, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 7502, 7503, 7504,
+      7505, 7510, 7511, 7512, 7513, 6806, 6807, 6808, 6809, 6810, 6811, 6812, 6813,
+      6815, 6816, 6817, 6819, 6823, 6828, 6831, 6840, 6845, 6875, 6879, 6881, 6888,
+      6889, 6947, 6950, 6956, 6957, 6959, 10006, 10008, 10009, 10010, 10011, 10016,
+      10017, 10018, 10019, 10020, 10021, 10022, 10023, 10024, 10029, 10030, 10031,
+      10032, 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042,
+      10044, 10045, 10046, 10051, 10052, 10053, 10054, 10055, 10056, 10057, 10060,
+      10066, 10069, 6820, 6821, 6822, 13333, 13334, 13335, 13336, 13337, 13338, 13339,
+      13340, 13341, 13351, 13352, 13353, 13359, 13361, 13362, 13363, 13366, 13367,
+      13368, 13369, 13370, 13371, 13375, 13376, 5700, 5702, 13400, 13401, 13402, 13403,
+      13404, 13406, 13407, 13408, 13410, 13417, 13418, 13419, 13420, 13422, 13425,
+      13427, 13428, 13429, 13430, 13431, 13433, 13434, 13436, 13437, 13326, 13330,
+      13331, 5717, 13442, 13451, 13452, 13455, 13456, 13457, 13458, 13459, 13460,
+      13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 1104, 1106, 1107, 1108,
+      1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1119, 1120, 1121, 1122,
+      1123, 1124, 1125, 1126, 1127, 1128, 1129, 13812, 13813, 6740, 1490, 1491]
+  need_gradient: true
+  table_id: 0
+- async_pull: true
+  class: DenseInputAccessor
+  input:
+  - name: fc_1.w_0
+    shape: [3672, 511]
+  - name: fc_1.b_0
+    shape: [511]
+  - name: fc_2.w_0
+    shape: [511, 255]
+  - name: fc_2.b_0
+    shape: [255]
+  - name: fc_3.w_0
+    shape: [255, 127]
+  - name: fc_3.b_0
+    shape: [127]
+  - name: fc_4.w_0
+    shape: [127, 127]
+  - name: fc_4.b_0
+    shape: [127]
+  - name: fc_5.w_0
+    shape: [127, 127]
+  - name: fc_5.b_0
+    shape: [127]
+  - name: ctr.w_0
+    shape: [127, 1]
+  - name: ctr.b_0
+    shape: [1]
+  need_gradient: true
+  table_id: 3
+- class: LabelInputAccessor
+  input:
+  - label_name: label_ctr
+    output_name: ctr.tmp_2
+    shape: [-1, 1]
+inputs:
+- name: cvm_input
+  shape: [-1, 3672]
+labels:
+- name: label_ctr
+  shape: [-1, 1]
+loss: loss_ctr
+outputs:
+- name: ctr.tmp_2
+  shape: [-1, 1]

paddle/fluid/train/custom_trainer/feed/scripts/start_feed_trainer.sh

-#!bash
+#!/bin/bash
 export LD_LIBRARY_PATH=LD_LIBRARY_PATH:./so
-./bin/feed_trainer
+./bin/feed_trainer "$@"

paddle/fluid/train/custom_trainer/feed/scripts/update.py

+#!/usr/bin/env python
+#-*- coding:utf-8 -*-
+
+"""
+This is an example of network building
+"""
+
+from __future__ import print_function, division
+import paddle
+from paddle import fluid
+
+def sparse_cvm_dim(sparse_info):
+    return sparse_info['slot_dim'] * len(sparse_info['slots'])
+
+def inference():
+    """Build inference network(without loss and optimizer)
+
+    Returns:
+        list<Variable>: inputs
+        and
+        list<Variable>: outputs
+    """
+    sparse_cvm = { "name": "cvm_input", "slot_dim" : 9, "slots": [6048,6002,6145,6202,6201,6121,6738,6119,6146,6120,6147,6122,6123,6118,6142,6143,6008,6148,6151,6127,6144,6094,6083,6952,6739,6150,6109,6003,6099,6149,6129,6203,6153,6152,6128,6106,6251,7082,7515,6951,6949,7080,6066,7507,6186,6007,7514,6125,7506,10001,6006,7023,6085,10000,6098,6250,6110,6124,6090,6082,6067,6101,6004,6191,7075,6948,6157,6126,6188,7077,6070,6111,6087,6103,6107,6194,6156,6005,6247,6814,6158,7122,6058,6189,7058,6059,6115,7079,7081,6833,7024,6108,13342,13345,13412,13343,13350,13346,13409,6009,6011,6012,6013,6014,6015,6019,6023,6024,6027,6029,6031,6050,6060,6068,6069,6089,6095,6105,6112,6130,6131,6132,6134,6161,6162,6163,6166,6182,6183,6185,6190,6212,6213,6231,6233,6234,6236,6238,6239,6240,6241,6242,6243,6244,6245,6354,7002,7005,7008,7010,7012,7013,7015,7016,7017,7018,7019,7020,7045,7046,7048,7049,7052,7054,7056,7064,7066,7076,7078,7083,7084,7085,7086,7087,7088,7089,7090,7099,7100,7101,7102,7103,7104,7105,7109,7124,7126,7136,7142,7143,7144,7145,7146,7147,7148,7150,7151,7152,7153,7154,7155,7156,7157,7047,7050,6253,6254,6255,6256,6257,6259,6260,6261,7170,7185,7186,6751,6755,6757,6759,6760,6763,6764,6765,6766,6767,6768,6769,6770,7502,7503,7504,7505,7510,7511,7512,7513,6806,6807,6808,6809,6810,6811,6812,6813,6815,6816,6817,6819,6823,6828,6831,6840,6845,6875,6879,6881,6888,6889,6947,6950,6956,6957,6959,10006,10008,10009,10010,10011,10016,10017,10018,10019,10020,10021,10022,10023,10024,10029,10030,10031,10032,10033,10034,10035,10036,10037,10038,10039,10040,10041,10042,10044,10045,10046,10051,10052,10053,10054,10055,10056,10057,10060,10066,10069,6820,6821,6822,13333,13334,13335,13336,13337,13338,13339,13340,13341,13351,13352,13353,13359,13361,13362,13363,13366,13367,13368,13369,13370,13371,13375,13376,5700,5702,13400,13401,13402,13403,13404,13406,13407,13408,13410,13417,13418,13419,13420,13422,13425,13427,13428,13429,13430,13431,13433,13434,13436,13437,13326,13330,13331,5717,13442,13451,13452,13455,13456,13457,13458,13459,13460,13461,13462,13463,13464,13465,13466,13467,13468,1104,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,13812,13813,6740,1490,1491]} 
+    # TODO: build network here
+    cvm_input = fluid.layers.data(name='cvm_input', shape=[sparse_cvm_dim(sparse_cvm)], dtype='float32', stop_gradient=False)
+
+    net = cvm_input
+    net = fluid.layers.fc(net, 511, act='relu', name='fc_1')
+    net = fluid.layers.fc(net, 255, act='relu', name='fc_2')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_3')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_4')
+    net = fluid.layers.fc(net, 127, act='relu', name='fc_5')
+
+    ctr_output = fluid.layers.fc(net, 1, act='sigmoid', name='ctr')
+
+    accessors = [
+        { "class": "AbacusSparseUpdateAccessor", "input": "sparses", "table_id": 0, "need_gradient": True},
+        { "class": "DenseInputAccessor", "input": "vars", "table_id": 3, "need_gradient": True, "async_pull": True},
+        { "class": "LabelInputAccessor", "input": "labels"}
+    ]
+
+    return accessors, [sparse_cvm], [cvm_input], [ctr_output]
+
+def loss_function(ctr_output):
+    """
+    Args:
+        *outputs: the second result of inference()
+
+    Returns:
+        Variable: loss
+        and
+        list<Variable>: labels
+    """
+    # TODO: calc loss here
+
+    label = fluid.layers.data(name='label_ctr', shape=ctr_output.shape, dtype='float32')
+    loss = fluid.layers.square_error_cost(input=ctr_output, label=label)
+    loss = fluid.layers.mean(loss, name='loss_ctr')
+
+    return loss, [label]

paddle/fluid/train/custom_trainer/feed/tool/format_newcate_hotnews.awk

+#!/bin/awk -f
+{
+    if ($1 !~ /^([0-9a-zA-Z])+$/ || $2 !~ /^([0-9])+$/ || $3 !~ /^([0-9])+$/) {
+        next;
+    }
+    show = $2;
+    clk = $3;
+    if (clk > show) {
+        clk = show;
+    }
+    for (i = 0; i < clk; i++) {
+        $2 = "1";
+        $3 = "1";
+        print $0;
+    }
+    for (i = 0; i < show - clk; i++) {
+        $2 = "1";
+        $3 = "0";
+        print $0;
+    }
+}

paddle/fluid/train/custom_trainer/feed/tool/gdbinit

+#                                                                                                        
+#   STL GDB evaluators/views/utilities - 1.03
+#
+#   The new GDB commands:                                                         
+# 	    are entirely non instrumental                                             
+# 	    do not depend on any "inline"(s) - e.g. size(), [], etc
+#       are extremely tolerant to debugger settings
+#                                                                                 
+#   This file should be "included" in .gdbinit as following:
+#   source stl-views.gdb or just paste it into your .gdbinit file
+#
+#   The following STL containers are currently supported:
+#
+#       std::vector<T> -- via pvector command
+#       std::list<T> -- via plist or plist_member command
+#       std::map<T,T> -- via pmap or pmap_member command
+#       std::multimap<T,T> -- via pmap or pmap_member command
+#       std::set<T> -- via pset command
+#       std::multiset<T> -- via pset command
+#       std::deque<T> -- via pdequeue command
+#       std::stack<T> -- via pstack command
+#       std::queue<T> -- via pqueue command
+#       std::priority_queue<T> -- via ppqueue command
+#       std::bitset<n> -- via pbitset command
+#       std::string -- via pstring command
+#       std::widestring -- via pwstring command
+#
+#   The end of this file contains (optional) C++ beautifiers
+#   Make sure your debugger supports $argc
+#
+#   Simple GDB Macros writen by Dan Marinescu (H-PhD) - License GPL
+#   Inspired by intial work of Tom Malnar, 
+#     Tony Novac (PhD) / Cornell / Stanford,
+#     Gilad Mishne (PhD) and Many Many Others.
+#   Contact: dan_c_marinescu@yahoo.com (Subject: STL)
+#
+#   Modified to work with g++ 4.3 by Anders Elton
+#   Also added _member functions, that instead of printing the entire class in map, prints a member.
+
+
+
+#
+# std::vector<>
+#
+
+define pvector
+	if $argc == 0
+		help pvector
+	else
+		set $size = $arg0._M_impl._M_finish - $arg0._M_impl._M_start
+		set $capacity = $arg0._M_impl._M_end_of_storage - $arg0._M_impl._M_start
+		set $size_max = $size - 1
+	end
+	if $argc == 1
+		set $i = 0
+		while $i < $size
+			printf "elem[%u]: ", $i
+			p *($arg0._M_impl._M_start + $i)
+			set $i++
+		end
+	end
+	if $argc == 2
+		set $idx = $arg1
+		if $idx < 0 || $idx > $size_max
+			printf "idx1, idx2 are not in acceptable range: [0..%u].\n", $size_max
+		else
+			printf "elem[%u]: ", $idx
+			p *($arg0._M_impl._M_start + $idx)
+		end
+	end
+	if $argc == 3
+	  set $start_idx = $arg1
+	  set $stop_idx = $arg2
+	  if $start_idx > $stop_idx
+	    set $tmp_idx = $start_idx
+	    set $start_idx = $stop_idx
+	    set $stop_idx = $tmp_idx
+	  end
+	  if $start_idx < 0 || $stop_idx < 0 || $start_idx > $size_max || $stop_idx > $size_max
+	    printf "idx1, idx2 are not in acceptable range: [0..%u].\n", $size_max
+	  else
+	    set $i = $start_idx
+		while $i <= $stop_idx
+			printf "elem[%u]: ", $i
+			p *($arg0._M_impl._M_start + $i)
+			set $i++
+		end
+	  end
+	end
+	if $argc > 0
+		printf "Vector size = %u\n", $size
+		printf "Vector capacity = %u\n", $capacity
+		printf "Element "
+		whatis $arg0._M_impl._M_start
+	end
+end
+
+document pvector
+	Prints std::vector<T> information.
+	Syntax: pvector <vector> <idx1> <idx2>
+	Note: idx, idx1 and idx2 must be in acceptable range [0..<vector>.size()-1].
+	Examples:
+	pvector v - Prints vector content, size, capacity and T typedef
+	pvector v 0 - Prints element[idx] from vector
+	pvector v 1 2 - Prints elements in range [idx1..idx2] from vector
+end 
+
+#
+# std::list<>
+#
+
+define plist
+	if $argc == 0
+		help plist
+	else
+		set $head = &$arg0._M_impl._M_node
+		set $current = $arg0._M_impl._M_node._M_next
+		set $size = 0
+		while $current != $head
+			if $argc == 2
+				printf "elem[%u]: ", $size
+				p *($arg1*)($current + 1)
+			end
+			if $argc == 3
+				if $size == $arg2
+					printf "elem[%u]: ", $size
+					p *($arg1*)($current + 1)
+				end
+			end
+			set $current = $current._M_next
+			set $size++
+		end
+		printf "List size = %u \n", $size
+		if $argc == 1
+			printf "List "
+			whatis $arg0
+			printf "Use plist <variable_name> <element_type> to see the elements in the list.\n"
+		end
+	end
+end
+
+document plist
+	Prints std::list<T> information.
+	Syntax: plist <list> <T> <idx>: Prints list size, if T defined all elements or just element at idx
+	Examples:
+	plist l - prints list size and definition
+	plist l int - prints all elements and list size
+	plist l int 2 - prints the third element in the list (if exists) and list size
+end
+
+define plist_member
+	if $argc == 0
+		help plist_member
+	else
+		set $head = &$arg0._M_impl._M_node
+		set $current = $arg0._M_impl._M_node._M_next
+		set $size = 0
+		while $current != $head
+			if $argc == 3
+				printf "elem[%u]: ", $size
+				p (*($arg1*)($current + 1)).$arg2
+			end
+			if $argc == 4
+				if $size == $arg3
+					printf "elem[%u]: ", $size
+					p (*($arg1*)($current + 1)).$arg2
+				end
+			end
+			set $current = $current._M_next
+			set $size++
+		end
+		printf "List size = %u \n", $size
+		if $argc == 1
+			printf "List "
+			whatis $arg0
+			printf "Use plist_member <variable_name> <element_type> <member> to see the elements in the list.\n"
+		end
+	end
+end
+
+document plist_member
+	Prints std::list<T> information.
+	Syntax: plist <list> <T> <idx>: Prints list size, if T defined all elements or just element at idx
+	Examples:
+	plist_member l int member - prints all elements and list size
+	plist_member l int member 2 - prints the third element in the list (if exists) and list size
+end
+
+
+#
+# std::map and std::multimap
+#
+
+define pmap
+	if $argc == 0
+		help pmap
+	else
+		set $tree = $arg0
+		set $i = 0
+		set $node = $tree._M_t._M_impl._M_header._M_left
+		set $end = $tree._M_t._M_impl._M_header
+		set $tree_size = $tree._M_t._M_impl._M_node_count
+		if $argc == 1
+			printf "Map "
+			whatis $tree
+			printf "Use pmap <variable_name> <left_element_type> <right_element_type> to see the elements in the map.\n"
+		end
+		if $argc == 3
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				printf "elem[%u].left: ", $i
+				p *($arg1*)$value
+				set $value = $value + sizeof($arg1)
+				printf "elem[%u].right: ", $i
+				p *($arg2*)$value
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+		end
+		if $argc == 4
+			set $idx = $arg3
+			set $ElementsFound = 0
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				if *($arg1*)$value == $idx
+					printf "elem[%u].left: ", $i
+					p *($arg1*)$value
+					set $value = $value + sizeof($arg1)
+					printf "elem[%u].right: ", $i
+					p *($arg2*)$value
+					set $ElementsFound++
+				end
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+			printf "Number of elements found = %u\n", $ElementsFound
+		end
+		if $argc == 5
+			set $idx1 = $arg3
+			set $idx2 = $arg4
+			set $ElementsFound = 0
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				set $valueLeft = *($arg1*)$value
+				set $valueRight = *($arg2*)($value + sizeof($arg1))
+				if $valueLeft == $idx1 && $valueRight == $idx2
+					printf "elem[%u].left: ", $i
+					p $valueLeft
+					printf "elem[%u].right: ", $i
+					p $valueRight
+					set $ElementsFound++
+				end
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+			printf "Number of elements found = %u\n", $ElementsFound
+		end
+		printf "Map size = %u\n", $tree_size
+	end
+end
+
+document pmap
+	Prints std::map<TLeft and TRight> or std::multimap<TLeft and TRight> information. Works for std::multimap as well.
+	Syntax: pmap <map> <TtypeLeft> <TypeRight> <valLeft> <valRight>: Prints map size, if T defined all elements or just element(s) with val(s)
+	Examples:
+	pmap m - prints map size and definition
+	pmap m int int - prints all elements and map size
+	pmap m int int 20 - prints the element(s) with left-value = 20 (if any) and map size
+	pmap m int int 20 200 - prints the element(s) with left-value = 20 and right-value = 200 (if any) and map size
+end
+
+
+define pmap_member
+	if $argc == 0
+		help pmap_member
+	else
+		set $tree = $arg0
+		set $i = 0
+		set $node = $tree._M_t._M_impl._M_header._M_left
+		set $end = $tree._M_t._M_impl._M_header
+		set $tree_size = $tree._M_t._M_impl._M_node_count
+		if $argc == 1
+			printf "Map "
+			whatis $tree
+			printf "Use pmap <variable_name> <left_element_type> <right_element_type> to see the elements in the map.\n"
+		end
+		if $argc == 5
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				printf "elem[%u].left: ", $i
+				p (*($arg1*)$value).$arg2
+				set $value = $value + sizeof($arg1)
+				printf "elem[%u].right: ", $i
+				p (*($arg3*)$value).$arg4
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+		end
+		if $argc == 6
+			set $idx = $arg5
+			set $ElementsFound = 0
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				if *($arg1*)$value == $idx
+					printf "elem[%u].left: ", $i
+					p (*($arg1*)$value).$arg2
+					set $value = $value + sizeof($arg1)
+					printf "elem[%u].right: ", $i
+					p (*($arg3*)$value).$arg4
+					set $ElementsFound++
+				end
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+			printf "Number of elements found = %u\n", $ElementsFound
+		end
+		printf "Map size = %u\n", $tree_size
+	end
+end
+
+document pmap_member
+	Prints std::map<TLeft and TRight> or std::multimap<TLeft and TRight> information. Works for std::multimap as well.
+	Syntax: pmap <map> <TtypeLeft> <TypeRight> <valLeft> <valRight>: Prints map size, if T defined all elements or just element(s) with val(s)
+	Examples:
+	pmap_member m class1 member1 class2 member2 - prints class1.member1 : class2.member2
+	pmap_member m class1 member1 class2 member2 lvalue - prints class1.member1 : class2.member2 where class1 == lvalue
+end
+
+
+#
+# std::set and std::multiset
+#
+
+define pset
+	if $argc == 0
+		help pset
+	else
+		set $tree = $arg0
+		set $i = 0
+		set $node = $tree._M_t._M_impl._M_header._M_left
+		set $end = $tree._M_t._M_impl._M_header
+		set $tree_size = $tree._M_t._M_impl._M_node_count
+		if $argc == 1
+			printf "Set "
+			whatis $tree
+			printf "Use pset <variable_name> <element_type> to see the elements in the set.\n"
+		end
+		if $argc == 2
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				printf "elem[%u]: ", $i
+				p *($arg1*)$value
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+		end
+		if $argc == 3
+			set $idx = $arg2
+			set $ElementsFound = 0
+			while $i < $tree_size
+				set $value = (void *)($node + 1)
+				if *($arg1*)$value == $idx
+					printf "elem[%u]: ", $i
+					p *($arg1*)$value
+					set $ElementsFound++
+				end
+				if $node._M_right != 0
+					set $node = $node._M_right
+					while $node._M_left != 0
+						set $node = $node._M_left
+					end
+				else
+					set $tmp_node = $node._M_parent
+					while $node == $tmp_node._M_right
+						set $node = $tmp_node
+						set $tmp_node = $tmp_node._M_parent
+					end
+					if $node._M_right != $tmp_node
+						set $node = $tmp_node
+					end
+				end
+				set $i++
+			end
+			printf "Number of elements found = %u\n", $ElementsFound
+		end
+		printf "Set size = %u\n", $tree_size
+	end
+end
+
+document pset
+	Prints std::set<T> or std::multiset<T> information. Works for std::multiset as well.
+	Syntax: pset <set> <T> <val>: Prints set size, if T defined all elements or just element(s) having val
+	Examples:
+	pset s - prints set size and definition
+	pset s int - prints all elements and the size of s
+	pset s int 20 - prints the element(s) with value = 20 (if any) and the size of s
+end
+
+
+
+#
+# std::dequeue
+#
+
+define pdequeue
+	if $argc == 0
+		help pdequeue
+	else
+		set $size = 0
+		set $start_cur = $arg0._M_impl._M_start._M_cur
+		set $start_last = $arg0._M_impl._M_start._M_last
+		set $start_stop = $start_last
+		while $start_cur != $start_stop
+			p *$start_cur
+			set $start_cur++
+			set $size++
+		end
+		set $finish_first = $arg0._M_impl._M_finish._M_first
+		set $finish_cur = $arg0._M_impl._M_finish._M_cur
+		set $finish_last = $arg0._M_impl._M_finish._M_last
+		if $finish_cur < $finish_last
+			set $finish_stop = $finish_cur
+		else
+			set $finish_stop = $finish_last
+		end
+		while $finish_first != $finish_stop
+			p *$finish_first
+			set $finish_first++
+			set $size++
+		end
+		printf "Dequeue size = %u\n", $size
+	end
+end
+
+document pdequeue
+	Prints std::dequeue<T> information.
+	Syntax: pdequeue <dequeue>: Prints dequeue size, if T defined all elements
+	Deque elements are listed "left to right" (left-most stands for front and right-most stands for back)
+	Example:
+	pdequeue d - prints all elements and size of d
+end
+
+
+
+#
+# std::stack
+#
+
+define pstack
+	if $argc == 0
+		help pstack
+	else
+		set $start_cur = $arg0.c._M_impl._M_start._M_cur
+		set $finish_cur = $arg0.c._M_impl._M_finish._M_cur
+		set $size = $finish_cur - $start_cur
+        set $i = $size - 1
+        while $i >= 0
+            p *($start_cur + $i)
+            set $i--
+        end
+		printf "Stack size = %u\n", $size
+	end
+end
+
+document pstack
+	Prints std::stack<T> information.
+	Syntax: pstack <stack>: Prints all elements and size of the stack
+	Stack elements are listed "top to buttom" (top-most element is the first to come on pop)
+	Example:
+	pstack s - prints all elements and the size of s
+end
+
+
+
+#
+# std::queue
+#
+
+define pqueue
+	if $argc == 0
+		help pqueue
+	else
+		set $start_cur = $arg0.c._M_impl._M_start._M_cur
+		set $finish_cur = $arg0.c._M_impl._M_finish._M_cur
+		set $size = $finish_cur - $start_cur
+        set $i = 0
+        while $i < $size
+            p *($start_cur + $i)
+            set $i++
+        end
+		printf "Queue size = %u\n", $size
+	end
+end
+
+document pqueue
+	Prints std::queue<T> information.
+	Syntax: pqueue <queue>: Prints all elements and the size of the queue
+	Queue elements are listed "top to bottom" (top-most element is the first to come on pop)
+	Example:
+	pqueue q - prints all elements and the size of q
+end
+
+
+
+#
+# std::priority_queue
+#
+
+define ppqueue
+	if $argc == 0
+		help ppqueue
+	else
+		set $size = $arg0.c._M_impl._M_finish - $arg0.c._M_impl._M_start
+		set $capacity = $arg0.c._M_impl._M_end_of_storage - $arg0.c._M_impl._M_start
+		set $i = $size - 1
+		while $i >= 0
+			p *($arg0.c._M_impl._M_start + $i)
+			set $i--
+		end
+		printf "Priority queue size = %u\n", $size
+		printf "Priority queue capacity = %u\n", $capacity
+	end
+end
+
+document ppqueue
+	Prints std::priority_queue<T> information.
+	Syntax: ppqueue <priority_queue>: Prints all elements, size and capacity of the priority_queue
+	Priority_queue elements are listed "top to buttom" (top-most element is the first to come on pop)
+	Example:
+	ppqueue pq - prints all elements, size and capacity of pq
+end
+
+
+
+#
+# std::bitset
+#
+
+define pbitset
+	if $argc == 0
+		help pbitset
+	else
+        p /t $arg0._M_w
+	end
+end
+
+document pbitset
+	Prints std::bitset<n> information.
+	Syntax: pbitset <bitset>: Prints all bits in bitset
+	Example:
+	pbitset b - prints all bits in b
+end
+
+
+
+#
+# std::string
+#
+
+define pstring
+	if $argc == 0
+		help pstring
+	else
+		printf "String \t\t\t= \"%s\"\n", $arg0._M_data()
+		printf "String size/length \t= %u\n", $arg0._M_rep()._M_length
+		printf "String capacity \t= %u\n", $arg0._M_rep()._M_capacity
+		printf "String ref-count \t= %d\n", $arg0._M_rep()._M_refcount
+	end
+end
+
+document pstring
+	Prints std::string information.
+	Syntax: pstring <string>
+	Example:
+	pstring s - Prints content, size/length, capacity and ref-count of string s
+end 
+
+#
+# std::wstring
+#
+
+define pwstring
+	if $argc == 0
+		help pwstring
+	else
+		call printf("WString \t\t= \"%ls\"\n", $arg0._M_data())
+		printf "WString size/length \t= %u\n", $arg0._M_rep()._M_length
+		printf "WString capacity \t= %u\n", $arg0._M_rep()._M_capacity
+		printf "WString ref-count \t= %d\n", $arg0._M_rep()._M_refcount
+	end
+end
+
+document pwstring
+	Prints std::wstring information.
+	Syntax: pwstring <wstring>
+	Example:
+	pwstring s - Prints content, size/length, capacity and ref-count of wstring s
+end 
+
+#
+# C++ related beautifiers (optional)
+#
+
+set print pretty on
+set print object on
+set print static-members on
+set print vtbl on
+set print demangle on
+set demangle-style gnu-v3
+set print sevenbit-strings off
+
+set follow-fork-mode child
+set detach-on-fork off

paddle/fluid/train/custom_trainer/feed/tool/ins_weight.py

+#!/usr/bin/python
+import sys
+import re
+import math
+
+del_text_slot = True
+g_ratio = 1
+w_ratio = 0.01
+slots_str = "6048 6145 6202 6201 6121 6119 6146 6120 6147 6122 6123 6118 6142 6143 6008 6148 6151 6127 6144 6150 6109 6003 6096 6149 6129 6203 6153 6152 6128 6106 6251 7082 7515 7080 6066 7507 6186 6007 7514 6054 6125 7506 10001 6006 6080 7023 6085 10000 6250 6110 6124 6090 6082 6067 7516 6101 6004 6191 6188 6070 6194 6247 6814 7512 10007 6058 6189 6059 7517 10005 7510 7024 7502 7503 6183 7511 6060 6806 7504 6185 6810 6248 10004 6815 6182 10068 6069 6073 6196 6816 7513 6071 6809 6072 6817 6190 7505 6813 6192 6807 6808 6195 6826 6184 6197 6068 6812 7107 6811 6823 6824 6819 6818 6821 6822 6820 6094 6083 6952 6099 6951 6949 6098 7075 6948 6157 6126 7077 6111 6087 6103 6107 6156 6005 6158 7122 6155 7058 6115 7079 7081 6833 6108 6840 6837 7147 7129 6097 6231 6957 7145 6956 7143 6130 7149 7142 6212 6827 7144 6089 6161 7055 6233 6105 7057 6237 6828 6850 6163 7124 6354 6162 7146 6830 7123 6160 6235 7056 6081 6841 6132 6954 6131 6236 6831 6845 6832 6953 6839 6950 7125 7054 6138 6166 6076 6851 6353 7076 7148 6858 6842 6860 7126 6829 6835 7078 6866 6869 6871 7052 6134 6855 6947 6862 6215 6852 7128 6092 6112 6213 6232 6863 6113 6165 6214 6216 6873 6865 6870 6077 6234 6861 6164 6217 7127 6218 6962 7053 7051 6961 6002 6738 6739 10105 7064 6751 6770 7100 6014 6765 6755 10021 10022 6010 10056 6011 6756 10055 6768 10024 6023 10003 6769 10002 6767 6759 10018 6024 6064 6012 6050 10042 6168 6253 10010 10020 6015 6018 10033 10041 10039 10031 10016 6764 7083 7152 7066 6171 7150 7085 6255 10044 10008 7102 6167 6240 6238 6095 10017 10046 6019 6031 6763 6256 6169 6254 10034 7108 7186 6257 10019 6757 10040 6025 7019 7086 10029 10011 7104 6261 6013 6766 10106 7105 7153 7089 6057 7134 7151 7045 7005 7008 7101 6035 7137 10023 6036 6172 7099 7087 6239 7185 6170 10006 6243 6350 7103 7090 7157 6259 7171 6875 7084 7154 6242 6260 7155 7017 7048 7156 6959 7047 10053 7135 6244 7136 10030 7063 6760 7016 7065 7179 6881 7018 6876 10081 10052 10054 10038 6886 10069 7004 10051 7007 7109 10057 6029 6888 10009 6889 7021 10047 6245 6878 10067 6879 6884 7180 7182 10071 7002 6880 6890 6887 10061 6027 6877 6892 10060 6893 7050 10036 7049 10012 10025 7012 7183 10058 7181 10086 6891 6258 6894 6883 7046 6037 7106 10043 10048 10045 10087 6885 10013 10028 7187 10037 10035 10050 6895 7011 7170 7172 10026 10063 10095 10082 10084 6960 10092 10075 6038 7010 7015 10015 10027 10064 7184 10014 10059 7013 7020 10072 10066 10080 6896 10083 10090 6039 10049 7164 7165 10091 10099 6963 7166 10079 10103 7006 7009 7169 6034 7028 7029 7030 7034 7035 7036 7040 7041 7042 10032 6009 6241 7003 7014 7088 13326 13330 13331 13352 13353 6198"
+slot_whitelist = slots_str.split(" ")
+
+def calc_ins_weight(params, label):
+    """calc ins weight"""
+    global g_ratio
+    global w_ratio
+    slots = []
+    s_clk_num = 0
+    s_show_num = 0
+    active = 0
+    attclk_num = 0
+    attshow_num = 0
+    attclk_avg = 0
+    for items in params:
+        if len(items) != 2:
+            continue
+        slot_name = items[0]
+        slot_val = items[1]
+        if slot_name not in slots:
+            slots.append(slot_name)
+        if slot_name == "session_click_num":
+            s_clk_num = int(slot_val)
+        if slot_name == "session_show_num":
+            s_show_num = int(slot_val)
+        if slot_name == "activity":
+            active = float(slot_val) / 10000.0
+    w = 1
+    # for inactive user 
+    if active >= 0 and active < 0.4 and s_show_num >=0 and s_show_num < 20:
+        w = math.log(w_ratio * (420 - (active * 50 + 1) * (s_show_num + 1)) + math.e)
+        if label == "0":
+            w = 1 + (w - 1) * g_ratio
+    return w
+
+def filter_whitelist_slot(tmp_line):
+    terms = tmp_line.split()
+    line = "%s %s %s" % (terms[0], terms[1], terms[2])
+    for item in terms[3:]:
+        feasign = item.split(':')
+        if len(feasign) == 2 and \
+            feasign[1] in slot_whitelist:
+            line = "%s %s" %(line, item)
+    return line
+
+def get_sample_type(line):
+    # vertical_type = 20
+    # if line.find("13038012583501790:6738") > 0:
+    #     return 30
+    # vertical_type = 0/5/1/2/9/11/13/16/29/-1
+    if (line.find("7408512894065610:6738") > 0) or \
+        (line.find("8815887816424655:6738") > 0) or \
+        (line.find("7689987878537419:6738") > 0) or \
+        (line.find("7971462863009228:6738") > 0) or \
+        (line.find("9941787754311891:6738") > 0) or \
+        (line.find("10504737723255509:6738") > 0) or \
+        (line.find("11067687692199127:6738") > 0) or \
+        (line.find("11912112645614554:6738") > 0) or \
+        (line.find("15571287443748071:6738") > 0) or \
+        (line.find("7127025017546227:6738") > 0): 
+        return 20
+    return -1
+
+def main():
+    """ins adjust"""
+    global del_text_slot
+    for l in sys.stdin:
+        l = l.rstrip("\n")
+        items = l.split(" ")
+        if len(items) < 3:
+            continue
+        label = items[2]
+        lines = l.split("\t")
+        line = lines[0]
+        # streaming ins include all ins, sample_type only handle NEWS ins
+        sample_type = -1
+        if 'NEWS' in l:
+            sample_type = get_sample_type(line)
+        #line = filter_whitelist_slot(tmp_line)
+        if len(lines) >= 4:
+            if 'VIDEO' in lines[3]:
+                continue
+            params = lines[2]
+            params = params.split(" ")
+            m = [tuple(i.split(":")) for i in params]
+            if m is None or len(m) == 0:
+                if sample_type > 0:
+                    print "%s $%s *1" % (line, sample_type)
+                else:
+                    print "%s *1" % line
+                sys.stdout.flush()
+                continue
+            weight = calc_ins_weight(m, label)
+            if sample_type > 0:
+                print "%s $%s *%s" % (line, sample_type, weight)
+            else:
+                print "%s *%s" % (line, weight)
+            sys.stdout.flush()
+        else:
+            if sample_type > 0:
+                print "%s $%s *1" % (line, sample_type)
+            else:
+                print "%s *1" % line
+            sys.stdout.flush()
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "0":
+            del_text_slot = False
+        if len(sys.argv) > 2:
+            g_ratio = float(sys.argv[2])
+        if len(sys.argv) > 3:
+            w_ratio = float(sys.argv[3])
+    main()

paddle/fluid/train/custom_trainer/feed/tool/xbox_compressor_mf.py

+#!/usr/bin/python
+"""
+xbox model compressor
+"""
+
+import sys
+import math
+import time
+import re
+
+#WISE
+#SHOW_COMPRESS_RATIO : 8192
+#CLICK_COMPRESS_RATIO : 8192
+#LR_COMPRESS_RATIO : 1048576
+#MIO_COMPRESS_RATIO:8192
+
+#PC
+#MIO_COMPRESS_RATIO : 1024
+#SHOW_COMPRESS_RATIO : 128
+#CLICK_COMPRESS_RATIO : 1024
+#LR_COMPRESS_RATIO : 8192
+
+#STAMP_COL = 2
+SHOW_COL = 3
+CLICK_COL = 4
+LR_W_COL = 5
+LR_G2SUM_COL = 6
+FM_COL = 9
+
+#DAY_SPAN = 300
+
+#show clk lr = float
+SHOW_RATIO = 1
+#SHOW_RATIO = 1024
+CLK_RATIO = 8
+#CLK_RATIO = 1024
+LR_RATIO = 1024
+MF_RATIO = 1024
+
+base_update_threshold=0.965
+base_xbox_clk_cof=1
+base_xbox_nonclk_cof=0.2
+
+def as_num(x):
+    y='{:.5f}'.format(x)
+    return(y)
+
+def compress_show(xx):
+    """
+    compress show
+    """
+    preci = SHOW_RATIO
+
+    x = float(xx)
+    return str(int(math.floor(x * preci + 0.5)))
+
+
+def compress_clk(xx):
+    """
+    compress clk
+    """
+    preci = CLK_RATIO
+
+    x = float(xx)
+    clk = int(math.floor(x * preci + 0.5))
+    if clk == 0:
+        return ""
+    return str(clk)
+
+
+def compress_lr(xx):
+    """
+    compress lr
+    """
+    preci = LR_RATIO
+
+    x = float(xx)
+    lr = int(math.floor(x * preci + 0.5))
+    if lr == 0:
+        return ""
+    return str(lr)
+
+def compress_mf(xx):
+    """
+    compress mf
+    """
+    preci = MF_RATIO
+
+    x = float(xx)
+    return int(math.floor(x * preci + 0.5))
+
+
+def show_clk_score(show, clk):
+    """
+    calculate show_clk score
+    """
+    return (show - clk) * 0.2 + clk
+
+
+for l in sys.stdin:
+    cols = re.split(r'\s+', l.strip())
+    key = cols[0].strip()
+
+    #day = int(cols[STAMP_COL].strip())
+    #cur_day = int(time.time()/3600/24)
+    #if (day + DAY_SPAN) <= cur_day:
+    #    continue
+
+    # cvm features
+    show = cols[SHOW_COL]
+    click = cols[CLICK_COL]
+    pred = ""
+
+    f_show = float(show)
+    f_clk = float(click)
+    """
+    if f_show != 0:
+        show_log = math.log(f_show)
+    else:
+        show_log = 0
+
+    if f_clk != 0:
+        click_log =  math.log(f_clk) - show_log
+    else:
+        click_log = 0
+    """
+    show_log = f_show
+    click_log = f_clk
+    #print f_show, f_clk
+    #if show_clk_score(f_show, f_clk) < base_update_threshold:
+    #    continue
+
+    #show = compress_show(show)
+    show = compress_show(show_log)
+    #clk = compress_clk(click)
+    clk = compress_clk(click_log)
+
+    # personal lr weight
+    lr_w = cols[LR_W_COL].strip()
+    lr_wei = compress_lr(lr_w)
+
+    # fm weight
+    fm_wei = []
+    fm_sum = 0
+    if len(cols) > 7:
+    #fm_dim = int(cols[FM_COL].strip())
+    #if fm_dim != 0:
+        for v in xrange(FM_COL, len(cols), 1):
+            mf_v = compress_mf(cols[v])
+            #print mf_v
+            fm_wei.append(str(mf_v))
+            fm_sum += (mf_v * mf_v)
+
+    sys.stdout.write("%s\t%s\t%s\t%s" % (key, show, clk, pred))
+    sys.stdout.write("\t")
+    sys.stdout.write("%s" % lr_wei)
+    if len(fm_wei) > 0 and fm_sum > 0:
+        sys.stdout.write("\t%s" % "\t".join(fm_wei))
+    else:
+        sys.stdout.write("\t[\t]")
+    sys.stdout.write("\n")
+

paddle/fluid/train/custom_trainer/feed/tool/xbox_decompressor_mf.awk

+#!/bin/awk -f
+{
+    OFS="\t";
+    SHOW_RATIO = 1;
+    CLK_RATIO = 8;
+    LR_RATIO = 1024;
+    MF_RATIO = 1024;
+}
+
+function decompress_show(x) {
+    x = x * 1.0 / SHOW_RATIO;
+    return x;
+}
+
+function decompress_clk(x) {
+    if (x == "") {
+        x = 0;
+    }
+    x = x * 1.0 / CLK_RATIO;
+    return x;
+}
+
+function decompress_lr(x) {
+    return x * 1.0 / LR_RATIO;
+}
+
+function decompress_mf(x) {
+    return x * 1.0 / MF_RATIO;
+}
+
+function show_clk_sore(show, clk, nonclk_coeff, clk_coeff) {
+    return (show - clk) * nonclk_coeff + clk * clk_coeff;
+}
+
+#key, show, clk, pred, lr_w, mf_w or [\t]
+{
+    l=split($0, a, "\t");
+
+    show = decompress_show(a[2]);
+    click = decompress_clk(a[3]);
+    lr = decompress_lr(a[5]);
+    printf("%s\t0\t0\t%s\t%s\t%s\t0\t", a[1], show, click, lr);
+    if (l == 7) {
+        printf("0\n");
+    } else {
+        printf("%d", l-5)
+        for(i = 6; i <= l; i++) {
+            printf("\t%s", decompress_mf(a[i]));
+        }
+        printf("\t0\n");
+    }
+}

paddle/fluid/train/custom_trainer/feed/trainer_context.h

@@ -2,8 +2,8 @@
 #include <string>
 #include <memory>
 #include <vector>
-#include <yaml-cpp/yaml.h>
 #include "paddle/fluid/platform/place.h"
+#include "paddle/fluid/train/custom_trainer/feed/common/yaml_helper.h"
 #include "paddle/fluid/train/custom_trainer/feed/common/pslib_warpper.h"
 #include "paddle/fluid/train/custom_trainer/feed/common/runtime_environment.h"
 
@@ -17,35 +17,43 @@ class Dataset;
 class FileSystem;
 class EpochAccessor;
 
+const uint32_t SecondsPerMin = 60;
+const uint32_t SecondsPerHour = 3600;
+const uint32_t SecondsPerDay = 24 * 3600;
+
 enum class ModelSaveWay {
     ModelSaveTrainCheckpoint = 0,
     ModelSaveInferenceDelta = 1,
     ModelSaveInferenceBase = 2
 };
 
-class TableMeta {
+class SignCacheDict {
 public:
-    TableMeta() {}
-    ~TableMeta() {}
-    int table_id() {
-        return _id;
+    int32_t sign2index(uint64_t sign) {
+        return -1;
+    }
+
+    uint64_t index2sign(int32_t index) {
+        return 0;
     }
-private:
-    int _id;
 };
 
 class TrainerContext {
 public:
-YAML::Node trainer_config;
-paddle::platform::CPUPlace cpu_place;
-
-std::shared_ptr<PSlib> pslib;
-std::shared_ptr<Dataset> dataset;                          //训练样本
-std::shared_ptr<FileSystem> file_system;                   //文件操作辅助类
-std::vector<TableMeta> params_table_list;                  //参数表
-std::shared_ptr<EpochAccessor> epoch_accessor;             //训练轮次控制
-std::shared_ptr<RuntimeEnvironment> environment;           //运行环境
-std::vector<std::shared_ptr<Process>> process_list;        //训练流程
+    inline paddle::ps::PSClient* ps_client() {
+        return pslib->ps_client();
+    }
+
+    YAML::Node trainer_config;
+    paddle::platform::CPUPlace cpu_place;
+
+    std::shared_ptr<PSlib> pslib;
+    std::shared_ptr<Dataset> dataset;                          //训练样本
+    std::shared_ptr<FileSystem> file_system;                   //文件操作辅助类
+    std::shared_ptr<EpochAccessor> epoch_accessor;             //训练轮次控制
+    std::shared_ptr<RuntimeEnvironment> environment;           //运行环境
+    std::vector<std::shared_ptr<Process>> process_list;        //训练流程
+    std::shared_ptr<SignCacheDict> cache_dict;                 //大模型cache词典
 };
 
 }  // namespace feed

paddle/fluid/train/custom_trainer/feed/unit_test/test_create_programs.cc

 #include <iostream>
 #include <fstream>
 #include <gtest/gtest.h>
+#include <cstdlib>
+#include <cmath>
 
 #include "paddle/fluid/train/custom_trainer/feed/executor/executor.h"
 #include "paddle/fluid/framework/tensor_util.h"
@@ -51,6 +53,11 @@ public:
         context_ptr = nullptr;
     }
 
+    float random(float min_x = 0.0, float max_x = 1.0) {
+        float r = static_cast<float>(rand()) / RAND_MAX;
+        return min_x + (max_x - min_x) * r;
+    }
+
     std::shared_ptr<TrainerContext> context_ptr;
 };
 
@@ -61,32 +68,43 @@ TEST_F(CreateProgramsTest, example_network) {
     auto config = YAML::Load(string::format_string("{thread_num: 2, startup_program: %s, main_program: %s}", startup_program_path, main_program_path));
     auto model_desc = YAML::LoadFile(model_desc_path);
     ASSERT_EQ(0, executor->initialize(config, context_ptr));
-    
+
     std::string input_name = "cvm_input";
+    std::string loss_name = "loss_ctr";
+    std::string label_name = "label_ctr";
+
+    // loss
+    ASSERT_TRUE(model_desc["loss"]);
+    ASSERT_EQ(loss_name, model_desc["loss"].as<std::string>());
+
+    // input
     ASSERT_TRUE(model_desc["inputs"]);
     ASSERT_EQ(1, model_desc["inputs"].size());
     ASSERT_TRUE(model_desc["inputs"][0]["name"]);
     ASSERT_TRUE(model_desc["inputs"][0]["shape"]);
     ASSERT_EQ(input_name, model_desc["inputs"][0]["name"].as<std::string>());
-    std::vector<int> input_shape = model_desc["inputs"][0]["shape"].as<std::vector<int>>(std::vector<int>());
+    auto input_shape = model_desc["inputs"][0]["shape"].as<std::vector<int>>(std::vector<int>());
     ASSERT_EQ(2, input_shape.size());
     ASSERT_EQ(-1, input_shape[0]);
     ASSERT_EQ(4488, input_shape[1]);
 
-    ASSERT_TRUE(model_desc["loss_all"]);
-    auto loss_all_name = model_desc["loss_all"].as<std::string>();
+    // label
+    ASSERT_TRUE(model_desc["labels"]);
+    ASSERT_EQ(1, model_desc["labels"].size());
+    ASSERT_TRUE(model_desc["labels"][0]["name"]);
+    ASSERT_TRUE(model_desc["labels"][0]["shape"]);
+    ASSERT_EQ(label_name, model_desc["labels"][0]["name"].as<std::string>());
+    auto label_shape = model_desc["labels"][0]["shape"].as<std::vector<int>>(std::vector<int>());
+    ASSERT_EQ(2, label_shape.size());
+    ASSERT_EQ(-1, label_shape[0]);
+    ASSERT_EQ(1, label_shape[1]);
 
     ASSERT_TRUE(model_desc["outputs"]);
     ASSERT_EQ(1, model_desc["outputs"].size());
     ASSERT_TRUE(model_desc["outputs"][0]["name"]);
     ASSERT_TRUE(model_desc["outputs"][0]["shape"]);
-    ASSERT_TRUE(model_desc["outputs"][0]["label_name"]);
-    ASSERT_TRUE(model_desc["outputs"][0]["loss_name"]);
-
-    auto ctr_output_label_name = model_desc["outputs"][0]["label_name"].as<std::string>();
-    auto ctr_output_loss_name = model_desc["outputs"][0]["loss_name"].as<std::string>();
-    auto ctr_output_name = model_desc["outputs"][0]["name"].as<std::string>();
-    std::vector<int> output_shape = model_desc["outputs"][0]["shape"].as<std::vector<int>>(std::vector<int>());
+    auto output_name = model_desc["outputs"][0]["name"].as<std::string>();
+    auto output_shape = model_desc["outputs"][0]["shape"].as<std::vector<int>>(std::vector<int>());
     ASSERT_EQ(2, output_shape.size());
     ASSERT_EQ(-1, output_shape[0]);
     ASSERT_EQ(1, output_shape[1]);
@@ -94,7 +112,7 @@ TEST_F(CreateProgramsTest, example_network) {
     paddle::framework::Scope scope;
     executor->initialize_scope(&scope);
     auto input_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, input_name);
-    auto label_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, ctr_output_label_name);
+    auto label_var = ScopeHelper::mutable_var<::paddle::framework::LoDTensor>(&scope, label_name);
     ASSERT_NE(nullptr, input_var);
     ASSERT_NE(nullptr, label_var);
 
@@ -102,28 +120,26 @@ TEST_F(CreateProgramsTest, example_network) {
     auto input_data = input_var->mutable_data<float>(context_ptr->cpu_place);
     ASSERT_NE(nullptr, input_data);
     for (int i = 0; i < input_shape[1]; ++i) {
-        input_data[i] = 0.1;
+        input_data[i] = random();
     }
 
     label_var->Resize({1, 1});
     auto label_data = label_var->mutable_data<float>(context_ptr->cpu_place);
     ASSERT_NE(nullptr, label_data);
-    label_data[0] = 0.5;
+    label_data[0] = random();
 
     ASSERT_EQ(0, executor->run(&scope));
 
-    auto loss_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, ctr_output_loss_name);
+    auto loss_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, loss_name);
     auto loss = loss_var.data<float>()[0];
 
-    auto loss_all_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, loss_all_name);
-    auto loss_all = loss_all_var.data<float>()[0];
-
-    auto ctr_output_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, ctr_output_name);
-    auto ctr_output = ctr_output_var.data<float>()[0];
+    auto output_var = ScopeHelper::var<::paddle::framework::LoDTensor>(&scope, output_name);
+    auto output = output_var.data<float>()[0];
 
-    std::cout << "loss: " << loss << std::endl;
-    std::cout << "ctr_output: " << ctr_output << std::endl;
-    ASSERT_NEAR(loss, loss_all, 1e-9);
+    LOG(INFO) << "loss: " << loss << std::endl;
+    LOG(INFO) << "label: " << label_data[0] << std::endl;
+    LOG(INFO) << "output: " << output << std::endl;
+    ASSERT_NEAR(loss, pow(output - label_data[0], 2), 1e-8);
 }
 
 }  // namespace feed

paddle/fluid/train/custom_trainer/feed/unit_test/test_datareader.cc

@@ -194,14 +194,14 @@ TEST_F(DataReaderTest, LineDataReader_FileSystem) {
             "file_system:\n"
             "    class: AutoFileSystem\n"
             "    file_systems:\n"
-            "        'afs:': &HDFS \n"
+            "        'afs': &HDFS \n"
             "            class: HadoopFileSystem\n"
             "            hdfs_command: 'hadoop fs'\n"
             "            ugis:\n"
             "                'default': 'feed_video,D3a0z8'\n"
             "                'xingtian.afs.baidu.com:9902': 'feed_video,D3a0z8'\n"
             "            \n"
-            "        'hdfs:': *HDFS\n");
+            "        'hdfs': *HDFS\n");
     ASSERT_EQ(0, data_reader->initialize(config, context_ptr));
     {
         auto data_file_list = data_reader->data_file_list(test_data_dir);