add some doc

paddle/fluid/framework/data_set.cc

@@ -24,6 +24,7 @@
 namespace paddle {
 namespace framework {
 
+// constructor
 template <typename T>
 DatasetImpl<T>::DatasetImpl() {
   thread_num_ = 1;
@@ -31,37 +32,24 @@ DatasetImpl<T>::DatasetImpl() {
   file_idx_ = 0;
 }
 
+// set filelist, file_idx_ will reset to zero.
 template <typename T>
 void DatasetImpl<T>::SetFileList(const std::vector<std::string>& filelist) {
   VLOG(3) << "filelist size: " << filelist.size();
   filelist_ = filelist;
   file_idx_ = 0;
-  /*
-  int file_cnt = filelist_.size();
-  if (thread_num_ > file_cnt) {
-    VLOG(1) << "DataSet thread num = " << thread_num_
-            << ", file num = " << file_cnt
-            << ". Changing DataSet thread num = " << file_cnt;
-    thread_num_ = file_cnt;
-  }*/
 }
 
-// buggy here, a user should set filelist first before this function
-// not user friendly
+// set expect thread num. actually it may change
 template <typename T>
 void DatasetImpl<T>::SetThreadNum(int thread_num) {
   VLOG(3) << "SetThreadNum thread_num=" << thread_num;
-  //int file_cnt = filelist_.size();
-  /*
-  if (file_cnt != 0 && thread_num > file_cnt) {
-    VLOG(3) << "DataSet thread num = " << thread_num
-            << ", file num = " << file_cnt
-            << ". Changing DataSet thread num = " << file_cnt;
-    thread_num = file_cnt;
-  }*/
   thread_num_ = thread_num;
 }
 
+// if you run distributed, and want to do global shuffle,
+// set this before global shuffle.
+// be sure you call CreateReaders before SetTrainerNum
 template <typename T>
 void DatasetImpl<T>::SetTrainerNum(int trainer_num) {
   trainer_num_ = trainer_num;
@@ -86,12 +74,16 @@ void DatasetImpl<T>::SetDataFeedDesc(const std::string& data_feed_desc_str) {
                                                 &data_feed_desc_);
 }
 
+// readers_.size() may not be equal to thread_num_,
+// it changes when filelist_.size() < thread_num_
 template <typename T>
 std::vector<std::shared_ptr<paddle::framework::DataFeed>>&
 DatasetImpl<T>::GetReaders() {
   return readers_;
 }
 
+// load data into memory, Dataset hold this memory,
+// which will later be fed into readers' channel
 template <typename T>
 void DatasetImpl<T>::LoadIntoMemory() {
   VLOG(3) << "DatasetImpl<T>::LoadIntoMemory() begin";
@@ -114,6 +106,7 @@ void DatasetImpl<T>::LoadIntoMemory() {
           << ", cost time=" << timeline.ElapsedSec() << " seconds";
 }
 
+// do local shuffle
 template <typename T>
 void DatasetImpl<T>::LocalShuffle() {
   VLOG(3) << "DatasetImpl<T>::LocalShuffle() begin";

paddle/fluid/framework/data_set.h

@@ -26,6 +26,16 @@
 namespace paddle {
 namespace framework {
 
+// Dataset is a abstract class, which defines user interfaces
+// Example Usage:
+//    Dataset* dataset = DatasetFactory::CreateDataset("InMemoryDataset")
+//    dataset->SetFileList(std::vector<std::string>{"a.txt", "b.txt"})
+//    dataset->SetThreadNum(1)
+//    dataset->CreateReaders();
+//    dataset->SetDataFeedDesc(your_data_feed_desc);
+//    dataset->LoadIntoMemory();
+//    dataset->SetTrainerNum(2);
+//    dataset->GlobalShuffle();
 class Dataset {
  public:
   Dataset() {}
@@ -53,6 +63,8 @@ class Dataset {
                                 const std::string& msg) = 0;
 };
 
+// DatasetImpl is the implementation of Dataset,
+// it holds memory data if user calls load_into_memory
 template <typename T>
 class DatasetImpl : public Dataset {
  public:
@@ -95,6 +107,7 @@ class DatasetImpl : public Dataset {
   std::mutex mutex_for_pick_file_;
 };
 
+// use std::vector<MultiSlotType> as data type
 class MultiSlotDataset : public DatasetImpl<std::vector<MultiSlotType>> {
  public:
   MultiSlotDataset() {}

paddle/fluid/framework/fleet/fleet_wrapper.h

@@ -146,7 +146,9 @@ class FleetWrapper {
 
  private:
   static std::shared_ptr<FleetWrapper> s_instance_;
+#ifdef PADDLE_WITH_PSLIB
   std::map<uint64_t, std::vector<paddle::ps::Region>> _regions;
+#endif
 
  protected:
   static bool is_initialized_;