feature/refactor storage code (#54)

visualdl/storage/storage.cc

+#include "visualdl/storage/storage.h"
+
+namespace visualdl {
+
+Storage::Storage() {
+  dir_ = std::make_shared<std::string>();
+  data_ = std::make_shared<storage::Storage>();
+  tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
+  modes_ = std::make_shared<std::set<std::string>>();
+  time_t t;
+  time(&t);
+  data_->set_timestamp(t);
+}
+
+Storage::Storage(const Storage& other)
+    : data_(other.data_), tablets_(other.tablets_), modes_(other.modes_) {
+  dir_ = other.dir_;
+}
+
+void Storage::AddMode(const std::string& x) {
+  // avoid duplicate modes.
+  if (modes_->count(x) != 0) return;
+  *data_->add_modes() = x;
+  modes_->insert(x);
+  WRITE_GUARD
+}
+
+Tablet Storage::AddTablet(const std::string& x) {
+  CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
+  (*tablets_)[x] = storage::Tablet();
+  AddTag(x);
+  LOG(INFO) << "really add tag " << x;
+  // WRITE_GUARD
+  PersistToDisk();
+  return Tablet(&(*tablets_)[x], this);
+}
+
+void Storage::PersistToDisk() { PersistToDisk(*dir_); }
+
+void Storage::PersistToDisk(const std::string& dir) {
+  CHECK(!dir.empty()) << "dir should be set.";
+  fs::TryRecurMkdir(dir);
+
+  fs::SerializeToFile(*data_, meta_path(dir));
+  for (auto tag : data_->tags()) {
+    auto it = tablets_->find(tag);
+    CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
+    fs::SerializeToFile(it->second, tablet_path(dir, tag));
+  }
+}
+
+std::vector<std::string> StorageReader::all_tags() {
+  storage::Storage storage;
+  Reload(storage);
+  return std::vector<std::string>(storage.tags().begin(), storage.tags().end());
+}
+
+std::vector<std::string> StorageReader::tags(Tablet::Type component) {
+  auto tags = all_tags();
+  auto it =
+      std::remove_if(tags.begin(), tags.end(), [&](const std::string& tag) {
+        auto tb = tablet(tag);
+        return tb.type() != component;
+      });
+  tags.resize(it - tags.begin());
+  return tags;
+}
+
+std::vector<std::string> StorageReader::modes() {
+  storage::Storage storage;
+  Reload(storage);
+  return std::vector<std::string>(storage.modes().begin(),
+                                  storage.modes().end());
+}
+
+TabletReader StorageReader::tablet(const std::string& tag) const {
+  auto path = tablet_path(dir_, tag);
+  storage::Tablet tablet;
+  fs::DeSerializeFromFile(&tablet, path);
+  return TabletReader(tablet);
+}
+
+}  // namespace visualdl

visualdl/storage/storage.h

@@ -20,10 +20,13 @@ static std::string meta_path(const std::string& dir) {
   return dir + "/" + meta_file_name;
 }
 static std::string tablet_path(const std::string& dir, const std::string& tag) {
-  CHECK(!dir.empty()) << "dir should be set first";
+  CHECK(!dir.empty()) << "dir is empty";
   return dir + "/" + tag;
 }
 
+// A simple logic to sync storage between memory and disk. Each writing
+// operation will trigger an `Inc`, and check whether `ToSync`, if true, write
+// memory to disk.
 struct SimpleSyncMeta {
   void Inc() { counter++; }
 
@@ -33,78 +36,37 @@ struct SimpleSyncMeta {
   int cycle;
 };
 
-/*
- * Helper for operations on storage::Storage.
- */
+// Helper class for operations on storage::Storage.
 struct Storage {
   DECL_GUARD(Storage)
 
   mutable SimpleSyncMeta meta;
 
-  Storage() {
-    dir_ = std::make_shared<std::string>();
-    data_ = std::make_shared<storage::Storage>();
-    tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
-    modes_ = std::make_shared<std::set<std::string>>();
-    time_t t;
-    time(&t);
-    data_->set_timestamp(t);
-  }
-  Storage(const Storage& other)
-      : data_(other.data_), tablets_(other.tablets_), modes_(other.modes_) {
-    dir_ = other.dir_;
-  }
+  Storage();
+  Storage(const Storage& other);
 
-  // write operations
-  void AddMode(const std::string& x) {
-    // avoid duplicate modes.
-    if (modes_->count(x) != 0) return;
-    *data_->add_modes() = x;
-    modes_->insert(x);
-    WRITE_GUARD
-  }
+  // Add a mode. Mode is similar to TB's FileWriter, It can be "train" or "test"
+  // or something else.
+  void AddMode(const std::string& x);
 
-  Tablet AddTablet(const std::string& x) {
-    CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
-    (*tablets_)[x] = storage::Tablet();
-    AddTag(x);
-    LOG(INFO) << "really add tag " << x;
-    // WRITE_GUARD
-    PersistToDisk();
-    return Tablet(&(*tablets_)[x], this);
-  }
+  // Add a tablet which tag is `x`.
+  Tablet AddTablet(const std::string& x);
 
+  // Set storage's directory.
   void SetDir(const std::string& dir) { *dir_ = dir; }
   std::string dir() const { return *dir_; }
-  void PersistToDisk() {
-    CHECK(!dir_->empty()) << "dir should be set.";
-    fs::TryRecurMkdir(*dir_);
-
-    fs::SerializeToFile(*data_, meta_path(*dir_));
-    for (auto tag : data_->tags()) {
-      auto it = tablets_->find(tag);
-      CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
-      fs::SerializeToFile(it->second, tablet_path(*dir_, tag));
-    }
-  }
-  /*
-   * Save memory to disk.
-   */
-  void PersistToDisk(const std::string& dir) {
-    CHECK(!dir.empty()) << "dir should be set.";
-    fs::TryRecurMkdir(dir);
-
-    fs::SerializeToFile(*data_, meta_path(dir));
-    for (auto tag : data_->tags()) {
-      auto it = tablets_->find(tag);
-      CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
-      fs::SerializeToFile(it->second, tablet_path(dir, tag));
-    }
-  }
 
+  // Save content in memory to `dir_`.
+  void PersistToDisk();
+
+  // Save content in memory to `dir`.
+  void PersistToDisk(const std::string& dir);
+
+  // A trick help to retrieve the storage's `SimpleSyncMeta`.
   Storage* parent() { return this; }
 
 protected:
+  // Add a tag which content is `x`.
   void AddTag(const std::string& x) {
     *data_->add_tags() = x;
     WRITE_GUARD
@@ -117,44 +79,24 @@ private:
   std::shared_ptr<std::set<std::string>> modes_;
 };
 
-/*
- * Storage reader, each interface will trigger a read.
- */
+// Storage reader, each method will trigger a reading from disk.
 struct StorageReader {
   StorageReader(const std::string& dir) : dir_(dir) {}
 
-  // read operations
-  std::vector<std::string> all_tags() {
-    storage::Storage storage;
-    Reload(storage);
-    return std::vector<std::string>(storage.tags().begin(),
-                                    storage.tags().end());
-  }
-  std::vector<std::string> tags(Tablet::Type component) {
-    auto tags = all_tags();
-    auto it =
-        std::remove_if(tags.begin(), tags.end(), [&](const std::string& tag) {
-          auto tb = tablet(tag);
-          return tb.type() != component;
-        });
-    tags.resize(it - tags.begin());
-    return tags;
-  }
-  std::vector<std::string> modes() {
-    storage::Storage storage;
-    Reload(storage);
-    return std::vector<std::string>(storage.modes().begin(),
-                                    storage.modes().end());
-  }
+  // Get all tags of the storage.
+  std::vector<std::string> all_tags();
 
-  TabletReader tablet(const std::string& tag) const {
-    auto path = tablet_path(dir_, tag);
-    storage::Tablet tablet;
-    fs::DeSerializeFromFile(&tablet, path);
-    return TabletReader(tablet);
-  }
+  // Get all the tags of the tablet of the kind of `component`.
+  std::vector<std::string> tags(Tablet::Type component);
+
+  // Get all the modes of the storage.
+  std::vector<std::string> modes();
+
+  // Get a tablet whose tag is `tag`.
+  TabletReader tablet(const std::string& tag) const;
 
 protected:
+  // Load meta from disk to memory.
   void Reload(storage::Storage& storage) {
     const std::string path = meta_path(dir_);
     fs::DeSerializeFromFile(&storage, path);