Only write the modified tablets to file system. (#304)

* Only write the modified tablets to file system.
Move some functions definitions from headers to cc files.

* Persist the data in the destructor
Use unordered_set first.

* Use -1 for the unknown.

visualdl/storage/storage.cc

@@ -21,6 +21,7 @@ Storage::Storage() {
   data_ = std::make_shared<storage::Storage>();
   tablets_ = std::make_shared<std::map<std::string, storage::Tablet>>();
   modes_ = std::make_shared<std::set<std::string>>();
+  modified_tablet_set_ = std::unordered_set<std::string>();
   time_t t;
   time(&t);
   data_->set_timestamp(t);
@@ -31,6 +32,10 @@ Storage::Storage(const Storage& other)
   dir_ = other.dir_;
 }
 
+Storage::~Storage() {
+    PersistToDisk();
+}
+
 void Storage::AddMode(const std::string& x) {
   // avoid duplicate modes.
   if (modes_->count(x) != 0) return;
@@ -43,11 +48,20 @@ Tablet Storage::AddTablet(const std::string& x) {
   CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed";
   (*tablets_)[x] = storage::Tablet();
   AddTag(x);
+  MarkTabletModified(x);
   // WRITE_GUARD
   PersistToDisk();
   return Tablet(&(*tablets_)[x], this);
 }
 
+void Storage::SetDir(const std::string& dir) {
+    *dir_ = dir;
+}
+
+std::string Storage::dir() const {
+    return *dir_;
+}
+
 void Storage::PersistToDisk() { PersistToDisk(*dir_); }
 
 void Storage::PersistToDisk(const std::string& dir) {
@@ -56,12 +70,29 @@ void Storage::PersistToDisk(const std::string& dir) {
 
   fs::SerializeToFile(*data_, meta_path(dir));
   for (auto tag : data_->tags()) {
-    auto it = tablets_->find(tag);
-    CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
-    fs::SerializeToFile(it->second, tablet_path(dir, tag));
+    if (modified_tablet_set_.count(tag) > 0){
+        auto it = tablets_->find(tag);
+        CHECK(it != tablets_->end()) << "tag " << tag << " not exist.";
+        fs::SerializeToFile(it->second, tablet_path(dir, tag));
+    }
   }
+  modified_tablet_set_.clear();
 }
 
+Storage* Storage::parent() {
+    return this;
+}
+
+void Storage::MarkTabletModified(const std::string tag) {
+    modified_tablet_set_.insert(tag);
+}
+
+void Storage::AddTag(const std::string& x) {
+    *data_->add_tags() = x;
+    WRITE_GUARD
+  }
+
+// StorageReader
 std::vector<std::string> StorageReader::all_tags() {
   storage::Storage storage;
   Reload(storage);

visualdl/storage/storage.h

@@ -58,7 +58,7 @@ struct Storage {
 
   Storage();
   Storage(const Storage& other);
-
+  ~Storage();
   // Add a mode. Mode is similar to TB's FileWriter, It can be "train" or "test"
   // or something else.
   void AddMode(const std::string& x);
@@ -67,8 +67,8 @@ struct Storage {
   Tablet AddTablet(const std::string& x);
 
   // Set storage's directory.
-  void SetDir(const std::string& dir) { *dir_ = dir; }
-  std::string dir() const { return *dir_; }
+  void SetDir(const std::string& dir);
+  std::string dir() const;
 
   // Save content in memory to `dir_`.
   void PersistToDisk();
@@ -77,20 +77,20 @@ struct Storage {
   void PersistToDisk(const std::string& dir);
 
   // A trick help to retrieve the storage's `SimpleSyncMeta`.
-  Storage* parent() { return this; }
+  Storage* parent();
+
+  void MarkTabletModified(const std::string tag);
 
 protected:
   // Add a tag which content is `x`.
-  void AddTag(const std::string& x) {
-    *data_->add_tags() = x;
-    WRITE_GUARD
-  }
+  void AddTag(const std::string& x);
 
 private:
   std::shared_ptr<std::string> dir_;
   std::shared_ptr<std::map<std::string, storage::Tablet>> tablets_;
   std::shared_ptr<storage::Storage> data_;
   std::shared_ptr<std::set<std::string>> modes_;
+  std::unordered_set<std::string> modified_tablet_set_;
 };
 
 // Storage reader, each method will trigger a reading from disk.

visualdl/storage/tablet.cc

@@ -13,9 +13,25 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "visualdl/storage/tablet.h"
+#include "visualdl/storage/storage.h"
 
 namespace visualdl {
 
+void Tablet::SetTag(const std::string& mode, const std::string& tag) {
+    auto internal_tag = mode + "/" + tag;
+    string::TagEncode(internal_tag);
+    internal_encoded_tag_ = internal_tag;
+    data_->set_tag(internal_tag);
+    WRITE_GUARD
+}
+
+Record Tablet::AddRecord() {
+    parent()->MarkTabletModified(internal_encoded_tag_);
+    IncTotalRecords();
+    WRITE_GUARD
+    return Record(data_->add_records(), parent());
+}
+
 TabletReader Tablet::reader() { return TabletReader(*data_); }
 
 }  // namespace visualdl

visualdl/storage/tablet.h

@@ -29,11 +29,11 @@ struct TabletReader;
  * Tablet is a helper for operations on storage::Tablet.
  */
 struct Tablet {
-  enum Type { kScalar = 0, kHistogram = 1, kImage = 2 };
+  enum Type { kScalar = 0, kHistogram = 1, kImage = 2, kUnknown = -1};
 
   DECL_GUARD(Tablet);
 
-  Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent) {}
+  Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent), internal_encoded_tag_("") {}
 
   static Type type(const std::string& name) {
     if (name == "scalar") {
@@ -46,6 +46,7 @@ struct Tablet {
       return kImage;
     }
     LOG(ERROR) << "unknown component: " << name;
+    return kUnknown;
   }
 
   // write operations.
@@ -59,18 +60,8 @@ struct Tablet {
     WRITE_GUARD
   }
 
-  void SetTag(const std::string& mode, const std::string& tag) {
-    auto internal_tag = mode + "/" + tag;
-    string::TagEncode(internal_tag);
-    data_->set_tag(internal_tag);
-    WRITE_GUARD
-  }
-
-  Record AddRecord() {
-    IncTotalRecords();
-    WRITE_GUARD
-    return Record(data_->add_records(), parent());
-  }
+  void SetTag(const std::string& mode, const std::string& tag);
+  Record AddRecord();
 
   template <typename T>
   Entry MutableMeta() {
@@ -102,6 +93,7 @@ struct Tablet {
 private:
   Storage* x_;
   storage::Tablet* data_{nullptr};
+  std::string internal_encoded_tag_;
 };
 
 /*