refactor the serialize or deserialize to/from file

visualdl/backend/storage/storage.cc

@@ -6,12 +6,17 @@
 
 namespace visualdl {
 
-std::string GenPathFromTag(const std::string &dir, const std::string &tag) {
-  return dir + "/" + tag;
-}
-
 const std::string StorageBase::meta_file_name = "storage.meta";
 
+std::string StorageBase::meta_path() const {
+  CHECK(!storage_.dir().empty()) << "storage.dir should be set first";
+  return storage_.dir() + "/" + meta_file_name;
+}
+std::string StorageBase::tablet_path(const std::string &tag) const {
+  CHECK(!storage_.dir().empty()) << "storage.dir should be set first";
+  return storage_.dir() + "/" + tag;
+}
+
 storage::Tablet *MemoryStorage::NewTablet(const std::string &tag,
                                           int num_samples) {
   auto it = tablets_.find(tag);
@@ -32,38 +37,38 @@ storage::Tablet *MemoryStorage::tablet(const std::string &tag) {
   return &it->second;
 }
 
+// TODO add some checksum to avoid unnecessary saving
 void MemoryStorage::PersistToDisk() const {
+  LOG(INFO) << "inside dir " << storage_.dir() << " "
+            << (!storage_.dir().empty());
+  CHECK(!storage_.dir().empty()) << "storage's dir should be set first";
+  LOG(INFO) << "after check dir";
   VLOG(3) << "persist storage to disk path " << storage_.dir();
   // make a directory if not exist
   fs::TryMkdir(storage_.dir());
   // write storage out
-  CHECK(!storage_.dir().empty()) << "storage's dir should be set first";
-  const auto meta_path = storage_.dir() + "/" + meta_file_name;
-  fs::Write(meta_path, fs::Serialize(storage_));
+  fs::SerializeToFile(storage_, meta_path());
   // write all the tablets
   for (auto tag : storage_.tags()) {
-    auto path = GenPathFromTag(storage_.dir(), tag);
     auto it = tablets_.find(tag);
     CHECK(it != tablets_.end());
-    fs::Write(path, fs::Serialize(it->second));
+    fs::SerializeToFile(it->second, tablet_path(tag));
   }
 }
 
+// TODO add some checksum to avoid unnecessary loading
 void MemoryStorage::LoadFromDisk(const std::string &dir) {
   VLOG(3) << "load storage from disk path " << dir;
   CHECK(!dir.empty()) << "dir is empty";
+  storage_.set_dir(dir);
   // load storage
-  const auto meta_path = dir + "/" + meta_file_name;
-  auto buf = fs::Read(meta_path);
-  CHECK(fs::DeSerialize(&storage_, buf))
-      << "failed to parse protobuf loaded from " << meta_path;
+  CHECK(fs::DeSerializeFromFile(&storage_, meta_path()))
+      << "parse from " << meta_path() << " failed";
 
   // load all the tablets
   for (int i = 0; i < storage_.tags_size(); i++) {
-    std::string tag = storage_.tags(i);
-    auto path = GenPathFromTag(storage_.dir(), tag);
-    CHECK(tablets_[tag].ParseFromString(fs::Read(path)))
-        << "failed to parse protobuf text loaded from " << path;
+    auto tag = storage_.tags(i);
+    CHECK(fs::DeSerializeFromFile(&tablets_[tag], tablet_path(tag)));
   }
 }
 

visualdl/backend/storage/storage.h

@@ -34,6 +34,9 @@ public:
     storage_.set_dir(dir);
   }
 
+  std::string meta_path() const;
+  std::string tablet_path(const std::string &tag) const;
+
   /*
    * Create a new Tablet storage.
    */
@@ -78,6 +81,18 @@ public:
 
   void LoadFromDisk(const std::string &dir) override;
 
+  /*
+   * Create a thread which will keep reading the latest data from the disk to
+   * memory.
+   */
+  void StartReadService();
+
+  /*
+   * Create a thread which will keep writing the latest changes from memory to
+   * disk.
+   */
+  void StartWriteSerice();
+
 private:
   std::map<std::string, storage::Tablet> tablets_;
 };

visualdl/backend/storage/storage_test.cc

@@ -32,6 +32,8 @@ TEST_F(MemoryStorageTest, AddTablet) {
 }
 
 TEST_F(MemoryStorageTest, PersistToDisk) {
+  storage_.SetStorage("./tmp");
+  CHECK(!storage_.data().dir().empty());
   string tag = "add%20tag0";
   storage_.NewTablet(tag, -1);
 

visualdl/backend/utils/filesystem.h

@@ -30,6 +30,18 @@ bool DeSerialize(T* proto, const std::string buf, bool human_readable = false) {
   return proto->ParseFromString(buf);
 }
 
+template <typename T>
+bool SerializeToFile(const T& proto, const std::string& path) {
+  std::ofstream file(path, std::ios::binary);
+  return proto.SerializeToOstream(&file);
+}
+
+template <typename T>
+bool DeSerializeFromFile(T* proto, const std::string& path) {
+  std::ifstream file(path, std::ios::binary);
+  return proto->ParseFromIstream(&file);
+}
+
 void TryMkdir(const std::string& dir) {
   VLOG(1) << "try to mkdir " << dir;
   struct stat st = {0};

visualdl/backend/utils/log.h

+#ifndef VISUALDL_BACKEND_UTILS_LOG_H
+#define VISUALDL_BACKEND_UTILS_LOG_H
+
+#include <stdexcept>
+namespace visualdl {
+
+namespace log {
+
+class NotImplementedException : public std::logic_error {
+public:
+  NotImplementedException() : std::logic_error{"Function not implemented"} {}
+};
+
+}  // namespace log
+
+}  // namespace visualdl
+
+#endif