add Storage Interface and MemoryStorage baisc implementation

visualdl/backend/storage/storage.cc

@@ -2,9 +2,61 @@
 #include <fstream>
 
 #include "visualdl/backend/storage/storage.h"
+#include "visualdl/backend/utils/filesystem.h"
 
 namespace visualdl {
 
+std::string GenPathFromTag(const std::string &dir, const std::string &tag) {
+  return dir + "/" + tag;
+}
+
+const std::string StorageBase::meta_file_name = "storage.meta";
+
+storage::Tablet *MemoryStorage::NewTablet(const std::string &tag,
+                                          int num_samples) {
+  auto it = tablets_.find(tag);
+  if (it == tablets_.end()) {
+    // create new tablet
+    tablets_[tag] = storage::Tablet();
+    *storage_.add_tags() = tag;
+  } else {
+    return &it->second;
+  }
+  return &tablets_[tag];
+}
+
+storage::Tablet *MemoryStorage::tablet(const std::string &tag) {
+  auto it = tablets_.find(tag);
+  CHECK(it != tablets_.end()) << "tablet tagged as " << tag << " not exists";
+  return &it->second;
+}
+
+void MemoryStorage::PersistToDisk() const {
+  // write storage out
+  const auto meta_path = storage_.dir() + "/" + meta_file_name;
+  fs::Write(meta_path, storage_.SerializeAsString());
+  // write all the tablets
+  for (auto tag : storage_.tags()) {
+    auto path = GenPathFromTag(storage_.dir(), tag);
+    auto it = tablets_.find(tag);
+    CHECK(it != tablets_.end());
+    fs::Write(path, it->second.SerializeAsString());
+  }
+}
+
+void MemoryStorage::LoadFromDisk(const std::string &dir) {
+  // load storage
+  const auto meta_path = storage_.dir() + "/" + meta_file_name;
+  storage_.ParseFromString(fs::Read(meta_path));
+
+  // load all the tablets
+  for (auto tag : storage_.tags()) {
+    auto path = GenPathFromTag(storage_.dir(), tag);
+    tablets_[tag];
+    tablets_[tag].ParseFromString(fs::Read(path));
+  }
+}
+
 storage::Tablet *Storage::Add(const std::string &tag, int num_samples) {
   auto *tablet = &(*proto_.mutable_tablets())[tag];
   tablet->set_num_samples(num_samples);

visualdl/backend/storage/storage.h

 #ifndef VISUALDL_STORAGE_H
 #define VISUALDL_STORAGE_H
 
-#include <string>
 #include <time.h>
+#include <map>
+#include <string>
 
 #include "visualdl/backend/storage/storage.pb.h"
 
 namespace visualdl {
 
+/*
+ * Generate a tablet path in disk from its tag.
+ */
+inline std::string GenPathFromTag(const std::string &dir,
+                                  const std::string &tag);
+
+/*
+ * Storage Interface. The might be a bunch of implementations, for example, a
+ * MemStorage that keep a copy of all the taplets in memory, can be changed with
+ * a higher performance; a DiskStorage that keep all the data in disk, apply to
+ * the scenerios where memory consumption should be considered.
+ */
+class StorageBase {
+public:
+  const static std::string meta_file_name;
+
+  void SetStorage(const std::string &dir) {
+    time_t t;
+    time(&t);
+    storage_.set_timestamp(t);
+    storage_.set_dir(dir);
+  }
+
+  /*
+   * Create a new Tablet storage.
+   */
+  virtual storage::Tablet *NewTablet(const std::string &tag,
+                                     int num_samples) = 0;
+
+  /*
+   * Get a tablet from memory, this can be viewed as a cache, if the storage is
+   * in disk, a hash map in memory will first load the corresponding Tablet
+   * Protobuf from disk and hold all the changes.
+   */
+  virtual storage::Tablet *tablet(const std::string &tag) = 0;
+
+  /*
+   * Persist the data from cache to disk. Both the memory storage or disk
+   * storage should write changes to disk for persistence.
+   */
+  virtual void PersistToDisk() const = 0;
+
+  /*
+   * Load data from disk.
+   */
+  virtual void LoadFromDisk(const std::string &dir) = 0;
+
+protected:
+  storage::Storage storage_;
+};
+
+/*
+ * Storage in Memory, that will support quick edits on data.
+ */
+class MemoryStorage final : public StorageBase {
+public:
+  storage::Tablet *NewTablet(const std::string &tag, int num_samples) override;
+
+  storage::Tablet *tablet(const std::string &tag) override;
+
+  void PersistToDisk() const override;
+
+  void LoadFromDisk(const std::string &dir) override;
+
+private:
+  std::map<std::string, storage::Tablet> tablets_;
+};
+
 class Storage final {
 public:
   Storage() {

visualdl/backend/storage/storage.proto

@@ -130,7 +130,9 @@ The Storage stores all the records.
 */
 message Storage {
   // tags to Tablet, should be thread safe if fix the keys after initialization.
+  // TODO to delete in the new storage interface.
   map<string, Tablet> tablets = 1;
+  repeated string tags = 4;
   string dir = 2;
   int64 timestamp = 3;
 }