diff --git a/visualdl/backend/storage/storage.cc b/visualdl/backend/storage/storage.cc index ca45d80142443ba86b4ed208242ed9b44b64c078..0350b5c3f26b4414fe644cb723e6bc6a328df030 100644 --- a/visualdl/backend/storage/storage.cc +++ b/visualdl/backend/storage/storage.cc @@ -6,12 +6,17 @@ namespace visualdl { -std::string GenPathFromTag(const std::string &dir, const std::string &tag) { - return dir + "/" + tag; -} - const std::string StorageBase::meta_file_name = "storage.meta"; +std::string StorageBase::meta_path() const { + CHECK(!storage_.dir().empty()) << "storage.dir should be set first"; + return storage_.dir() + "/" + meta_file_name; +} +std::string StorageBase::tablet_path(const std::string &tag) const { + CHECK(!storage_.dir().empty()) << "storage.dir should be set first"; + return storage_.dir() + "/" + tag; +} + storage::Tablet *MemoryStorage::NewTablet(const std::string &tag, int num_samples) { auto it = tablets_.find(tag); @@ -32,38 +37,38 @@ storage::Tablet *MemoryStorage::tablet(const std::string &tag) { return &it->second; } +// TODO add some checksum to avoid unnecessary saving void MemoryStorage::PersistToDisk() const { + LOG(INFO) << "inside dir " << storage_.dir() << " " + << (!storage_.dir().empty()); + CHECK(!storage_.dir().empty()) << "storage's dir should be set first"; + LOG(INFO) << "after check dir"; VLOG(3) << "persist storage to disk path " << storage_.dir(); // make a directory if not exist fs::TryMkdir(storage_.dir()); // write storage out - CHECK(!storage_.dir().empty()) << "storage's dir should be set first"; - const auto meta_path = storage_.dir() + "/" + meta_file_name; - fs::Write(meta_path, fs::Serialize(storage_)); + fs::SerializeToFile(storage_, meta_path()); // write all the tablets for (auto tag : storage_.tags()) { - auto path = GenPathFromTag(storage_.dir(), tag); auto it = tablets_.find(tag); CHECK(it != tablets_.end()); - fs::Write(path, fs::Serialize(it->second)); + fs::SerializeToFile(it->second, tablet_path(tag)); } } +// TODO add some checksum to avoid unnecessary loading void MemoryStorage::LoadFromDisk(const std::string &dir) { VLOG(3) << "load storage from disk path " << dir; CHECK(!dir.empty()) << "dir is empty"; + storage_.set_dir(dir); // load storage - const auto meta_path = dir + "/" + meta_file_name; - auto buf = fs::Read(meta_path); - CHECK(fs::DeSerialize(&storage_, buf)) - << "failed to parse protobuf loaded from " << meta_path; + CHECK(fs::DeSerializeFromFile(&storage_, meta_path())) + << "parse from " << meta_path() << " failed"; // load all the tablets for (int i = 0; i < storage_.tags_size(); i++) { - std::string tag = storage_.tags(i); - auto path = GenPathFromTag(storage_.dir(), tag); - CHECK(tablets_[tag].ParseFromString(fs::Read(path))) - << "failed to parse protobuf text loaded from " << path; + auto tag = storage_.tags(i); + CHECK(fs::DeSerializeFromFile(&tablets_[tag], tablet_path(tag))); } } diff --git a/visualdl/backend/storage/storage.h b/visualdl/backend/storage/storage.h index d9e8ae29a2481a7301b8e6740763abb91c76889c..12de7718c18d50fbe2164aafc4a6eb80e465b51f 100644 --- a/visualdl/backend/storage/storage.h +++ b/visualdl/backend/storage/storage.h @@ -34,6 +34,9 @@ public: storage_.set_dir(dir); } + std::string meta_path() const; + std::string tablet_path(const std::string &tag) const; + /* * Create a new Tablet storage. */ @@ -78,6 +81,18 @@ public: void LoadFromDisk(const std::string &dir) override; + /* + * Create a thread which will keep reading the latest data from the disk to + * memory. + */ + void StartReadService(); + + /* + * Create a thread which will keep writing the latest changes from memory to + * disk. + */ + void StartWriteSerice(); + private: std::map tablets_; }; diff --git a/visualdl/backend/storage/storage_test.cc b/visualdl/backend/storage/storage_test.cc index 6090d71f4b4ac88379f9eacce3a01f01434ecff9..91c2a59d47ed38f8f6a6eb5affc27305171bbf92 100644 --- a/visualdl/backend/storage/storage_test.cc +++ b/visualdl/backend/storage/storage_test.cc @@ -32,6 +32,8 @@ TEST_F(MemoryStorageTest, AddTablet) { } TEST_F(MemoryStorageTest, PersistToDisk) { + storage_.SetStorage("./tmp"); + CHECK(!storage_.data().dir().empty()); string tag = "add%20tag0"; storage_.NewTablet(tag, -1); diff --git a/visualdl/backend/utils/filesystem.h b/visualdl/backend/utils/filesystem.h index e30640b97bbf9509b78cc3ef55e7569bfd9bf7c8..9a509407c12816f77a2a97f1d2ed5e2972a7c531 100644 --- a/visualdl/backend/utils/filesystem.h +++ b/visualdl/backend/utils/filesystem.h @@ -30,6 +30,18 @@ bool DeSerialize(T* proto, const std::string buf, bool human_readable = false) { return proto->ParseFromString(buf); } +template +bool SerializeToFile(const T& proto, const std::string& path) { + std::ofstream file(path, std::ios::binary); + return proto.SerializeToOstream(&file); +} + +template +bool DeSerializeFromFile(T* proto, const std::string& path) { + std::ifstream file(path, std::ios::binary); + return proto->ParseFromIstream(&file); +} + void TryMkdir(const std::string& dir) { VLOG(1) << "try to mkdir " << dir; struct stat st = {0}; diff --git a/visualdl/backend/utils/log.h b/visualdl/backend/utils/log.h new file mode 100644 index 0000000000000000000000000000000000000000..9aed0806d905079b09e37734224bbf2ddb3b376c --- /dev/null +++ b/visualdl/backend/utils/log.h @@ -0,0 +1,18 @@ +#ifndef VISUALDL_BACKEND_UTILS_LOG_H +#define VISUALDL_BACKEND_UTILS_LOG_H + +#include +namespace visualdl { + +namespace log { + +class NotImplementedException : public std::logic_error { +public: + NotImplementedException() : std::logic_error{"Function not implemented"} {} +}; + +} // namespace log + +} // namespace visualdl + +#endif