diff --git a/visualdl/backend/storage/storage.cc b/visualdl/backend/storage/storage.cc index 7a27448c7b9b773897acbe3910c34b98129fd60e..8200e22cc694b7ba19c00bf097b0ee3261b35315 100644 --- a/visualdl/backend/storage/storage.cc +++ b/visualdl/backend/storage/storage.cc @@ -2,9 +2,61 @@ #include #include "visualdl/backend/storage/storage.h" +#include "visualdl/backend/utils/filesystem.h" namespace visualdl { +std::string GenPathFromTag(const std::string &dir, const std::string &tag) { + return dir + "/" + tag; +} + +const std::string StorageBase::meta_file_name = "storage.meta"; + +storage::Tablet *MemoryStorage::NewTablet(const std::string &tag, + int num_samples) { + auto it = tablets_.find(tag); + if (it == tablets_.end()) { + // create new tablet + tablets_[tag] = storage::Tablet(); + *storage_.add_tags() = tag; + } else { + return &it->second; + } + return &tablets_[tag]; +} + +storage::Tablet *MemoryStorage::tablet(const std::string &tag) { + auto it = tablets_.find(tag); + CHECK(it != tablets_.end()) << "tablet tagged as " << tag << " not exists"; + return &it->second; +} + +void MemoryStorage::PersistToDisk() const { + // write storage out + const auto meta_path = storage_.dir() + "/" + meta_file_name; + fs::Write(meta_path, storage_.SerializeAsString()); + // write all the tablets + for (auto tag : storage_.tags()) { + auto path = GenPathFromTag(storage_.dir(), tag); + auto it = tablets_.find(tag); + CHECK(it != tablets_.end()); + fs::Write(path, it->second.SerializeAsString()); + } +} + +void MemoryStorage::LoadFromDisk(const std::string &dir) { + // load storage + const auto meta_path = storage_.dir() + "/" + meta_file_name; + storage_.ParseFromString(fs::Read(meta_path)); + + // load all the tablets + for (auto tag : storage_.tags()) { + auto path = GenPathFromTag(storage_.dir(), tag); + tablets_[tag]; + tablets_[tag].ParseFromString(fs::Read(path)); + } +} + storage::Tablet *Storage::Add(const std::string &tag, int num_samples) { auto *tablet = &(*proto_.mutable_tablets())[tag]; tablet->set_num_samples(num_samples); diff --git a/visualdl/backend/storage/storage.h b/visualdl/backend/storage/storage.h index a623eb0f9449ed35ebc78d5d6237504f99d5df8a..b8009be61f1e5c8d5d6ba4463ad3803f47f2ff65 100644 --- a/visualdl/backend/storage/storage.h +++ b/visualdl/backend/storage/storage.h @@ -1,13 +1,82 @@ #ifndef VISUALDL_STORAGE_H #define VISUALDL_STORAGE_H -#include #include +#include +#include #include "visualdl/backend/storage/storage.pb.h" namespace visualdl { +/* + * Generate a tablet path in disk from its tag. + */ +inline std::string GenPathFromTag(const std::string &dir, + const std::string &tag); + +/* + * Storage Interface. The might be a bunch of implementations, for example, a + * MemStorage that keep a copy of all the taplets in memory, can be changed with + * a higher performance; a DiskStorage that keep all the data in disk, apply to + * the scenerios where memory consumption should be considered. + */ +class StorageBase { +public: + const static std::string meta_file_name; + + void SetStorage(const std::string &dir) { + time_t t; + time(&t); + storage_.set_timestamp(t); + storage_.set_dir(dir); + } + + /* + * Create a new Tablet storage. + */ + virtual storage::Tablet *NewTablet(const std::string &tag, + int num_samples) = 0; + + /* + * Get a tablet from memory, this can be viewed as a cache, if the storage is + * in disk, a hash map in memory will first load the corresponding Tablet + * Protobuf from disk and hold all the changes. + */ + virtual storage::Tablet *tablet(const std::string &tag) = 0; + + /* + * Persist the data from cache to disk. Both the memory storage or disk + * storage should write changes to disk for persistence. + */ + virtual void PersistToDisk() const = 0; + + /* + * Load data from disk. + */ + virtual void LoadFromDisk(const std::string &dir) = 0; + +protected: + storage::Storage storage_; +}; + +/* + * Storage in Memory, that will support quick edits on data. + */ +class MemoryStorage final : public StorageBase { +public: + storage::Tablet *NewTablet(const std::string &tag, int num_samples) override; + + storage::Tablet *tablet(const std::string &tag) override; + + void PersistToDisk() const override; + + void LoadFromDisk(const std::string &dir) override; + +private: + std::map tablets_; +}; + class Storage final { public: Storage() { @@ -66,6 +135,6 @@ private: storage::Storage proto_; }; -} // namespace visualdl +} // namespace visualdl -#endif // VISUALDL_STORAGE_H +#endif // VISUALDL_STORAGE_H diff --git a/visualdl/backend/storage/storage.proto b/visualdl/backend/storage/storage.proto index a38cb3ce3cf5e2b9523bbd7cb8167ef91770b1dd..e587e011327540476cff69535f16357d750b74a5 100644 --- a/visualdl/backend/storage/storage.proto +++ b/visualdl/backend/storage/storage.proto @@ -130,7 +130,9 @@ The Storage stores all the records. */ message Storage { // tags to Tablet, should be thread safe if fix the keys after initialization. + // TODO to delete in the new storage interface. map tablets = 1; + repeated string tags = 4; string dir = 2; int64 timestamp = 3; -} \ No newline at end of file +}