From 429c35b88ab8bf16c547e937ae4960a0f3c3d2a0 Mon Sep 17 00:00:00 2001 From: superjom Date: Fri, 24 Nov 2017 15:38:10 +0800 Subject: [PATCH] refactor Storage Interface --- CMakeLists.txt | 3 +- visualdl/backend/logic/im.cc | 21 +++---- visualdl/backend/logic/im.h | 20 ++++-- visualdl/backend/logic/im_test.cc | 18 ++++-- visualdl/backend/logic/sdk.h | 2 +- visualdl/backend/storage/storage.cc | 79 +++++------------------- visualdl/backend/storage/storage.h | 63 ++----------------- visualdl/backend/storage/storage_test.cc | 46 ++++++++++++++ visualdl/backend/utils/filesystem.h | 68 ++++++++++++++++++++ 9 files changed, 175 insertions(+), 145 deletions(-) create mode 100644 visualdl/backend/storage/storage_test.cc create mode 100644 visualdl/backend/utils/filesystem.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b3eec5e1..58b9f76d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,5 +26,6 @@ set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE) add_executable(vl_test ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc + ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage_test.cc ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc) -target_link_libraries(vl_test storage im gtest glog protobuf gflags) +target_link_libraries(vl_test storage im gtest glog protobuf gflags pthread) diff --git a/visualdl/backend/logic/im.cc b/visualdl/backend/logic/im.cc index 12fa5cd5..e33ac276 100644 --- a/visualdl/backend/logic/im.cc +++ b/visualdl/backend/logic/im.cc @@ -26,24 +26,21 @@ int ReserviorSample(int num_samples, int num_records) { } void InformationMaintainer::SetPersistDest(const std::string &path) { - CHECK(storage_.mutable_data()->dir().empty()) + CHECK(storage_->mutable_data()->dir().empty()) << "duplicate set storage's path"; - storage_.mutable_data()->set_dir(path); + storage_->mutable_data()->set_dir(path); } storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag, int num_samples) { - auto *tablet = storage_.Find(tag); - if (!tablet) { - tablet = storage_.Add(tag, num_samples); - } + auto tablet = storage_->NewTablet(tag, num_samples); return tablet; } void InformationMaintainer::AddRecord(const std::string &tag, const storage::Record &data) { - auto *tablet = storage_.Find(tag); - CHECK(tablet); + auto *tablet = storage_->tablet(tag); + CHECK(tablet) << "no tablet called " << tag; auto num_records = tablet->total_records(); const auto num_samples = tablet->num_samples(); @@ -59,9 +56,9 @@ void InformationMaintainer::AddRecord(const std::string &tag, storage::Record *record; if (offset >= num_records) { - record = storage_.NewRecord(tag); + record = tablet->add_records(); } else { - record = storage_.GetRecord(tag, offset); + record = tablet->mutable_records(offset); } *record = data; @@ -76,10 +73,10 @@ void InformationMaintainer::Clear() { } void InformationMaintainer::PersistToDisk() { - CHECK(!storage_.data().dir().empty()) << "path of storage should be set"; + CHECK(!storage_->data().dir().empty()) << "path of storage should be set"; // TODO make dir first // MakeDir(storage_.data().dir()); - storage_.Save(storage_.data().dir() + "/storage.pb"); + storage_->PersistToDisk(); } } // namespace visualdl diff --git a/visualdl/backend/logic/im.h b/visualdl/backend/logic/im.h index dfb73a83..d96aa4d5 100644 --- a/visualdl/backend/logic/im.h +++ b/visualdl/backend/logic/im.h @@ -1,6 +1,8 @@ #ifndef VISUALDL_BACKEND_LOGIC_IM_H #define VISUALDL_BACKEND_LOGIC_IM_H +#include +#include #include #include "visualdl/backend/storage/storage.h" @@ -13,7 +15,15 @@ namespace visualdl { */ class InformationMaintainer final { public: - InformationMaintainer() {} + InformationMaintainer(StorageBase::Type type = StorageBase::Type::kMemory) { + switch (type) { + case StorageBase::Type::kMemory: + storage_.reset(new MemoryStorage); + break; + default: + CHECK(false) << "Unsupported storage kind " << type; + } + } static InformationMaintainer &Global() { static InformationMaintainer *x = new InformationMaintainer(); @@ -46,12 +56,12 @@ public: */ void PersistToDisk(); - Storage &storage() { return storage_; } + StorageBase &storage() { return *storage_; } private: - Storage storage_; + std::unique_ptr storage_; }; -} // namespace visualdl +} // namespace visualdl -#endif // VISUALDL_BACKEND_LOGIC_IM_H +#endif // VISUALDL_BACKEND_LOGIC_IM_H diff --git a/visualdl/backend/logic/im_test.cc b/visualdl/backend/logic/im_test.cc index 8c8896ab..183853e5 100644 --- a/visualdl/backend/logic/im_test.cc +++ b/visualdl/backend/logic/im_test.cc @@ -11,19 +11,25 @@ protected: InformationMaintainer &im = InformationMaintainer::Global(); }; -TEST_F(ImTester, AddTablet) { im.AddTablet("tag0", 20); } +TEST_F(ImTester, AddTablet) { + im.Clear(); + im.AddTablet("tag0", 20); +} TEST_F(ImTester, AddRecord) { - storage::Record rcd; - rcd.set_dtype(storage::DataType::kInt32s); + im.Clear(); + + im.AddTablet("tag0", 20); for (int i = 0; i < 100; i++) { + storage::Record rcd; + rcd.set_dtype(storage::DataType::kInt32s); for (int j = 0; j < 10; j++) { - rcd.mutable_data()->add_i32s(i * 20 + j); + rcd.add_data()->add_i32s(i * 20 + j); } im.AddRecord("tag0", rcd); } - ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL); + ASSERT_EQ(im.storage().tablet("tag0")->records_size(), 100UL); } -} // namespace visualdl +} // namespace visualdl diff --git a/visualdl/backend/logic/sdk.h b/visualdl/backend/logic/sdk.h index d463e3cd..5162fee8 100644 --- a/visualdl/backend/logic/sdk.h +++ b/visualdl/backend/logic/sdk.h @@ -98,7 +98,7 @@ public: InformationMaintainer::Global().storage().mutable_data()); } TabletHelper tablet(const std::string &tag) { - return TabletHelper(InformationMaintainer::Global().storage().Find(tag)); + return TabletHelper(InformationMaintainer::Global().storage().tablet(tag)); } TabletHelper AddTablet(const std::string &tag, int num_samples) { return TabletHelper( diff --git a/visualdl/backend/storage/storage.cc b/visualdl/backend/storage/storage.cc index 8200e22c..ca45d801 100644 --- a/visualdl/backend/storage/storage.cc +++ b/visualdl/backend/storage/storage.cc @@ -18,6 +18,7 @@ storage::Tablet *MemoryStorage::NewTablet(const std::string &tag, if (it == tablets_.end()) { // create new tablet tablets_[tag] = storage::Tablet(); + tablets_[tag].set_tag(tag); *storage_.add_tags() = tag; } else { return &it->second; @@ -32,84 +33,38 @@ storage::Tablet *MemoryStorage::tablet(const std::string &tag) { } void MemoryStorage::PersistToDisk() const { + VLOG(3) << "persist storage to disk path " << storage_.dir(); + // make a directory if not exist + fs::TryMkdir(storage_.dir()); // write storage out + CHECK(!storage_.dir().empty()) << "storage's dir should be set first"; const auto meta_path = storage_.dir() + "/" + meta_file_name; - fs::Write(meta_path, storage_.SerializeAsString()); + fs::Write(meta_path, fs::Serialize(storage_)); // write all the tablets for (auto tag : storage_.tags()) { auto path = GenPathFromTag(storage_.dir(), tag); auto it = tablets_.find(tag); CHECK(it != tablets_.end()); - fs::Write(path, it->second.SerializeAsString()); + fs::Write(path, fs::Serialize(it->second)); } } void MemoryStorage::LoadFromDisk(const std::string &dir) { + VLOG(3) << "load storage from disk path " << dir; + CHECK(!dir.empty()) << "dir is empty"; // load storage - const auto meta_path = storage_.dir() + "/" + meta_file_name; - storage_.ParseFromString(fs::Read(meta_path)); + const auto meta_path = dir + "/" + meta_file_name; + auto buf = fs::Read(meta_path); + CHECK(fs::DeSerialize(&storage_, buf)) + << "failed to parse protobuf loaded from " << meta_path; // load all the tablets - for (auto tag : storage_.tags()) { + for (int i = 0; i < storage_.tags_size(); i++) { + std::string tag = storage_.tags(i); auto path = GenPathFromTag(storage_.dir(), tag); - tablets_[tag]; - tablets_[tag].ParseFromString(fs::Read(path)); - } -} - -storage::Tablet *Storage::Add(const std::string &tag, int num_samples) { - auto *tablet = &(*proto_.mutable_tablets())[tag]; - tablet->set_num_samples(num_samples); - return tablet; -} - -storage::Tablet *Storage::Find(const std::string &tag) { - auto it = proto_.mutable_tablets()->find(tag); - if (it != proto_.tablets().end()) { - return &it->second; + CHECK(tablets_[tag].ParseFromString(fs::Read(path))) + << "failed to parse protobuf text loaded from " << path; } - return nullptr; -} - -storage::Record *Storage::NewRecord(const std::string &tag) { - auto *tablet = Find(tag); - CHECK(tablet) << "Tablet" << tag << " should be create first"; - auto *record = tablet->mutable_records()->Add(); - // increase num_records - int num_records = tablet->total_records(); - tablet->set_total_records(num_records + 1); - return record; -} -storage::Record *Storage::GetRecord(const std::string &tag, int offset) { - auto *tablet = Find(tag); - CHECK(tablet) << "Tablet" << tag << " should be create first"; - - auto num_records = tablet->total_records(); - CHECK_LT(offset, num_records) << "invalid offset"; - return tablet->mutable_records()->Mutable(offset); -} - -void Storage::Save(const std::string &path) const { - std::ofstream file(path, file.binary | file.out); - CHECK(file.is_open()) << "can't open path " << path; - auto str = Serialize(); - file.write(str.c_str(), str.size()); -} - -void Storage::Load(const std::string &path) { - std::ifstream file(path, file.binary); - CHECK(file.is_open()) << "can't open path " << path; - size_t size = file.tellg(); - std::string buffer(size, ' '); - file.seekg(0); - file.read(&buffer[0], size); - DeSerialize(buffer); -} - -std::string Storage::Serialize() const { return proto_.SerializeAsString(); } - -void Storage::DeSerialize(const std::string &data) { - proto_.ParseFromString(data); } } // namespace visualdl diff --git a/visualdl/backend/storage/storage.h b/visualdl/backend/storage/storage.h index b8009be6..d9e8ae29 100644 --- a/visualdl/backend/storage/storage.h +++ b/visualdl/backend/storage/storage.h @@ -25,6 +25,8 @@ class StorageBase { public: const static std::string meta_file_name; + enum Type { kMemory = 0, kDisk = 1 }; + void SetStorage(const std::string &dir) { time_t t; time(&t); @@ -56,6 +58,9 @@ public: */ virtual void LoadFromDisk(const std::string &dir) = 0; + storage::Storage *mutable_data() { return &storage_; } + const storage::Storage &data() { return storage_; } + protected: storage::Storage storage_; }; @@ -77,64 +82,6 @@ private: std::map tablets_; }; -class Storage final { -public: - Storage() { - // set time stamp - time_t time0; - time(&time0); - proto_.set_timestamp(time0); - } - - /* - * Add a new tablet named `tag`, the newly added instance will be returned. - */ - storage::Tablet *Add(const std::string &tag, int num_samples); - - /* - * Search the tablet named `tag`, if not exist, return nullptr. - */ - storage::Tablet *Find(const std::string &tag); - - /* - * Append a new record to the tail of tablet. - */ - storage::Record *NewRecord(const std::string &tag); - - /* - * Get a record at `offset`, if the offset is not valid, yield a failed CHECK. - */ - storage::Record *GetRecord(const std::string &tag, int offset); - - /* - * Serialize this object to string and save it to a file. - */ - void Save(const std::string &path) const; - - /* - * Load the Protobuf message from a file. - */ - void Load(const std::string &path); - - storage::Storage *mutable_data() { return &proto_; } - - const storage::Storage &data() { return proto_; } - -protected: - /* - * Serialize the Storage instance to string. - */ - std::string Serialize() const; - - /* - * De-serialize from a string and update this Storage instance. - */ - void DeSerialize(const std::string &data); - -private: - storage::Storage proto_; -}; - } // namespace visualdl #endif // VISUALDL_STORAGE_H diff --git a/visualdl/backend/storage/storage_test.cc b/visualdl/backend/storage/storage_test.cc new file mode 100644 index 00000000..6090d71f --- /dev/null +++ b/visualdl/backend/storage/storage_test.cc @@ -0,0 +1,46 @@ +#include "visualdl/backend/storage/storage.h" + +#include +#include + +namespace visualdl { +using namespace std; + +class MemoryStorageTest : public ::testing::Test { +public: + void SetUp() override { storage_.SetStorage("./tmp"); } + + MemoryStorage storage_; +}; + +TEST_F(MemoryStorageTest, SetStorage) { + string dir = "./tmp"; + storage_.SetStorage(dir); + + ASSERT_EQ(storage_.data().dir(), dir); +} + +TEST_F(MemoryStorageTest, AddTablet) { + // TODO need to escape tag as name + string tag = "add%20tag0"; + storage_.NewTablet(tag, -1); + + auto* tablet = storage_.tablet(tag); + + ASSERT_TRUE(tablet != nullptr); + ASSERT_EQ(tablet->tag(), tag); +} + +TEST_F(MemoryStorageTest, PersistToDisk) { + string tag = "add%20tag0"; + storage_.NewTablet(tag, -1); + + storage_.PersistToDisk(); + + MemoryStorage other; + other.LoadFromDisk("./tmp"); + ASSERT_EQ(other.data().SerializeAsString(), + storage_.data().SerializeAsString()); +} + +} // namespace visualdl diff --git a/visualdl/backend/utils/filesystem.h b/visualdl/backend/utils/filesystem.h new file mode 100644 index 00000000..e30640b9 --- /dev/null +++ b/visualdl/backend/utils/filesystem.h @@ -0,0 +1,68 @@ +#ifndef VISUALDL_BACKEND_UTILS_FILESYSTEM_H +#define VISUALDL_BACKEND_UTILS_FILESYSTEM_H + +#include +#include +#include +#include +#include + +namespace visualdl { + +namespace fs { + +template +std::string Serialize(const T& proto, bool human_readable = false) { + if (human_readable) { + std::string buffer; + google::protobuf::TextFormat::PrintToString(proto, &buffer); + return buffer; + } + return proto.SerializeAsString(); +} + +template +bool DeSerialize(T* proto, const std::string buf, bool human_readable = false) { + // NOTE human_readable not valid + if (human_readable) { + return google::protobuf::TextFormat::ParseFromString(buf, proto); + } + return proto->ParseFromString(buf); +} + +void TryMkdir(const std::string& dir) { + VLOG(1) << "try to mkdir " << dir; + struct stat st = {0}; + if (stat(dir.c_str(), &st) == -1) { + ::mkdir(dir.c_str(), 0700); + } +} + +inline void Write(const std::string& path, + const std::string& buffer, + std::ios::openmode open_mode = std::ios::binary) { + VLOG(1) << "write to path " << path; + std::ofstream file(path, open_mode); + CHECK(file.is_open()) << "failed to open " << path; + file.write(buffer.c_str(), buffer.size()); + file.close(); +} + +inline std::string Read(const std::string& path, + std::ios::openmode open_mode = std::ios::binary) { + VLOG(1) << "read from path " << path; + std::string buffer; + std::ifstream file(path, open_mode | std::ios::ate); + CHECK(file.is_open()) << "failed to open " << path; + size_t size = file.tellg(); + file.seekg(0); + buffer.resize(size); + file.read(&buffer[0], size); + return buffer; +} + +} // namespace fs + +} // namespace visualdl + +#endif // VISUALDL_BACKEND_UTILS_FILESYSTEM_H -- GitLab