diff --git a/CMakeLists.txt b/CMakeLists.txt index bb45c59ecdfd3ed89094b5bf19a87578b5ee825c..04937a2840c0dcfa7b5ab1f9e2bc7358e05c8e02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,19 +1,32 @@ -cmake_minimum_required(VERSION 3.8) +cmake_minimum_required(VERSION 3.5) project(VisualDL) set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_FLAGS "-fPIC") include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(thirdparty/local/include) -link_directories(thirdparty/local/lib) -add_subdirectory(thirdparty/pybind11-2.2.1) - -set(SOURCE_FILES - visualdl/backend/storage/storage.cc visualdl/backend/storage/storage.h - visualdl/backend/storage/storage.pb.h - visualdl/backend/storage/storage.pb.cc - ) - -add_library(storage visualdl/backend/storage/storage.cc - visualdl/backend/storage/storage.pb.cc) -add_executable(VisualDL ${SOURCE_FILES}) \ No newline at end of file +include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include) +include_directories(/usr/include/python2.7) + +set(PYBIND11_CPP_STANDARD -std=c++11) +add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/pybind11-2.2.1) + +link_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/lib) + +add_library(storage + ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.cc + ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.pb.cc) +add_library(c_api ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/c_api.cc) +add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc) +add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc) + +pybind11_add_module(core + ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc) +target_link_libraries(core PRIVATE pybind11::module im storage sdk protobuf glog) +set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +add_executable(vl_test + ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc + ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc) +target_link_libraries(vl_test storage im gtest glog protobuf gflags) + diff --git a/visualdl/.clang-format b/visualdl/.clang-format new file mode 100644 index 0000000000000000000000000000000000000000..9ba433b17362424973626470d930356c2173dd84 --- /dev/null +++ b/visualdl/.clang-format @@ -0,0 +1,28 @@ +# This file is used by clang-format to autoformat paddle source code +# +# The clang-format is part of llvm toolchain. +# It need to install llvm and clang to format source code style. +# +# The basic usage is, +# clang-format -i -style=file PATH/TO/SOURCE/CODE +# +# The -style=file implicit use ".clang-format" file located in one of +# parent directory. +# The -i means inplace change. +# +# The document of clang-format is +# http://clang.llvm.org/docs/ClangFormat.html +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +--- +Language: Cpp +BasedOnStyle: Google +IndentWidth: 2 +TabWidth: 2 +ContinuationIndentWidth: 4 +AccessModifierOffset: -2 # The private/protected/public has no indent in class +Standard: Cpp11 +AllowAllParametersOfDeclarationOnNextLine: true +BinPackParameters: false +BinPackArguments: false +... + diff --git a/visualdl/backend/logic/im.cc b/visualdl/backend/logic/im.cc new file mode 100644 index 0000000000000000000000000000000000000000..241ff469cb80faba724adbc49e7f179c7b28aabd --- /dev/null +++ b/visualdl/backend/logic/im.cc @@ -0,0 +1,71 @@ +#include +#include + +#include "visualdl/backend/logic/im.h" + +namespace visualdl { + +/* + * @num_samples: number of instances to sample + * @size: counter of the records. + * @returns: id of the instance to replace, if drop this instance, return -1. + */ +int ReserviorSample(int num_samples, int num_records) { + if (num_records <= num_samples) { + return num_records; + } + + std::srand(std::time(0)); + float prob = static_cast(std::rand()) / RAND_MAX; + float receive_prob = static_cast(num_samples) / num_records; + if (prob < receive_prob) { + int offset2replace = std::rand() % num_samples; + return offset2replace; + } + return -1; +} + +void InformationMaintainer::SetPersistDest(const std::string &path) { + CHECK(storage_.mutable_data()->dir().empty()) + << "duplicate set storage's path"; + storage_.mutable_data()->set_dir(path); +} + +storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag, + int num_samples) { + auto *tablet = storage_.Find(tag); + if (!tablet) { + tablet = storage_.Add(tag, num_samples); + } + return tablet; +} + +void InformationMaintainer::AddRecord(const std::string &tag, + const storage::Record &data) { + auto *tablet = storage_.Find(tag); + CHECK(tablet); + + auto num_records = tablet->num_records(); + const auto num_samples = tablet->num_samples(); + const auto offset = ReserviorSample(num_samples, num_records + 1); + if (offset < 0) + return; + + storage::Record *record; + if (offset >= num_records) { + record = storage_.NewRecord(tag); + } else { + record = storage_.GetRecord(tag, offset); + } + + *record = data; + + tablet->set_num_records(num_records + 1); +} + +void InformationMaintainer::PersistToDisk() { + CHECK(!storage_.data().dir().empty()) << "path of storage should be set"; + storage_.Save(storage_.data().dir()); +} + +} // namespace visualdl diff --git a/visualdl/backend/logic/im.h b/visualdl/backend/logic/im.h new file mode 100644 index 0000000000000000000000000000000000000000..b94102db08efb032f37e0e837bd379bb9f830569 --- /dev/null +++ b/visualdl/backend/logic/im.h @@ -0,0 +1,53 @@ +#ifndef VISUALDL_BACKEND_LOGIC_IM_H +#define VISUALDL_BACKEND_LOGIC_IM_H + +#include + +#include "visualdl/backend/storage/storage.h" + +namespace visualdl { + +/* + * Maintain the Storage singleton in memory, pre-compute some the statical + * information to help visualizaton. + */ +class InformationMaintainer final { +public: + InformationMaintainer() {} + + static InformationMaintainer &Global() { + static InformationMaintainer *x = new InformationMaintainer(); + return *x; + } + + /* + * Set the disk path to store the Storage object. + */ + void SetPersistDest(const std::string &path); + + storage::Tablet *AddTablet(const std::string &tag, int num_samples); + + /* + * @tag: tag of the target Tablet. + * @type: type of target Tablet. + * @data: storage Record. + * + * NOTE pass in the serialized protobuf message will trigger copying, but + * simpler to support different Tablet data formats. + */ + void AddRecord(const std::string &tag, const storage::Record &record); + + /* + * Save the Storage Protobuf to disk. + */ + void PersistToDisk(); + + Storage &storage() { return storage_; } + +private: + Storage storage_; +}; + +} // namespace visualdl + +#endif // VISUALDL_BACKEND_LOGIC_IM_H diff --git a/visualdl/backend/logic/im_test.cc b/visualdl/backend/logic/im_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..8c8896aba646a6cde9ae63109623268263ff6113 --- /dev/null +++ b/visualdl/backend/logic/im_test.cc @@ -0,0 +1,29 @@ +#include "visualdl/backend/logic/im.h" + +#include "gtest/gtest.h" + +namespace visualdl { + +class ImTester : public ::testing::Test { +protected: + void SetUp() override {} + + InformationMaintainer &im = InformationMaintainer::Global(); +}; + +TEST_F(ImTester, AddTablet) { im.AddTablet("tag0", 20); } + +TEST_F(ImTester, AddRecord) { + storage::Record rcd; + rcd.set_dtype(storage::DataType::kInt32s); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 10; j++) { + rcd.mutable_data()->add_i32s(i * 20 + j); + } + im.AddRecord("tag0", rcd); + } + + ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL); +} + +} // namespace visualdl diff --git a/visualdl/backend/logic/pybind.cc b/visualdl/backend/logic/pybind.cc new file mode 100644 index 0000000000000000000000000000000000000000..242c25d3e1a80a7704dcfddbaa62775ad1dbde20 --- /dev/null +++ b/visualdl/backend/logic/pybind.cc @@ -0,0 +1,52 @@ +#include +#include + +#include "visualdl/backend/logic/sdk.h" + +namespace py = pybind11; +namespace vs = visualdl; + +PYBIND11_MODULE(core, m) { + m.doc() = "visualdl python core API"; + + py::class_(m, "Tablet") + // interfaces for components + .def("add_scalar_int32", + &vs::TabletHelper::AddScalarRecord, + "add a scalar int32 record", + pybind11::arg("id"), + pybind11::arg("value")) + .def("add_scalar_int64", + &vs::TabletHelper::AddScalarRecord, + "add a scalar int64 record", + pybind11::arg("id"), + pybind11::arg("value")) + .def("add_scalar_float", &vs::TabletHelper::AddScalarRecord) + .def("add_scalar_double", &vs::TabletHelper::AddScalarRecord) + // other member setter and getter + .def("record_buffer", &vs::TabletHelper::record_buffer) + .def("records_size", &vs::TabletHelper::records_size) + .def("buffer", &vs::TabletHelper::buffer) + .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer) + .def("set_buffer", + (void (vs::TabletHelper::*)(const std::string&)) & + vs::TabletHelper::SetBuffer); + + py::class_(m, "Storage") + .def("timestamp", &vs::StorageHelper::timestamp) + .def("dir", &vs::StorageHelper::dir) + .def("set_dir", &vs::StorageHelper::SetDir) + .def("tablets_size", &vs::StorageHelper::tablets_size) + .def("buffer", &vs::StorageHelper::buffer) + .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer) + .def("set_buffer", + (void (vs::StorageHelper::*)(const std::string&)) & + vs::StorageHelper::SetBuffer); + + py::class_(m, "Im") + .def("storage", &vs::ImHelper::storage) + .def("tablet", &vs::ImHelper::tablet) + .def("add_tablet", &vs::ImHelper::AddTablet); + + m.def("im", &vs::get_im, "global information-maintainer object."); +} diff --git a/visualdl/backend/logic/sdk.cc b/visualdl/backend/logic/sdk.cc new file mode 100644 index 0000000000000000000000000000000000000000..10406da6d7373c7efb54390cabdca8bdbffa7119 --- /dev/null +++ b/visualdl/backend/logic/sdk.cc @@ -0,0 +1,35 @@ +#include +#include "visualdl/backend/logic/sdk.h" + +namespace visualdl { + +#define IMPL_ENTRY_SET_OR_ADD(method__, ctype__, dtype__, opr__) \ + template <> void Entry::method__(ctype__ v) { \ + entry->set_dtype(storage::DataType::dtype__); \ + entry->opr__(v); \ + } + +IMPL_ENTRY_SET_OR_ADD(Set, int32_t, kInt32, set_i32); +IMPL_ENTRY_SET_OR_ADD(Set, int64_t, kInt64, set_i64); +IMPL_ENTRY_SET_OR_ADD(Set, bool, kBool, set_b); +IMPL_ENTRY_SET_OR_ADD(Set, float, kFloat, set_f); +IMPL_ENTRY_SET_OR_ADD(Set, double, kDouble, set_d); +IMPL_ENTRY_SET_OR_ADD(Add, int32_t, kInt32s, add_i32s); +IMPL_ENTRY_SET_OR_ADD(Add, int64_t, kInt64s, add_i64s); +IMPL_ENTRY_SET_OR_ADD(Add, bool, kBools, add_bs); +IMPL_ENTRY_SET_OR_ADD(Add, float, kFloats, add_fs); +IMPL_ENTRY_SET_OR_ADD(Add, double, kDoubles, add_ds); + +std::string StorageHelper::human_readable_buffer() const { + std::string buffer; + google::protobuf::TextFormat::PrintToString(*data_, &buffer); + return buffer; +} + +std::string TabletHelper::human_readable_buffer() const { + std::string buffer; + google::protobuf::TextFormat::PrintToString(*data_, &buffer); + return buffer; +} + +} // namespace visualdl diff --git a/visualdl/backend/logic/sdk.h b/visualdl/backend/logic/sdk.h new file mode 100644 index 0000000000000000000000000000000000000000..c99958a7e11c129f860a983813f11076875788ce --- /dev/null +++ b/visualdl/backend/logic/sdk.h @@ -0,0 +1,79 @@ +#ifndef VISUALDL_BACKEND_LOGIC_SDK_H +#define VISUALDL_BACKEND_LOGIC_SDK_H +#include "visualdl/backend/logic/im.h" + +#include + +namespace visualdl { + +class TabletHelper { +public: + // method for each components + template + void AddScalarRecord(int id, T value); + + // basic member getter and setter + std::string record_buffer(int idx) const { return data_->records(idx).SerializeAsString(); } + size_t records_size() const { return data_->records_size(); } + std::string buffer() const { return data_->SerializeAsString(); } + std::string human_readable_buffer() const; + void SetBuffer(const storage::Tablet &t) { *data_ = t; } + void SetBuffer(const std::string &b) { data_->ParseFromString(b); } + + // constructor that enable concurrency. + TabletHelper(storage::Tablet *t) : data_(t) {} + // data updater that resuage of one instance. + TabletHelper &operator()(storage::Tablet *t) { data_ = t; return *this; } + +private: + storage::Tablet *data_; +}; + +class StorageHelper { +public: + StorageHelper(storage::Storage *s) : data_(s) {} + StorageHelper &operator()(storage::Storage *s) { + data_ = s; + return *this; + } + + void SetBuffer(const storage::Storage &buffer) { *data_ = buffer; } + void SetBuffer(const std::string &buffer) { data_->ParseFromString(buffer); } + void SetDir(const std::string &dir) { data_->set_dir(dir); } + + int64_t timestamp() const { return data_->timestamp(); } + std::string dir() const { return data_->dir(); } + int tablets_size() const { return data_->tablets_size(); } + std::string buffer() const { return data_->SerializeAsString(); } + std::string human_readable_buffer() const; + +private: + storage::Storage *data_; +}; + +class ImHelper { +public: + ImHelper() {} + + StorageHelper storage() { + return StorageHelper( + InformationMaintainer::Global().storage().mutable_data()); + } + TabletHelper tablet(const std::string &tag) { + return TabletHelper(InformationMaintainer::Global().storage().Find(tag)); + } + TabletHelper AddTablet(const std::string &tag, int num_samples) { + return TabletHelper( + InformationMaintainer::Global().AddTablet(tag, num_samples)); + } +}; + +static ImHelper& get_im() { + static ImHelper im; + return im; +} + +} // namespace visualdl + +#include "visualdl/backend/logic/sdk.hpp" +#endif // VISUALDL_BACKEND_LOGIC_SDK_H diff --git a/visualdl/backend/logic/sdk.hpp b/visualdl/backend/logic/sdk.hpp new file mode 100644 index 0000000000000000000000000000000000000000..517abe3f8a1dbbd0aa307a5b1cc41224dd167ee3 --- /dev/null +++ b/visualdl/backend/logic/sdk.hpp @@ -0,0 +1,33 @@ +#include "visualdl/backend/logic/im.h" + +namespace visualdl { + +/* + * Utility helper for storage::Entry. + */ +template struct Entry { + // use pointer to avoid copy + storage::Entry *entry{nullptr}; + + Entry(storage::Entry *entry) : entry(entry) {} + + /* + * Set a single value. + */ + void Set(T v); + + /* + * Add a value to repeated message field. + */ + void Add(T v); +}; + +template +void TabletHelper::AddScalarRecord(int id, T value) { + auto* record = data_->add_records(); + record->set_id(id); + Entry entry_helper(record->mutable_data()); + entry_helper.Set(value); +} + +} // namespace visualdl diff --git a/visualdl/backend/storage/storage.cc b/visualdl/backend/storage/storage.cc index 5af049d9a79351d76c45d586a238de037ad24806..5a89aa49b60abef9720ae29a1ecff8c7a29798c8 100644 --- a/visualdl/backend/storage/storage.cc +++ b/visualdl/backend/storage/storage.cc @@ -5,18 +5,38 @@ namespace visualdl { -storage::Tablet *Storage::Add(const std::string &tag) { - return &proto_.mutable_tablets()->at(tag); +storage::Tablet *Storage::Add(const std::string &tag, int num_samples) { + auto *tablet = &(*proto_.mutable_tablets())[tag]; + tablet->set_num_samples(num_samples); + return tablet; } -const storage::Tablet *Storage::Find(const std::string &tag) const { - auto it = proto_.tablets().find(tag); +storage::Tablet *Storage::Find(const std::string &tag) { + auto it = proto_.mutable_tablets()->find(tag); if (it != proto_.tablets().end()) { return &it->second; } return nullptr; } +storage::Record *Storage::NewRecord(const std::string &tag) { + auto *tablet = Find(tag); + CHECK(tablet) << "Tablet" << tag << " should be create first"; + auto *record = tablet->mutable_records()->Add(); + // increase num_records + int num_records = tablet->num_records(); + tablet->set_num_records(num_records + 1); + return record; +} +storage::Record *Storage::GetRecord(const std::string &tag, int offset) { + auto *tablet = Find(tag); + CHECK(tablet) << "Tablet" << tag << " should be create first"; + + auto num_records = tablet->num_records(); + CHECK_LT(offset, num_records) << "invalid offset"; + return tablet->mutable_records()->Mutable(offset); +} + void Storage::Save(const std::string &path) const { std::ofstream file(path, file.binary | file.out); CHECK(file.is_open()) << "can't open path " << path; @@ -34,12 +54,10 @@ void Storage::Load(const std::string &path) { DeSerialize(buffer); } -std::string Storage::Serialize() const { - return proto_.SerializeAsString(); -} +std::string Storage::Serialize() const { return proto_.SerializeAsString(); } void Storage::DeSerialize(const std::string &data) { proto_.ParseFromString(data); } -} // namespace visualdl +} // namespace visualdl diff --git a/visualdl/backend/storage/storage.h b/visualdl/backend/storage/storage.h index aec08a872dcb03ffb99d80b4b2dbb877c1a4da2e..a623eb0f9449ed35ebc78d5d6237504f99d5df8a 100644 --- a/visualdl/backend/storage/storage.h +++ b/visualdl/backend/storage/storage.h @@ -9,24 +9,33 @@ namespace visualdl { class Storage final { - public: - /* - * There should be only one Storage instance in memory. - */ - Storage &Global() { - static Storage *instance = new Storage(); - return *instance; +public: + Storage() { + // set time stamp + time_t time0; + time(&time0); + proto_.set_timestamp(time0); } /* * Add a new tablet named `tag`, the newly added instance will be returned. */ - storage::Tablet *Add(const std::string &tag); + storage::Tablet *Add(const std::string &tag, int num_samples); /* * Search the tablet named `tag`, if not exist, return nullptr. */ - const storage::Tablet *Find(const std::string &tag) const; + storage::Tablet *Find(const std::string &tag); + + /* + * Append a new record to the tail of tablet. + */ + storage::Record *NewRecord(const std::string &tag); + + /* + * Get a record at `offset`, if the offset is not valid, yield a failed CHECK. + */ + storage::Record *GetRecord(const std::string &tag, int offset); /* * Serialize this object to string and save it to a file. @@ -38,7 +47,11 @@ class Storage final { */ void Load(const std::string &path); - protected: + storage::Storage *mutable_data() { return &proto_; } + + const storage::Storage &data() { return proto_; } + +protected: /* * Serialize the Storage instance to string. */ @@ -49,17 +62,10 @@ class Storage final { */ void DeSerialize(const std::string &data); - Storage() { - // set time stamp - time_t time0; - time(&time0); - proto_.set_timestamp(time0); - } - - private: +private: storage::Storage proto_; }; } // namespace visualdl -#endif //VISUALDL_STORAGE_H +#endif // VISUALDL_STORAGE_H diff --git a/visualdl/backend/storage/storage.proto b/visualdl/backend/storage/storage.proto index 3cc1aa0c544c3041829270e546d3b88d6ae85af7..2b5729f83769f5ac73bc2bb38d72e6fce9f01f29 100644 --- a/visualdl/backend/storage/storage.proto +++ b/visualdl/backend/storage/storage.proto @@ -16,6 +16,8 @@ enum DataType { kStrings = 9; kInt32s = 10; kBools = 11; + + kUnknown = 12; } // A data array, which type is `type`. diff --git a/visualdl/backend/test.cc b/visualdl/backend/test.cc new file mode 100644 index 0000000000000000000000000000000000000000..1659be89ee1a2fe0dbb8b1388f3a2a4c9860ef3c --- /dev/null +++ b/visualdl/backend/test.cc @@ -0,0 +1,6 @@ +#include "gtest/gtest.h" + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/visualdl/backend/test.py b/visualdl/backend/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8755ae7ff8a23587756fa8a8b76d568365c1744c --- /dev/null +++ b/visualdl/backend/test.py @@ -0,0 +1,48 @@ +import sys +import unittest + +sys.path.append('../../build') +import core + +im = core.im() + + +class StorageTester(unittest.TestCase): + def setUp(self): + self.storage = im.storage() + + def test_size(self): + self.assertEqual(self.storage.tablets_size(), 0) + im.add_tablet("tag0", 100) + self.assertEqual(self.storage.tablets_size(), 1) + + for i in range(1, 11): + im.add_tablet("tag%d" % i, 100) + self.assertEqual(self.storage.tablets_size(), 11) + + def test_timestamp(self): + print self.storage.timestamp() + + def test_dir(self): + dir = "./1.txt" + self.storage.set_dir(dir) + self.assertEqual(dir, self.storage.dir()) + + def test_human_readable_buffer(self): + print self.storage.human_readable_buffer() + + +class TabletTester(unittest.TestCase): + def setUp(self): + self.tablet = im.add_tablet("tag101", 20) + + def test_add_scalar(self): + self.tablet.add_scalar_float(1, 0.3) + self.assertEqual(self.tablet.records_size(), 1) + + def test_human_readable_buffer(self): + print self.tablet.human_readable_buffer() + + +if __name__ == '__main__': + unittest.main()