diff --git a/CMakeLists.txt b/CMakeLists.txt index f944d7d22d6ce5ccac813928d75cde5a4fb5e47f..04937a2840c0dcfa7b5ab1f9e2bc7358e05c8e02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,29 +1,32 @@ -cmake_minimum_required(VERSION 3.8) +cmake_minimum_required(VERSION 3.5) project(VisualDL) set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_FLAGS "-fPIC") include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(thirdparty/local/include) -add_subdirectory(thirdparty/pybind11-2.2.1) +include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include) +include_directories(/usr/include/python2.7) -link_directories(thirdparty/local/lib) +set(PYBIND11_CPP_STANDARD -std=c++11) +add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/pybind11-2.2.1) -set(SOURCE_FILES - visualdl/backend/storage/storage.cc visualdl/backend/storage/storage.h - visualdl/backend/storage/storage.pb.h - visualdl/backend/storage/storage.pb.cc - ) +link_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/lib) -add_library(storage visualdl/backend/storage/storage.cc - visualdl/backend/storage/storage.pb.cc) -add_library(c_api visualdl/backend/logic/c_api.cc) -add_library(sdk visualdl/backend/logic/sdk.cc) -add_library(im visualdl/backend/logic/im.cc) +add_library(storage + ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.cc + ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.pb.cc) +add_library(c_api ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/c_api.cc) +add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc) +add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc) + +pybind11_add_module(core + ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc) +target_link_libraries(core PRIVATE pybind11::module im storage sdk protobuf glog) +set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE) add_executable(vl_test - visualdl/backend/test.cc - visualdl/backend/logic/im_test.cc) + ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc + ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc) target_link_libraries(vl_test storage im gtest glog protobuf gflags) -add_executable(VisualDL ${SOURCE_FILES}) diff --git a/visualdl/.clang-format b/visualdl/.clang-format new file mode 100644 index 0000000000000000000000000000000000000000..9ba433b17362424973626470d930356c2173dd84 --- /dev/null +++ b/visualdl/.clang-format @@ -0,0 +1,28 @@ +# This file is used by clang-format to autoformat paddle source code +# +# The clang-format is part of llvm toolchain. +# It need to install llvm and clang to format source code style. +# +# The basic usage is, +# clang-format -i -style=file PATH/TO/SOURCE/CODE +# +# The -style=file implicit use ".clang-format" file located in one of +# parent directory. +# The -i means inplace change. +# +# The document of clang-format is +# http://clang.llvm.org/docs/ClangFormat.html +# http://clang.llvm.org/docs/ClangFormatStyleOptions.html +--- +Language: Cpp +BasedOnStyle: Google +IndentWidth: 2 +TabWidth: 2 +ContinuationIndentWidth: 4 +AccessModifierOffset: -2 # The private/protected/public has no indent in class +Standard: Cpp11 +AllowAllParametersOfDeclarationOnNextLine: true +BinPackParameters: false +BinPackArguments: false +... + diff --git a/visualdl/backend/logic/im.cc b/visualdl/backend/logic/im.cc index 5f567daa4dbffcf1a2678cdd4e5794150100bf2c..241ff469cb80faba724adbc49e7f179c7b28aabd 100644 --- a/visualdl/backend/logic/im.cc +++ b/visualdl/backend/logic/im.cc @@ -41,9 +41,7 @@ storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag, } void InformationMaintainer::AddRecord(const std::string &tag, - storage::Tablet::Type type, - const std::string &data, - storage::DataType dtype) { + const storage::Record &data) { auto *tablet = storage_.Find(tag); CHECK(tablet); @@ -60,8 +58,7 @@ void InformationMaintainer::AddRecord(const std::string &tag, record = storage_.GetRecord(tag, offset); } - record->set_dtype(dtype); - record->ParseFromString(data); + *record = data; tablet->set_num_records(num_records + 1); } diff --git a/visualdl/backend/logic/im.h b/visualdl/backend/logic/im.h index ce85042d024ea333d229b46028860b785e9e9f90..b94102db08efb032f37e0e837bd379bb9f830569 100644 --- a/visualdl/backend/logic/im.h +++ b/visualdl/backend/logic/im.h @@ -30,14 +30,12 @@ public: /* * @tag: tag of the target Tablet. * @type: type of target Tablet. - * @data: serialized protobuf message. + * @data: storage Record. * * NOTE pass in the serialized protobuf message will trigger copying, but * simpler to support different Tablet data formats. */ - void AddRecord(const std::string &tag, storage::Tablet::Type type, - const std::string &data, - storage::DataType dtype = storage::DataType::kUnknown); + void AddRecord(const std::string &tag, const storage::Record &record); /* * Save the Storage Protobuf to disk. diff --git a/visualdl/backend/logic/im_test.cc b/visualdl/backend/logic/im_test.cc index 83bd900330cb80ff4de9c53ad3dedc3f77349d25..8c8896aba646a6cde9ae63109623268263ff6113 100644 --- a/visualdl/backend/logic/im_test.cc +++ b/visualdl/backend/logic/im_test.cc @@ -20,8 +20,7 @@ TEST_F(ImTester, AddRecord) { for (int j = 0; j < 10; j++) { rcd.mutable_data()->add_i32s(i * 20 + j); } - im.AddRecord("tag0", storage::Tablet::Type::Tablet_Type_kGraph, - rcd.SerializeAsString()); + im.AddRecord("tag0", rcd); } ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL); diff --git a/visualdl/backend/logic/pybind.cc b/visualdl/backend/logic/pybind.cc new file mode 100644 index 0000000000000000000000000000000000000000..242c25d3e1a80a7704dcfddbaa62775ad1dbde20 --- /dev/null +++ b/visualdl/backend/logic/pybind.cc @@ -0,0 +1,52 @@ +#include +#include + +#include "visualdl/backend/logic/sdk.h" + +namespace py = pybind11; +namespace vs = visualdl; + +PYBIND11_MODULE(core, m) { + m.doc() = "visualdl python core API"; + + py::class_(m, "Tablet") + // interfaces for components + .def("add_scalar_int32", + &vs::TabletHelper::AddScalarRecord, + "add a scalar int32 record", + pybind11::arg("id"), + pybind11::arg("value")) + .def("add_scalar_int64", + &vs::TabletHelper::AddScalarRecord, + "add a scalar int64 record", + pybind11::arg("id"), + pybind11::arg("value")) + .def("add_scalar_float", &vs::TabletHelper::AddScalarRecord) + .def("add_scalar_double", &vs::TabletHelper::AddScalarRecord) + // other member setter and getter + .def("record_buffer", &vs::TabletHelper::record_buffer) + .def("records_size", &vs::TabletHelper::records_size) + .def("buffer", &vs::TabletHelper::buffer) + .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer) + .def("set_buffer", + (void (vs::TabletHelper::*)(const std::string&)) & + vs::TabletHelper::SetBuffer); + + py::class_(m, "Storage") + .def("timestamp", &vs::StorageHelper::timestamp) + .def("dir", &vs::StorageHelper::dir) + .def("set_dir", &vs::StorageHelper::SetDir) + .def("tablets_size", &vs::StorageHelper::tablets_size) + .def("buffer", &vs::StorageHelper::buffer) + .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer) + .def("set_buffer", + (void (vs::StorageHelper::*)(const std::string&)) & + vs::StorageHelper::SetBuffer); + + py::class_(m, "Im") + .def("storage", &vs::ImHelper::storage) + .def("tablet", &vs::ImHelper::tablet) + .def("add_tablet", &vs::ImHelper::AddTablet); + + m.def("im", &vs::get_im, "global information-maintainer object."); +} diff --git a/visualdl/backend/logic/sdk.cc b/visualdl/backend/logic/sdk.cc new file mode 100644 index 0000000000000000000000000000000000000000..10406da6d7373c7efb54390cabdca8bdbffa7119 --- /dev/null +++ b/visualdl/backend/logic/sdk.cc @@ -0,0 +1,35 @@ +#include +#include "visualdl/backend/logic/sdk.h" + +namespace visualdl { + +#define IMPL_ENTRY_SET_OR_ADD(method__, ctype__, dtype__, opr__) \ + template <> void Entry::method__(ctype__ v) { \ + entry->set_dtype(storage::DataType::dtype__); \ + entry->opr__(v); \ + } + +IMPL_ENTRY_SET_OR_ADD(Set, int32_t, kInt32, set_i32); +IMPL_ENTRY_SET_OR_ADD(Set, int64_t, kInt64, set_i64); +IMPL_ENTRY_SET_OR_ADD(Set, bool, kBool, set_b); +IMPL_ENTRY_SET_OR_ADD(Set, float, kFloat, set_f); +IMPL_ENTRY_SET_OR_ADD(Set, double, kDouble, set_d); +IMPL_ENTRY_SET_OR_ADD(Add, int32_t, kInt32s, add_i32s); +IMPL_ENTRY_SET_OR_ADD(Add, int64_t, kInt64s, add_i64s); +IMPL_ENTRY_SET_OR_ADD(Add, bool, kBools, add_bs); +IMPL_ENTRY_SET_OR_ADD(Add, float, kFloats, add_fs); +IMPL_ENTRY_SET_OR_ADD(Add, double, kDoubles, add_ds); + +std::string StorageHelper::human_readable_buffer() const { + std::string buffer; + google::protobuf::TextFormat::PrintToString(*data_, &buffer); + return buffer; +} + +std::string TabletHelper::human_readable_buffer() const { + std::string buffer; + google::protobuf::TextFormat::PrintToString(*data_, &buffer); + return buffer; +} + +} // namespace visualdl diff --git a/visualdl/backend/logic/sdk.h b/visualdl/backend/logic/sdk.h new file mode 100644 index 0000000000000000000000000000000000000000..c99958a7e11c129f860a983813f11076875788ce --- /dev/null +++ b/visualdl/backend/logic/sdk.h @@ -0,0 +1,79 @@ +#ifndef VISUALDL_BACKEND_LOGIC_SDK_H +#define VISUALDL_BACKEND_LOGIC_SDK_H +#include "visualdl/backend/logic/im.h" + +#include + +namespace visualdl { + +class TabletHelper { +public: + // method for each components + template + void AddScalarRecord(int id, T value); + + // basic member getter and setter + std::string record_buffer(int idx) const { return data_->records(idx).SerializeAsString(); } + size_t records_size() const { return data_->records_size(); } + std::string buffer() const { return data_->SerializeAsString(); } + std::string human_readable_buffer() const; + void SetBuffer(const storage::Tablet &t) { *data_ = t; } + void SetBuffer(const std::string &b) { data_->ParseFromString(b); } + + // constructor that enable concurrency. + TabletHelper(storage::Tablet *t) : data_(t) {} + // data updater that resuage of one instance. + TabletHelper &operator()(storage::Tablet *t) { data_ = t; return *this; } + +private: + storage::Tablet *data_; +}; + +class StorageHelper { +public: + StorageHelper(storage::Storage *s) : data_(s) {} + StorageHelper &operator()(storage::Storage *s) { + data_ = s; + return *this; + } + + void SetBuffer(const storage::Storage &buffer) { *data_ = buffer; } + void SetBuffer(const std::string &buffer) { data_->ParseFromString(buffer); } + void SetDir(const std::string &dir) { data_->set_dir(dir); } + + int64_t timestamp() const { return data_->timestamp(); } + std::string dir() const { return data_->dir(); } + int tablets_size() const { return data_->tablets_size(); } + std::string buffer() const { return data_->SerializeAsString(); } + std::string human_readable_buffer() const; + +private: + storage::Storage *data_; +}; + +class ImHelper { +public: + ImHelper() {} + + StorageHelper storage() { + return StorageHelper( + InformationMaintainer::Global().storage().mutable_data()); + } + TabletHelper tablet(const std::string &tag) { + return TabletHelper(InformationMaintainer::Global().storage().Find(tag)); + } + TabletHelper AddTablet(const std::string &tag, int num_samples) { + return TabletHelper( + InformationMaintainer::Global().AddTablet(tag, num_samples)); + } +}; + +static ImHelper& get_im() { + static ImHelper im; + return im; +} + +} // namespace visualdl + +#include "visualdl/backend/logic/sdk.hpp" +#endif // VISUALDL_BACKEND_LOGIC_SDK_H diff --git a/visualdl/backend/logic/sdk.hpp b/visualdl/backend/logic/sdk.hpp new file mode 100644 index 0000000000000000000000000000000000000000..517abe3f8a1dbbd0aa307a5b1cc41224dd167ee3 --- /dev/null +++ b/visualdl/backend/logic/sdk.hpp @@ -0,0 +1,33 @@ +#include "visualdl/backend/logic/im.h" + +namespace visualdl { + +/* + * Utility helper for storage::Entry. + */ +template struct Entry { + // use pointer to avoid copy + storage::Entry *entry{nullptr}; + + Entry(storage::Entry *entry) : entry(entry) {} + + /* + * Set a single value. + */ + void Set(T v); + + /* + * Add a value to repeated message field. + */ + void Add(T v); +}; + +template +void TabletHelper::AddScalarRecord(int id, T value) { + auto* record = data_->add_records(); + record->set_id(id); + Entry entry_helper(record->mutable_data()); + entry_helper.Set(value); +} + +} // namespace visualdl diff --git a/visualdl/backend/test.cc b/visualdl/backend/test.cc new file mode 100644 index 0000000000000000000000000000000000000000..1659be89ee1a2fe0dbb8b1388f3a2a4c9860ef3c --- /dev/null +++ b/visualdl/backend/test.cc @@ -0,0 +1,6 @@ +#include "gtest/gtest.h" + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/visualdl/backend/test.py b/visualdl/backend/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8755ae7ff8a23587756fa8a8b76d568365c1744c --- /dev/null +++ b/visualdl/backend/test.py @@ -0,0 +1,48 @@ +import sys +import unittest + +sys.path.append('../../build') +import core + +im = core.im() + + +class StorageTester(unittest.TestCase): + def setUp(self): + self.storage = im.storage() + + def test_size(self): + self.assertEqual(self.storage.tablets_size(), 0) + im.add_tablet("tag0", 100) + self.assertEqual(self.storage.tablets_size(), 1) + + for i in range(1, 11): + im.add_tablet("tag%d" % i, 100) + self.assertEqual(self.storage.tablets_size(), 11) + + def test_timestamp(self): + print self.storage.timestamp() + + def test_dir(self): + dir = "./1.txt" + self.storage.set_dir(dir) + self.assertEqual(dir, self.storage.dir()) + + def test_human_readable_buffer(self): + print self.storage.human_readable_buffer() + + +class TabletTester(unittest.TestCase): + def setUp(self): + self.tablet = im.add_tablet("tag101", 20) + + def test_add_scalar(self): + self.tablet.add_scalar_float(1, 0.3) + self.assertEqual(self.tablet.records_size(), 1) + + def test_human_readable_buffer(self): + print self.tablet.human_readable_buffer() + + +if __name__ == '__main__': + unittest.main()