diff --git a/CMakeLists.txt b/CMakeLists.txt index 57f68632e961aa668a7fdc342830d90f45477a2d..9f8d4cb30cb5f42fe1b7689b3f3216a252a0231e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,14 +38,15 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/python) add_executable(vl_test ${PROJECT_SOURCE_DIR}/visualdl/test.cc ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc + ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h ) -target_link_libraries(vl_test storage im gtest glog protobuf gflags pthread) +target_link_libraries(vl_test entry sdk storage im gtest glog protobuf gflags pthread) enable_testing () -add_custom_target(test_init COMMAND ./init_test.sh $CMAKE_BINARY_DIR) +add_custom_target(test_init COMMAND $CMAKE_BINARY_DIR) add_test(NAME vstest COMMAND ./vl_test) set_target_properties(vl_test PROPERTIES DEPENDS test_init) diff --git a/visualdl/logic/CMakeLists.txt b/visualdl/logic/CMakeLists.txt index 046e40489b714dcdc881ab06145c933eaf97e989..77c8e61a20a3587b7527a74869f71422bc65db4d 100644 --- a/visualdl/logic/CMakeLists.txt +++ b/visualdl/logic/CMakeLists.txt @@ -2,11 +2,10 @@ add_library(im ${PROJECT_SOURCE_DIR}/visualdl/logic/im.cc) add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk.cc) add_dependencies(im storage_proto) -add_dependencies(sdk storage_proto) -#add_dependencies(sdk storage_proto) +add_dependencies(sdk entry storage storage_proto) ## pybind -#add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc) -#add_dependencies(core pybind python im storage sdk protobuf glog) -#target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog) -#set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so") +add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc) +add_dependencies(core pybind python im storage sdk protobuf glog) +target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog) +set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so") diff --git a/visualdl/logic/pybind.cc b/visualdl/logic/pybind.cc index 69d40ca039e606d9d20a3d327347b665c1010480..7d6cb89dd1a7aa56bb82c266c57c991c608dbdf0 100644 --- a/visualdl/logic/pybind.cc +++ b/visualdl/logic/pybind.cc @@ -6,84 +6,66 @@ namespace py = pybind11; namespace vs = visualdl; +namespace cp = visualdl::components; PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of VisualDL"); - // m.doc() = "visualdl python core API"; - py::class_(m, "Tablet") - // other member setter and getter - .def("record_buffer", &vs::TabletHelper::record_buffer) - .def("records_size", &vs::TabletHelper::records_size) - .def("buffer", &vs::TabletHelper::buffer) - .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer) - .def("set_buffer", - (void (vs::TabletHelper::*)(const std::string&)) & - vs::TabletHelper::SetBuffer) - // scalar interface - .def("as_int32_scalar", - [](vs::TabletHelper& self, vs::ImHelper& im) { - return vs::components::ScalarHelper(self, &im.handler()); - }) - .def("as_int64_scalar", - [](vs::TabletHelper& self, vs::ImHelper& im) { - return vs::components::ScalarHelper(&self.data(), - &im.handler()); - }) - .def("as_float_scalar", - [](vs::TabletHelper& self, vs::ImHelper& im) { - return vs::components::ScalarHelper(&self.data(), - &im.handler()); - }) - .def("as_double_scalar", [](vs::TabletHelper& self, vs::ImHelper& im) { - return vs::components::ScalarHelper(&self.data(), - &im.handler()); - }); +#define ADD_SCALAR(T) \ + py::class_>(m, "ScalarReader__" #T) \ + .def("records", &cp::ScalarReader::records) \ + .def("timestamps", &cp::ScalarReader::timestamps) \ + .def("captions", &cp::ScalarReader::captions); + ADD_SCALAR(int); + ADD_SCALAR(float); + ADD_SCALAR(double); + ADD_SCALAR(int64_t); +#undef ADD_SCALAR - py::class_(m, "Storage") - .def("timestamp", &vs::StorageHelper::timestamp) - .def("dir", &vs::StorageHelper::dir) - .def("set_dir", &vs::StorageHelper::SetDir) - .def("tablets_size", &vs::StorageHelper::tablets_size) - .def("buffer", &vs::StorageHelper::buffer) - .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer) - .def("set_buffer", - (void (vs::StorageHelper::*)(const std::string&)) & - vs::StorageHelper::SetBuffer); +#define ADD_SCALAR_WRITER(T) \ + py::class_>(m, "ScalarWriter__" #T) \ + .def("set_caption", &cp::Scalar::SetCaption) \ + .def("add_record", &cp::Scalar::AddRecord); + ADD_SCALAR_WRITER(int); + ADD_SCALAR_WRITER(float); + ADD_SCALAR_WRITER(double); +#undef ADD_SCALAR_WRITER - py::class_(m, "Im") - .def("__init__", - [](vs::ImHelper& instance) { new (&instance) vs::ImHelper(); }) - .def("storage", &vs::ImHelper::storage) - .def("tablet", &vs::ImHelper::tablet) - .def("add_tablet", &vs::ImHelper::AddTablet) - .def("persist_to_disk", &vs::ImHelper::PersistToDisk) - .def("clear_tablets", &vs::ImHelper::ClearTablets) - .def("start_read_service", - &vs::ImHelper::StartReadService, - "start a thread to maintain read service") - .def("start_write_service", - &vs::ImHelper::StartWriteSerice, - "start a thread to maintain write service") - .def("stop_service", - &vs::ImHelper::StopService, - "stop the service thread"); +#define ADD_SCALAR(T) \ + .def("get_scalar_" #T, [](vs::Reader& self, const std::string& tag) { \ + auto tablet = self.tablet(tag); \ + return vs::components::ScalarReader(std::move(tablet)); \ + }) + py::class_(m, "Reader") + .def( + "__init__", + [](vs::Reader& instance, + const std::string& mode, + const std::string& dir) { new (&instance) vs::Reader(mode, dir); }) + // clang-format off + ADD_SCALAR(float) + ADD_SCALAR(double) + ADD_SCALAR(int) +// clang-format on +#undef ADD_SCALAR -// interfaces for components begin +#define ADD_SCALAR(T) \ + .def("new_scalar_" #T, [](vs::Writer& self, const std::string& tag) { \ + auto tablet = self.AddTablet(tag); \ + return cp::Scalar(tablet); \ + }) -// different data type of scalar conponent -#define ADD_SCALAR_TYPED_INTERFACE(T, name__) \ - py::class_>(m, #name__) \ - .def("add_record", &vs::components::ScalarHelper::AddRecord) \ - .def("set_captions", &vs::components::ScalarHelper::SetCaptions) \ - .def("get_records", &vs::components::ScalarHelper::GetRecords) \ - .def("get_captions", &vs::components::ScalarHelper::GetCaptions) \ - .def("get_ids", &vs::components::ScalarHelper::GetIds) \ - .def("get_record_size", &vs::components::ScalarHelper::GetSize) \ - .def("get_timestamps", &vs::components::ScalarHelper::GetTimestamps); - ADD_SCALAR_TYPED_INTERFACE(int32_t, ScalarInt32); - ADD_SCALAR_TYPED_INTERFACE(int64_t, ScalarInt64); - ADD_SCALAR_TYPED_INTERFACE(float, ScalarFloat); - ADD_SCALAR_TYPED_INTERFACE(double, ScalarDouble); -#undef ADD_SCALAR_TYPED_INTERFACE -} + py::class_(m, "Writer") + .def( + "__init__", + [](vs::Writer& instance, + const std::string& mode, + const std::string& dir) { new (&instance) vs::Writer(mode, dir); }) + // clang-format off + ADD_SCALAR(float) + ADD_SCALAR(double) + ADD_SCALAR(int) +// clang-format on +#undef ADD_SCALAR + +} // end pybind diff --git a/visualdl/logic/sdk.cc b/visualdl/logic/sdk.cc index 11b4393752781dc96b2ba0d8409467f7a390d86e..f1760320052dcb0fee3975917a4b925fca0c20be 100644 --- a/visualdl/logic/sdk.cc +++ b/visualdl/logic/sdk.cc @@ -54,6 +54,12 @@ template size_t ScalarReader::size() const { return reader_.total_records(); } + +template class Scalar; +template class Scalar; +template class Scalar; +template class Scalar; + } // namespace components } // namespace visualdl diff --git a/visualdl/logic/sdk.h b/visualdl/logic/sdk.h index e75fd7d17053e7d6e769d97debce97de164f5834..f6d9b557e642a267c60ecf2c1df1ab032e36c7b3 100644 --- a/visualdl/logic/sdk.h +++ b/visualdl/logic/sdk.h @@ -3,8 +3,45 @@ #include "visualdl/storage/storage.h" #include "visualdl/storage/tablet.h" - +#include "visualdl/utils/string.h" namespace visualdl { + +class Writer { +public: + Writer(const std::string& mode, const std::string& dir) : mode_(mode) { + storage_.SetDir(dir); + } + + Tablet AddTablet(const std::string& tag) { + // TODO(ChunweiYan) add string check here. + auto tmp = mode_ + "/" + tag; + string::TagEncode(tmp); + return storage_.AddTablet(tmp); + } + + Storage& storage() { return storage_; } + +private: + Storage storage_; + std::string mode_; +}; + +class Reader { +public: + Reader(const std::string& mode, const std::string& dir) + : mode_(mode), reader_(dir) {} + + TabletReader tablet(const std::string& tag) { + auto tmp = mode_ + "/" + tag; + string::TagEncode(tmp); + return reader_.tablet(tmp); + } + +private: + StorageReader reader_; + std::string mode_; +}; + namespace components { /* diff --git a/visualdl/logic/sdk_test.cc b/visualdl/logic/sdk_test.cc index 566b3f002f2a3b240867e26cb0298865832de2ae..71e87e9e2780d0968c67da0f4863c7061fa40e28 100644 --- a/visualdl/logic/sdk_test.cc +++ b/visualdl/logic/sdk_test.cc @@ -4,95 +4,29 @@ namespace visualdl { -struct ScalarTestHelper { - ImHelper _rim; - ImHelper _wim; - ImHelper rim = _rim.AsMode("train"); - ImHelper wim = _wim.AsMode("train"); - const std::string dir = "./tmp/sdk_test.test"; - - void operator()(std::function read, std::function write) { - wim.StartWriteSerice(dir, 200); - write(); - - // should wait for the write service create log's path - std::this_thread::sleep_for(std::chrono::milliseconds(400)); - rim.StartReadService(dir, 100); - // should wait for the read service to load "tag0" tablet into memory - std::this_thread::sleep_for(std::chrono::milliseconds(600)); - read(); - } -}; - -TEST(Scalar, set_caption) { - ScalarTestHelper helper; - - const std::vector captions({"train", "test"}); - - auto write = [&] { - auto tablet = helper.wim.AddTablet("tag0", -1); - components::ScalarHelper scalar(tablet, &helper.wim.handler()); - - scalar.SetCaptions(captions); - }; - - auto read = [&] { - auto mytablet = helper.rim.tablet("tag0"); - components::ScalarHelper myscalar(mytablet, &helper.rim.handler()); - auto mycaptions = myscalar.GetCaptions(); - - ASSERT_EQ(captions, mycaptions); - }; - - helper(read, write); -} - -TEST(Scalar, add_records) { - ScalarTestHelper helper; - - const std::vector captions({"train", "test"}); - - const size_t nsteps = 100; - - auto write = [&] { - auto tablet = helper.wim.AddTablet("tag0", -1); - components::ScalarHelper scalar(tablet, &helper.wim.handler()); - - scalar.SetCaptions(captions); - - for (int i = 0; i < nsteps; i++) { - scalar.AddRecord(i * 10, std::vector({(float)i, (float)i + 1})); - } - }; - - auto read = [&] { - auto mytablet = helper.rim.tablet("tag0"); - components::ScalarHelper myscalar(mytablet, &helper.rim.handler()); - - auto records = myscalar.GetRecords(); - ASSERT_EQ(records.size(), nsteps); - - for (int i = 0; i < nsteps; i++) { - ASSERT_EQ(records[i], std::vector({(float)i, (float)i + 1})); - } - }; - - helper(read, write); -} - -TEST(Scalar, mode) { - ScalarTestHelper helper; - auto train_wim = helper.wim.AsMode("train"); - - auto write = [&] { - auto tablet = train_wim.AddTablet("tag1", -1); - components::ScalarHelper scalar(tablet, &train_wim.handler()); - - scalar.SetCaptions(std::vector({"train"})); - scalar.AddRecord(10, std::vector({0.1})); - }; - - auto reader = [&] {}; +TEST(Scalar, write) { + const auto dir = "./tmp/sdk_test"; + Storage storage; + // write disk every time + storage.meta.cycle = 1; + storage.SetDir(dir); + auto tablet = storage.AddTablet("scalar0"); + components::Scalar scalar(tablet); + scalar.SetCaption("train"); + scalar.AddRecord(0, std::vector({12})); + + // read from disk + StorageReader reader(dir); + auto scalar_reader = reader.tablet("scalar0"); + auto captioins = scalar_reader.captions(); + ASSERT_EQ(captioins.size(), 1); + ASSERT_EQ(captioins.front(), "train"); + ASSERT_EQ(scalar_reader.total_records(), 1); + auto record = scalar_reader.record(0); + // check the first entry of first record + auto vs = record.data(0).GetMulti(); + ASSERT_EQ(vs.size(), 1); + ASSERT_EQ(vs.front(), 12); } } // namespace visualdl diff --git a/visualdl/storage/CMakeLists.txt b/visualdl/storage/CMakeLists.txt index 68ffe14fb36b4f09029485e99b988edd169edd23..3a0b36caca84e6a446269d27d65ea07260aa79fb 100644 --- a/visualdl/storage/CMakeLists.txt +++ b/visualdl/storage/CMakeLists.txt @@ -4,14 +4,12 @@ add_library(storage_proto ${PROTO_SRCS}) add_dependencies(storage_proto protobuf) ## add storage as target -#add_library(storage storage.cc storage.h ${PROTO_SRCS} ${PROTO_HDRS}) add_library(entry entry.cc entry.h ${PROTO_SRCS} ${PROTO_HDRS}) add_library(tablet tablet.cc tablet.h ${PROTO_SRCS} ${PROTO_HDRS}) add_library(record record.cc record.h ${PROTO_SRCS} ${PROTO_HDRS}) add_library(storage storage.cc storage.h ${PROTO_SRCS} ${PROTO_HDRS}) add_dependencies(entry storage_proto) +add_dependencies(record storage_proto entry) add_dependencies(tablet storage_proto) -add_dependencies(record storage_proto) add_dependencies(storage storage_proto) -#add_dependencies(storage storage_proto) diff --git a/visualdl/storage/entry.h b/visualdl/storage/entry.h index d22f32c2c0409eac16b5af4a50dfbfc79da88d3f..6516efb3045e17e7203af8ac67d3d5a9d3b5534b 100644 --- a/visualdl/storage/entry.h +++ b/visualdl/storage/entry.h @@ -19,9 +19,9 @@ struct Entry { storage::Entry* entry{nullptr}; Entry() {} - explicit Entry(storage::Entry* entry, void* parent) + explicit Entry(storage::Entry* entry, Storage* parent) : entry(entry), x_(parent) {} - void operator()(storage::Entry* entry, void* parent) { + void operator()(storage::Entry* entry, Storage* parent) { this->entry = entry; x_ = parent; } diff --git a/visualdl/storage/storage.h b/visualdl/storage/storage.h index 97a891777d572920e3a2f3378f94ba108d3bef48..d81b7d787a82f38016ccf0d1edf618ce59bdb17a 100644 --- a/visualdl/storage/storage.h +++ b/visualdl/storage/storage.h @@ -54,9 +54,10 @@ struct Storage { } Tablet AddTablet(const std::string& x) { - AddTag(x); CHECK(tablets_.count(x) == 0) << "tablet [" << x << "] has existed"; tablets_[x] = storage::Tablet(); + AddTag(x); + LOG(INFO) << "really add tag " << x; WRITE_GUARD return Tablet(&tablets_[x], this); } @@ -83,7 +84,6 @@ struct Storage { protected: void AddTag(const std::string& x) { - WRITE_GUARD *data_->add_tags() = x; } diff --git a/visualdl/storage/storage_test.cc b/visualdl/storage/storage_test.cc index 8603bf3c0b1bba92d9abb9faba2ff35eecb4c595..47f13e8d11e6a20caa16b2fa0a4c2d2b37f70ae9 100644 --- a/visualdl/storage/storage_test.cc +++ b/visualdl/storage/storage_test.cc @@ -8,7 +8,7 @@ class StorageTest : public ::testing::Test { public: void SetUp() { storage.SetDir("./tmp/storage_test"); - storage.meta.cycle = 2; + storage.meta.cycle = 1; } Storage storage; @@ -20,6 +20,9 @@ TEST_F(StorageTest, main) { auto tag0 = storage.AddTablet("tag0"); auto tag1 = storage.AddTablet("tag1"); + auto record = tag0.AddRecord(); + auto entry = record.AddData(); + entry.Set(12); StorageReader reader("./tmp/storage_test"); auto modes = reader.Modes();