finish sdk

CMakeLists.txt

@@ -38,14 +38,15 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/python)
 add_executable(vl_test
         ${PROJECT_SOURCE_DIR}/visualdl/test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h
         ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h
         )
-target_link_libraries(vl_test storage im gtest glog protobuf gflags pthread)
+target_link_libraries(vl_test entry sdk storage im gtest glog protobuf gflags pthread)
 
 enable_testing ()
 
-add_custom_target(test_init COMMAND ./init_test.sh $CMAKE_BINARY_DIR)
+add_custom_target(test_init COMMAND $CMAKE_BINARY_DIR)
 add_test(NAME vstest COMMAND ./vl_test)
 set_target_properties(vl_test PROPERTIES DEPENDS test_init)

visualdl/logic/CMakeLists.txt

@@ -2,11 +2,10 @@
 add_library(im ${PROJECT_SOURCE_DIR}/visualdl/logic/im.cc)
 add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk.cc)
 add_dependencies(im storage_proto)
-add_dependencies(sdk storage_proto)
-#add_dependencies(sdk storage_proto)
+add_dependencies(sdk entry storage storage_proto)
 
 ## pybind
-#add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc)
-#add_dependencies(core pybind python im storage sdk protobuf glog)
-#target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog)
-#set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")
+add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc)
+add_dependencies(core pybind python im storage sdk protobuf glog)
+target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog)
+set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")

visualdl/logic/pybind.cc

@@ -6,84 +6,66 @@
 
 namespace py = pybind11;
 namespace vs = visualdl;
+namespace cp = visualdl::components;
 
 PYBIND11_PLUGIN(core) {
   py::module m("core", "C++ core of VisualDL");
-  //  m.doc() = "visualdl python core API";
 
-  py::class_<vs::TabletHelper>(m, "Tablet")
-      // other member setter and getter
-      .def("record_buffer", &vs::TabletHelper::record_buffer)
-      .def("records_size", &vs::TabletHelper::records_size)
-      .def("buffer", &vs::TabletHelper::buffer)
-      .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer)
-      .def("set_buffer",
-           (void (vs::TabletHelper::*)(const std::string&)) &
-               vs::TabletHelper::SetBuffer)
-      // scalar interface
-      .def("as_int32_scalar",
-           [](vs::TabletHelper& self, vs::ImHelper& im) {
-             return vs::components::ScalarHelper<int32_t>(self, &im.handler());
-           })
-      .def("as_int64_scalar",
-           [](vs::TabletHelper& self, vs::ImHelper& im) {
-             return vs::components::ScalarHelper<int64_t>(&self.data(),
-                                                          &im.handler());
-           })
-      .def("as_float_scalar",
-           [](vs::TabletHelper& self, vs::ImHelper& im) {
-             return vs::components::ScalarHelper<float>(&self.data(),
-                                                        &im.handler());
-           })
-      .def("as_double_scalar", [](vs::TabletHelper& self, vs::ImHelper& im) {
-        return vs::components::ScalarHelper<double>(&self.data(),
-                                                    &im.handler());
-      });
+#define ADD_SCALAR(T)                                      \
+  py::class_<cp::ScalarReader<T>>(m, "ScalarReader__" #T)  \
+      .def("records", &cp::ScalarReader<T>::records)       \
+      .def("timestamps", &cp::ScalarReader<T>::timestamps) \
+      .def("captions", &cp::ScalarReader<T>::captions);
+  ADD_SCALAR(int);
+  ADD_SCALAR(float);
+  ADD_SCALAR(double);
+  ADD_SCALAR(int64_t);
+#undef ADD_SCALAR
 
-  py::class_<vs::StorageHelper>(m, "Storage")
-      .def("timestamp", &vs::StorageHelper::timestamp)
-      .def("dir", &vs::StorageHelper::dir)
-      .def("set_dir", &vs::StorageHelper::SetDir)
-      .def("tablets_size", &vs::StorageHelper::tablets_size)
-      .def("buffer", &vs::StorageHelper::buffer)
-      .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer)
-      .def("set_buffer",
-           (void (vs::StorageHelper::*)(const std::string&)) &
-               vs::StorageHelper::SetBuffer);
+#define ADD_SCALAR_WRITER(T)                          \
+  py::class_<cp::Scalar<T>>(m, "ScalarWriter__" #T)   \
+      .def("set_caption", &cp::Scalar<T>::SetCaption) \
+      .def("add_record", &cp::Scalar<T>::AddRecord);
+  ADD_SCALAR_WRITER(int);
+  ADD_SCALAR_WRITER(float);
+  ADD_SCALAR_WRITER(double);
+#undef ADD_SCALAR_WRITER
 
-  py::class_<vs::ImHelper>(m, "Im")
-      .def("__init__",
-           [](vs::ImHelper& instance) { new (&instance) vs::ImHelper(); })
-      .def("storage", &vs::ImHelper::storage)
-      .def("tablet", &vs::ImHelper::tablet)
-      .def("add_tablet", &vs::ImHelper::AddTablet)
-      .def("persist_to_disk", &vs::ImHelper::PersistToDisk)
-      .def("clear_tablets", &vs::ImHelper::ClearTablets)
-      .def("start_read_service",
-           &vs::ImHelper::StartReadService,
-           "start a thread to maintain read service")
-      .def("start_write_service",
-           &vs::ImHelper::StartWriteSerice,
-           "start a thread to maintain write service")
-      .def("stop_service",
-           &vs::ImHelper::StopService,
-           "stop the service thread");
+#define ADD_SCALAR(T)                                                   \
+  .def("get_scalar_" #T, [](vs::Reader& self, const std::string& tag) { \
+    auto tablet = self.tablet(tag);                                     \
+    return vs::components::ScalarReader<T>(std::move(tablet));          \
+  })
+  py::class_<vs::Reader>(m, "Reader")
+      .def(
+          "__init__",
+          [](vs::Reader& instance,
+             const std::string& mode,
+             const std::string& dir) { new (&instance) vs::Reader(mode, dir); })
+      // clang-format off
+    ADD_SCALAR(float)
+    ADD_SCALAR(double)
+    ADD_SCALAR(int)
+// clang-format on
+#undef ADD_SCALAR
 
-// interfaces for components begin
+#define ADD_SCALAR(T)                                                   \
+  .def("new_scalar_" #T, [](vs::Writer& self, const std::string& tag) { \
+    auto tablet = self.AddTablet(tag);                                  \
+    return cp::Scalar<T>(tablet);                                       \
+  })
 
-// different data type of scalar conponent
-#define ADD_SCALAR_TYPED_INTERFACE(T, name__)                             \
-  py::class_<vs::components::ScalarHelper<T>>(m, #name__)                 \
-      .def("add_record", &vs::components::ScalarHelper<T>::AddRecord)     \
-      .def("set_captions", &vs::components::ScalarHelper<T>::SetCaptions) \
-      .def("get_records", &vs::components::ScalarHelper<T>::GetRecords)   \
-      .def("get_captions", &vs::components::ScalarHelper<T>::GetCaptions) \
-      .def("get_ids", &vs::components::ScalarHelper<T>::GetIds)           \
-      .def("get_record_size", &vs::components::ScalarHelper<T>::GetSize)  \
-      .def("get_timestamps", &vs::components::ScalarHelper<T>::GetTimestamps);
-  ADD_SCALAR_TYPED_INTERFACE(int32_t, ScalarInt32);
-  ADD_SCALAR_TYPED_INTERFACE(int64_t, ScalarInt64);
-  ADD_SCALAR_TYPED_INTERFACE(float, ScalarFloat);
-  ADD_SCALAR_TYPED_INTERFACE(double, ScalarDouble);
-#undef ADD_SCALAR_TYPED_INTERFACE
-}
+  py::class_<vs::Writer>(m, "Writer")
+      .def(
+          "__init__",
+          [](vs::Writer& instance,
+             const std::string& mode,
+             const std::string& dir) { new (&instance) vs::Writer(mode, dir); })
+      // clang-format off
+    ADD_SCALAR(float)
+    ADD_SCALAR(double)
+    ADD_SCALAR(int)
+// clang-format on
+#undef ADD_SCALAR
+
+}  // end pybind

visualdl/logic/sdk.cc

@@ -54,6 +54,12 @@ template <typename T>
 size_t ScalarReader<T>::size() const {
   return reader_.total_records();
 }
+
+template class Scalar<int>;
+template class Scalar<int64_t>;
+template class Scalar<float>;
+template class Scalar<double>;
+
 }  // namespace components
 
 }  // namespace visualdl

visualdl/logic/sdk.h

@@ -3,8 +3,45 @@
 
 #include "visualdl/storage/storage.h"
 #include "visualdl/storage/tablet.h"
-
+#include "visualdl/utils/string.h"
 namespace visualdl {
+
+class Writer {
+public:
+  Writer(const std::string& mode, const std::string& dir) : mode_(mode) {
+    storage_.SetDir(dir);
+  }
+
+  Tablet AddTablet(const std::string& tag) {
+    // TODO(ChunweiYan) add string check here.
+    auto tmp = mode_ + "/" + tag;
+    string::TagEncode(tmp);
+    return storage_.AddTablet(tmp);
+  }
+
+  Storage& storage() { return storage_; }
+
+private:
+  Storage storage_;
+  std::string mode_;
+};
+
+class Reader {
+public:
+  Reader(const std::string& mode, const std::string& dir)
+      : mode_(mode), reader_(dir) {}
+
+  TabletReader tablet(const std::string& tag) {
+    auto tmp = mode_ + "/" + tag;
+    string::TagEncode(tmp);
+    return reader_.tablet(tmp);
+  }
+
+private:
+  StorageReader reader_;
+  std::string mode_;
+};
+
 namespace components {
 
 /*

visualdl/logic/sdk_test.cc

@@ -4,95 +4,29 @@
 
 namespace visualdl {
 
-struct ScalarTestHelper {
-  ImHelper _rim;
-  ImHelper _wim;
-  ImHelper rim = _rim.AsMode("train");
-  ImHelper wim = _wim.AsMode("train");
-  const std::string dir = "./tmp/sdk_test.test";
-
-  void operator()(std::function<void()> read, std::function<void()> write) {
-    wim.StartWriteSerice(dir, 200);
-    write();
-
-    // should wait for the write service create log's path
-    std::this_thread::sleep_for(std::chrono::milliseconds(400));
-    rim.StartReadService(dir, 100);
-    // should wait for the read service to load "tag0" tablet into memory
-    std::this_thread::sleep_for(std::chrono::milliseconds(600));
-    read();
-  }
-};
-
-TEST(Scalar, set_caption) {
-  ScalarTestHelper helper;
-
-  const std::vector<std::string> captions({"train", "test"});
-
-  auto write = [&] {
-    auto tablet = helper.wim.AddTablet("tag0", -1);
-    components::ScalarHelper<float> scalar(tablet, &helper.wim.handler());
-
-    scalar.SetCaptions(captions);
-  };
-
-  auto read = [&] {
-    auto mytablet = helper.rim.tablet("tag0");
-    components::ScalarHelper<float> myscalar(mytablet, &helper.rim.handler());
-    auto mycaptions = myscalar.GetCaptions();
-
-    ASSERT_EQ(captions, mycaptions);
-  };
-
-  helper(read, write);
-}
-
-TEST(Scalar, add_records) {
-  ScalarTestHelper helper;
-
-  const std::vector<std::string> captions({"train", "test"});
-
-  const size_t nsteps = 100;
-
-  auto write = [&] {
-    auto tablet = helper.wim.AddTablet("tag0", -1);
-    components::ScalarHelper<float> scalar(tablet, &helper.wim.handler());
-
-    scalar.SetCaptions(captions);
-
-    for (int i = 0; i < nsteps; i++) {
-      scalar.AddRecord(i * 10, std::vector<float>({(float)i, (float)i + 1}));
-    }
-  };
-
-  auto read = [&] {
-    auto mytablet = helper.rim.tablet("tag0");
-    components::ScalarHelper<float> myscalar(mytablet, &helper.rim.handler());
-
-    auto records = myscalar.GetRecords();
-    ASSERT_EQ(records.size(), nsteps);
-
-    for (int i = 0; i < nsteps; i++) {
-      ASSERT_EQ(records[i], std::vector<float>({(float)i, (float)i + 1}));
-    }
-  };
-
-  helper(read, write);
-}
-
-TEST(Scalar, mode) {
-  ScalarTestHelper helper;
-  auto train_wim = helper.wim.AsMode("train");
-
-  auto write = [&] {
-    auto tablet = train_wim.AddTablet("tag1", -1);
-    components::ScalarHelper<float> scalar(tablet, &train_wim.handler());
-
-    scalar.SetCaptions(std::vector<std::string>({"train"}));
-    scalar.AddRecord(10, std::vector<float>({0.1}));
-  };
-
-  auto reader = [&] {};
+TEST(Scalar, write) {
+  const auto dir = "./tmp/sdk_test";
+  Storage storage;
+  // write disk every time
+  storage.meta.cycle = 1;
+  storage.SetDir(dir);
+  auto tablet = storage.AddTablet("scalar0");
+  components::Scalar<int> scalar(tablet);
+  scalar.SetCaption("train");
+  scalar.AddRecord(0, std::vector<int>({12}));
+
+  // read from disk
+  StorageReader reader(dir);
+  auto scalar_reader = reader.tablet("scalar0");
+  auto captioins = scalar_reader.captions();
+  ASSERT_EQ(captioins.size(), 1);
+  ASSERT_EQ(captioins.front(), "train");
+  ASSERT_EQ(scalar_reader.total_records(), 1);
+  auto record = scalar_reader.record(0);
+  // check the first entry of first record
+  auto vs = record.data<int>(0).GetMulti();
+  ASSERT_EQ(vs.size(), 1);
+  ASSERT_EQ(vs.front(), 12);
 }
 
 }  // namespace visualdl

visualdl/storage/CMakeLists.txt

@@ -4,14 +4,12 @@ add_library(storage_proto ${PROTO_SRCS})
 add_dependencies(storage_proto protobuf)
 
 ## add storage as target
-#add_library(storage storage.cc storage.h ${PROTO_SRCS} ${PROTO_HDRS})
 add_library(entry entry.cc entry.h ${PROTO_SRCS} ${PROTO_HDRS})
 add_library(tablet tablet.cc tablet.h ${PROTO_SRCS} ${PROTO_HDRS})
 add_library(record record.cc record.h ${PROTO_SRCS} ${PROTO_HDRS})
 add_library(storage storage.cc storage.h ${PROTO_SRCS} ${PROTO_HDRS})
 
 add_dependencies(entry storage_proto)
+add_dependencies(record storage_proto entry)
 add_dependencies(tablet storage_proto)
-add_dependencies(record storage_proto)
 add_dependencies(storage storage_proto)
-#add_dependencies(storage storage_proto)

visualdl/storage/entry.h

@@ -19,9 +19,9 @@ struct Entry {
   storage::Entry* entry{nullptr};
 
   Entry() {}
-  explicit Entry(storage::Entry* entry, void* parent)
+  explicit Entry(storage::Entry* entry, Storage* parent)
       : entry(entry), x_(parent) {}
-  void operator()(storage::Entry* entry, void* parent) {
+  void operator()(storage::Entry* entry, Storage* parent) {
     this->entry = entry;
     x_ = parent;
   }

visualdl/storage/storage.h

@@ -54,9 +54,10 @@ struct Storage {
   }
 
   Tablet AddTablet(const std::string& x) {
-    AddTag(x);
     CHECK(tablets_.count(x) == 0) << "tablet [" << x << "] has existed";
     tablets_[x] = storage::Tablet();
+    AddTag(x);
+    LOG(INFO) << "really add tag " << x;
     WRITE_GUARD
     return Tablet(&tablets_[x], this);
   }
@@ -83,7 +84,6 @@ struct Storage {
 
 protected:
   void AddTag(const std::string& x) {
-    WRITE_GUARD
     *data_->add_tags() = x;
   }
 

visualdl/storage/storage_test.cc

@@ -8,7 +8,7 @@ class StorageTest : public ::testing::Test {
 public:
   void SetUp() {
     storage.SetDir("./tmp/storage_test");
-    storage.meta.cycle = 2;
+    storage.meta.cycle = 1;
   }
 
   Storage storage;
@@ -20,6 +20,9 @@ TEST_F(StorageTest, main) {
 
   auto tag0 = storage.AddTablet("tag0");
   auto tag1 = storage.AddTablet("tag1");
+  auto record = tag0.AddRecord();
+  auto entry = record.AddData<int>();
+  entry.Set(12);
 
   StorageReader reader("./tmp/storage_test");
   auto modes = reader.Modes();