refactor summary.py

CMakeLists.txt

@@ -48,4 +48,6 @@ target_link_libraries(vl_test storage sdk im gtest glog protobuf gflags pthread)
 
 enable_testing ()
 
-add_test(NAME vstest COMMAND vl_test)
+add_custom_target(test_init COMMAND ./init_test.sh $CMAKE_BINARY_DIR)
+add_test(NAME vstest COMMAND ./vl_test)
+set_target_properties(vl_test PROPERTIES DEPENDS test_init)

visualdl/logic/CMakeLists.txt

@@ -7,3 +7,4 @@ add_dependencies(sdk storage_proto)
 add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc)
 add_dependencies(core pybind python im storage sdk protobuf glog)
 target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog)
+set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")

visualdl/logic/pybind.cc

@@ -22,19 +22,22 @@ PYBIND11_PLUGIN(core) {
                vs::TabletHelper::SetBuffer)
       // scalar interface
       .def("as_int32_scalar",
-           [](const vs::TabletHelper& self) {
-             return vs::components::ScalarHelper<int32_t>(&self.data());
+           [](vs::TabletHelper& self, vs::ImHelper& im) {
+             return vs::components::ScalarHelper<int32_t>(self, &im.handler());
            })
       .def("as_int64_scalar",
-           [](const vs::TabletHelper& self) {
-             return vs::components::ScalarHelper<int64_t>(&self.data());
+           [](vs::TabletHelper& self, vs::ImHelper& im) {
+             return vs::components::ScalarHelper<int64_t>(&self.data(),
+                                                          &im.handler());
            })
       .def("as_float_scalar",
-           [](const vs::TabletHelper& self) {
-             return vs::components::ScalarHelper<float>(&self.data());
+           [](vs::TabletHelper& self, vs::ImHelper& im) {
+             return vs::components::ScalarHelper<float>(&self.data(),
+                                                        &im.handler());
            })
-      .def("as_double_scalar", [](const vs::TabletHelper& self) {
-        return vs::components::ScalarHelper<double>(&self.data());
+      .def("as_double_scalar", [](vs::TabletHelper& self, vs::ImHelper& im) {
+        return vs::components::ScalarHelper<double>(&self.data(),
+                                                    &im.handler());
       });
 
   py::class_<vs::StorageHelper>(m, "Storage")
@@ -49,6 +52,8 @@ PYBIND11_PLUGIN(core) {
                vs::StorageHelper::SetBuffer);
 
   py::class_<vs::ImHelper>(m, "Im")
+      .def("__init__",
+           [](vs::ImHelper& instance) { new (&instance) vs::ImHelper(); })
       .def("storage", &vs::ImHelper::storage)
       .def("tablet", &vs::ImHelper::tablet)
       .def("add_tablet", &vs::ImHelper::AddTablet)

visualdl/python/summary.py

@@ -6,26 +6,15 @@ import core
 
 dtypes = ("float", "double", "int32", "int64")
 
-
-def set_storage(dir):
-    '''
-    :param dir: str
-        directory of summary to write log.
-    :return: None
-    '''
-    core.im().storage().set_dir(dir)
-
-
-def set_readable_storage(dir):
-    core.start_read_service(dir)
-
-
-def set_writable_storage(dir):
-    core.start_write_service(dir)
-
-
-def stop_service():
-    core.stop_threads()
+def IM(dir, mode="read", msecs=500):
+    im = core.Im()
+    READ = "read"
+    WRITE = "write"
+    if mode == READ:
+        im.start_read_service(dir, msecs)
+    else:
+        im.start_write_service(dir, msecs)
+    return im
 
 
 class _Scalar(object):
@@ -93,7 +82,7 @@ class _Scalar(object):
         return self._core_object.get_record_size()
 
 
-def scalar(tag, dtype='float'):
+def scalar(im, tag, dtype='float'):
     '''
     create a scalar component.
 
@@ -105,24 +94,24 @@ def scalar(tag, dtype='float'):
     '''
     assert dtype in dtypes, "invalid dtype(%s), should be one of %s" % (
         dtype, str(dtypes))
-    tablet = core.im().add_tablet(tag, -1)
+    tablet = im.add_tablet(tag, -1)
     dtype2obj = {
         'float': tablet.as_float_scalar,
         'double': tablet.as_double_scalar,
         'int32': tablet.as_int32_scalar,
         'int64': tablet.as_int64_scalar,
     }
-    obj = dtype2obj[dtype]()
+    obj = dtype2obj[dtype](im)
     return _Scalar(obj)
 
 
-def read_scalar(tag, dtype='float'):
-    tablet = core.im().tablet(tag)
-    dtype2obj = {
-        'float': tablet.as_float_scalar,
-        'double': tablet.as_double_scalar,
-        'int32': tablet.as_int32_scalar,
-        'int64': tablet.as_int64_scalar,
-    }
-    obj = dtype2obj[dtype]()
-    return _Scalar(obj)
+# def read_scalar(tag, dtype='float'):
+#     tablet = core.im().tablet(tag)
+#     dtype2obj = {
+#         'float': tablet.as_float_scalar,
+#         'double': tablet.as_double_scalar,
+#         'int32': tablet.as_int32_scalar,
+#         'int64': tablet.as_int64_scalar,
+#     }
+#     obj = dtype2obj[dtype]()
+#     return _Scalar(obj)

visualdl/python/test_summary.py

@@ -8,10 +8,9 @@ once_flag = False
 
 class ScalarTester(unittest.TestCase):
     def setUp(self):
-        summary.set_storage("tmp_dir")
-        global once_flag
-        self.scalar = summary.scalar("scalar0")
-        if not once_flag:
+        dir = "tmp/3.test"
+        self.im = summary.IM("write", dir)
+        self.scalar = summary.scalar(self.im, "scalar0")
         self.py_captions = ["train cost", "test cost"]
         self.scalar.set_captions(self.py_captions)
 
@@ -22,9 +21,8 @@ class ScalarTester(unittest.TestCase):
             id = i * 10
             self.py_records.append(record)
             self.py_ids.append(id)
-            if not once_flag:
             self.scalar.add(id, record)
-        once_flag = True
+
 
     def test_records(self):
         self.assertEqual(self.scalar.size, len(self.py_records))

visualdl/storage/storage.cc

@@ -64,7 +64,6 @@ void MemoryStorage::LoadFromDisk(const std::string &dir) {
   // load storage
   CHECK(fs::DeSerializeFromFile(&storage_, meta_path(dir)))
       << "parse from " << meta_path(dir) << " failed";
-
   // load all the tablets
   for (int i = 0; i < storage_.tags_size(); i++) {
     auto tag = storage_.tags(i);

visualdl/utils/filesystem.h

@@ -39,6 +39,7 @@ bool SerializeToFile(const T& proto, const std::string& path) {
 template <typename T>
 bool DeSerializeFromFile(T* proto, const std::string& path) {
   std::ifstream file(path, std::ios::binary);
+  CHECK(file.is_open()) << "open " << path << " failed";
   return proto->ParseFromIstream(&file);
 }