From 67b43463cbeb9ced21158ebf7a856d1926be4116 Mon Sep 17 00:00:00 2001 From: Superjom Date: Sun, 17 Dec 2017 11:40:11 +0800 Subject: [PATCH] refactor summary.py --- CMakeLists.txt | 4 ++- visualdl/logic/CMakeLists.txt | 1 + visualdl/logic/pybind.cc | 21 ++++++++----- visualdl/python/summary.py | 55 +++++++++++++-------------------- visualdl/python/test_summary.py | 16 +++++----- visualdl/storage/storage.cc | 1 - visualdl/utils/filesystem.h | 1 + 7 files changed, 47 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4efc6748..126529b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,4 +48,6 @@ target_link_libraries(vl_test storage sdk im gtest glog protobuf gflags pthread) enable_testing () -add_test(NAME vstest COMMAND vl_test) +add_custom_target(test_init COMMAND ./init_test.sh $CMAKE_BINARY_DIR) +add_test(NAME vstest COMMAND ./vl_test) +set_target_properties(vl_test PROPERTIES DEPENDS test_init) diff --git a/visualdl/logic/CMakeLists.txt b/visualdl/logic/CMakeLists.txt index 7f021af7..6b88d931 100644 --- a/visualdl/logic/CMakeLists.txt +++ b/visualdl/logic/CMakeLists.txt @@ -7,3 +7,4 @@ add_dependencies(sdk storage_proto) add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc) add_dependencies(core pybind python im storage sdk protobuf glog) target_link_libraries(core PRIVATE pybind python im storage sdk protobuf glog) +set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so") diff --git a/visualdl/logic/pybind.cc b/visualdl/logic/pybind.cc index 09e29a3b..69d40ca0 100644 --- a/visualdl/logic/pybind.cc +++ b/visualdl/logic/pybind.cc @@ -22,19 +22,22 @@ PYBIND11_PLUGIN(core) { vs::TabletHelper::SetBuffer) // scalar interface .def("as_int32_scalar", - [](const vs::TabletHelper& self) { - return vs::components::ScalarHelper(&self.data()); + [](vs::TabletHelper& self, vs::ImHelper& im) { + return vs::components::ScalarHelper(self, &im.handler()); }) .def("as_int64_scalar", - [](const vs::TabletHelper& self) { - return vs::components::ScalarHelper(&self.data()); + [](vs::TabletHelper& self, vs::ImHelper& im) { + return vs::components::ScalarHelper(&self.data(), + &im.handler()); }) .def("as_float_scalar", - [](const vs::TabletHelper& self) { - return vs::components::ScalarHelper(&self.data()); + [](vs::TabletHelper& self, vs::ImHelper& im) { + return vs::components::ScalarHelper(&self.data(), + &im.handler()); }) - .def("as_double_scalar", [](const vs::TabletHelper& self) { - return vs::components::ScalarHelper(&self.data()); + .def("as_double_scalar", [](vs::TabletHelper& self, vs::ImHelper& im) { + return vs::components::ScalarHelper(&self.data(), + &im.handler()); }); py::class_(m, "Storage") @@ -49,6 +52,8 @@ PYBIND11_PLUGIN(core) { vs::StorageHelper::SetBuffer); py::class_(m, "Im") + .def("__init__", + [](vs::ImHelper& instance) { new (&instance) vs::ImHelper(); }) .def("storage", &vs::ImHelper::storage) .def("tablet", &vs::ImHelper::tablet) .def("add_tablet", &vs::ImHelper::AddTablet) diff --git a/visualdl/python/summary.py b/visualdl/python/summary.py index 13b9400c..19a459b6 100644 --- a/visualdl/python/summary.py +++ b/visualdl/python/summary.py @@ -6,26 +6,15 @@ import core dtypes = ("float", "double", "int32", "int64") - -def set_storage(dir): - ''' - :param dir: str - directory of summary to write log. - :return: None - ''' - core.im().storage().set_dir(dir) - - -def set_readable_storage(dir): - core.start_read_service(dir) - - -def set_writable_storage(dir): - core.start_write_service(dir) - - -def stop_service(): - core.stop_threads() +def IM(dir, mode="read", msecs=500): + im = core.Im() + READ = "read" + WRITE = "write" + if mode == READ: + im.start_read_service(dir, msecs) + else: + im.start_write_service(dir, msecs) + return im class _Scalar(object): @@ -93,7 +82,7 @@ class _Scalar(object): return self._core_object.get_record_size() -def scalar(tag, dtype='float'): +def scalar(im, tag, dtype='float'): ''' create a scalar component. @@ -105,24 +94,24 @@ def scalar(tag, dtype='float'): ''' assert dtype in dtypes, "invalid dtype(%s), should be one of %s" % ( dtype, str(dtypes)) - tablet = core.im().add_tablet(tag, -1) + tablet = im.add_tablet(tag, -1) dtype2obj = { 'float': tablet.as_float_scalar, 'double': tablet.as_double_scalar, 'int32': tablet.as_int32_scalar, 'int64': tablet.as_int64_scalar, } - obj = dtype2obj[dtype]() + obj = dtype2obj[dtype](im) return _Scalar(obj) -def read_scalar(tag, dtype='float'): - tablet = core.im().tablet(tag) - dtype2obj = { - 'float': tablet.as_float_scalar, - 'double': tablet.as_double_scalar, - 'int32': tablet.as_int32_scalar, - 'int64': tablet.as_int64_scalar, - } - obj = dtype2obj[dtype]() - return _Scalar(obj) +# def read_scalar(tag, dtype='float'): +# tablet = core.im().tablet(tag) +# dtype2obj = { +# 'float': tablet.as_float_scalar, +# 'double': tablet.as_double_scalar, +# 'int32': tablet.as_int32_scalar, +# 'int64': tablet.as_int64_scalar, +# } +# obj = dtype2obj[dtype]() +# return _Scalar(obj) diff --git a/visualdl/python/test_summary.py b/visualdl/python/test_summary.py index c3caab75..e7189807 100644 --- a/visualdl/python/test_summary.py +++ b/visualdl/python/test_summary.py @@ -8,12 +8,11 @@ once_flag = False class ScalarTester(unittest.TestCase): def setUp(self): - summary.set_storage("tmp_dir") - global once_flag - self.scalar = summary.scalar("scalar0") - if not once_flag: - self.py_captions = ["train cost", "test cost"] - self.scalar.set_captions(self.py_captions) + dir = "tmp/3.test" + self.im = summary.IM("write", dir) + self.scalar = summary.scalar(self.im, "scalar0") + self.py_captions = ["train cost", "test cost"] + self.scalar.set_captions(self.py_captions) self.py_records = [] self.py_ids = [] @@ -22,9 +21,8 @@ class ScalarTester(unittest.TestCase): id = i * 10 self.py_records.append(record) self.py_ids.append(id) - if not once_flag: - self.scalar.add(id, record) - once_flag = True + self.scalar.add(id, record) + def test_records(self): self.assertEqual(self.scalar.size, len(self.py_records)) diff --git a/visualdl/storage/storage.cc b/visualdl/storage/storage.cc index 81431545..ec028647 100644 --- a/visualdl/storage/storage.cc +++ b/visualdl/storage/storage.cc @@ -64,7 +64,6 @@ void MemoryStorage::LoadFromDisk(const std::string &dir) { // load storage CHECK(fs::DeSerializeFromFile(&storage_, meta_path(dir))) << "parse from " << meta_path(dir) << " failed"; - // load all the tablets for (int i = 0; i < storage_.tags_size(); i++) { auto tag = storage_.tags(i); diff --git a/visualdl/utils/filesystem.h b/visualdl/utils/filesystem.h index 59301351..1a416749 100644 --- a/visualdl/utils/filesystem.h +++ b/visualdl/utils/filesystem.h @@ -39,6 +39,7 @@ bool SerializeToFile(const T& proto, const std::string& path) { template bool DeSerializeFromFile(T* proto, const std::string& path) { std::ifstream file(path, std::ios::binary); + CHECK(file.is_open()) << "open " << path << " failed"; return proto->ParseFromIstream(&file); } -- GitLab