add read and write to test_summary.py

visualdl/python/test_summary.py

 import summary
 import numpy as np
 import unittest
+import random
 import time
 
 once_flag = False
@@ -8,14 +9,21 @@ once_flag = False
 
 class ScalarTester(unittest.TestCase):
     def setUp(self):
-        dir = "tmp/3.test"
-        self.im = summary.IM("write", dir)
-        self.scalar = summary.scalar(self.im, "scalar0")
+        self.dir = "tmp/summary.test"
+        # clean path
+        try:
+            os.rmdir(self.dir)
+        except:
+            pass
+        self.im = summary.IM(self.dir, "write", 200)
+        self.tablet_name = "scalar0"
+        self.scalar = summary.scalar(self.im, self.tablet_name)
         self.py_captions = ["train cost", "test cost"]
         self.scalar.set_captions(self.py_captions)
 
         self.py_records = []
         self.py_ids = []
+        # write
         for i in range(10):
             record = [0.1 * i, 0.2 * i]
             id = i * 10
@@ -23,7 +31,6 @@ class ScalarTester(unittest.TestCase):
             self.py_ids.append(id)
             self.scalar.add(id, record)
 
-
     def test_records(self):
         self.assertEqual(self.scalar.size, len(self.py_records))
         for i, record in enumerate(self.scalar.records):
@@ -37,6 +44,58 @@ class ScalarTester(unittest.TestCase):
     def test_captions(self):
         self.assertEqual(self.scalar.captions, self.py_captions)
 
+    def test_read_records(self):
+        time.sleep(1)
+        im = summary.IM(self.dir, "read", 200)
+        time.sleep(1)
+        scalar = summary.scalar(im, self.tablet_name)
+        records = scalar.records
+        self.assertEqual(len(self.py_records), scalar.size)
+        for i, record in enumerate(self.scalar.records):
+            self.assertTrue(np.isclose(record, records[i]).all())
+
+    def test_read_ids(self):
+        time.sleep(0.6)
+        im = summary.IM(self.dir, "read", msecs=200)
+        time.sleep(0.6)
+        scalar = summary.scalar(im, self.tablet_name)
+        self.assertEqual(len(self.py_ids), scalar.size)
+        for i, id in enumerate(scalar.ids):
+            self.assertEqual(self.py_ids[i], id)
+
+    def test_read_captions(self):
+        time.sleep(0.6)
+        im = summary.IM(self.dir, "read", msecs=200)
+        time.sleep(0.6)
+        scalar = summary.scalar(im, self.tablet_name)
+        self.assertEqual(scalar.captions, self.py_captions)
+
+    def test_mix_read_write(self):
+        write_im = summary.IM(self.dir, "write", msecs=200)
+        time.sleep(0.6)
+        read_im = summary.IM(self.dir, "read", msecs=200)
+
+        scalar_writer = summary.scalar(write_im, self.tablet_name)
+        scalar_reader = summary.scalar(read_im, self.tablet_name)
+
+        scalar_writer.set_captions(["train cost", "test cost"])
+        for i in range(1000):
+            scalar_writer.add(i, [random.random(), random.random()])
+
+        scalar_reader.records
+
+        for i in range(500):
+            scalar_writer.add(i, [random.random(), random.random()])
+
+        scalar_reader.records
+
+        for i in range(500):
+            scalar_writer.add(i, [random.random(), random.random()])
+
+        for i in range(10):
+            scalar_reader.records
+            scalar_reader.captions
+
 
 if __name__ == '__main__':
     unittest.main()

visualdl/storage/storage.cc

@@ -46,9 +46,9 @@ void MemoryStorage::PersistToDisk(const std::string &dir) {
   // make a directory if not exist
   fs::TryRecurMkdir(dir);
   // write storage out
-  LOG(INFO) << "to serize meta to dir " << dir;
+  VLOG(2) << "to serize meta to dir " << dir;
   fs::SerializeToFile(storage_, meta_path(dir));
-  LOG(INFO) << "serize meta to dir " << dir;
+  VLOG(2) << "serize meta to dir " << dir;
   // write all the tablets
   for (auto tag : storage_.tags()) {
     auto it = tablets_.find(tag);
@@ -77,7 +77,7 @@ void MemoryStorage::StartReadService(const std::string &dir,
   CHECK(executor_ != nullptr);
   CHECK(!dir.empty()) << "dir should be set first";
   cc::PeriodExector::task_t task = [dir, this, handler] {
-    LOG(INFO) << "loading from " << dir;
+    VLOG(1) << "loading from " << dir;
     if (handler != nullptr) {
       std::lock_guard<std::mutex> _(*handler);
       LoadFromDisk(dir);
@@ -87,7 +87,7 @@ void MemoryStorage::StartReadService(const std::string &dir,
     return true;
   };
   // executor_.Start();
-  LOG(INFO) << "push read task";
+  VLOG(1) << "push read task";
   (*executor_)(std::move(task), msecs);
 }
 
@@ -99,7 +99,7 @@ void MemoryStorage::StartWriteService(const std::string &dir,
   storage_.set_dir(dir);
   // executor_.Start();
   cc::PeriodExector::task_t task = [dir, handler, this] {
-    LOG(INFO) << "persist to disk";
+    VLOG(2) << "persist to disk";
     if (handler != nullptr) {
       std::lock_guard<std::mutex> _(*handler);
       PersistToDisk(dir);

visualdl/utils/filesystem.h

@@ -44,7 +44,7 @@ bool DeSerializeFromFile(T* proto, const std::string& path) {
 }
 
 void TryMkdir(const std::string& dir) {
-  LOG(INFO) << "try to mkdir " << dir;
+  VLOG(1) << "try to mkdir " << dir;
   struct stat st = {0};
   if (stat(dir.c_str(), &st) == -1) {
     ::mkdir(dir.c_str(), 0700);