From beb1e3b5944dbf9abdf7bedec4eb99c3117a2001 Mon Sep 17 00:00:00 2001 From: superjom Date: Mon, 18 Dec 2017 11:42:14 +0800 Subject: [PATCH] add read and write to test_summary.py --- visualdl/python/test_summary.py | 67 +++++++++++++++++++++++++++++++-- visualdl/storage/storage.cc | 10 ++--- visualdl/utils/filesystem.h | 2 +- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/visualdl/python/test_summary.py b/visualdl/python/test_summary.py index e7189807..d4ce733b 100644 --- a/visualdl/python/test_summary.py +++ b/visualdl/python/test_summary.py @@ -1,6 +1,7 @@ import summary import numpy as np import unittest +import random import time once_flag = False @@ -8,14 +9,21 @@ once_flag = False class ScalarTester(unittest.TestCase): def setUp(self): - dir = "tmp/3.test" - self.im = summary.IM("write", dir) - self.scalar = summary.scalar(self.im, "scalar0") + self.dir = "tmp/summary.test" + # clean path + try: + os.rmdir(self.dir) + except: + pass + self.im = summary.IM(self.dir, "write", 200) + self.tablet_name = "scalar0" + self.scalar = summary.scalar(self.im, self.tablet_name) self.py_captions = ["train cost", "test cost"] self.scalar.set_captions(self.py_captions) self.py_records = [] self.py_ids = [] + # write for i in range(10): record = [0.1 * i, 0.2 * i] id = i * 10 @@ -23,7 +31,6 @@ class ScalarTester(unittest.TestCase): self.py_ids.append(id) self.scalar.add(id, record) - def test_records(self): self.assertEqual(self.scalar.size, len(self.py_records)) for i, record in enumerate(self.scalar.records): @@ -37,6 +44,58 @@ class ScalarTester(unittest.TestCase): def test_captions(self): self.assertEqual(self.scalar.captions, self.py_captions) + def test_read_records(self): + time.sleep(1) + im = summary.IM(self.dir, "read", 200) + time.sleep(1) + scalar = summary.scalar(im, self.tablet_name) + records = scalar.records + self.assertEqual(len(self.py_records), scalar.size) + for i, record in enumerate(self.scalar.records): + self.assertTrue(np.isclose(record, records[i]).all()) + + def test_read_ids(self): + time.sleep(0.6) + im = summary.IM(self.dir, "read", msecs=200) + time.sleep(0.6) + scalar = summary.scalar(im, self.tablet_name) + self.assertEqual(len(self.py_ids), scalar.size) + for i, id in enumerate(scalar.ids): + self.assertEqual(self.py_ids[i], id) + + def test_read_captions(self): + time.sleep(0.6) + im = summary.IM(self.dir, "read", msecs=200) + time.sleep(0.6) + scalar = summary.scalar(im, self.tablet_name) + self.assertEqual(scalar.captions, self.py_captions) + + def test_mix_read_write(self): + write_im = summary.IM(self.dir, "write", msecs=200) + time.sleep(0.6) + read_im = summary.IM(self.dir, "read", msecs=200) + + scalar_writer = summary.scalar(write_im, self.tablet_name) + scalar_reader = summary.scalar(read_im, self.tablet_name) + + scalar_writer.set_captions(["train cost", "test cost"]) + for i in range(1000): + scalar_writer.add(i, [random.random(), random.random()]) + + scalar_reader.records + + for i in range(500): + scalar_writer.add(i, [random.random(), random.random()]) + + scalar_reader.records + + for i in range(500): + scalar_writer.add(i, [random.random(), random.random()]) + + for i in range(10): + scalar_reader.records + scalar_reader.captions + if __name__ == '__main__': unittest.main() diff --git a/visualdl/storage/storage.cc b/visualdl/storage/storage.cc index b29b0ceb..9d1d393c 100644 --- a/visualdl/storage/storage.cc +++ b/visualdl/storage/storage.cc @@ -46,9 +46,9 @@ void MemoryStorage::PersistToDisk(const std::string &dir) { // make a directory if not exist fs::TryRecurMkdir(dir); // write storage out - LOG(INFO) << "to serize meta to dir " << dir; + VLOG(2) << "to serize meta to dir " << dir; fs::SerializeToFile(storage_, meta_path(dir)); - LOG(INFO) << "serize meta to dir " << dir; + VLOG(2) << "serize meta to dir " << dir; // write all the tablets for (auto tag : storage_.tags()) { auto it = tablets_.find(tag); @@ -77,7 +77,7 @@ void MemoryStorage::StartReadService(const std::string &dir, CHECK(executor_ != nullptr); CHECK(!dir.empty()) << "dir should be set first"; cc::PeriodExector::task_t task = [dir, this, handler] { - LOG(INFO) << "loading from " << dir; + VLOG(1) << "loading from " << dir; if (handler != nullptr) { std::lock_guard _(*handler); LoadFromDisk(dir); @@ -87,7 +87,7 @@ void MemoryStorage::StartReadService(const std::string &dir, return true; }; // executor_.Start(); - LOG(INFO) << "push read task"; + VLOG(1) << "push read task"; (*executor_)(std::move(task), msecs); } @@ -99,7 +99,7 @@ void MemoryStorage::StartWriteService(const std::string &dir, storage_.set_dir(dir); // executor_.Start(); cc::PeriodExector::task_t task = [dir, handler, this] { - LOG(INFO) << "persist to disk"; + VLOG(2) << "persist to disk"; if (handler != nullptr) { std::lock_guard _(*handler); PersistToDisk(dir); diff --git a/visualdl/utils/filesystem.h b/visualdl/utils/filesystem.h index ed97d03b..58438f26 100644 --- a/visualdl/utils/filesystem.h +++ b/visualdl/utils/filesystem.h @@ -44,7 +44,7 @@ bool DeSerializeFromFile(T* proto, const std::string& path) { } void TryMkdir(const std::string& dir) { - LOG(INFO) << "try to mkdir " << dir; + VLOG(1) << "try to mkdir " << dir; struct stat st = {0}; if (stat(dir.c_str(), &st) == -1) { ::mkdir(dir.c_str(), 0700); -- GitLab