diff --git a/CMakeLists.txt b/CMakeLists.txt index e18ab2fdd5478a6602c0ff80b2daa9c5ed777fe0..fe5210ed1cbe3027e3ef2ec38b7de8ff7fd64760 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,8 +30,6 @@ include(external/python) # find python and set path include_directories(${PROJECT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -# TODO(ChunweiYan) debug, remote latter -#include_directories(/home/superjom/project/VisualDL/build/third_party/eigen3/src/extern_eigen3) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/storage) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/logic) @@ -40,6 +38,7 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/python) add_executable(vl_test ${PROJECT_SOURCE_DIR}/visualdl/test.cc ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc + ${PROJECT_SOURCE_DIR}/visualdl/logic/histogram_test.cc ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc ${PROJECT_SOURCE_DIR}/visualdl/utils/test_image.cc diff --git a/server/visualdl/lib.py b/server/visualdl/lib.py index c08500379e08a07be363393251b9c5cf951a9cd5..00fd820f7cca3a633181ddbbfe37d10a981b7fa0 100644 --- a/server/visualdl/lib.py +++ b/server/visualdl/lib.py @@ -7,18 +7,16 @@ from tempfile import NamedTemporaryFile import numpy as np from PIL import Image -import storage - def get_modes(storage): return storage.modes() -def get_scalar_tags(storage, mode): +def get_tags(storage, component): result = {} for mode in storage.modes(): with storage.mode(mode) as reader: - tags = reader.tags('scalar') + tags = reader.tags(component) if tags: result[mode] = {} for tag in tags: @@ -29,6 +27,10 @@ def get_scalar_tags(storage, mode): return result +def get_scalar_tags(storage): + return get_tags(storage, 'scalar') + + def get_scalar(storage, mode, tag, num_records=300): assert num_records > 1 @@ -143,6 +145,38 @@ def get_invididual_image(storage, mode, tag, step_index, max_size=80): return tempfile +def get_histogram_tags(storage): + return get_tags(storage, 'histogram') + + +def get_histogram(storage, mode, tag): + with storage.mode(mode) as reader: + histogram = reader.histogram(tag) + res = [] + + for i in xrange(histogram.num_records()): + try: + # some bug with protobuf, some times may overflow + record = histogram.record(i) + except: + continue + + res.append([]) + py_record = res[-1] + py_record.append(record.timestamp()) + py_record.append(record.step()) + py_record.append([]) + + data = py_record[-1] + for j in xrange(record.num_instances()): + instance = record.instance(j) + data.append( + [instance.left(), + instance.right(), + instance.frequency()]) + return res + + if __name__ == '__main__': reader = storage.LogReader('./tmp/mock') tags = get_image_tags(reader) diff --git a/server/visualdl/lib_test.py b/server/visualdl/lib_test.py index 16324fc0868fbb979f3610d7cfe3b522a6157646..d3bbf7aa9d3a4e3932bf3697e44fe775d47248a2 100644 --- a/server/visualdl/lib_test.py +++ b/server/visualdl/lib_test.py @@ -3,13 +3,15 @@ import unittest import lib import storage -from storage_mock import add_image, add_scalar + +import pprint +from storage_mock import add_scalar, add_image, add_histogram class LibTest(unittest.TestCase): def setUp(self): dir = "./tmp/mock" - writer = storage.LogWriter(dir, sync_cycle=20) + writer = storage.LogWriter(dir, sync_cycle=10) add_scalar(writer, "train", "layer/scalar0/min", 1000, 1) add_scalar(writer, "test", "layer/scalar0/min", 1000, 10) @@ -22,23 +24,26 @@ class LibTest(unittest.TestCase): add_image(writer, "train", "layer/image0", 7, 10, 1) add_image(writer, "test", "layer/image0", 7, 10, 3) - add_image(writer, "train", "layer/image1", 7, 10, 1, shape=[30,30,2]) - add_image(writer, "test", "layer/image1", 7, 10, 1, shape=[30,30,2]) + add_image(writer, "train", "layer/image1", 7, 10, 1, shape=[30, 30, 2]) + add_image(writer, "test", "layer/image1", 7, 10, 1, shape=[30, 30, 2]) + + add_histogram(writer, "train", "layer/histogram0", 100) + add_histogram(writer, "test", "layer/histogram0", 100) self.reader = storage.LogReader(dir) def test_modes(self): modes = lib.get_modes(self.reader) - self.assertEqual(sorted(modes), sorted(["default", "train", "test", "valid"])) + self.assertEqual( + sorted(modes), sorted(["default", "train", "test", "valid"])) def test_scalar(self): - - for mode in "train test valid".split(): - tags = lib.get_scalar_tags(self.reader, mode) - print 'scalar tags:' - pprint.pprint(tags) - self.assertEqual(len(tags), 3) - self.assertEqual(sorted(tags.keys()), sorted("train test valid".split())) + tags = lib.get_scalar_tags(self.reader) + print 'scalar tags:' + pprint.pprint(tags) + self.assertEqual(len(tags), 3) + self.assertEqual( + sorted(tags.keys()), sorted("train test valid".split())) def test_image(self): tags = lib.get_image_tags(self.reader) @@ -47,9 +52,17 @@ class LibTest(unittest.TestCase): tags = lib.get_image_tag_steps(self.reader, 'train', 'layer/image0/0') pprint.pprint(tags) - image = lib.get_invididual_image(self.reader, "train", 'layer/image0/0', 2) + image = lib.get_invididual_image(self.reader, "train", + 'layer/image0/0', 2) print image + def test_histogram(self): + tags = lib.get_histogram_tags(self.reader) + self.assertEqual(len(tags), 2) + + res = lib.get_histogram(self.reader, "train", "layer/histogram0") + pprint.pprint(res) + if __name__ == '__main__': unittest.main() diff --git a/server/visualdl/storage_mock.py b/server/visualdl/storage_mock.py index 18bb31bcbb7020319905d94b086ac8a052d0506e..c16b11895902aed7a150de4147d1a29f5466d7c0 100644 --- a/server/visualdl/storage_mock.py +++ b/server/visualdl/storage_mock.py @@ -35,15 +35,8 @@ def add_image(writer, image_writer.set_sample(index, shape, list(data)) image_writer.finish_sampling() - -if __name__ == '__main__': - add_scalar("train", "layer/scalar0/min", 1000, 1) - add_scalar("test", "layer/scalar0/min", 1000, 10) - add_scalar("valid", "layer/scalar0/min", 1000, 10) - - add_scalar("train", "layer/scalar0/max", 1000, 1) - add_scalar("test", "layer/scalar0/max", 1000, 10) - add_scalar("valid", "layer/scalar0/max", 1000, 10) - - add_image("train", "layer/image0", 7, 10, 1) - add_image("test", "layer/image0", 7, 10, 3) +def add_histogram(writer, mode, tag, num_buckets): + with writer.mode(mode) as writer: + histogram = writer.histogram(tag, num_buckets) + for i in range(10): + histogram.add_record(i, np.random.normal(0.1 + i * 0.01, size=1000)) diff --git a/server/visualdl/visual_dl.py b/server/visualdl/visual_dl.py index eb89668d56b6781b7208cb4a2e5b3abaeb1a72a1..9b6c68e71bf0d48b330c2d44659f180ef6657cfe 100644 --- a/server/visualdl/visual_dl.py +++ b/server/visualdl/visual_dl.py @@ -98,8 +98,7 @@ def scalar_tags(): if is_debug: result = mock_tags.data() else: - result = lib.get_scalar_tags(log_reader, mode) - print 'scalar tags (mode: %s)' % mode, result + result = lib.get_scalar_tags(log_reader) result = gen_result(0, "", result) return Response(json.dumps(result), mimetype='application/json') @@ -108,7 +107,14 @@ def scalar_tags(): def image_tags(): mode = request.args.get('run') result = lib.get_image_tags(log_reader) - print 'image tags (mode: %s)'%mode, result + result = gen_result(0, "", result) + return Response(json.dumps(result), mimetype='application/json') + + +@app.route("/data/plugin/histograms/tags") +def histogram_tags(): + mode = request.args.get('run') + result = lib.get_histogram_tags(log_reader) result = gen_result(0, "", result) return Response(json.dumps(result), mimetype='application/json') @@ -151,6 +157,15 @@ def individual_image(): return response +@app.route('/data/plugin/histograms/histograms') +def histogram(): + run = request.args.get('run') + tag = request.args.get('tag') + result = lib.get_histogram(log_reader, run, tag) + result = gen_result(0, "", result) + return Response(json.dumps(result), mimetype='application/json') + + @app.route('/data/plugin/graphs/graph') def graph(): model_json = graph.load_model("") diff --git a/visualdl/logic/histogram.h b/visualdl/logic/histogram.h new file mode 100644 index 0000000000000000000000000000000000000000..b64dce85eac8eaaedcabcc3c4c04f60433a89e85 --- /dev/null +++ b/visualdl/logic/histogram.h @@ -0,0 +1,107 @@ +#ifndef VISUALDL_LOGIC_HISTOGRAM_H +#define VISUALDL_LOGIC_HISTOGRAM_H + +#include +#include +#include +#include + +namespace visualdl { + +// An interface to retrieve records of a histogram. +template +struct HistogramRecord { + struct Instance { + T left; + T right; + int32_t frequency; + }; + + uint64_t timestamp; + int step; + + HistogramRecord(uint64_t timestamp, + int step, + T left, + T right, + std::vector&& frequency) + : timestamp(timestamp), + step(step), + left(left), + right(right), + frequency(frequency), + span_(float(right - left) / frequency.size()) {} + + Instance instance(int i) const { + CHECK_LT(i, frequency.size()); + Instance res; + res.left = left + span_ * i; + res.right = res.left + span_; + res.frequency = frequency[i]; + return res; + } + + size_t num_instances() const { return frequency.size(); } + +private: + T span_; + T left; + T right; + std::vector frequency; +}; + +// Create a histogram with default(10%) set of bucket boundaries. +// The buckets cover the range from min to max. +template +struct HistogramBuilder { + HistogramBuilder(int num_buckets) : num_buckets_(num_buckets) {} + + void operator()(const std::vector& data) { + CHECK_GE(data.size(), num_buckets_); + + UpdateBoundary(data); + CreateBuckets(data); + } + + T left_boundary{std::numeric_limits::max()}; + T right_boundary{std::numeric_limits::min()}; + std::vector buckets; + + void Get(size_t n, T* left, T* right, int* frequency) { + CHECK(!buckets.empty()) << "need to CreateBuckets first."; + CHECK_LT(n, num_buckets_) << "n out of range."; + *left = left_boundary + span_ * n; + *right = *left + span_; + *frequency = buckets[n]; + } + +private: + // Get the left and right boundaries. + void UpdateBoundary(const std::vector& data) { + for (auto v : data) { + if (v > right_boundary) + right_boundary = v; + else if (v < left_boundary) + left_boundary = v; + } + } + + // Create `num_buckets` buckets. + void CreateBuckets(const std::vector& data) { + span_ = (float)right_boundary / num_buckets_ - + (float)left_boundary / num_buckets_; + buckets.resize(num_buckets_); + + for (auto v : data) { + int offset = std::min(int((v - left_boundary) / span_), num_buckets_ - 1); + buckets[offset]++; + } + } + + float span_; + int num_buckets_; +}; + +} // namespace visualdl + +#endif diff --git a/visualdl/logic/histogram_test.cc b/visualdl/logic/histogram_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..3bb7ac630414488549d47d391bd2e61a03f954aa --- /dev/null +++ b/visualdl/logic/histogram_test.cc @@ -0,0 +1,25 @@ +#include "visualdl/logic/histogram.h" + +#include +#include + +using namespace std; +using namespace visualdl; + +TEST(HistogramBuilder, build) { + const int size = 3000; + std::vector data(size); + for (auto& v : data) { + v = (float)rand() / RAND_MAX - 0.5; + } + + HistogramBuilder builder(100); + builder(data); + + float left, right; + int frequency; + for (int i = 0; i < 100; i++) { + builder.Get(i, &left, &right, &frequency); + ASSERT_GT(frequency, 0); + } +} diff --git a/visualdl/logic/im.cc b/visualdl/logic/im.cc index 41653d585d65a2e0387704427697d43bb70e8402..ce1821f984a502785f2ae2760b73673d78234f1f 100644 --- a/visualdl/logic/im.cc +++ b/visualdl/logic/im.cc @@ -32,13 +32,6 @@ void SimpleWriteSyncGuard::Sync() { template class SimpleWriteSyncGuard; template class SimpleWriteSyncGuard; template class SimpleWriteSyncGuard; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>; -template class SimpleWriteSyncGuard>>; -template class SimpleWriteSyncGuard>; +template class SimpleWriteSyncGuard; } // namespace visualdl diff --git a/visualdl/logic/pybind.cc b/visualdl/logic/pybind.cc index 449c5f9935aa8dbfe817596f376ef38bd51db3a0..bfaee7f001704fd908ab955bb5f899693354c2b4 100644 --- a/visualdl/logic/pybind.cc +++ b/visualdl/logic/pybind.cc @@ -8,14 +8,15 @@ namespace py = pybind11; namespace vs = visualdl; namespace cp = visualdl::components; +#define ADD_FULL_TYPE_IMPL(CODE) \ + CODE(int32_t); \ + CODE(int64_t); \ + CODE(float); \ + CODE(double); + PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of VisualDL"); -#define READER_ADD_SCALAR(T) \ - .def("get_scalar_" #T, [](vs::LogReader& self, const std::string& tag) { \ - auto tablet = self.tablet(tag); \ - return vs::components::ScalarReader(std::move(tablet)); \ - }) py::class_(m, "LogReader") .def("__init__", [](vs::LogReader& instance, const std::string& dir) { @@ -25,23 +26,35 @@ PYBIND11_PLUGIN(core) { .def("set_mode", &vs::LogReader::SetMode) .def("modes", [](vs::LogReader& self) { return self.storage().modes(); }) .def("tags", &vs::LogReader::tags) - // clang-format off + +// clang-format off + #define READER_ADD_SCALAR(T) \ + .def("get_scalar_" #T, [](vs::LogReader& self, const std::string& tag) { \ + auto tablet = self.tablet(tag); \ + return vs::components::ScalarReader(std::move(tablet)); \ + }) READER_ADD_SCALAR(float) READER_ADD_SCALAR(double) READER_ADD_SCALAR(int) + #undef READER_ADD_SCALAR + + #define READER_ADD_HISTOGRAM(T) \ + .def("get_histogram_" #T, [](vs::LogReader& self, const std::string& tag) { \ + auto tablet = self.tablet(tag); \ + return vs::components::HistogramReader(std::move(tablet)); \ + }) + READER_ADD_HISTOGRAM(float) + READER_ADD_HISTOGRAM(double) + READER_ADD_HISTOGRAM(int) + #undef READER_ADD_HISTOGRAM + // clang-format on .def("get_image", [](vs::LogReader& self, const std::string& tag) { auto tablet = self.tablet(tag); return vs::components::ImageReader(self.mode(), tablet); }); -#undef READER_ADD_SCALAR - -#define WRITER_ADD_SCALAR(T) \ - .def("new_scalar_" #T, [](vs::LogWriter& self, const std::string& tag) { \ - auto tablet = self.AddTablet(tag); \ - return cp::Scalar(tablet); \ - }) + // clang-format on py::class_(m, "LogWriter") .def("__init__", [](vs::LogWriter& instance, const std::string& dir, int sync_cycle) { @@ -49,10 +62,24 @@ PYBIND11_PLUGIN(core) { }) .def("set_mode", &vs::LogWriter::SetMode) .def("as_mode", &vs::LogWriter::AsMode) - // clang-format off +// clang-format off + #define WRITER_ADD_SCALAR(T) \ + .def("new_scalar_" #T, [](vs::LogWriter& self, const std::string& tag) { \ + auto tablet = self.AddTablet(tag); \ + return cp::Scalar(tablet); \ + }) + #define WRITER_ADD_HISTOGRAM(T) \ + .def("new_histogram_" #T, \ + [](vs::LogWriter& self, const std::string& tag, int num_buckets) { \ + auto tablet = self.AddTablet(tag); \ + return cp::Histogram(tablet, num_buckets); \ + }) WRITER_ADD_SCALAR(float) WRITER_ADD_SCALAR(double) WRITER_ADD_SCALAR(int) + WRITER_ADD_HISTOGRAM(float) + WRITER_ADD_HISTOGRAM(double) + WRITER_ADD_HISTOGRAM(int) // clang-format on .def("new_image", [](vs::LogWriter& self, @@ -108,7 +135,44 @@ PYBIND11_PLUGIN(core) { .def("record", &cp::ImageReader::record) .def("timestamp", &cp::ImageReader::timestamp); - // .def("data", &cp::ImageReader::data) - // .def("shape", &cp::ImageReader::shape); +#define ADD_HISTOGRAM_WRITER(T) \ + py::class_>(m, "HistogramWriter__" #T) \ + .def("add_record", &cp::Histogram::AddRecord); + ADD_FULL_TYPE_IMPL(ADD_HISTOGRAM_WRITER) +#undef ADD_HISTOGRAM_WRITER + +#define ADD_HISTOGRAM_RECORD_INSTANCE(T) \ + py::class_::Instance>(m, "HistogramInstance__" #T) \ + .def("left", \ + [](typename vs::HistogramRecord::Instance& self) { \ + return self.left; \ + }) \ + .def("right", \ + [](typename vs::HistogramRecord::Instance& self) { \ + return self.right; \ + }) \ + .def("frequency", [](typename vs::HistogramRecord::Instance& self) { \ + return self.frequency; \ + }); + + ADD_FULL_TYPE_IMPL(ADD_HISTOGRAM_RECORD_INSTANCE) +#undef ADD_HISTOGRAM_RECORD_INSTANCE + +#define ADD_HISTOGRAM_RECORD(T) \ + py::class_>(m, "HistogramRecord__" #T) \ + .def("step", [](vs::HistogramRecord& self) { return self.step; }) \ + .def("timestamp", \ + [](vs::HistogramRecord& self) { return self.timestamp; }) \ + .def("instance", &vs::HistogramRecord::instance) \ + .def("num_instances", &vs::HistogramRecord::num_instances); + ADD_FULL_TYPE_IMPL(ADD_HISTOGRAM_RECORD) +#undef ADD_HISTOGRAM_RECORD + +#define ADD_HISTOGRAM_READER(T) \ + py::class_>(m, "HistogramReader__" #T) \ + .def("num_records", &cp::HistogramReader::num_records) \ + .def("record", &cp::HistogramReader::record); + ADD_FULL_TYPE_IMPL(ADD_HISTOGRAM_READER) +#undef ADD_HISTOGRAM_READER } // end pybind diff --git a/visualdl/logic/sdk.cc b/visualdl/logic/sdk.cc index 993776fe053f2221d81434650556b09f75ab2b6f..dcf0f405a77f74d3cd81fe168dccc18709ef0339 100644 --- a/visualdl/logic/sdk.cc +++ b/visualdl/logic/sdk.cc @@ -1,6 +1,8 @@ #include "visualdl/logic/sdk.h" +#include "visualdl/logic/histogram.h" #include "visualdl/utils/image.h" +#include "visualdl/utils/macro.h" namespace visualdl { @@ -10,7 +12,7 @@ template std::vector ScalarReader::records() const { std::vector res; for (int i = 0; i < reader_.total_records(); i++) { - res.push_back(reader_.record(i).template data(0).Get()); + res.push_back(reader_.record(i).data(0).template Get()); } return res; } @@ -44,11 +46,6 @@ size_t ScalarReader::size() const { return reader_.total_records(); } -template class ScalarReader; -template class ScalarReader; -template class ScalarReader; -template class ScalarReader; - void Image::StartSampling() { if (!ToSampleThisStep()) return; @@ -60,7 +57,7 @@ void Image::StartSampling() { // resize record for (int i = 0; i < num_samples_; i++) { - step_.AddData(); + step_.AddData(); } num_records_ = 0; } @@ -129,11 +126,14 @@ void Image::SetSample(int index, !is_same_type::value, "value_t should not use int64_t field, this type is used to store shape"); - // set meta with hack - Entry meta; - meta.set_parent(entry.parent()); - meta.entry = entry.entry; - meta.SetMulti(shape); + // set meta. + entry.SetMulti(shape); + + // // set meta with hack + // Entry meta; + // meta.set_parent(entry.parent()); + // meta.entry = entry.entry; + // meta.SetMulti(shape); } std::string ImageReader::caption() { @@ -149,18 +149,56 @@ std::string ImageReader::caption() { ImageReader::ImageRecord ImageReader::record(int offset, int index) { ImageRecord res; auto record = reader_.record(offset); - auto data_entry = record.data>(index); - auto shape_entry = record.data(index); - auto data_str = data_entry.GetRaw(); + auto entry = record.data(index); + auto data_str = entry.GetRaw(); std::transform(data_str.begin(), data_str.end(), std::back_inserter(res.data), [](byte_t i) { return (int)(i); }); - res.shape = shape_entry.GetMulti(); + res.shape = entry.GetMulti(); res.step_id = record.id(); return res; } +template +void Histogram::AddRecord(int step, const std::vector& data) { + HistogramBuilder builder(num_buckets_); + builder(data); + + auto record = writer_.AddRecord(); + record.SetId(step); + time_t time = std::time(nullptr); + record.SetTimeStamp(time); + // set frequencies. + auto entry = record.AddData(); + entry.SetMulti(builder.buckets); + // Serialize left and right boundaries. + std::string boundaries_str = std::to_string(builder.left_boundary) + " " + + std::to_string(builder.right_boundary); + entry.SetRaw(boundaries_str); +} + +template +HistogramRecord HistogramReader::record(int i) { + CHECK_LT(i, reader_.total_records()); + auto r = reader_.record(i); + auto d = r.data(0); + auto boundaries_str = d.GetRaw(); + std::stringstream ss(boundaries_str); + T left, right; + ss >> left >> right; + + auto frequency = d.GetMulti(); + auto timestamp = r.timestamp(); + auto step = r.id(); + + return HistogramRecord(timestamp, step, left, right, std::move(frequency)); +} + +DECL_BASIC_TYPES_CLASS_IMPL(class, ScalarReader) +DECL_BASIC_TYPES_CLASS_IMPL(struct, Histogram) +DECL_BASIC_TYPES_CLASS_IMPL(struct, HistogramReader) + } // namespace components } // namespace visualdl diff --git a/visualdl/logic/sdk.h b/visualdl/logic/sdk.h index d7e5b9d94182be8d51b33a28226cc8ca535bb617..cabc08929c797d378f6c9629c174921d73d2b28e 100644 --- a/visualdl/logic/sdk.h +++ b/visualdl/logic/sdk.h @@ -1,9 +1,11 @@ #ifndef VISUALDL_LOGIC_SDK_H #define VISUALDL_LOGIC_SDK_H +#include "visualdl/logic/histogram.h" #include "visualdl/storage/storage.h" #include "visualdl/storage/tablet.h" #include "visualdl/utils/string.h" + namespace visualdl { const static std::string kDefaultMode{"default"}; @@ -82,7 +84,8 @@ public: std::vector tags(const std::string& component) { auto type = Tablet::type(component); auto tags = reader_.tags(type); - CHECK(!tags.empty()); + CHECK(!tags.empty()) << "component " << component + << " has no taged records"; std::vector res; for (const auto& tag : tags) { if (TagMatchMode(tag, mode_)) { @@ -130,10 +133,10 @@ struct Scalar { void AddRecord(int id, T value) { auto record = tablet_.AddRecord(); record.SetId(id); + auto entry = record.AddData(); time_t time = std::time(nullptr); record.SetTimeStamp(time); - auto entry = record.template AddData(); entry.Set(value); } @@ -262,6 +265,32 @@ private: std::string mode_; }; +template +struct Histogram { + Histogram(Tablet tablet, int num_buckets) + : writer_(tablet), num_buckets_(num_buckets) { + writer_.SetType(Tablet::Type::kHistogram); + } + + void AddRecord(int step, const std::vector& data); + +private: + int num_buckets_; + Tablet writer_; +}; + +template +struct HistogramReader { + HistogramReader(TabletReader tablet) : reader_(tablet) {} + + size_t num_records() { return reader_.total_records(); } + + HistogramRecord record(int i); + +private: + TabletReader reader_; +}; + } // namespace components } // namespace visualdl diff --git a/visualdl/logic/sdk_test.cc b/visualdl/logic/sdk_test.cc index 2ed373f6ec5b0cb28718c307c795e670d44b1dd4..b60c3e7eb7784b3624e40bd9cea068802ef13b08 100644 --- a/visualdl/logic/sdk_test.cc +++ b/visualdl/logic/sdk_test.cc @@ -81,6 +81,22 @@ TEST(Image, test) { CHECK_EQ(image2read.num_records(), num_steps); } +TEST(Histogram, AddRecord) { + const auto dir = "./tmp/sdk_test.histogram"; + LogWriter writer__(dir, 1); + auto writer = writer__.AsMode("train"); + + auto tablet = writer.AddTablet("histogram0"); + components::Histogram histogram(tablet, 10); + + std::vector data(1000); + for (auto& v : data) { + v = (float)rand() / RAND_MAX; + } + + histogram.AddRecord(10, data); +} + TEST(Scalar, more_than_one_mode) { const auto dir = "./tmp/sdk_multi_mode"; LogWriter log(dir, 20); diff --git a/visualdl/python/storage.py b/visualdl/python/storage.py index b7bed55050790393c132a1bc08d0072ce3da632e..c78b3b6c453030f90dde96db77d0c944dc8e5f25 100644 --- a/visualdl/python/storage.py +++ b/visualdl/python/storage.py @@ -40,6 +40,14 @@ class LogReader(object): def image(self, tag): return self.reader.get_image(tag) + def histogram(self, tag, type='float'): + type2scalar = { + 'float': self.reader.get_histogram_float, + 'double': self.reader.get_histogram_double, + 'int': self.reader.get_histogram_int, + } + return type2scalar[type](tag) + def __enter__(self): return self @@ -65,6 +73,9 @@ class LogWriter(object): return LogWriter.cur_mode def scalar(self, tag, type='float'): + ''' + Create a scalar component. + ''' type2scalar = { 'float': self.writer.new_scalar_float, 'double': self.writer.new_scalar_double, @@ -73,8 +84,22 @@ class LogWriter(object): return type2scalar[type](tag) def image(self, tag, num_samples, step_cycle): + ''' + Create an image component. + ''' return self.writer.new_image(tag, num_samples, step_cycle) + def histogram(self, tag, num_buckets, type='float'): + ''' + Create a histogram component. + ''' + types = { + 'float': self.writer.new_histogram_float, + 'double': self.writer.new_histogram_double, + 'int': self.writer.new_histogram_int, + } + return types[type](tag, num_buckets) + def __enter__(self): return self diff --git a/visualdl/storage/entry.cc b/visualdl/storage/entry.cc index f2f2d0c644599bbc904cea682edb4be8d1a9f5b0..d65e61595e523360f21622d2f0550ddebd1807f0 100644 --- a/visualdl/storage/entry.cc +++ b/visualdl/storage/entry.cc @@ -4,7 +4,7 @@ namespace visualdl { #define IMPL_ENTRY_SET_OR_ADD(method__, ctype__, dtype__, opr__) \ template <> \ - void Entry::method__(ctype__ v) { \ + void Entry::method__(ctype__ v) { \ entry->set_dtype(storage::DataType::dtype__); \ entry->opr__(v); \ WRITE_GUARD \ @@ -12,7 +12,7 @@ namespace visualdl { #define IMPL_ENTRY_SETMUL(ctype__, dtype__, field__) \ template <> \ - void Entry::SetMulti(const std::vector& vs) { \ + void Entry::SetMulti(const std::vector& vs) { \ entry->set_dtype(storage::DataType::dtype__); \ entry->clear_##field__(); \ for (auto v : vs) { \ @@ -22,14 +22,14 @@ namespace visualdl { } template <> -void Entry>::Set(std::vector v) { +void Entry::Set>(std::vector v) { entry->set_dtype(storage::DataType::kBytes); entry->set_y(std::string(v.begin(), v.end())); WRITE_GUARD } template <> -void Entry>::Add(std::vector v) { +void Entry::Add>(std::vector v) { entry->set_dtype(storage::DataType::kBytess); *entry->add_ys() = std::string(v.begin(), v.end()); WRITE_GUARD @@ -56,7 +56,7 @@ IMPL_ENTRY_SETMUL(bool, kBool, bs); #define IMPL_ENTRY_GET(T, fieldname__) \ template <> \ - T EntryReader::Get() const { \ + T EntryReader::Get() const { \ return data_.fieldname__(); \ } @@ -68,14 +68,14 @@ IMPL_ENTRY_GET(std::string, s); IMPL_ENTRY_GET(bool, b); template <> -std::vector EntryReader>::Get() const { +std::vector EntryReader::Get>() const { const auto& y = data_.y(); return std::vector(y.begin(), y.end()); } #define IMPL_ENTRY_GET_MULTI(T, fieldname__) \ template <> \ - std::vector EntryReader::GetMulti() const { \ + std::vector EntryReader::GetMulti() const { \ return std::vector(data_.fieldname__().begin(), \ data_.fieldname__().end()); \ } @@ -87,16 +87,4 @@ IMPL_ENTRY_GET_MULTI(double, ds); IMPL_ENTRY_GET_MULTI(std::string, ss); IMPL_ENTRY_GET_MULTI(bool, bs); -template class Entry; -template class Entry; -template class Entry; -template class Entry; -template class Entry>; - -template class EntryReader; -template class EntryReader; -template class EntryReader; -template class EntryReader; -template class EntryReader>; - } // namespace visualdl diff --git a/visualdl/storage/entry.h b/visualdl/storage/entry.h index 343fa310da02a58e3d56b9011ae693ef48be2245..9ce3299171ff49e4c6d6d4ceaa59065f9e5da5f6 100644 --- a/visualdl/storage/entry.h +++ b/visualdl/storage/entry.h @@ -14,15 +14,14 @@ using byte_t = unsigned char; /* * Utility helper for storage::Entry. */ -template struct Entry { - DECL_GUARD(Entry) + DECL_GUARD(Entry) // use pointer to avoid copy storage::Entry* entry{nullptr}; Entry() {} Entry(storage::Entry* entry, Storage* parent) : entry(entry), x_(parent) {} - Entry(const Entry& other) : entry(other.entry), x_(other.x_) {} + Entry(const Entry& other) : entry(other.entry), x_(other.x_) {} void operator()(storage::Entry* entry, Storage* parent) { this->entry = entry; @@ -30,13 +29,16 @@ struct Entry { } // Set a single value. + template void Set(T v); void SetRaw(const std::string& bytes) { entry->set_y(bytes); } // Add a value to repeated message field. + template void Add(T v); + template void SetMulti(const std::vector& v); Storage* parent() { return x_; } @@ -46,12 +48,13 @@ private: Storage* x_; }; -template struct EntryReader { EntryReader(storage::Entry x) : data_(x) {} // Get a single value. + template T Get() const; // Get repeated field. + template std::vector GetMulti() const; std::string GetRaw() { return data_.y(); } diff --git a/visualdl/storage/record.h b/visualdl/storage/record.h index 31fae1bb978bd6575386497102e29b9f796644a3..15878cef486a965d31d4d5427a073f56963e00b2 100644 --- a/visualdl/storage/record.h +++ b/visualdl/storage/record.h @@ -51,20 +51,19 @@ struct Record { } template - Entry MutableMeta() { - return Entry(data_->mutable_meta(), parent()); + Entry MutableMeta() { + return Entry(data_->mutable_meta(), parent()); } - template - Entry AddData() { + Entry AddData() { WRITE_GUARD - return Entry(data_->add_data(), parent()); + return Entry(data_->add_data(), parent()); } template - Entry MutableData(int i) { + Entry MutableData(int i) { WRITE_GUARD - return Entry(data_->mutable_data(i), parent()); + return Entry(data_->mutable_data(i), parent()); } Storage* parent() { return x_; } @@ -80,19 +79,13 @@ struct RecordReader { // read operations size_t data_size() const { return data_.data_size(); } - template - EntryReader data(int i) { - return EntryReader(data_.data(i)); - } + EntryReader data(int i) { return EntryReader(data_.data(i)); } int64_t timestamp() const { return data_.timestamp(); } int64_t id() const { return data_.id(); } Record::Dtype dtype() const { return (Record::Dtype)data_.dtype(); } - template - Entry meta() const { - return data_.meta(); - } + EntryReader meta() const { return data_.meta(); } private: storage::Record data_; diff --git a/visualdl/storage/storage_test.cc b/visualdl/storage/storage_test.cc index e39e8970d7c9adc0ef30d51e297b19624a816418..4cd6b573676995af779ccc24bbef74457a60effa 100644 --- a/visualdl/storage/storage_test.cc +++ b/visualdl/storage/storage_test.cc @@ -21,7 +21,7 @@ TEST_F(StorageTest, main) { auto tag0 = storage.AddTablet("tag0"); auto tag1 = storage.AddTablet("tag1"); auto record = tag0.AddRecord(); - auto entry = record.AddData(); + auto entry = record.AddData(); entry.Set(12); StorageReader reader("./tmp/storage_test"); diff --git a/visualdl/storage/tablet.h b/visualdl/storage/tablet.h index 17ca6ef53ac73b92f5764afa772f718588e99bf7..7999678540eda1c19fcb8e66925947cac18073d0 100644 --- a/visualdl/storage/tablet.h +++ b/visualdl/storage/tablet.h @@ -60,8 +60,8 @@ struct Tablet { } template - Entry MutableMeta() { - Entry x(data_->mutable_meta(), parent()); + Entry MutableMeta() { + Entry x(data_->mutable_meta(), parent()); } void SetCaptions(const std::vector& xs) { @@ -104,8 +104,8 @@ struct TabletReader { int32_t num_samples() const { return data_.num_samples(); } RecordReader record(int i) const { return RecordReader(data_.records(i)); } template - EntryReader meta() const { - return EntryReader(data_.meta()); + EntryReader meta() const { + return EntryReader(data_.meta()); } std::vector captions() const { std::vector x(data_.captions().begin(), diff --git a/visualdl/utils/image.h b/visualdl/utils/image.h index 19323cf1592844de526146e6749c3024b56bb1a6..ad345f95c1e8ec1db6b250c34bfc50ab48910302 100644 --- a/visualdl/utils/image.h +++ b/visualdl/utils/image.h @@ -67,6 +67,7 @@ static void NormalizeImage(Uint8Image* image, scale = (image_max < kZeroThreshold ? 0.0f : 255.0f) / image_max; offset = 0.0f; } + // Transform image, turning nonfinite values to bad_color for (int i = 0; i < depth; i++) { auto tmp = scale * values.row(i).array() + offset; diff --git a/visualdl/utils/macro.h b/visualdl/utils/macro.h new file mode 100644 index 0000000000000000000000000000000000000000..f39d9beaf7be3b1652c90041f16f790942663b6c --- /dev/null +++ b/visualdl/utils/macro.h @@ -0,0 +1,10 @@ +#ifndef VISUALDL_UTILS_MACRO_H +#define VISUALDL_UTILS_MACRO_H + +#define DECL_BASIC_TYPES_CLASS_IMPL(class__, name__) \ + template class__ name__; \ + template class__ name__; \ + template class__ name__; \ + template class__ name__; + +#endif