Merge branch 'develop' of github.com:VisualDL/VisualDL into feature/pre-commit-backend-autostyle

CMakeLists.txt

-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.5)
 project(VisualDL)
 
 set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_FLAGS "-fPIC")
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-include_directories(thirdparty/local/include)
-link_directories(thirdparty/local/lib)
-add_subdirectory(thirdparty/pybind11-2.2.1)
-
-set(SOURCE_FILES
-        visualdl/backend/storage/storage.cc visualdl/backend/storage/storage.h
-        visualdl/backend/storage/storage.pb.h
-        visualdl/backend/storage/storage.pb.cc
-        )
-
-add_library(storage visualdl/backend/storage/storage.cc
-            visualdl/backend/storage/storage.pb.cc)
-add_executable(VisualDL ${SOURCE_FILES})
+include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include)
+include_directories(/usr/include/python2.7)
+
+set(PYBIND11_CPP_STANDARD -std=c++11)
+add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/pybind11-2.2.1)
+
+link_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/lib)
+
+add_library(storage
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.pb.cc)
+add_library(c_api ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/c_api.cc)
+add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc)
\ No newline at end of file
+add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc)
+
+pybind11_add_module(core
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc)
+target_link_libraries(core PRIVATE pybind11::module im storage sdk protobuf glog)
+set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+
+add_executable(vl_test
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc)
+target_link_libraries(vl_test storage im gtest glog protobuf gflags)
+

visualdl/.clang-format

+# This file is used by clang-format to autoformat paddle source code
+#
+# The clang-format is part of llvm toolchain.
+# It need to install llvm and clang to format source code style.
+#
+# The basic usage is,
+#   clang-format -i -style=file PATH/TO/SOURCE/CODE
+#
+# The -style=file implicit use ".clang-format" file located in one of 
+# parent directory. 
+# The -i means inplace change.
+#
+# The document of clang-format is 
+#   http://clang.llvm.org/docs/ClangFormat.html
+#   http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+---
+Language:        Cpp
+BasedOnStyle:  Google
+IndentWidth:     2
+TabWidth:        2
+ContinuationIndentWidth: 4
+AccessModifierOffset: -2  # The private/protected/public has no indent in class
+Standard:  Cpp11 
+AllowAllParametersOfDeclarationOnNextLine: true
+BinPackParameters: false
+BinPackArguments: false
+...
+

visualdl/backend/logic/im.cc

+#include <ctime>
+#include <glog/logging.h>
+
+#include "visualdl/backend/logic/im.h"
+
+namespace visualdl {
+
+/*
+ * @num_samples: number of instances to sample
+ * @size: counter of the records.
+ * @returns: id of the instance to replace, if drop this instance, return -1.
+ */
+int ReserviorSample(int num_samples, int num_records) {
+  if (num_records <= num_samples) {
+    return num_records;
+  }
+
+  std::srand(std::time(0));
+  float prob = static_cast<float>(std::rand()) / RAND_MAX;
+  float receive_prob = static_cast<float>(num_samples) / num_records;
+  if (prob < receive_prob) {
+    int offset2replace = std::rand() % num_samples;
+    return offset2replace;
+  }
+  return -1;
+}
+
+void InformationMaintainer::SetPersistDest(const std::string &path) {
+  CHECK(storage_.mutable_data()->dir().empty())
+      << "duplicate set storage's path";
+  storage_.mutable_data()->set_dir(path);
+}
+
+storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag,
+                                                  int num_samples) {
+  auto *tablet = storage_.Find(tag);
+  if (!tablet) {
+    tablet = storage_.Add(tag, num_samples);
+  }
+  return tablet;
+}
+
+void InformationMaintainer::AddRecord(const std::string &tag,
+                                      const storage::Record &data) {
+  auto *tablet = storage_.Find(tag);
+  CHECK(tablet);
+
+  auto num_records = tablet->num_records();
+  const auto num_samples = tablet->num_samples();
+  const auto offset = ReserviorSample(num_samples, num_records + 1);
+  if (offset < 0)
+    return;
+
+  storage::Record *record;
+  if (offset >= num_records) {
+    record = storage_.NewRecord(tag);
+  } else {
+    record = storage_.GetRecord(tag, offset);
+  }
+
+  *record = data;
+
+  tablet->set_num_records(num_records + 1);
+}
+
+void InformationMaintainer::PersistToDisk() {
+  CHECK(!storage_.data().dir().empty()) << "path of storage should be set";
+  storage_.Save(storage_.data().dir());
+}
+
+} // namespace visualdl

visualdl/backend/logic/im.h

+#ifndef VISUALDL_BACKEND_LOGIC_IM_H
+#define VISUALDL_BACKEND_LOGIC_IM_H
+
+#include <string>
+
+#include "visualdl/backend/storage/storage.h"
+
+namespace visualdl {
+
+/*
+ * Maintain the Storage singleton in memory, pre-compute some the statical
+ * information to help visualizaton.
+ */
+class InformationMaintainer final {
+public:
+  InformationMaintainer() {}
+
+  static InformationMaintainer &Global() {
+    static InformationMaintainer *x = new InformationMaintainer();
+    return *x;
+  }
+
+  /*
+   * Set the disk path to store the Storage object.
+   */
+  void SetPersistDest(const std::string &path);
+
+  storage::Tablet *AddTablet(const std::string &tag, int num_samples);
+
+  /*
+   * @tag: tag of the target Tablet.
+   * @type: type of target Tablet.
+   * @data: storage Record.
+   *
+   * NOTE pass in the serialized protobuf message will trigger copying, but
+   * simpler to support different Tablet data formats.
+   */
+  void AddRecord(const std::string &tag, const storage::Record &record);
+
+  /*
+   * Save the Storage Protobuf to disk.
+   */
+  void PersistToDisk();
+
+  Storage &storage() { return storage_; }
+
+private:
+  Storage storage_;
+};
+
+} // namespace visualdl
+
+#endif // VISUALDL_BACKEND_LOGIC_IM_H

visualdl/backend/logic/im_test.cc

+#include "visualdl/backend/logic/im.h"
+
+#include "gtest/gtest.h"
+
+namespace visualdl {
+
+class ImTester : public ::testing::Test {
+protected:
+  void SetUp() override {}
+
+  InformationMaintainer &im = InformationMaintainer::Global();
+};
+
+TEST_F(ImTester, AddTablet) { im.AddTablet("tag0", 20); }
+
+TEST_F(ImTester, AddRecord) {
+  storage::Record rcd;
+  rcd.set_dtype(storage::DataType::kInt32s);
+  for (int i = 0; i < 100; i++) {
+    for (int j = 0; j < 10; j++) {
+      rcd.mutable_data()->add_i32s(i * 20 + j);
+    }
+    im.AddRecord("tag0", rcd);
+  }
+
+  ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL);
+}
+
+} // namespace visualdl

visualdl/backend/logic/pybind.cc

+#include <ctype.h>
+#include <pybind11/pybind11.h>
+
+#include "visualdl/backend/logic/sdk.h"
+
+namespace py = pybind11;
+namespace vs = visualdl;
+
+PYBIND11_MODULE(core, m) {
+  m.doc() = "visualdl python core API";
+
+  py::class_<vs::TabletHelper>(m, "Tablet")
+      // interfaces for components
+      .def("add_scalar_int32",
+           &vs::TabletHelper::AddScalarRecord<int32_t>,
+           "add a scalar int32 record",
+           pybind11::arg("id"),
+           pybind11::arg("value"))
+      .def("add_scalar_int64",
+           &vs::TabletHelper::AddScalarRecord<int64_t>,
+           "add a scalar int64 record",
+           pybind11::arg("id"),
+           pybind11::arg("value"))
+      .def("add_scalar_float", &vs::TabletHelper::AddScalarRecord<float>)
+      .def("add_scalar_double", &vs::TabletHelper::AddScalarRecord<double>)
+      // other member setter and getter
+      .def("record_buffer", &vs::TabletHelper::record_buffer)
+      .def("records_size", &vs::TabletHelper::records_size)
+      .def("buffer", &vs::TabletHelper::buffer)
+      .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer)
+      .def("set_buffer",
+           (void (vs::TabletHelper::*)(const std::string&)) &
+               vs::TabletHelper::SetBuffer);
+
+  py::class_<vs::StorageHelper>(m, "Storage")
+      .def("timestamp", &vs::StorageHelper::timestamp)
+      .def("dir", &vs::StorageHelper::dir)
+      .def("set_dir", &vs::StorageHelper::SetDir)
+      .def("tablets_size", &vs::StorageHelper::tablets_size)
+      .def("buffer", &vs::StorageHelper::buffer)
+      .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer)
+      .def("set_buffer",
+           (void (vs::StorageHelper::*)(const std::string&)) &
+               vs::StorageHelper::SetBuffer);
+
+  py::class_<vs::ImHelper>(m, "Im")
+      .def("storage", &vs::ImHelper::storage)
+      .def("tablet", &vs::ImHelper::tablet)
+      .def("add_tablet", &vs::ImHelper::AddTablet);
+
+  m.def("im", &vs::get_im, "global information-maintainer object.");
+}

visualdl/backend/logic/sdk.cc

+#include <google/protobuf/text_format.h>
+#include "visualdl/backend/logic/sdk.h"
+
+namespace visualdl {
+
+#define IMPL_ENTRY_SET_OR_ADD(method__, ctype__, dtype__, opr__)               \
+  template <> void Entry<ctype__>::method__(ctype__ v) {                       \
+    entry->set_dtype(storage::DataType::dtype__);                              \
+    entry->opr__(v);                                                           \
+  }
+
+IMPL_ENTRY_SET_OR_ADD(Set, int32_t, kInt32, set_i32);
+IMPL_ENTRY_SET_OR_ADD(Set, int64_t, kInt64, set_i64);
+IMPL_ENTRY_SET_OR_ADD(Set, bool, kBool, set_b);
+IMPL_ENTRY_SET_OR_ADD(Set, float, kFloat, set_f);
+IMPL_ENTRY_SET_OR_ADD(Set, double, kDouble, set_d);
+IMPL_ENTRY_SET_OR_ADD(Add, int32_t, kInt32s, add_i32s);
+IMPL_ENTRY_SET_OR_ADD(Add, int64_t, kInt64s, add_i64s);
+IMPL_ENTRY_SET_OR_ADD(Add, bool, kBools, add_bs);
+IMPL_ENTRY_SET_OR_ADD(Add, float, kFloats, add_fs);
+IMPL_ENTRY_SET_OR_ADD(Add, double, kDoubles, add_ds);
+
+std::string StorageHelper::human_readable_buffer() const {
+  std::string buffer;
+  google::protobuf::TextFormat::PrintToString(*data_, &buffer);
+  return buffer;
+}
+
+std::string TabletHelper::human_readable_buffer() const {
+  std::string buffer;
+  google::protobuf::TextFormat::PrintToString(*data_, &buffer);
+  return buffer;
+}
+
+} // namespace visualdl

visualdl/backend/logic/sdk.h

+#ifndef VISUALDL_BACKEND_LOGIC_SDK_H
+#define VISUALDL_BACKEND_LOGIC_SDK_H
+#include "visualdl/backend/logic/im.h"
+
+#include <map>
+
+namespace visualdl {
+
+class TabletHelper {
+public:
+  // method for each components
+  template <typename T>
+  void AddScalarRecord(int id, T value);
+
+  // basic member getter and setter
+  std::string record_buffer(int idx) const { return data_->records(idx).SerializeAsString(); }
+  size_t records_size() const { return data_->records_size(); }
+  std::string buffer() const { return data_->SerializeAsString(); }
+  std::string human_readable_buffer() const;
+  void SetBuffer(const storage::Tablet &t) { *data_ = t; }
+  void SetBuffer(const std::string &b) { data_->ParseFromString(b); }
+
+  // constructor that enable concurrency.
+  TabletHelper(storage::Tablet *t) : data_(t) {}
+  // data updater that resuage of one instance.
+  TabletHelper &operator()(storage::Tablet *t) { data_ = t; return *this; }
+
+private:
+  storage::Tablet *data_;
+};
+
+class StorageHelper {
+public:
+  StorageHelper(storage::Storage *s) : data_(s) {}
+  StorageHelper &operator()(storage::Storage *s) {
+    data_ = s;
+    return *this;
+  }
+
+  void SetBuffer(const storage::Storage &buffer) { *data_ = buffer; }
+  void SetBuffer(const std::string &buffer) { data_->ParseFromString(buffer); }
+  void SetDir(const std::string &dir) { data_->set_dir(dir); }
+
+  int64_t timestamp() const { return data_->timestamp(); }
+  std::string dir() const { return data_->dir(); }
+  int tablets_size() const { return data_->tablets_size(); }
+  std::string buffer() const { return data_->SerializeAsString(); }
+  std::string human_readable_buffer() const;
+
+private:
+  storage::Storage *data_;
+};
+
+class ImHelper {
+public:
+  ImHelper() {}
+
+  StorageHelper storage() {
+    return StorageHelper(
+        InformationMaintainer::Global().storage().mutable_data());
+  }
+  TabletHelper tablet(const std::string &tag) {
+    return TabletHelper(InformationMaintainer::Global().storage().Find(tag));
+  }
+  TabletHelper AddTablet(const std::string &tag, int num_samples) {
+    return TabletHelper(
+        InformationMaintainer::Global().AddTablet(tag, num_samples));
+  }
+};
+
+static ImHelper& get_im() {
+  static ImHelper im;
+  return im;
+}
+
+} // namespace visualdl
+
+#include "visualdl/backend/logic/sdk.hpp"
+#endif  // VISUALDL_BACKEND_LOGIC_SDK_H

visualdl/backend/logic/sdk.hpp

+#include "visualdl/backend/logic/im.h"
+
+namespace visualdl {
+
+/*
+ * Utility helper for storage::Entry.
+ */
+template <typename T> struct Entry {
+  // use pointer to avoid copy
+  storage::Entry *entry{nullptr};
+
+  Entry(storage::Entry *entry) : entry(entry) {}
+
+  /*
+   * Set a single value.
+   */
+  void Set(T v);
+
+  /*
+   * Add a value to repeated message field.
+   */
+  void Add(T v);
+};
+
+template <typename T>
+void TabletHelper::AddScalarRecord(int id, T value) {
+  auto* record = data_->add_records();
+  record->set_id(id);
+  Entry<T> entry_helper(record->mutable_data());
+  entry_helper.Set(value);
+}
+
+} // namespace visualdl

visualdl/backend/storage/storage.cc

@@ -5,18 +5,38 @@
 
 namespace visualdl {
 
-storage::Tablet *Storage::Add(const std::string &tag) {
-  return &proto_.mutable_tablets()->at(tag);
+storage::Tablet *Storage::Add(const std::string &tag, int num_samples) {
+  auto *tablet = &(*proto_.mutable_tablets())[tag];
+  tablet->set_num_samples(num_samples);
+  return tablet;
 }
 
-const storage::Tablet *Storage::Find(const std::string &tag) const {
-  auto it = proto_.tablets().find(tag);
+storage::Tablet *Storage::Find(const std::string &tag) {
+  auto it = proto_.mutable_tablets()->find(tag);
   if (it != proto_.tablets().end()) {
     return &it->second;
   }
   return nullptr;
 }
 
+storage::Record *Storage::NewRecord(const std::string &tag) {
+  auto *tablet = Find(tag);
+  CHECK(tablet) << "Tablet" << tag << " should be create first";
+  auto *record = tablet->mutable_records()->Add();
+  // increase num_records
+  int num_records = tablet->num_records();
+  tablet->set_num_records(num_records + 1);
+  return record;
+}
+storage::Record *Storage::GetRecord(const std::string &tag, int offset) {
+  auto *tablet = Find(tag);
+  CHECK(tablet) << "Tablet" << tag << " should be create first";
+
+  auto num_records = tablet->num_records();
+  CHECK_LT(offset, num_records) << "invalid offset";
+  return tablet->mutable_records()->Mutable(offset);
+}
+
 void Storage::Save(const std::string &path) const {
   std::ofstream file(path, file.binary | file.out);
   CHECK(file.is_open()) << "can't open path " << path;
@@ -34,12 +54,10 @@ void Storage::Load(const std::string &path) {
   DeSerialize(buffer);
 }
 
-std::string Storage::Serialize() const {
-  return proto_.SerializeAsString();
-}
+std::string Storage::Serialize() const { return proto_.SerializeAsString(); }
 
 void Storage::DeSerialize(const std::string &data) {
   proto_.ParseFromString(data);
 }
 
-}  // namespace visualdl
+} // namespace visualdl

visualdl/backend/storage/storage.h

@@ -9,24 +9,33 @@
 namespace visualdl {
 
 class Storage final {
- public:
-  /*
-   * There should be only one Storage instance in memory.
-   */
-  Storage &Global() {
-    static Storage *instance = new Storage();
-    return *instance;
+public:
+  Storage() {
+    // set time stamp
+    time_t time0;
+    time(&time0);
+    proto_.set_timestamp(time0);
   }
 
   /*
    * Add a new tablet named `tag`, the newly added instance will be returned.
    */
-  storage::Tablet *Add(const std::string &tag);
+  storage::Tablet *Add(const std::string &tag, int num_samples);
 
   /*
    * Search the tablet named `tag`, if not exist, return nullptr.
    */
-  const storage::Tablet *Find(const std::string &tag) const;
+  storage::Tablet *Find(const std::string &tag);
+
+  /*
+   * Append a new record to the tail of tablet.
+   */
+  storage::Record *NewRecord(const std::string &tag);
+
+  /*
+   * Get a record at `offset`, if the offset is not valid, yield a failed CHECK.
+   */
+  storage::Record *GetRecord(const std::string &tag, int offset);
 
   /*
    * Serialize this object to string and save it to a file.
@@ -38,7 +47,11 @@ class Storage final {
    */
   void Load(const std::string &path);
 
- protected:
+  storage::Storage *mutable_data() { return &proto_; }
+
+  const storage::Storage &data() { return proto_; }
+
+protected:
   /*
    * Serialize the Storage instance to string.
    */
@@ -49,17 +62,10 @@ class Storage final {
    */
   void DeSerialize(const std::string &data);
 
-  Storage() {
-    // set time stamp
-    time_t time0;
-    time(&time0);
-    proto_.set_timestamp(time0);
-  }
-
- private:
+private:
   storage::Storage proto_;
 };
 
 } // namespace visualdl
 
-#endif //VISUALDL_STORAGE_H
+#endif // VISUALDL_STORAGE_H

visualdl/backend/storage/storage.proto

@@ -16,6 +16,8 @@ enum DataType {
     kStrings = 9;
     kInt32s = 10;
     kBools = 11;
+
+    kUnknown = 12;
 }
 
 // A data array, which type is `type`.

visualdl/backend/test.cc

+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
\ No newline at end of file

visualdl/backend/test.py

+import sys
+import unittest
+
+sys.path.append('../../build')
+import core
+
+im = core.im()
+
+
+class StorageTester(unittest.TestCase):
+    def setUp(self):
+        self.storage = im.storage()
+
+    def test_size(self):
+        self.assertEqual(self.storage.tablets_size(), 0)
+        im.add_tablet("tag0", 100)
+        self.assertEqual(self.storage.tablets_size(), 1)
+
+        for i in range(1, 11):
+            im.add_tablet("tag%d" % i, 100)
+        self.assertEqual(self.storage.tablets_size(), 11)
+
+    def test_timestamp(self):
+        print self.storage.timestamp()
+
+    def test_dir(self):
+        dir = "./1.txt"
+        self.storage.set_dir(dir)
+        self.assertEqual(dir, self.storage.dir())
+
+    def test_human_readable_buffer(self):
+        print self.storage.human_readable_buffer()
+
+
+class TabletTester(unittest.TestCase):
+    def setUp(self):
+        self.tablet = im.add_tablet("tag101", 20)
+
+    def test_add_scalar(self):
+        self.tablet.add_scalar_float(1, 0.3)
+        self.assertEqual(self.tablet.records_size(), 1)
+
+    def test_human_readable_buffer(self):
+        print self.tablet.human_readable_buffer()
+
+
+if __name__ == '__main__':
+    unittest.main()