feature/logic_layer_pybind

CMakeLists.txt

-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.5)
 project(VisualDL)
 
 set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_FLAGS "-fPIC")
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
-include_directories(thirdparty/local/include)
-add_subdirectory(thirdparty/pybind11-2.2.1)
+include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include)
+include_directories(/usr/include/python2.7)
 
-link_directories(thirdparty/local/lib)
+set(PYBIND11_CPP_STANDARD -std=c++11)
+add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/pybind11-2.2.1)
 
-set(SOURCE_FILES
-        visualdl/backend/storage/storage.cc visualdl/backend/storage/storage.h
-        visualdl/backend/storage/storage.pb.h
-        visualdl/backend/storage/storage.pb.cc
-        )
+link_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/lib)
 
-add_library(storage visualdl/backend/storage/storage.cc
-            visualdl/backend/storage/storage.pb.cc)
-add_library(c_api visualdl/backend/logic/c_api.cc)
-add_library(sdk visualdl/backend/logic/sdk.cc)
-add_library(im visualdl/backend/logic/im.cc)
+add_library(storage
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage.pb.cc)
+add_library(c_api ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/c_api.cc)
+add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc)
+add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc)
+
+pybind11_add_module(core
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc)
+target_link_libraries(core PRIVATE pybind11::module im storage sdk protobuf glog)
+set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 
 add_executable(vl_test
-        visualdl/backend/test.cc
-        visualdl/backend/logic/im_test.cc)
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc)
 target_link_libraries(vl_test storage im gtest glog protobuf gflags)
-add_executable(VisualDL ${SOURCE_FILES})
 

visualdl/.clang-format

+# This file is used by clang-format to autoformat paddle source code
+#
+# The clang-format is part of llvm toolchain.
+# It need to install llvm and clang to format source code style.
+#
+# The basic usage is,
+#   clang-format -i -style=file PATH/TO/SOURCE/CODE
+#
+# The -style=file implicit use ".clang-format" file located in one of 
+# parent directory. 
+# The -i means inplace change.
+#
+# The document of clang-format is 
+#   http://clang.llvm.org/docs/ClangFormat.html
+#   http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+---
+Language:        Cpp
+BasedOnStyle:  Google
+IndentWidth:     2
+TabWidth:        2
+ContinuationIndentWidth: 4
+AccessModifierOffset: -2  # The private/protected/public has no indent in class
+Standard:  Cpp11 
+AllowAllParametersOfDeclarationOnNextLine: true
+BinPackParameters: false
+BinPackArguments: false
+...
+

visualdl/backend/logic/im.cc

@@ -41,9 +41,7 @@ storage::Tablet *InformationMaintainer::AddTablet(const std::string &tag,
 }
 
 void InformationMaintainer::AddRecord(const std::string &tag,
-                                      storage::Tablet::Type type,
-                                      const std::string &data,
-                                      storage::DataType dtype) {
+                                      const storage::Record &data) {
   auto *tablet = storage_.Find(tag);
   CHECK(tablet);
 
@@ -60,8 +58,7 @@ void InformationMaintainer::AddRecord(const std::string &tag,
     record = storage_.GetRecord(tag, offset);
   }
 
-  record->set_dtype(dtype);
-  record->ParseFromString(data);
+  *record = data;
 
   tablet->set_num_records(num_records + 1);
 }

visualdl/backend/logic/im.h

@@ -30,14 +30,12 @@ public:
   /*
    * @tag: tag of the target Tablet.
    * @type: type of target Tablet.
-   * @data: serialized protobuf message.
+   * @data: storage Record.
    *
    * NOTE pass in the serialized protobuf message will trigger copying, but
    * simpler to support different Tablet data formats.
    */
-  void AddRecord(const std::string &tag, storage::Tablet::Type type,
-                 const std::string &data,
-                 storage::DataType dtype = storage::DataType::kUnknown);
+  void AddRecord(const std::string &tag, const storage::Record &record);
 
   /*
    * Save the Storage Protobuf to disk.

visualdl/backend/logic/im_test.cc

@@ -20,8 +20,7 @@ TEST_F(ImTester, AddRecord) {
     for (int j = 0; j < 10; j++) {
       rcd.mutable_data()->add_i32s(i * 20 + j);
     }
-    im.AddRecord("tag0", storage::Tablet::Type::Tablet_Type_kGraph,
-                 rcd.SerializeAsString());
+    im.AddRecord("tag0", rcd);
   }
 
   ASSERT_EQ(im.storage().Find("tag0")->records_size(), 20UL);

visualdl/backend/logic/pybind.cc

+#include <ctype.h>
+#include <pybind11/pybind11.h>
+
+#include "visualdl/backend/logic/sdk.h"
+
+namespace py = pybind11;
+namespace vs = visualdl;
+
+PYBIND11_MODULE(core, m) {
+  m.doc() = "visualdl python core API";
+
+  py::class_<vs::TabletHelper>(m, "Tablet")
+      // interfaces for components
+      .def("add_scalar_int32",
+           &vs::TabletHelper::AddScalarRecord<int32_t>,
+           "add a scalar int32 record",
+           pybind11::arg("id"),
+           pybind11::arg("value"))
+      .def("add_scalar_int64",
+           &vs::TabletHelper::AddScalarRecord<int64_t>,
+           "add a scalar int64 record",
+           pybind11::arg("id"),
+           pybind11::arg("value"))
+      .def("add_scalar_float", &vs::TabletHelper::AddScalarRecord<float>)
+      .def("add_scalar_double", &vs::TabletHelper::AddScalarRecord<double>)
+      // other member setter and getter
+      .def("record_buffer", &vs::TabletHelper::record_buffer)
+      .def("records_size", &vs::TabletHelper::records_size)
+      .def("buffer", &vs::TabletHelper::buffer)
+      .def("human_readable_buffer", &vs::TabletHelper::human_readable_buffer)
+      .def("set_buffer",
+           (void (vs::TabletHelper::*)(const std::string&)) &
+               vs::TabletHelper::SetBuffer);
+
+  py::class_<vs::StorageHelper>(m, "Storage")
+      .def("timestamp", &vs::StorageHelper::timestamp)
+      .def("dir", &vs::StorageHelper::dir)
+      .def("set_dir", &vs::StorageHelper::SetDir)
+      .def("tablets_size", &vs::StorageHelper::tablets_size)
+      .def("buffer", &vs::StorageHelper::buffer)
+      .def("human_readable_buffer", &vs::StorageHelper::human_readable_buffer)
+      .def("set_buffer",
+           (void (vs::StorageHelper::*)(const std::string&)) &
+               vs::StorageHelper::SetBuffer);
+
+  py::class_<vs::ImHelper>(m, "Im")
+      .def("storage", &vs::ImHelper::storage)
+      .def("tablet", &vs::ImHelper::tablet)
+      .def("add_tablet", &vs::ImHelper::AddTablet);
+
+  m.def("im", &vs::get_im, "global information-maintainer object.");
+}

visualdl/backend/logic/sdk.cc

+#include <google/protobuf/text_format.h>
+#include "visualdl/backend/logic/sdk.h"
+
+namespace visualdl {
+
+#define IMPL_ENTRY_SET_OR_ADD(method__, ctype__, dtype__, opr__)               \
+  template <> void Entry<ctype__>::method__(ctype__ v) {                       \
+    entry->set_dtype(storage::DataType::dtype__);                              \
+    entry->opr__(v);                                                           \
+  }
+
+IMPL_ENTRY_SET_OR_ADD(Set, int32_t, kInt32, set_i32);
+IMPL_ENTRY_SET_OR_ADD(Set, int64_t, kInt64, set_i64);
+IMPL_ENTRY_SET_OR_ADD(Set, bool, kBool, set_b);
+IMPL_ENTRY_SET_OR_ADD(Set, float, kFloat, set_f);
+IMPL_ENTRY_SET_OR_ADD(Set, double, kDouble, set_d);
+IMPL_ENTRY_SET_OR_ADD(Add, int32_t, kInt32s, add_i32s);
+IMPL_ENTRY_SET_OR_ADD(Add, int64_t, kInt64s, add_i64s);
+IMPL_ENTRY_SET_OR_ADD(Add, bool, kBools, add_bs);
+IMPL_ENTRY_SET_OR_ADD(Add, float, kFloats, add_fs);
+IMPL_ENTRY_SET_OR_ADD(Add, double, kDoubles, add_ds);
+
+std::string StorageHelper::human_readable_buffer() const {
+  std::string buffer;
+  google::protobuf::TextFormat::PrintToString(*data_, &buffer);
+  return buffer;
+}
+
+std::string TabletHelper::human_readable_buffer() const {
+  std::string buffer;
+  google::protobuf::TextFormat::PrintToString(*data_, &buffer);
+  return buffer;
+}
+
+} // namespace visualdl

visualdl/backend/logic/sdk.h

+#ifndef VISUALDL_BACKEND_LOGIC_SDK_H
+#define VISUALDL_BACKEND_LOGIC_SDK_H
+#include "visualdl/backend/logic/im.h"
+
+#include <map>
+
+namespace visualdl {
+
+class TabletHelper {
+public:
+  // method for each components
+  template <typename T>
+  void AddScalarRecord(int id, T value);
+
+  // basic member getter and setter
+  std::string record_buffer(int idx) const { return data_->records(idx).SerializeAsString(); }
+  size_t records_size() const { return data_->records_size(); }
+  std::string buffer() const { return data_->SerializeAsString(); }
+  std::string human_readable_buffer() const;
+  void SetBuffer(const storage::Tablet &t) { *data_ = t; }
+  void SetBuffer(const std::string &b) { data_->ParseFromString(b); }
+
+  // constructor that enable concurrency.
+  TabletHelper(storage::Tablet *t) : data_(t) {}
+  // data updater that resuage of one instance.
+  TabletHelper &operator()(storage::Tablet *t) { data_ = t; return *this; }
+
+private:
+  storage::Tablet *data_;
+};
+
+class StorageHelper {
+public:
+  StorageHelper(storage::Storage *s) : data_(s) {}
+  StorageHelper &operator()(storage::Storage *s) {
+    data_ = s;
+    return *this;
+  }
+
+  void SetBuffer(const storage::Storage &buffer) { *data_ = buffer; }
+  void SetBuffer(const std::string &buffer) { data_->ParseFromString(buffer); }
+  void SetDir(const std::string &dir) { data_->set_dir(dir); }
+
+  int64_t timestamp() const { return data_->timestamp(); }
+  std::string dir() const { return data_->dir(); }
+  int tablets_size() const { return data_->tablets_size(); }
+  std::string buffer() const { return data_->SerializeAsString(); }
+  std::string human_readable_buffer() const;
+
+private:
+  storage::Storage *data_;
+};
+
+class ImHelper {
+public:
+  ImHelper() {}
+
+  StorageHelper storage() {
+    return StorageHelper(
+        InformationMaintainer::Global().storage().mutable_data());
+  }
+  TabletHelper tablet(const std::string &tag) {
+    return TabletHelper(InformationMaintainer::Global().storage().Find(tag));
+  }
+  TabletHelper AddTablet(const std::string &tag, int num_samples) {
+    return TabletHelper(
+        InformationMaintainer::Global().AddTablet(tag, num_samples));
+  }
+};
+
+static ImHelper& get_im() {
+  static ImHelper im;
+  return im;
+}
+
+} // namespace visualdl
+
+#include "visualdl/backend/logic/sdk.hpp"
+#endif  // VISUALDL_BACKEND_LOGIC_SDK_H

visualdl/backend/logic/sdk.hpp

+#include "visualdl/backend/logic/im.h"
+
+namespace visualdl {
+
+/*
+ * Utility helper for storage::Entry.
+ */
+template <typename T> struct Entry {
+  // use pointer to avoid copy
+  storage::Entry *entry{nullptr};
+
+  Entry(storage::Entry *entry) : entry(entry) {}
+
+  /*
+   * Set a single value.
+   */
+  void Set(T v);
+
+  /*
+   * Add a value to repeated message field.
+   */
+  void Add(T v);
+};
+
+template <typename T>
+void TabletHelper::AddScalarRecord(int id, T value) {
+  auto* record = data_->add_records();
+  record->set_id(id);
+  Entry<T> entry_helper(record->mutable_data());
+  entry_helper.Set(value);
+}
+
+} // namespace visualdl

visualdl/backend/test.cc

+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
\ No newline at end of file

visualdl/backend/test.py

+import sys
+import unittest
+
+sys.path.append('../../build')
+import core
+
+im = core.im()
+
+
+class StorageTester(unittest.TestCase):
+    def setUp(self):
+        self.storage = im.storage()
+
+    def test_size(self):
+        self.assertEqual(self.storage.tablets_size(), 0)
+        im.add_tablet("tag0", 100)
+        self.assertEqual(self.storage.tablets_size(), 1)
+
+        for i in range(1, 11):
+            im.add_tablet("tag%d" % i, 100)
+        self.assertEqual(self.storage.tablets_size(), 11)
+
+    def test_timestamp(self):
+        print self.storage.timestamp()
+
+    def test_dir(self):
+        dir = "./1.txt"
+        self.storage.set_dir(dir)
+        self.assertEqual(dir, self.storage.dir())
+
+    def test_human_readable_buffer(self):
+        print self.storage.human_readable_buffer()
+
+
+class TabletTester(unittest.TestCase):
+    def setUp(self):
+        self.tablet = im.add_tablet("tag101", 20)
+
+    def test_add_scalar(self):
+        self.tablet.add_scalar_float(1, 0.3)
+        self.assertEqual(self.tablet.records_size(), 1)
+
+    def test_human_readable_buffer(self):
+        print self.tablet.human_readable_buffer()
+
+
+if __name__ == '__main__':
+    unittest.main()