init concurrency

CMakeLists.txt

@@ -20,12 +20,16 @@ add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/sdk.cc)
 add_library(im ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im.cc)
 
 pybind11_add_module(core
-        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc)
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/pybind.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/utils/filesystem.h
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/utils/concurrency.h
+        )
 target_link_libraries(core PRIVATE pybind11::module im storage sdk protobuf glog)
 set_target_properties(core PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
 
 add_executable(vl_test
         ${PROJECT_SOURCE_DIR}/visualdl/backend/test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/backend/storage/storage_test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/backend/utils/test_concurrency.cc
         ${PROJECT_SOURCE_DIR}/visualdl/backend/logic/im_test.cc)
 target_link_libraries(vl_test storage im gtest glog protobuf gflags pthread)

visualdl/backend/logic/im.h

@@ -10,8 +10,19 @@
 namespace visualdl {
 
 /*
- * Maintain the Storage singleton in memory, pre-compute some the statical
- * information to help visualizaton.
+ * InformationMaintainer(IM) maintain the Storage singleton in memory,
+ * pre-compute some the statistical information to help visualizaton.
+ *
+ * There should be two processes and each have an IM, one is the web server
+ * which hold one IM to read the storage, the other is the SDK(python or C++),
+ * it will get an IM to write latest changes to storage.
+ *
+ * An IM have an underlying Storage object, which might be a memory based
+ * storage or a disk based one, both has the same interfaces those defined by
+ * class StorageBase.
+ *
+ * The SDK's IM will maintain the changes and periodically write to disk, and
+ * the web server's IM will periodically read latest storage from disk.
  */
 class InformationMaintainer final {
 public:

visualdl/backend/utils/concurrency.h

+#ifndef VISUALDL_BACKEND_UTILS_CONCURRENCY_H
+#define VISUALDL_BACKEND_UTILS_CONCURRENCY_H
+
+#include <chrono>
+#include <memory>
+#include <thread>
+#include <vector>
+
+namespace visualdl {
+namespace cc {
+
+/*
+ * Run a task every `duration` milliseconds.
+ * Each evoke will start a thread to do this asynchronously.
+ */
+struct PeriodExector {
+  using task_t = std::function<void()>;
+  using duration_t = std::chrono::milliseconds;
+
+  PeriodExector& Global() {
+    static PeriodExector exec;
+    return exec;
+  }
+
+  void Quit() {
+    // TODO use some conditonal variable to help quit immediately.
+    quit = true;
+  }
+
+  void operator()(task_t&& task, duration_t duration) {
+    auto task_wrapper = [&, task] {
+      while (!quit) {
+        task();
+        std::this_thread::sleep_for(duration);
+      }
+    };
+    threads_.emplace_back(std::thread(std::move(task_wrapper)));
+  }
+
+  ~PeriodExector() {
+    for (auto& t : threads_) {
+      if (t.joinable()) {
+        t.join();
+      }
+    }
+  }
+
+private:
+  bool quit = false;
+  std::vector<std::thread> threads_;
+};
+
+}  // namespace cc
+}  // namespace visualdl
+
+#endif

visualdl/backend/utils/test_concurrency.cc

+#include <gtest/gtest.h>
+
+namespace visualdl {}  // namespace visualdl
\ No newline at end of file