Merge branch 'feature/add_image_component-eigen_support_transform' into feature/add_image_component

CMakeLists.txt

@@ -23,13 +23,15 @@ include(external/zlib)      # download, build, install zlib
 include(external/gflags)    # download, build, install gflags
 include(external/glog)      # download, build, install glog
 include(external/gtest)     # download, build, install gtest
+include(external/eigen)     # download eigen
 include(external/pybind11)  # download pybind11
 include(external/protobuf)  # download, build, install protobuf
 include(external/python)    # find python and set path
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
-include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include)
+# TODO(ChunweiYan) debug, remote latter
+#include_directories(/home/superjom/project/VisualDL/build/third_party/eigen3/src/extern_eigen3)
 
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/storage)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/logic)
@@ -37,9 +39,10 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/python)
 
 add_executable(vl_test
         ${PROJECT_SOURCE_DIR}/visualdl/test.cc
-        ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/utils/test_image.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h
         ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h
         )

cmake/external/eigen.cmake

+INCLUDE(ExternalProject)
+
+SET(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3)
+
+INCLUDE_DIRECTORIES(${EIGEN_SOURCE_DIR}/src/extern_eigen3)
+
+ExternalProject_Add(
+    extern_eigen3
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    GIT_REPOSITORY  "https://github.com/RLovelett/eigen.git"
+    GIT_TAG         "master"
+    PREFIX          ${EIGEN_SOURCE_DIR}
+    UPDATE_COMMAND  ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND     ""
+    INSTALL_COMMAND   ""
+    TEST_COMMAND      ""
+)
+
+if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
+    set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/eigen3_dummy.c)
+    file(WRITE ${dummyfile} "const char * dummy_eigen3 = \"${dummyfile}\";")
+    add_library(eigen3 STATIC ${dummyfile})
+else()
+    add_library(eigen3 INTERFACE)
+endif()
+
+add_dependencies(eigen3 extern_eigen3)
+
+LIST(APPEND external_project_dependencies eigen3)

visualdl/logic/CMakeLists.txt

 add_library(im ${PROJECT_SOURCE_DIR}/visualdl/logic/im.cc)
-add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk.cc)
+add_library(sdk ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk.cc ${PROJECT_SOURCE_DIR}/visualdl/utils/image.h)
 add_dependencies(im storage_proto)
 add_dependencies(sdk entry storage storage_proto)
 
 ## pybind
 add_library(core SHARED ${PROJECT_SOURCE_DIR}/visualdl/logic/pybind.cc)
-add_dependencies(core pybind python im entry tablet storage sdk protobuf glog)
+add_dependencies(core pybind python im entry tablet storage sdk protobuf glog eigen3)
 target_link_libraries(core PRIVATE pybind entry python im tablet storage sdk protobuf glog)
 set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")

visualdl/logic/sdk.cc

 #include "visualdl/logic/sdk.h"
 
+#include "visualdl/utils/image.h"
+
 namespace visualdl {
 
 namespace components {
@@ -101,7 +103,7 @@ void Image::SetSample(int index,
                       const std::vector<value_t>& data) {
   // production
   int size = std::accumulate(
-      shape.begin(), shape.end(), 1., [](float a, float b) { return a * b; });
+      shape.begin(), shape.end(), 1., [](int a, int b) { return a * b; });
   CHECK_GT(size, 0);
   CHECK_EQ(shape.size(), 3)
       << "shape should be something like (width, height, num_channel)";
@@ -111,7 +113,6 @@ void Image::SetSample(int index,
   CHECK_LT(index, num_samples_);
   CHECK_LE(index, num_records_);
 
-  // set data
   auto entry = step_.MutableData<std::vector<char>>(index);
   // trick to store int8 to protobuf
   std::vector<char> data_str(data.size());

visualdl/logic/sdk_test.cc

@@ -56,7 +56,7 @@ TEST(Image, test) {
   for (int step = 0; step < num_steps; step++) {
     image.StartSampling();
     for (int i = 0; i < 7; i++) {
-      vector<int64_t> shape({3, 5, 5});
+      vector<int64_t> shape({5, 5, 3});
       vector<float> data;
       for (int j = 0; j < 3 * 5 * 5; j++) {
         data.push_back(float(rand()) / RAND_MAX);

visualdl/python/test_storage.py

@@ -36,7 +36,7 @@ class StorageTest(unittest.TestCase):
         image_writer = self.writer.image(tag, 10, 1)
         num_passes = 10
         num_samples = 100
-        shape = [3, 10, 10]
+        shape = [10, 10, 3]
 
         for pass_ in xrange(num_passes):
             image_writer.start_sampling()