Add preliminary support for Windows (#461)

* Add preliminary support for Windows

    Known issues:

    1. No zlib support for protobuf
    2. Manual editting to core.vcxproj are required
    3. No build and packge script support
    4. Not tesetd, may break linux build

* Add build script and batch file for Windows

* Fix linux build and include batch file in wheel

* Fix Python 2.7 build

* Define NOMINMAX instead of preventing macro expansion

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+*.dll
 
 # C extensions
 *.so

CMakeLists.txt

@@ -53,6 +53,11 @@ if (NOT ON_RELEASE)
   add_definitions(-DVISUALDL_WITH_GLOG)
 endif(NOT ON_RELEASE)
 
+if (MSVC)
+  add_definitions(-DPROTOBUF_USE_DLLS)
+  add_definitions(-DNOMINMAX) # No min/max macros
+endif(MSVC)
+
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 

build.ps1

+# Build script for Windows
+# Requires: PowerShell, Python 3, Visual Studio 15 2017
+
+$TOP_DIR=$pwd.PATH
+$FRONTEND_DIR="$TOP_DIR/frontend"
+$BACKEND_DIR="$TOP_DIR/visualdl"
+$BUILD_DIR="$TOP_DIR/build"
+
+mkdir $BUILD_DIR -ErrorAction Ignore
+
+function check_duplicated($filename_format) {
+    $files = ls dist/$filename_format -ErrorAction Ignore
+    if (Get-TypeName($files.Length) -ne "FileInfo") {
+      Write-Error "dist have duplicate file for $filename_format, please clean and rerun"
+      exit(1)
+    }
+}
+
+function build_frontend() {
+    cd $FRONTEND_DIR
+    npm install
+    npm run build
+    foreach ($file_name in "manifest.*.js","index.*.js","vendor.*.js") {
+        echo $file_name
+        check_duplicated $file_name
+    }
+}
+
+function build_frontend_fake() {
+    cd $FRONTEND_DIR
+    mkdir -p dist
+}
+
+function build_backend() {
+    cd $BUILD_DIR
+    cmake -G "Visual Studio 15 2017 Win64" `
+        -DCMAKE_BUILD_TYPE=Release `
+        -DON_RELEASE=ON `
+        -DWITH_PYTHON3=ON `
+        -DWITH_TESTING=OFF `
+        -DBUILD_FOR_HOST:BOOL=YES ..
+    cmake --build . --config Release
+}
+
+function build_onnx_graph() {
+    $env:PATH = "$BUILD_DIR/third_party/protobuf/src/extern_protobuf-build/Release;" + $env:PATH
+    cd $TOP_DIR/visualdl/server/onnx
+    protoc onnx.proto --python_out .
+}
+
+function clean_env() {
+    rm -Recurse -Force -ErrorAction Ignore $TOP_DIR/visualdl/server/dist
+    rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/bdist*
+    rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/lib*
+    rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/temp*
+    rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/scripts*
+}
+
+function package() {
+    cp -Recurse $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/
+    cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl
+    cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl/python/
+}
+
+build_frontend
+clean_env
+build_backend
+build_onnx_graph
+package

cmake/external/protobuf.cmake

@@ -178,29 +178,55 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
         SET(OPTIONAL_CACHE_ARGS "-DZLIB_ROOT:STRING=${ZLIB_ROOT}")
     ENDIF()
 
-    ExternalProject_Add(
-        ${TARGET_NAME}
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        PREFIX          ${PROTOBUF_SOURCES_DIR}
-        UPDATE_COMMAND  ""
-        DEPENDS         zlib
-        GIT_REPOSITORY  "https://github.com/google/protobuf.git"
-        GIT_TAG         "2761122b810fe8861004ae785cc3ab39f384d342"
-        CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
-            ${OPTIONAL_ARGS}
-            -Dprotobuf_BUILD_TESTS=OFF
-            -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-            -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-            -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
-            -DCMAKE_INSTALL_LIBDIR=lib
-        CMAKE_CACHE_ARGS
-            -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
-            -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-            -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-            -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-            ${OPTIONAL_CACHE_ARGS}
-    )
+    IF(MSVC)
+        ExternalProject_Add(
+            ${TARGET_NAME}
+            ${EXTERNAL_PROJECT_LOG_ARGS}
+            PREFIX          ${PROTOBUF_SOURCES_DIR}
+            UPDATE_COMMAND  ""
+            URL https://github.com/google/protobuf/archive/v3.5.2.zip
+            CONFIGURE_COMMAND
+            ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
+                ${OPTIONAL_ARGS}
+                -G ${CMAKE_GENERATOR}
+                -Dprotobuf_BUILD_SHARED_LIBS=ON
+                -DCMAKE_CONFIGURATION_TYPES=${CMAKE_BUILD_TYPE}
+                -Dprotobuf_BUILD_TESTS=OFF
+                -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+                -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
+                -DCMAKE_INSTALL_LIBDIR=lib
+            CMAKE_CACHE_ARGS
+                -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
+                -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+                -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+                -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+                ${OPTIONAL_CACHE_ARGS}
+        )
+    ELSE()
+        ExternalProject_Add(
+            ${TARGET_NAME}
+            ${EXTERNAL_PROJECT_LOG_ARGS}
+            PREFIX          ${PROTOBUF_SOURCES_DIR}
+            UPDATE_COMMAND  ""
+            DEPENDS         zlib
+            URL https://github.com/google/protobuf/archive/v3.5.2.zip
+            CONFIGURE_COMMAND
+            ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
+                ${OPTIONAL_ARGS}
+                -Dprotobuf_BUILD_TESTS=OFF
+                -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+                -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
+                -DCMAKE_INSTALL_LIBDIR=lib
+            CMAKE_CACHE_ARGS
+                -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
+                -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+                -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+                -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+                ${OPTIONAL_CACHE_ARGS}
+        )
+    ENDIF(MSVC)
 ENDFUNCTION()
 
 SET(PROTOBUF_VERSION 3.5)

cmake/external/python.cmake

@@ -23,7 +23,11 @@ ELSE()
 ENDIF(WITH_PYTHON3)
 
 # Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE.
-ADD_LIBRARY(python SHARED IMPORTED GLOBAL)
+IF(MSVC)
+    ADD_LIBRARY(python STATIC IMPORTED GLOBAL)
+ELSE()
+    ADD_LIBRARY(python SHARED IMPORTED GLOBAL)
+ENDIF(MSVC)
 SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES})
 
 SET(py_env "")

setup.py

@@ -17,6 +17,7 @@ from __future__ import absolute_import
 
 import os
 import sys
+from sys import platform
 from distutils.spawn import find_executable
 from distutils import log
 import setuptools.command.build_py
@@ -44,6 +45,8 @@ LICENSE = readlines('LICENSE')[0].strip()
 # use memcache to reduce disk read frequency.
 install_requires = ['Flask', 'numpy', 'Pillow', 'protobuf', 'scipy']
 execute_requires = ['npm', 'node', 'bash', 'cmake', 'unzip']
+if platform == "win32":
+    execute_requires = ['node', 'powershell', 'cmake']
 
 
 def die(msg):
@@ -73,6 +76,8 @@ class BaseCommand(setuptools.Command):
 class build_py(setuptools.command.build_py.build_py):
     def run(self):
         cmd = ['bash', 'build.sh']
+        if platform == "win32":
+            cmd = ['powershell', '-NoProfile', './build.ps1']
         env = dict(os.environ)
         if MODE == "travis-CI":
             cmd.append('travis-CI')
@@ -94,6 +99,14 @@ packages = [
     'visualdl.server.onnx',
 ]
 
+libraries = ['core.so']
+if platform == 'win32':
+    libraries = ['core.pyd', 'libprotobuf.dll']
+
+scripts = ['visualdl/server/visualDL', 'demo/vdl_create_scratch_log']
+if platform == 'win32':
+    scripts.append('visualdl/server/visualDL.bat')
+
 setup(
     name="visualdl",
     version=VERSION_NUMBER,
@@ -106,10 +119,12 @@ setup(
     package_data={
         'visualdl.server':
         ['dist/*.js', 'dist/*.html', 'dist/fonts/*', 'dist/assets/*'],
-        'visualdl': ['core.so'],
-        'visualdl.python': ['core.so', 'dog.jpg', 'testing.wav']
+        'visualdl':
+        libraries,
+        'visualdl.python':
+        libraries + ['dog.jpg', 'testing.wav']
     },
     packages=packages,
     ext_modules=[Extension('_foo', ['stub.cc'])],
-    scripts=['visualdl/server/visualDL', 'demo/vdl_create_scratch_log'],
+    scripts=scripts,
     cmdclass=cmdclass)

stub.cc

+#include <Python.h>
+
+PyMODINIT_FUNC PyInit__foo() {
+#if PY_MAJOR_VERSION >= 3
+  return NULL;
+#else
+  return;
+#endif
+}

visualdl/logic/CMakeLists.txt

@@ -20,7 +20,7 @@ add_dependencies(sdk entry binary_record storage storage_proto eigen3)
 ## pybind
 set(OPTIONAL_LINK_FLAGS)
 
-if(NOT APPLE)
+if((NOT APPLE) AND (NOT WIN32))
   set(OPTIONAL_LINK_FLAGS "rt")
 endif()
 
@@ -33,13 +33,31 @@ else()
   add_dependencies(core pybind python im entry tablet storage sdk protobuf eigen3)
   target_link_libraries(core PRIVATE pybind entry binary_record python im tablet storage sdk protobuf ${OPTIONAL_LINK_FLAGS})
 endif()
-set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")
 
-add_custom_command(TARGET core POST_BUILD
-  COMMAND "${CMAKE_COMMAND}" -E copy
-     "${CMAKE_CURRENT_BINARY_DIR}/core.so"
-     "${VISUALDL_SOURCE_DIR}/visualdl/python/"
-  COMMAND "${CMAKE_COMMAND}" -E copy
-     "${CMAKE_CURRENT_BINARY_DIR}/core.so"
-     "${VISUALDL_SOURCE_DIR}/visualdl/"
-  COMMENT "Copying core.so to ${VISUALDL_SOURCE_DIR}/visualdl/python/")
+if (MSVC)
+  set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".pyd")
+  add_custom_command(TARGET core POST_BUILD
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd"
+      "${VISUALDL_SOURCE_DIR}/visualdl/python/"
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd"
+      "${VISUALDL_SOURCE_DIR}/visualdl/"
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll"
+      "${VISUALDL_SOURCE_DIR}/visualdl/python/"
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll"
+      "${VISUALDL_SOURCE_DIR}/visualdl/"
+    COMMENT "Copying core.pyd and libprotobuf.dll to ${VISUALDL_SOURCE_DIR}/visualdl/python/")
+else()
+  set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")
+  add_custom_command(TARGET core POST_BUILD
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${CMAKE_CURRENT_BINARY_DIR}/core.so"
+      "${VISUALDL_SOURCE_DIR}/visualdl/python/"
+    COMMAND "${CMAKE_COMMAND}" -E copy
+      "${CMAKE_CURRENT_BINARY_DIR}/core.so"
+      "${VISUALDL_SOURCE_DIR}/visualdl/"
+    COMMENT "Copying core.so to ${VISUALDL_SOURCE_DIR}/visualdl/python/")
+endif()

visualdl/logic/sdk.cc

@@ -15,6 +15,7 @@ limitations under the License. */
 #include "visualdl/logic/sdk.h"
 
 #include <cstdio>
+#include <numeric>
 
 #include "visualdl/logic/histogram.h"
 #include "visualdl/storage/binary_record.h"

visualdl/server/visualDL.bat

+@ECHO OFF
+setlocal DISABLEDELAYEDEXPANSION
+python %0\..\visualDL %*

visualdl/storage/CMakeLists.txt

+# A modified version of protobuf_generate_cpp, that does not use protobuf::protoc,
+# which are not supported on Windows.
+function(visualdl_protobuf_generate_cpp SRCS HDRS)
+  cmake_parse_arguments(protobuf "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN})
+  set(PROTO_FILES "${protobuf_UNPARSED_ARGUMENTS}")
+  if(NOT PROTO_FILES)
+    message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(protobuf_EXPORT_MACRO)
+    set(DLL_EXPORT_DECL "dllexport_decl=${protobuf_EXPORT_MACRO}:")
+  endif()
+
+  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${PROTO_FILES})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
+    set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
+  endif()
+
+  if(DEFINED Protobuf_IMPORT_DIRS)
+    foreach(DIR ${Protobuf_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  if (protobuf_DESCRIPTORS)
+    set(${protobuf_DESCRIPTORS})
+  endif()
+
+  foreach(FIL ${PROTO_FILES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)
+      get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+      if(FIL_DIR)
+        set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+      endif()
+    endif()
+
+    set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+    set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+    list(APPEND ${SRCS} "${_protobuf_protoc_src}")
+    list(APPEND ${HDRS} "${_protobuf_protoc_hdr}")
+
+    if(protobuf_DESCRIPTORS)
+      set(_protobuf_protoc_desc "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.desc")
+      set(_protobuf_protoc_flags "--descriptor_set_out=${_protobuf_protoc_desc}")
+      list(APPEND ${protobuf_DESCRIPTORS} "${_protobuf_protoc_desc}")
+    else()
+      set(_protobuf_protoc_desc "")
+      set(_protobuf_protoc_flags "")
+    endif()
+
+    add_custom_command(
+      OUTPUT "${_protobuf_protoc_src}"
+             "${_protobuf_protoc_hdr}"
+             ${_protobuf_protoc_desc}
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+               "--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}"
+               ${_protobuf_protoc_flags}
+               ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL} protobuf
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set(${SRCS} "${${SRCS}}" PARENT_SCOPE)
+  set(${HDRS} "${${HDRS}}" PARENT_SCOPE)
+  if(protobuf_DESCRIPTORS)
+    set(${protobuf_DESCRIPTORS} "${${protobuf_DESCRIPTORS}}" PARENT_SCOPE)
+  endif()
+endfunction()
+
 ## add storage_proto as target
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto)
+visualdl_protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto)
 add_library(storage_proto ${PROTO_SRCS})
 add_dependencies(storage_proto protobuf)
 

visualdl/storage/binary_record.h

@@ -43,7 +43,11 @@ struct BinaryRecord {
 
   void tofile() {
     fs::TryRecurMkdir(dir_);
+#ifdef _WIN32
+    std::fstream file(path_, std::ios_base::binary | std::ios_base::out);
+#else
     std::fstream file(path_, file.binary | file.out);
+#endif
     CHECK(file.is_open()) << "open " << path_ << " failed";
 
     size_t size = data_.size();
@@ -90,7 +94,11 @@ protected:
     // which impacts performance.
     if (!filename_.empty()) {
       std::string path = dir_ + "/" + filename_;
+#ifdef _WIN32
+      std::ifstream file(path, std::ios_base::binary);
+#else
       std::ifstream file(path, file.binary);
+#endif
       CHECK(file.is_open()) << " failed to open file " << path;
 
       size_t size;

visualdl/utils/filesystem.h

@@ -18,11 +18,16 @@ limitations under the License. */
 #include <google/protobuf/text_format.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <unistd.h>
 #include <fstream>
 
 #include "visualdl/utils/logging.h"
 
+#ifdef _WIN32
+#include <filesystem>
+#else
+#include <unistd.h>
+#endif
+
 namespace visualdl {
 
 namespace fs {
@@ -62,7 +67,11 @@ bool DeSerializeFromFile(T* proto, const std::string& path) {
 static void TryMkdir(const std::string& dir) {
   struct stat st = {0};
   if (stat(dir.c_str(), &st) == -1) {
+#ifdef _WIN32
+    std::experimental::filesystem::create_directory(dir);
+#else
     ::mkdir(dir.c_str(), 0700);
+#endif
   }
 }
 

visualdl/utils/image.h

@@ -16,6 +16,7 @@ limitations under the License. */
 #define VISUALDL_UTILS_IMAGE_H
 
 #include <Eigen/Core>
+#include <algorithm>
 #include <unsupported/Eigen/CXX11/Tensor>
 #include "visualdl/utils/logging.h"