From 19e24f76d96a095cda52ac14a617a2d402348a11 Mon Sep 17 00:00:00 2001 From: Oraoto Date: Thu, 14 Jun 2018 02:53:10 +0800 Subject: [PATCH] Add preliminary support for Windows (#461) * Add preliminary support for Windows Known issues: 1. No zlib support for protobuf 2. Manual editting to core.vcxproj are required 3. No build and packge script support 4. Not tesetd, may break linux build * Add build script and batch file for Windows * Fix linux build and include batch file in wheel * Fix Python 2.7 build * Define NOMINMAX instead of preventing macro expansion --- .gitignore | 1 + CMakeLists.txt | 5 ++ build.ps1 | 69 +++++++++++++++++++++++ cmake/external/protobuf.cmake | 72 ++++++++++++++++-------- cmake/external/python.cmake | 6 +- setup.py | 21 ++++++- stub.cc | 9 +++ visualdl/logic/CMakeLists.txt | 38 +++++++++---- visualdl/logic/sdk.cc | 1 + visualdl/server/visualDL.bat | 3 + visualdl/storage/CMakeLists.txt | 94 +++++++++++++++++++++++++++++++- visualdl/storage/binary_record.h | 8 +++ visualdl/utils/filesystem.h | 11 +++- visualdl/utils/image.h | 1 + 14 files changed, 300 insertions(+), 39 deletions(-) create mode 100644 build.ps1 create mode 100644 visualdl/server/visualDL.bat diff --git a/.gitignore b/.gitignore index d5c1dc27..9d3a0c2a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +*.dll # C extensions *.so diff --git a/CMakeLists.txt b/CMakeLists.txt index 45751386..9a0a6c6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,11 @@ if (NOT ON_RELEASE) add_definitions(-DVISUALDL_WITH_GLOG) endif(NOT ON_RELEASE) +if (MSVC) + add_definitions(-DPROTOBUF_USE_DLLS) + add_definitions(-DNOMINMAX) # No min/max macros +endif(MSVC) + include_directories(${PROJECT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 00000000..7c5b4d5e --- /dev/null +++ b/build.ps1 @@ -0,0 +1,69 @@ +# Build script for Windows +# Requires: PowerShell, Python 3, Visual Studio 15 2017 + +$TOP_DIR=$pwd.PATH +$FRONTEND_DIR="$TOP_DIR/frontend" +$BACKEND_DIR="$TOP_DIR/visualdl" +$BUILD_DIR="$TOP_DIR/build" + +mkdir $BUILD_DIR -ErrorAction Ignore + +function check_duplicated($filename_format) { + $files = ls dist/$filename_format -ErrorAction Ignore + if (Get-TypeName($files.Length) -ne "FileInfo") { + Write-Error "dist have duplicate file for $filename_format, please clean and rerun" + exit(1) + } +} + +function build_frontend() { + cd $FRONTEND_DIR + npm install + npm run build + foreach ($file_name in "manifest.*.js","index.*.js","vendor.*.js") { + echo $file_name + check_duplicated $file_name + } +} + +function build_frontend_fake() { + cd $FRONTEND_DIR + mkdir -p dist +} + +function build_backend() { + cd $BUILD_DIR + cmake -G "Visual Studio 15 2017 Win64" ` + -DCMAKE_BUILD_TYPE=Release ` + -DON_RELEASE=ON ` + -DWITH_PYTHON3=ON ` + -DWITH_TESTING=OFF ` + -DBUILD_FOR_HOST:BOOL=YES .. + cmake --build . --config Release +} + +function build_onnx_graph() { + $env:PATH = "$BUILD_DIR/third_party/protobuf/src/extern_protobuf-build/Release;" + $env:PATH + cd $TOP_DIR/visualdl/server/onnx + protoc onnx.proto --python_out . +} + +function clean_env() { + rm -Recurse -Force -ErrorAction Ignore $TOP_DIR/visualdl/server/dist + rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/bdist* + rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/lib* + rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/temp* + rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/scripts* +} + +function package() { + cp -Recurse $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/ + cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl + cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl/python/ +} + +build_frontend +clean_env +build_backend +build_onnx_graph +package diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 48e96581..76d16c25 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -178,29 +178,55 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) SET(OPTIONAL_CACHE_ARGS "-DZLIB_ROOT:STRING=${ZLIB_ROOT}") ENDIF() - ExternalProject_Add( - ${TARGET_NAME} - ${EXTERNAL_PROJECT_LOG_ARGS} - PREFIX ${PROTOBUF_SOURCES_DIR} - UPDATE_COMMAND "" - DEPENDS zlib - GIT_REPOSITORY "https://github.com/google/protobuf.git" - GIT_TAG "2761122b810fe8861004ae785cc3ab39f384d342" - CONFIGURE_COMMAND - ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake - ${OPTIONAL_ARGS} - -Dprotobuf_BUILD_TESTS=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR=lib - CMAKE_CACHE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR} - -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} - -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON - ${OPTIONAL_CACHE_ARGS} - ) + IF(MSVC) + ExternalProject_Add( + ${TARGET_NAME} + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${PROTOBUF_SOURCES_DIR} + UPDATE_COMMAND "" + URL https://github.com/google/protobuf/archive/v3.5.2.zip + CONFIGURE_COMMAND + ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake + ${OPTIONAL_ARGS} + -G ${CMAKE_GENERATOR} + -Dprotobuf_BUILD_SHARED_LIBS=ON + -DCMAKE_CONFIGURATION_TYPES=${CMAKE_BUILD_TYPE} + -Dprotobuf_BUILD_TESTS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=lib + CMAKE_CACHE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR} + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + ${OPTIONAL_CACHE_ARGS} + ) + ELSE() + ExternalProject_Add( + ${TARGET_NAME} + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${PROTOBUF_SOURCES_DIR} + UPDATE_COMMAND "" + DEPENDS zlib + URL https://github.com/google/protobuf/archive/v3.5.2.zip + CONFIGURE_COMMAND + ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake + ${OPTIONAL_ARGS} + -Dprotobuf_BUILD_TESTS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=lib + CMAKE_CACHE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR} + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + ${OPTIONAL_CACHE_ARGS} + ) + ENDIF(MSVC) ENDFUNCTION() SET(PROTOBUF_VERSION 3.5) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index c1a1bb0b..84a56afb 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -23,7 +23,11 @@ ELSE() ENDIF(WITH_PYTHON3) # Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE. -ADD_LIBRARY(python SHARED IMPORTED GLOBAL) +IF(MSVC) + ADD_LIBRARY(python STATIC IMPORTED GLOBAL) +ELSE() + ADD_LIBRARY(python SHARED IMPORTED GLOBAL) +ENDIF(MSVC) SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES}) SET(py_env "") diff --git a/setup.py b/setup.py index 8c2d95c6..72c2d919 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import os import sys +from sys import platform from distutils.spawn import find_executable from distutils import log import setuptools.command.build_py @@ -44,6 +45,8 @@ LICENSE = readlines('LICENSE')[0].strip() # use memcache to reduce disk read frequency. install_requires = ['Flask', 'numpy', 'Pillow', 'protobuf', 'scipy'] execute_requires = ['npm', 'node', 'bash', 'cmake', 'unzip'] +if platform == "win32": + execute_requires = ['node', 'powershell', 'cmake'] def die(msg): @@ -73,6 +76,8 @@ class BaseCommand(setuptools.Command): class build_py(setuptools.command.build_py.build_py): def run(self): cmd = ['bash', 'build.sh'] + if platform == "win32": + cmd = ['powershell', '-NoProfile', './build.ps1'] env = dict(os.environ) if MODE == "travis-CI": cmd.append('travis-CI') @@ -94,6 +99,14 @@ packages = [ 'visualdl.server.onnx', ] +libraries = ['core.so'] +if platform == 'win32': + libraries = ['core.pyd', 'libprotobuf.dll'] + +scripts = ['visualdl/server/visualDL', 'demo/vdl_create_scratch_log'] +if platform == 'win32': + scripts.append('visualdl/server/visualDL.bat') + setup( name="visualdl", version=VERSION_NUMBER, @@ -106,10 +119,12 @@ setup( package_data={ 'visualdl.server': ['dist/*.js', 'dist/*.html', 'dist/fonts/*', 'dist/assets/*'], - 'visualdl': ['core.so'], - 'visualdl.python': ['core.so', 'dog.jpg', 'testing.wav'] + 'visualdl': + libraries, + 'visualdl.python': + libraries + ['dog.jpg', 'testing.wav'] }, packages=packages, ext_modules=[Extension('_foo', ['stub.cc'])], - scripts=['visualdl/server/visualDL', 'demo/vdl_create_scratch_log'], + scripts=scripts, cmdclass=cmdclass) diff --git a/stub.cc b/stub.cc index e69de29b..563dc8d1 100644 --- a/stub.cc +++ b/stub.cc @@ -0,0 +1,9 @@ +#include + +PyMODINIT_FUNC PyInit__foo() { +#if PY_MAJOR_VERSION >= 3 + return NULL; +#else + return; +#endif +} diff --git a/visualdl/logic/CMakeLists.txt b/visualdl/logic/CMakeLists.txt index afb35315..a421d834 100644 --- a/visualdl/logic/CMakeLists.txt +++ b/visualdl/logic/CMakeLists.txt @@ -20,7 +20,7 @@ add_dependencies(sdk entry binary_record storage storage_proto eigen3) ## pybind set(OPTIONAL_LINK_FLAGS) -if(NOT APPLE) +if((NOT APPLE) AND (NOT WIN32)) set(OPTIONAL_LINK_FLAGS "rt") endif() @@ -33,13 +33,31 @@ else() add_dependencies(core pybind python im entry tablet storage sdk protobuf eigen3) target_link_libraries(core PRIVATE pybind entry binary_record python im tablet storage sdk protobuf ${OPTIONAL_LINK_FLAGS}) endif() -set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so") -add_custom_command(TARGET core POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E copy - "${CMAKE_CURRENT_BINARY_DIR}/core.so" - "${VISUALDL_SOURCE_DIR}/visualdl/python/" - COMMAND "${CMAKE_COMMAND}" -E copy - "${CMAKE_CURRENT_BINARY_DIR}/core.so" - "${VISUALDL_SOURCE_DIR}/visualdl/" - COMMENT "Copying core.so to ${VISUALDL_SOURCE_DIR}/visualdl/python/") +if (MSVC) + set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".pyd") + add_custom_command(TARGET core POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd" + "${VISUALDL_SOURCE_DIR}/visualdl/python/" + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd" + "${VISUALDL_SOURCE_DIR}/visualdl/" + COMMAND "${CMAKE_COMMAND}" -E copy + "${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll" + "${VISUALDL_SOURCE_DIR}/visualdl/python/" + COMMAND "${CMAKE_COMMAND}" -E copy + "${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll" + "${VISUALDL_SOURCE_DIR}/visualdl/" + COMMENT "Copying core.pyd and libprotobuf.dll to ${VISUALDL_SOURCE_DIR}/visualdl/python/") +else() + set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so") + add_custom_command(TARGET core POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_BINARY_DIR}/core.so" + "${VISUALDL_SOURCE_DIR}/visualdl/python/" + COMMAND "${CMAKE_COMMAND}" -E copy + "${CMAKE_CURRENT_BINARY_DIR}/core.so" + "${VISUALDL_SOURCE_DIR}/visualdl/" + COMMENT "Copying core.so to ${VISUALDL_SOURCE_DIR}/visualdl/python/") +endif() diff --git a/visualdl/logic/sdk.cc b/visualdl/logic/sdk.cc index 7058fad4..a5ec3bc0 100644 --- a/visualdl/logic/sdk.cc +++ b/visualdl/logic/sdk.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include "visualdl/logic/sdk.h" #include +#include #include "visualdl/logic/histogram.h" #include "visualdl/storage/binary_record.h" diff --git a/visualdl/server/visualDL.bat b/visualdl/server/visualDL.bat new file mode 100644 index 00000000..82a32755 --- /dev/null +++ b/visualdl/server/visualDL.bat @@ -0,0 +1,3 @@ +@ECHO OFF +setlocal DISABLEDELAYEDEXPANSION +python %0\..\visualDL %* diff --git a/visualdl/storage/CMakeLists.txt b/visualdl/storage/CMakeLists.txt index ba74609f..9aa7ed49 100644 --- a/visualdl/storage/CMakeLists.txt +++ b/visualdl/storage/CMakeLists.txt @@ -1,5 +1,97 @@ +# A modified version of protobuf_generate_cpp, that does not use protobuf::protoc, +# which are not supported on Windows. +function(visualdl_protobuf_generate_cpp SRCS HDRS) + cmake_parse_arguments(protobuf "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN}) + set(PROTO_FILES "${protobuf_UNPARSED_ARGUMENTS}") + if(NOT PROTO_FILES) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") + return() + endif() + + if(protobuf_EXPORT_MACRO) + set(DLL_EXPORT_DECL "dllexport_decl=${protobuf_EXPORT_MACRO}:") + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${PROTO_FILES}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + if (protobuf_DESCRIPTORS) + set(${protobuf_DESCRIPTORS}) + endif() + + foreach(FIL ${PROTO_FILES}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH) + get_filename_component(FIL_DIR ${FIL} DIRECTORY) + if(FIL_DIR) + set(FIL_WE "${FIL_DIR}/${FIL_WE}") + endif() + endif() + + set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + list(APPEND ${SRCS} "${_protobuf_protoc_src}") + list(APPEND ${HDRS} "${_protobuf_protoc_hdr}") + + if(protobuf_DESCRIPTORS) + set(_protobuf_protoc_desc "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.desc") + set(_protobuf_protoc_flags "--descriptor_set_out=${_protobuf_protoc_desc}") + list(APPEND ${protobuf_DESCRIPTORS} "${_protobuf_protoc_desc}") + else() + set(_protobuf_protoc_desc "") + set(_protobuf_protoc_flags "") + endif() + + add_custom_command( + OUTPUT "${_protobuf_protoc_src}" + "${_protobuf_protoc_hdr}" + ${_protobuf_protoc_desc} + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + "--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}" + ${_protobuf_protoc_flags} + ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} protobuf + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set(${SRCS} "${${SRCS}}" PARENT_SCOPE) + set(${HDRS} "${${HDRS}}" PARENT_SCOPE) + if(protobuf_DESCRIPTORS) + set(${protobuf_DESCRIPTORS} "${${protobuf_DESCRIPTORS}}" PARENT_SCOPE) + endif() +endfunction() + ## add storage_proto as target -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto) +visualdl_protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto) add_library(storage_proto ${PROTO_SRCS}) add_dependencies(storage_proto protobuf) diff --git a/visualdl/storage/binary_record.h b/visualdl/storage/binary_record.h index 8b8c42c6..3a01d076 100644 --- a/visualdl/storage/binary_record.h +++ b/visualdl/storage/binary_record.h @@ -43,7 +43,11 @@ struct BinaryRecord { void tofile() { fs::TryRecurMkdir(dir_); +#ifdef _WIN32 + std::fstream file(path_, std::ios_base::binary | std::ios_base::out); +#else std::fstream file(path_, file.binary | file.out); +#endif CHECK(file.is_open()) << "open " << path_ << " failed"; size_t size = data_.size(); @@ -90,7 +94,11 @@ protected: // which impacts performance. if (!filename_.empty()) { std::string path = dir_ + "/" + filename_; +#ifdef _WIN32 + std::ifstream file(path, std::ios_base::binary); +#else std::ifstream file(path, file.binary); +#endif CHECK(file.is_open()) << " failed to open file " << path; size_t size; diff --git a/visualdl/utils/filesystem.h b/visualdl/utils/filesystem.h index 46e9e13c..9ba9b94c 100644 --- a/visualdl/utils/filesystem.h +++ b/visualdl/utils/filesystem.h @@ -18,11 +18,16 @@ limitations under the License. */ #include #include #include -#include #include #include "visualdl/utils/logging.h" +#ifdef _WIN32 +#include +#else +#include +#endif + namespace visualdl { namespace fs { @@ -62,7 +67,11 @@ bool DeSerializeFromFile(T* proto, const std::string& path) { static void TryMkdir(const std::string& dir) { struct stat st = {0}; if (stat(dir.c_str(), &st) == -1) { +#ifdef _WIN32 + std::experimental::filesystem::create_directory(dir); +#else ::mkdir(dir.c_str(), 0700); +#endif } } diff --git a/visualdl/utils/image.h b/visualdl/utils/image.h index 307b5358..3c427dbf 100644 --- a/visualdl/utils/image.h +++ b/visualdl/utils/image.h @@ -16,6 +16,7 @@ limitations under the License. */ #define VISUALDL_UTILS_IMAGE_H #include +#include #include #include "visualdl/utils/logging.h" -- GitLab