提交 19e24f76 编写于 作者: O Oraoto 提交者: Jeff Wang

Add preliminary support for Windows (#461)

* Add preliminary support for Windows

    Known issues:

    1. No zlib support for protobuf
    2. Manual editting to core.vcxproj are required
    3. No build and packge script support
    4. Not tesetd, may break linux build

* Add build script and batch file for Windows

* Fix linux build and include batch file in wheel

* Fix Python 2.7 build

* Define NOMINMAX instead of preventing macro expansion
上级 f2b23b8e
......@@ -2,6 +2,7 @@
__pycache__/
*.py[cod]
*$py.class
*.dll
# C extensions
*.so
......
......@@ -53,6 +53,11 @@ if (NOT ON_RELEASE)
add_definitions(-DVISUALDL_WITH_GLOG)
endif(NOT ON_RELEASE)
if (MSVC)
add_definitions(-DPROTOBUF_USE_DLLS)
add_definitions(-DNOMINMAX) # No min/max macros
endif(MSVC)
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${CMAKE_CURRENT_BINARY_DIR})
......
# Build script for Windows
# Requires: PowerShell, Python 3, Visual Studio 15 2017
$TOP_DIR=$pwd.PATH
$FRONTEND_DIR="$TOP_DIR/frontend"
$BACKEND_DIR="$TOP_DIR/visualdl"
$BUILD_DIR="$TOP_DIR/build"
mkdir $BUILD_DIR -ErrorAction Ignore
function check_duplicated($filename_format) {
$files = ls dist/$filename_format -ErrorAction Ignore
if (Get-TypeName($files.Length) -ne "FileInfo") {
Write-Error "dist have duplicate file for $filename_format, please clean and rerun"
exit(1)
}
}
function build_frontend() {
cd $FRONTEND_DIR
npm install
npm run build
foreach ($file_name in "manifest.*.js","index.*.js","vendor.*.js") {
echo $file_name
check_duplicated $file_name
}
}
function build_frontend_fake() {
cd $FRONTEND_DIR
mkdir -p dist
}
function build_backend() {
cd $BUILD_DIR
cmake -G "Visual Studio 15 2017 Win64" `
-DCMAKE_BUILD_TYPE=Release `
-DON_RELEASE=ON `
-DWITH_PYTHON3=ON `
-DWITH_TESTING=OFF `
-DBUILD_FOR_HOST:BOOL=YES ..
cmake --build . --config Release
}
function build_onnx_graph() {
$env:PATH = "$BUILD_DIR/third_party/protobuf/src/extern_protobuf-build/Release;" + $env:PATH
cd $TOP_DIR/visualdl/server/onnx
protoc onnx.proto --python_out .
}
function clean_env() {
rm -Recurse -Force -ErrorAction Ignore $TOP_DIR/visualdl/server/dist
rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/bdist*
rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/lib*
rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/temp*
rm -Recurse -Force -ErrorAction Ignore $BUILD_DIR/scripts*
}
function package() {
cp -Recurse $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/
cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl
cp $BUILD_DIR/visualdl/logic/Release/core.pyd $TOP_DIR/visualdl/python/
}
build_frontend
clean_env
build_backend
build_onnx_graph
package
......@@ -178,14 +178,39 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
SET(OPTIONAL_CACHE_ARGS "-DZLIB_ROOT:STRING=${ZLIB_ROOT}")
ENDIF()
IF(MSVC)
ExternalProject_Add(
${TARGET_NAME}
${EXTERNAL_PROJECT_LOG_ARGS}
PREFIX ${PROTOBUF_SOURCES_DIR}
UPDATE_COMMAND ""
URL https://github.com/google/protobuf/archive/v3.5.2.zip
CONFIGURE_COMMAND
${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
${OPTIONAL_ARGS}
-G ${CMAKE_GENERATOR}
-Dprotobuf_BUILD_SHARED_LIBS=ON
-DCMAKE_CONFIGURATION_TYPES=${CMAKE_BUILD_TYPE}
-Dprotobuf_BUILD_TESTS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
CMAKE_CACHE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
${OPTIONAL_CACHE_ARGS}
)
ELSE()
ExternalProject_Add(
${TARGET_NAME}
${EXTERNAL_PROJECT_LOG_ARGS}
PREFIX ${PROTOBUF_SOURCES_DIR}
UPDATE_COMMAND ""
DEPENDS zlib
GIT_REPOSITORY "https://github.com/google/protobuf.git"
GIT_TAG "2761122b810fe8861004ae785cc3ab39f384d342"
URL https://github.com/google/protobuf/archive/v3.5.2.zip
CONFIGURE_COMMAND
${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
${OPTIONAL_ARGS}
......@@ -201,6 +226,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
${OPTIONAL_CACHE_ARGS}
)
ENDIF(MSVC)
ENDFUNCTION()
SET(PROTOBUF_VERSION 3.5)
......
......@@ -23,7 +23,11 @@ ELSE()
ENDIF(WITH_PYTHON3)
# Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE.
ADD_LIBRARY(python SHARED IMPORTED GLOBAL)
IF(MSVC)
ADD_LIBRARY(python STATIC IMPORTED GLOBAL)
ELSE()
ADD_LIBRARY(python SHARED IMPORTED GLOBAL)
ENDIF(MSVC)
SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES})
SET(py_env "")
......
......@@ -17,6 +17,7 @@ from __future__ import absolute_import
import os
import sys
from sys import platform
from distutils.spawn import find_executable
from distutils import log
import setuptools.command.build_py
......@@ -44,6 +45,8 @@ LICENSE = readlines('LICENSE')[0].strip()
# use memcache to reduce disk read frequency.
install_requires = ['Flask', 'numpy', 'Pillow', 'protobuf', 'scipy']
execute_requires = ['npm', 'node', 'bash', 'cmake', 'unzip']
if platform == "win32":
execute_requires = ['node', 'powershell', 'cmake']
def die(msg):
......@@ -73,6 +76,8 @@ class BaseCommand(setuptools.Command):
class build_py(setuptools.command.build_py.build_py):
def run(self):
cmd = ['bash', 'build.sh']
if platform == "win32":
cmd = ['powershell', '-NoProfile', './build.ps1']
env = dict(os.environ)
if MODE == "travis-CI":
cmd.append('travis-CI')
......@@ -94,6 +99,14 @@ packages = [
'visualdl.server.onnx',
]
libraries = ['core.so']
if platform == 'win32':
libraries = ['core.pyd', 'libprotobuf.dll']
scripts = ['visualdl/server/visualDL', 'demo/vdl_create_scratch_log']
if platform == 'win32':
scripts.append('visualdl/server/visualDL.bat')
setup(
name="visualdl",
version=VERSION_NUMBER,
......@@ -106,10 +119,12 @@ setup(
package_data={
'visualdl.server':
['dist/*.js', 'dist/*.html', 'dist/fonts/*', 'dist/assets/*'],
'visualdl': ['core.so'],
'visualdl.python': ['core.so', 'dog.jpg', 'testing.wav']
'visualdl':
libraries,
'visualdl.python':
libraries + ['dog.jpg', 'testing.wav']
},
packages=packages,
ext_modules=[Extension('_foo', ['stub.cc'])],
scripts=['visualdl/server/visualDL', 'demo/vdl_create_scratch_log'],
scripts=scripts,
cmdclass=cmdclass)
#include <Python.h>
PyMODINIT_FUNC PyInit__foo() {
#if PY_MAJOR_VERSION >= 3
return NULL;
#else
return;
#endif
}
......@@ -20,7 +20,7 @@ add_dependencies(sdk entry binary_record storage storage_proto eigen3)
## pybind
set(OPTIONAL_LINK_FLAGS)
if(NOT APPLE)
if((NOT APPLE) AND (NOT WIN32))
set(OPTIONAL_LINK_FLAGS "rt")
endif()
......@@ -33,9 +33,26 @@ else()
add_dependencies(core pybind python im entry tablet storage sdk protobuf eigen3)
target_link_libraries(core PRIVATE pybind entry binary_record python im tablet storage sdk protobuf ${OPTIONAL_LINK_FLAGS})
endif()
set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")
add_custom_command(TARGET core POST_BUILD
if (MSVC)
set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".pyd")
add_custom_command(TARGET core POST_BUILD
COMMAND "${CMAKE_COMMAND}" -E copy
"${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd"
"${VISUALDL_SOURCE_DIR}/visualdl/python/"
COMMAND "${CMAKE_COMMAND}" -E copy
"${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/core.pyd"
"${VISUALDL_SOURCE_DIR}/visualdl/"
COMMAND "${CMAKE_COMMAND}" -E copy
"${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll"
"${VISUALDL_SOURCE_DIR}/visualdl/python/"
COMMAND "${CMAKE_COMMAND}" -E copy
"${THIRD_PARTY_PATH}/install/protobuf/bin/libprotobuf.dll"
"${VISUALDL_SOURCE_DIR}/visualdl/"
COMMENT "Copying core.pyd and libprotobuf.dll to ${VISUALDL_SOURCE_DIR}/visualdl/python/")
else()
set_target_properties(core PROPERTIES PREFIX "" SUFFIX ".so")
add_custom_command(TARGET core POST_BUILD
COMMAND "${CMAKE_COMMAND}" -E copy
"${CMAKE_CURRENT_BINARY_DIR}/core.so"
"${VISUALDL_SOURCE_DIR}/visualdl/python/"
......@@ -43,3 +60,4 @@ add_custom_command(TARGET core POST_BUILD
"${CMAKE_CURRENT_BINARY_DIR}/core.so"
"${VISUALDL_SOURCE_DIR}/visualdl/"
COMMENT "Copying core.so to ${VISUALDL_SOURCE_DIR}/visualdl/python/")
endif()
......@@ -15,6 +15,7 @@ limitations under the License. */
#include "visualdl/logic/sdk.h"
#include <cstdio>
#include <numeric>
#include "visualdl/logic/histogram.h"
#include "visualdl/storage/binary_record.h"
......
@ECHO OFF
setlocal DISABLEDELAYEDEXPANSION
python %0\..\visualDL %*
# A modified version of protobuf_generate_cpp, that does not use protobuf::protoc,
# which are not supported on Windows.
function(visualdl_protobuf_generate_cpp SRCS HDRS)
cmake_parse_arguments(protobuf "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN})
set(PROTO_FILES "${protobuf_UNPARSED_ARGUMENTS}")
if(NOT PROTO_FILES)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
return()
endif()
if(protobuf_EXPORT_MACRO)
set(DLL_EXPORT_DECL "dllexport_decl=${protobuf_EXPORT_MACRO}:")
endif()
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
# Create an include path for each file specified
foreach(FIL ${PROTO_FILES})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
endif()
if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()
set(${SRCS})
set(${HDRS})
if (protobuf_DESCRIPTORS)
set(${protobuf_DESCRIPTORS})
endif()
foreach(FIL ${PROTO_FILES})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
if(FIL_DIR)
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
endif()
endif()
set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
list(APPEND ${SRCS} "${_protobuf_protoc_src}")
list(APPEND ${HDRS} "${_protobuf_protoc_hdr}")
if(protobuf_DESCRIPTORS)
set(_protobuf_protoc_desc "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.desc")
set(_protobuf_protoc_flags "--descriptor_set_out=${_protobuf_protoc_desc}")
list(APPEND ${protobuf_DESCRIPTORS} "${_protobuf_protoc_desc}")
else()
set(_protobuf_protoc_desc "")
set(_protobuf_protoc_flags "")
endif()
add_custom_command(
OUTPUT "${_protobuf_protoc_src}"
"${_protobuf_protoc_hdr}"
${_protobuf_protoc_desc}
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
"--cpp_out=${DLL_EXPORT_DECL}${CMAKE_CURRENT_BINARY_DIR}"
${_protobuf_protoc_flags}
${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL} protobuf
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()
set(${SRCS} "${${SRCS}}" PARENT_SCOPE)
set(${HDRS} "${${HDRS}}" PARENT_SCOPE)
if(protobuf_DESCRIPTORS)
set(${protobuf_DESCRIPTORS} "${${protobuf_DESCRIPTORS}}" PARENT_SCOPE)
endif()
endfunction()
## add storage_proto as target
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto)
visualdl_protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS storage.proto)
add_library(storage_proto ${PROTO_SRCS})
add_dependencies(storage_proto protobuf)
......
......@@ -43,7 +43,11 @@ struct BinaryRecord {
void tofile() {
fs::TryRecurMkdir(dir_);
#ifdef _WIN32
std::fstream file(path_, std::ios_base::binary | std::ios_base::out);
#else
std::fstream file(path_, file.binary | file.out);
#endif
CHECK(file.is_open()) << "open " << path_ << " failed";
size_t size = data_.size();
......@@ -90,7 +94,11 @@ protected:
// which impacts performance.
if (!filename_.empty()) {
std::string path = dir_ + "/" + filename_;
#ifdef _WIN32
std::ifstream file(path, std::ios_base::binary);
#else
std::ifstream file(path, file.binary);
#endif
CHECK(file.is_open()) << " failed to open file " << path;
size_t size;
......
......@@ -18,11 +18,16 @@ limitations under the License. */
#include <google/protobuf/text_format.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fstream>
#include "visualdl/utils/logging.h"
#ifdef _WIN32
#include <filesystem>
#else
#include <unistd.h>
#endif
namespace visualdl {
namespace fs {
......@@ -62,7 +67,11 @@ bool DeSerializeFromFile(T* proto, const std::string& path) {
static void TryMkdir(const std::string& dir) {
struct stat st = {0};
if (stat(dir.c_str(), &st) == -1) {
#ifdef _WIN32
std::experimental::filesystem::create_directory(dir);
#else
::mkdir(dir.c_str(), 0700);
#endif
}
}
......
......@@ -16,6 +16,7 @@ limitations under the License. */
#define VISUALDL_UTILS_IMAGE_H
#include <Eigen/Core>
#include <algorithm>
#include <unsupported/Eigen/CXX11/Tensor>
#include "visualdl/utils/logging.h"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册