From 34f8c9b6f595c0cfedb410a60fe82519b624b431 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Fri, 24 Aug 2018 12:34:24 +0800 Subject: [PATCH] windows port --- CMakeLists.txt | 16 ++++++-- cmake/configure.cmake | 5 +++ cmake/external/boost.cmake | 12 +++--- cmake/external/gflags.cmake | 11 +++++- cmake/external/glog.cmake | 7 ++++ cmake/external/openblas.cmake | 19 ++++++++-- cmake/external/protobuf.cmake | 23 ++++++++--- cmake/generic.cmake | 38 ++++++++++++++++++- cmake/inference_lib.cmake | 9 +++++ paddle/fluid/CMakeLists.txt | 6 ++- paddle/fluid/framework/CMakeLists.txt | 19 +++++++++- paddle/fluid/framework/data_type.h | 35 +++++++++++++++++ paddle/fluid/framework/eigen.h | 2 + paddle/fluid/framework/lod_tensor.cc | 17 ++++++++- paddle/fluid/framework/lod_tensor_test.cc | 2 + paddle/fluid/framework/rw_lock.h | 12 ++++++ .../inference/api/demo_ci/CMakeLists.txt | 2 + paddle/fluid/operators/CMakeLists.txt | 5 ++- paddle/fluid/operators/math/math_function.h | 1 + paddle/fluid/platform/CMakeLists.txt | 5 +++ paddle/fluid/platform/cpu_info.cc | 10 +++-- paddle/fluid/platform/device_tracer.h | 14 ++++++- paddle/fluid/platform/dynload/CMakeLists.txt | 2 + .../fluid/platform/dynload/dynamic_loader.cc | 3 +- paddle/fluid/platform/enforce.h | 24 ++++++++++-- paddle/fluid/platform/float16.h | 4 ++ paddle/fluid/platform/macros.h | 2 + paddle/fluid/platform/port.h | 1 + paddle/fluid/platform/profiler.h | 11 ++++++ paddle/fluid/pybind/CMakeLists.txt | 11 +++--- paddle/fluid/recordio/CMakeLists.txt | 1 + 31 files changed, 287 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba9af2cc7b7..13d545f0b3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,9 @@ message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}, version: " "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: " "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}") +if(WIN32) + set(CMAKE_STATIC_LIBRARY_PREFIX lib) +endif(WIN32) if(NOT CMAKE_CROSSCOMPILING) find_package(CUDA QUIET) @@ -171,7 +174,6 @@ include(external/python) # download, build, install python include(external/openblas) # download, build, install openblas include(external/mkldnn) # download, build, install mkldnn include(external/swig) # download, build, install swig -include(external/warpctc) # download, build, install warpctc include(external/boost) # download boost include(external/any) # download libn::any include(external/eigen) # download eigen3 @@ -179,6 +181,14 @@ include(external/pybind11) # download pybind11 include(external/cares) include(external/cub) +if (NOT WIN32) +# there is no official support of snappystream, warpctc, nccl, cupti in windows +include(external/snappy) # download snappy +include(external/snappystream) # download snappystream +include(external/warpctc) # download, build, install warpctc +include(cupti) +endif (NOT WIN32) + if(WITH_DISTRIBUTE) if(WITH_GRPC) include(external/grpc) @@ -200,8 +210,7 @@ if(WITH_BRPC_RDMA) endif() endif() -include(external/snappy) # download snappy -include(external/snappystream) + include(external/threadpool) if(WITH_GPU) @@ -214,7 +223,6 @@ endif() include(flags) # set paddle compile flags include(cudnn) # set cudnn libraries, must before configure -include(cupti) include(configure) # add paddle env configuration include(generic) # simplify cmake module include(package) # set paddle packages diff --git a/cmake/configure.cmake b/cmake/configure.cmake index ae90a529b1a..be99d9e4538 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -57,6 +57,11 @@ if(NOT CMAKE_CROSSCOMPILING) endif() endif() +if(WIN32) + # windows stupid compile option for all targets. + add_definitions(-D_XKEYCHECK_H) +endif(WIN32) + if(NOT WITH_GOLANG) add_definitions(-DPADDLE_WITHOUT_GOLANG) endif(NOT WITH_GOLANG) diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index 9bc4133f6a5..ede4b1f61f5 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -37,16 +37,17 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1) include_directories(${BOOST_INCLUDE_DIR}) set(COMMAND "wget --no-check-certificate ${BOOST_URL} -c -q -O ${BOOST_TAR}.tar.gz && tar zxf ${BOOST_TAR}.tar.gz") -if (NOT WIN32) -set(COMMAND "") -message(WARNING "Windows do not support automaticlly download and install boost. Please manually install it in the thrid_party/install/boost.") -endif(NOT WIN32) +#if (WIN32) +#set(COMMAND "") +#message(WARNING "Windows do not support automaticlly download and install boost. Please manually install it in the thrid_party/install/boost.") +#endif(WIN32) +if (NOT WIN32) ExternalProject_Add( ${BOOST_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} DOWNLOAD_DIR ${BOOST_DOWNLOAD_DIR} - DOWNLOAD_COMMAND + DOWNLOAD_COMMAND ${COMMAND} DOWNLOAD_NO_PROGRESS 1 PREFIX ${BOOST_SOURCES_DIR} CONFIGURE_COMMAND "" @@ -54,6 +55,7 @@ ExternalProject_Add( INSTALL_COMMAND "" UPDATE_COMMAND "" ) +endif(NOT WIN32) if (${CMAKE_VERSION} VERSION_LESS "3.3.0") set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/boost_dummy.c) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index a1d2d0f4468..cf58cc39762 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -18,7 +18,7 @@ SET(GFLAGS_SOURCES_DIR ${THIRD_PARTY_PATH}/gflags) SET(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags) SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) IF(WIN32) - set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) + set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) ELSE(WIN32) set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) ENDIF(WIN32) @@ -45,7 +45,13 @@ ExternalProject_Add( -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} ) - +IF(WIN32) + IF(NOT EXISTS "${GFLAGS_INSTALL_DIR}/lib/libgflags.lib") + add_custom_command(TARGET extern_gflags POST_BUILD + COMMAND cmake -E rename ${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib ${GFLAGS_INSTALL_DIR}/lib/libgflags.lib + ) + ENDIF() +ENDIF(WIN32) ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES}) ADD_DEPENDENCIES(gflags extern_gflags) @@ -60,3 +66,4 @@ IF(WITH_C_API) INSTALL(FILES ${GFLAGS_LIBRARIES} DESTINATION third_party/gflags/lib) ENDIF() ENDIF() + diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index ac0181e69cb..25ef2970ac5 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -60,6 +60,13 @@ ExternalProject_Add( -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} ) +IF(WIN32) + IF(NOT EXISTS "${GLOG_INSTALL_DIR}/lib/libglog.lib") + add_custom_command(TARGET extern_glog POST_BUILD + COMMAND cmake -E rename ${GLOG_INSTALL_DIR}/lib/glog.lib ${GLOG_INSTALL_DIR}/lib/libglog.lib + ) + ENDIF() +ENDIF(WIN32) ADD_LIBRARY(glog STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES}) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 56024edf5be..c3fbe4dbdb2 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -17,20 +17,29 @@ IF(USE_EIGEN_FOR_BLAS) ENDIF(USE_EIGEN_FOR_BLAS) INCLUDE(cblas) +# IF(WIN32 AND NOT ${CBLAS_FOUND}) + + IF(NOT ${CBLAS_FOUND}) + INCLUDE(ExternalProject) SET(CBLAS_SOURCES_DIR ${THIRD_PARTY_PATH}/openblas) SET(CBLAS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openblas) - SET(CBLAS_INC_DIR "${CBLAS_INSTALL_DIR}/include" CACHE PATH "openblas include directory." FORCE) + SET(CBLAS_INCLUDE_DIR "${CBLAS_INSTALL_DIR}/include" CACHE PATH "openblas include directory." FORCE) SET(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE FILEPATH "openblas library." FORCE) ADD_DEFINITIONS(-DPADDLE_USE_OPENBLAS) + IF (WIN32) + SET(CBLAS_FOUND true) + MESSAGE(WARNING, "In windows, openblas only support msvc build, please build it manually and put it at " ${CBLAS_INSTALL_DIR}) + ENDIF(WIN32) + IF (NOT WIN32) SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -Wno-unused-but-set-variable -Wno-unused-variable") SET(OPENBLAS_COMMIT "v0.2.20") @@ -69,7 +78,6 @@ IF(NOT ${CBLAS_FOUND}) ENDIF() SET(COMMON_ARGS CC=${OPENBLAS_CC} NO_SHARED=1 NO_LAPACK=1 libs) - ExternalProject_Add( extern_openblas ${EXTERNAL_PROJECT_LOG_ARGS} @@ -84,9 +92,11 @@ IF(NOT ${CBLAS_FOUND}) UPDATE_COMMAND "" CONFIGURE_COMMAND "" ) + ELSE() + ENDIF(NOT WIN32) SET(CBLAS_PROVIDER openblas) IF(WITH_C_API) - INSTALL(DIRECTORY ${CBLAS_INC_DIR} DESTINATION third_party/openblas) + INSTALL(DIRECTORY ${CBLAS_INCLUDE_DIR} DESTINATION third_party/openblas) # Because libopenblas.a is a symbolic link of another library, thus need to # install the whole directory. IF(ANDROID) @@ -107,7 +117,8 @@ IF(NOT ${CBLAS_FOUND}) ENDIF(NOT ${CBLAS_FOUND}) MESSAGE(STATUS "BLAS library: ${CBLAS_LIBRARIES}") -INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) +MESSAGE(STATUS "BLAS Include: ${CBLAS_INCLUDE_DIR}") +INCLUDE_DIRECTORIES(${CBLAS_INCLUDE_DIR}) # FIXME(gangliao): generate cblas target to track all high performance # linear algebra libraries for cc_library(xxx SRCS xxx.c DEPS cblas) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 2665996432b..550b0dada8e 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -14,11 +14,14 @@ INCLUDE(ExternalProject) # Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp +IF(NOT WIN32) FIND_PACKAGE(Protobuf QUIET) +ENDIF(NOT WIN32) macro(UNSET_VAR VAR_NAME) UNSET(${VAR_NAME} CACHE) UNSET(${VAR_NAME}) endmacro() + UNSET_VAR(PROTOBUF_INCLUDE_DIR) UNSET_VAR(PROTOBUF_FOUND) UNSET_VAR(PROTOBUF_PROTOC_EXECUTABLE) @@ -94,12 +97,14 @@ macro(PROMPT_PROTOBUF_LIB) SET(protobuf_DEPS ${ARGN}) MESSAGE(STATUS "Protobuf protoc executable: ${PROTOBUF_PROTOC_EXECUTABLE}") + MESSAGE(STATUS "Protobuf-lite library: ${PROTOBUF_LITE_LIBRARY}") MESSAGE(STATUS "Protobuf library: ${PROTOBUF_LIBRARY}") + MESSAGE(STATUS "Protoc library: ${PROTOBUF_PROTOC_LIBRARY}") MESSAGE(STATUS "Protobuf version: ${PROTOBUF_VERSION}") INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) # Assuming that all the protobuf libraries are of the same type. - IF(${PROTOBUF_LIBRARY} MATCHES "${CMAKE_STATIC_LIBRARY_SUFFIX}$") + IF(${PROTOBUF_LIBRARY} MATCHES ${CMAKE_STATIC_LIBRARY_SUFFIX}) SET(protobuf_LIBTYPE STATIC) ELSEIF(${PROTOBUF_LIBRARY} MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$") SET(protobuf_LIBTYPE SHARED) @@ -137,18 +142,25 @@ macro(SET_PROTOBUF_VERSION) endmacro() set(PROTOBUF_ROOT "" CACHE PATH "Folder contains protobuf") +IF (WIN32) + SET(PROTOBUF_ROOT ${THIRD_PARTY_PATH}/install/protobuf) + MESSAGE(WARNING, "In windows, protobuf only support msvc build, please build it manually and put it at " ${PROTOBUF_ROOT}) +ENDIF(WIN32) + if (NOT "${PROTOBUF_ROOT}" STREQUAL "") + find_path(PROTOBUF_INCLUDE_DIR google/protobuf/message.h PATHS ${PROTOBUF_ROOT}/include NO_DEFAULT_PATH) - find_library(PROTOBUF_LIBRARY protobuf PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) - find_library(PROTOBUF_LITE_LIBRARY protobuf-lite PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) - find_library(PROTOBUF_PROTOC_LIBRARY protoc PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) + find_library(PROTOBUF_LIBRARY protobuf libprotobuf.lib PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) + find_library(PROTOBUF_LITE_LIBRARY protobuf-lite libprotobuf-lite.lib PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) + find_library(PROTOBUF_PROTOC_LIBRARY protoc libprotoc.lib PATHS ${PROTOBUF_ROOT}/lib NO_DEFAULT_PATH) find_program(PROTOBUF_PROTOC_EXECUTABLE protoc PATHS ${PROTOBUF_ROOT}/bin NO_DEFAULT_PATH) if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOBUF_LITE_LIBRARY AND PROTOBUF_PROTOC_LIBRARY AND PROTOBUF_PROTOC_EXECUTABLE) message(STATUS "Using custom protobuf library in ${PROTOBUF_ROOT}.") + SET(PROTOBUF_FOUND true) SET_PROTOBUF_VERSION() PROMPT_PROTOBUF_LIB() else() - message(WARNING "Cannot find protobuf library in ${PROTOBUF_ROOT}.") + message(WARNING "Cannot find protobuf library in ${PROTOBUF_ROOT}") endif() endif() @@ -239,6 +251,7 @@ IF(CMAKE_CROSSCOMPILING) CACHE FILEPATH "protobuf executable." FORCE) ENDIF() + IF(NOT PROTOBUF_FOUND) build_protobuf(extern_protobuf FALSE) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 82c958073cb..6d230942321 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -148,7 +148,8 @@ function(merge_static_libs TARGET_NAME) COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles} ) - else() # general UNIX: use "ar" to extract objects and re-add to a common lib + endif(APPLE) + if(LINUX) # general UNIX: use "ar" to extract objects and re-add to a common lib set(target_DIR ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.dir) foreach(lib ${libs}) @@ -187,7 +188,36 @@ function(merge_static_libs TARGET_NAME) COMMAND ${CMAKE_AR} crs ${target_LIBNAME} `find ${target_DIR} -name '*.o'` COMMAND ${CMAKE_RANLIB} ${target_LIBNAME} WORKING_DIRECTORY ${target_DIR}) - endif() + endif(LINUX) + if(WIN32) # windows do not support gcc/nvcc combined compiling. Use msvc lib.exe to merge libs. + # Make the generated dummy source file depended on all static input + # libs. If input lib changes,the source file is touched + # which causes the desired effect (relink). + add_custom_command(OUTPUT ${target_SRCS} + COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} + DEPENDS ${libs}) + + # Generate dummy staic lib + file(WRITE ${target_SRCS} "const char *dummy_${TARGET_NAME} = \"${target_SRCS}\";") + add_library(${TARGET_NAME} STATIC ${target_SRCS}) + target_link_libraries(${TARGET_NAME} ${libs_deps}) + + foreach(lib ${libs}) + # Get the file names of the libraries to be merged + #if(NOT $ MATCHES "lib.*\\.lib") + # message("library" ${lib}) + # set(libfiles ${libfiles} lib$) + #else() + set(libfiles ${libfiles} $) + #endif() + endforeach() + + # windows cmd return error in clean env. + # COMMAND del "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${TARGET_NAME}.lib" + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND lib /OUT:${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.lib ${libfiles} + ) + endif(WIN32) endfunction(merge_static_libs) function(cc_library TARGET_NAME) @@ -195,6 +225,10 @@ function(cc_library TARGET_NAME) set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(cc_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(WIN32) + # add libxxx.lib prefix in windows + set(${TARGET_NAME}_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE STRING "output library name for target ${TARGET_NAME}") + endif(WIN32) if(cc_library_SRCS) if(cc_library_SHARED OR cc_library_shared) # build *.so add_library(${TARGET_NAME} SHARED ${cc_library_SRCS}) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 834ab5a9e52..bc36683a9fa 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -101,6 +101,7 @@ if(WITH_MKLDNN) ) endif() +if (NOT WIN32) if(NOT MOBILE_INFERENCE AND NOT RPI) set(dst_dir "${FLUID_INSTALL_DIR}/third_party/install/snappy") copy(snappy_lib @@ -120,15 +121,23 @@ if(NOT MOBILE_INFERENCE AND NOT RPI) DSTS ${dst_dir} ${dst_dir}/lib DEPS zlib) endif() +endif(NOT WIN32) # paddle fluid module set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid") set(dst_dir "${FLUID_INSTALL_DIR}/paddle/fluid") set(module "framework") +if (NOT WIN32) copy(framework_lib DEPS framework_py_proto SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/details/*.h ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h DSTS ${dst_dir}/${module} ${dst_dir}/${module}/details ${dst_dir}/${module} ) +else() +copy(framework_lib + SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/details/*.h ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h + DSTS ${dst_dir}/${module} ${dst_dir}/${module}/details ${dst_dir}/${module} +) +endif(NOT WIN32) set(module "memory") copy(memory_lib diff --git a/paddle/fluid/CMakeLists.txt b/paddle/fluid/CMakeLists.txt index 2577e59d9cf..ee1f655e25d 100644 --- a/paddle/fluid/CMakeLists.txt +++ b/paddle/fluid/CMakeLists.txt @@ -2,9 +2,13 @@ add_subdirectory(memory) add_subdirectory(platform) add_subdirectory(framework) add_subdirectory(operators) -add_subdirectory(pybind) add_subdirectory(string) + +if (NOT WIN32) +add_subdirectory(pybind) add_subdirectory(recordio) +endif(NOT WIN32) + if(WITH_INFERENCE) # NOTE: please add subdirectory inference at last. add_subdirectory(inference) diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index c587337b7df..f11b31baebd 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -1,5 +1,7 @@ +if (NOT WIN32) add_subdirectory(details) add_subdirectory(ir) +endif (NOT WIN32) # ddim lib proto_library(framework_proto SRCS framework.proto) @@ -28,8 +30,12 @@ if(WITH_GPU) else() cc_test(mixed_vector_test SRCS mixed_vector_test.cc DEPS place memory device_context tensor) endif() - +if (NOT WIN32) cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto recordio) +else() +cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto) +endif (NOT WIN32) + cc_test(lod_tensor_test SRCS lod_tensor_test.cc DEPS lod_tensor memory) nv_test(lod_tensor_gpu_test SRCS lod_tensor_test.cu DEPS lod_tensor) @@ -69,14 +75,22 @@ cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) cc_library(shape_inference SRCS shape_inference.cc DEPS ddim attribute device_context) + +if (NOT WIN32) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope glog shape_inference data_transform lod_tensor profiler) +else() +cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope glog + shape_inference data_transform lod_tensor) +endif(NOT WIN32) + cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context) cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS shape_inference op_info operator glog) cc_library(op_registry SRCS op_registry.cc DEPS op_proto_maker op_info operator glog proto_desc) nv_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) +if (NOT WIN32) py_proto_compile(framework_py_proto SRCS framework.proto) # Generate an empty __init__.py to make framework_py_proto as a valid python module. add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) @@ -86,6 +100,7 @@ add_custom_command(TARGET framework_py_proto POST_BUILD COMMAND cp *.py ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto/ COMMENT "Copy generated python proto into directory paddle/fluid/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endif(NOT WIN32) cc_library(lod_rank_table SRCS lod_rank_table.cc DEPS lod_tensor) @@ -118,7 +133,9 @@ cc_test(cow_ptr_tests SRCS details/cow_ptr_test.cc) # cc_test(channel_test SRCS channel_test.cc) cc_test(tuple_test SRCS tuple_test.cc ) +if (NOT WIN32) cc_test(rw_lock_test SRCS rw_lock_test.cc) +endif (NOT WIN32) # disable test temporarily. # TODO https://github.com/PaddlePaddle/Paddle/issues/11971 diff --git a/paddle/fluid/framework/data_type.h b/paddle/fluid/framework/data_type.h index 491413db8c8..8002a50206a 100644 --- a/paddle/fluid/framework/data_type.h +++ b/paddle/fluid/framework/data_type.h @@ -26,6 +26,7 @@ namespace framework { extern proto::VarType::Type ToDataType(std::type_index type); extern std::type_index ToTypeIndex(proto::VarType::Type type); +#if !defined(_WIN32) template inline void VisitDataType(proto::VarType::Type type, Visitor visitor) { switch (type) { @@ -57,6 +58,40 @@ inline void VisitDataType(proto::VarType::Type type, Visitor visitor) { PADDLE_THROW("Not supported %d", type); } } +#else +// the msvc compiler do not implement two-stage name lookup correctly. +template +inline void VisitDataType(proto::VarType::Type type, Visitor visitor) { + switch (type) { + case proto::VarType::FP16: + visitor.operator()(); + break; + case proto::VarType::FP32: + visitor.operator()(); + break; + case proto::VarType::FP64: + visitor.operator()(); + break; + case proto::VarType::INT32: + visitor.operator()(); + break; + case proto::VarType::INT64: + visitor.operator()(); + break; + case proto::VarType::BOOL: + visitor.operator()(); + break; + case proto::VarType::UINT8: + visitor.operator()(); + break; + case proto::VarType::INT16: + visitor.operator()(); + break; + default: + PADDLE_THROW("Not supported %d", type); + } +} +#endif // _WIN32 extern std::string DataTypeToString(const proto::VarType::Type type); extern size_t SizeOfType(std::type_index type); diff --git a/paddle/fluid/framework/eigen.h b/paddle/fluid/framework/eigen.h index 4ea1df655df..e23472cef2f 100644 --- a/paddle/fluid/framework/eigen.h +++ b/paddle/fluid/framework/eigen.h @@ -13,6 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +// for windows compile eigen with logging +#define GLOG_NO_ABBREVIATED_SEVERITIES #include "paddle/fluid/framework/tensor.h" #include "unsupported/Eigen/CXX11/Tensor" diff --git a/paddle/fluid/framework/lod_tensor.cc b/paddle/fluid/framework/lod_tensor.cc index 919029c38f2..0155465cea6 100644 --- a/paddle/fluid/framework/lod_tensor.cc +++ b/paddle/fluid/framework/lod_tensor.cc @@ -25,8 +25,10 @@ limitations under the License. */ #include "paddle/fluid/memory/memcpy.h" #include "paddle/fluid/memory/memory.h" +#if defined(_WIN32) #include "paddle/fluid/recordio/scanner.h" #include "paddle/fluid/recordio/writer.h" +#endif // _WIN32 namespace paddle { namespace framework { @@ -300,6 +302,7 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor, TensorFromStream(is, static_cast(tensor), dev_ctx); } +#if !defined(_WIN32) void WriteToRecordIO(recordio::Writer *writer, const std::vector &tensor, const platform::DeviceContext &dev_ctx) { @@ -329,7 +332,19 @@ bool ReadFromRecordIO(recordio::Scanner *scanner, return true; } - +#else +class Writer {}; +class Scanner {}; +void WriteToRecordIO(recordio::Writer *writer, + const std::vector &tensor, + const platform::DeviceContext &dev_ctx) { +} +bool ReadFromRecordIO(recordio::Scanner *scanner, + const platform::DeviceContext &dev_ctx, + std::vector *result_ptr) { + PADDLE_ENFORCE("windows didn't supported recordio!."); + return true;} +#endif // _WIN32 std::vector LoDTensor::SplitLoDTensor( const std::vector places) const { check_memory_size(); diff --git a/paddle/fluid/framework/lod_tensor_test.cc b/paddle/fluid/framework/lod_tensor_test.cc index cd50aaa2605..bb176aa14d1 100644 --- a/paddle/fluid/framework/lod_tensor_test.cc +++ b/paddle/fluid/framework/lod_tensor_test.cc @@ -274,6 +274,7 @@ TEST(LoD, ConvertToOffsetBasedLoD) { EXPECT_EQ(offset_lod, expected); } +#if !defined(_WIN32) template static void TestRecordIO() { LoDTensor tensor; @@ -320,6 +321,7 @@ TEST(LoDTensor, RecordIO) { TestRecordIO(); TestRecordIO(); } +#endif // !defined(_WIN32) } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/rw_lock.h b/paddle/fluid/framework/rw_lock.h index 1418fb5134f..70121316525 100644 --- a/paddle/fluid/framework/rw_lock.h +++ b/paddle/fluid/framework/rw_lock.h @@ -14,13 +14,16 @@ limitations under the License. */ #pragma once +#if !defined(_WIN32) #include +#endif // !_WIN32 #include "paddle/fluid/platform/enforce.h" namespace paddle { namespace framework { +#if !defined(_WIN32) struct RWLock { RWLock() { pthread_rwlock_init(&lock_, nullptr); } @@ -43,6 +46,15 @@ struct RWLock { private: pthread_rwlock_t lock_; }; +#else +// https://stackoverflow.com/questions/7125250/making-pthread-rwlock-wrlock-recursive +// In windows, rw_lock seems like a hack. Use empty object and do nothing. +struct RWLock { + void RDLock() {} + void WRLock() {} + void UNLock() {} +}; +#endif } // namespace framework } // namespace paddle diff --git a/paddle/fluid/inference/api/demo_ci/CMakeLists.txt b/paddle/fluid/inference/api/demo_ci/CMakeLists.txt index ba73a6eaa6f..a697218377e 100644 --- a/paddle/fluid/inference/api/demo_ci/CMakeLists.txt +++ b/paddle/fluid/inference/api/demo_ci/CMakeLists.txt @@ -23,9 +23,11 @@ include_directories("${PADDLE_LIB}") include_directories("${PADDLE_LIB}/third_party/install/protobuf/include") include_directories("${PADDLE_LIB}/third_party/install/glog/include") include_directories("${PADDLE_LIB}/third_party/install/gflags/include") +if (NOT WIN32) include_directories("${PADDLE_LIB}/third_party/install/snappy/include") include_directories("${PADDLE_LIB}/third_party/install/snappystream/include") include_directories("${PADDLE_LIB}/third_party/install/zlib/include") +endif(NOT WIN32) include_directories("${PADDLE_LIB}/third_party/boost") include_directories("${PADDLE_LIB}/third_party/eigen3") diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index 1e96222de29..8a7ac17e8a8 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -85,7 +85,7 @@ function(op_library TARGET) #remove windows unsupported op if (WIN32) - foreach(windows_unsupport_op "nccl_op" "gen_nccl_id_op") + foreach(windows_unsupport_op "nccl_op" "gen_nccl_id_op" "warpctc_op") if ("${TARGET}" STREQUAL "${windows_unsupport_op}") return() endif() @@ -317,8 +317,9 @@ foreach(src ${GENERAL_OPS}) endforeach() file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(logical_and);\nUSE_NO_KERNEL_OP(read_from_array);\n") - +if (NOT WIN32) add_subdirectory(reader) +endif(NOT WIN32) foreach(src ${READER_LIBRARY}) set(OP_LIBRARY ${src} ${OP_LIBRARY}) endforeach() diff --git a/paddle/fluid/operators/math/math_function.h b/paddle/fluid/operators/math/math_function.h index 7ec78d9ef8e..03bfe595b09 100644 --- a/paddle/fluid/operators/math/math_function.h +++ b/paddle/fluid/operators/math/math_function.h @@ -17,6 +17,7 @@ limitations under the License. */ #include "paddle/fluid/platform/dynload/mklml.h" #endif + #ifdef PADDLE_USE_OPENBLAS #include #endif diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt index f08c0e8e345..39f1eeb913d 100644 --- a/paddle/fluid/platform/CMakeLists.txt +++ b/paddle/fluid/platform/CMakeLists.txt @@ -1,3 +1,4 @@ +if (NOT WIN32) proto_library(profiler_proto SRCS profiler.proto DEPS framework_proto) py_proto_compile(profiler_py_proto SRCS profiler.proto) @@ -10,6 +11,7 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD COMMAND cp *.py ${PADDLE_BINARY_DIR}/python/paddle/fluid/proto/profiler COMMENT "Copy generated python proto into directory paddle/fluid/proto/profiler." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endif(NOT WIN32) if(WITH_GPU) nv_library(enforce SRCS enforce.cc) @@ -58,9 +60,12 @@ cc_test(init_test SRCS init_test.cc DEPS device_context) nv_test(cudnn_helper_test SRCS cudnn_helper_test.cc DEPS dynload_cuda) nv_test(transform_test SRCS transform_test.cu DEPS memory place device_context) + +if (NOT WIN32) cc_library(device_tracer SRCS device_tracer.cc DEPS boost profiler_proto framework_proto ${GPU_CTX_DEPS}) cc_library(profiler SRCS profiler.cc DEPS device_context device_tracer) cc_test(profiler_test SRCS profiler_test.cc DEPS profiler) +endif(NOT WIN32) nv_test(float16_gpu_test SRCS float16_test.cu DEPS lod_tensor) cc_test(float16_test SRCS float16_test.cc DEPS lod_tensor) diff --git a/paddle/fluid/platform/cpu_info.cc b/paddle/fluid/platform/cpu_info.cc index dc09f2657c8..cb02d6c4663 100644 --- a/paddle/fluid/platform/cpu_info.cc +++ b/paddle/fluid/platform/cpu_info.cc @@ -23,6 +23,7 @@ limitations under the License. */ #include #include #elif defined(_WIN32) +#define NOMINMAX #include #include #else @@ -35,16 +36,19 @@ limitations under the License. */ DEFINE_double(fraction_of_cpu_memory_to_use, 1, "Default use 100% of CPU memory for PaddlePaddle," "reserve the rest for page tables, etc"); - +#if !defined(_WIN32) DEFINE_uint64(initial_cpu_memory_in_mb, #ifdef PADDLE_WITH_MKLDNN /* Aligned with mozga-intel, MKLDNN need at least 5000 MB * to obtain the best performance*/ - 5000, + 5000ul, #else - 500, + 500ul, #endif "Initial CPU memory for PaddlePaddle, in MD unit."); +#else +DEFINE_uint64(initial_cpu_memory_in_mb, 500ul, "Initial CPU memory for PaddlePaddle, in MD unit."); +#endif // !defined(_WIN32) DEFINE_double( fraction_of_cuda_pinned_memory_to_use, 0.5, diff --git a/paddle/fluid/platform/device_tracer.h b/paddle/fluid/platform/device_tracer.h index 322996fb4f5..2aed4c6e83c 100644 --- a/paddle/fluid/platform/device_tracer.h +++ b/paddle/fluid/platform/device_tracer.h @@ -13,7 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#if !defined(_WIN32) #include +#else +#include +#endif // !_WIN32 + #include #include // NOLINT #include @@ -27,12 +32,17 @@ namespace platform { /////////////////////// // WARN: Under Development. Don't depend on it yet. ////////////////////// - +#if !defined(_WIN32) inline uint64_t PosixInNsec() { struct timeval tv; gettimeofday(&tv, nullptr); - return 1000 * (static_cast(tv.tv_sec) * 1000000 + tv.tv_usec); + return 1000 * (static_cast(tv.tv_sec) * 1000000 + tv.tv_usec); +} +#else + inline uint64_t PosixInNsec() { + return static_cast(0); } +#endif // !_WIN32 // DeviceTracer performs the following tasks: // 1. Register cuda callbacks for various events: kernel, memcpy, etc. diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt index 07159d4a12e..5939c500c94 100644 --- a/paddle/fluid/platform/dynload/CMakeLists.txt +++ b/paddle/fluid/platform/dynload/CMakeLists.txt @@ -16,7 +16,9 @@ if (CUPTI_FOUND) list(APPEND CUDA_SRCS cupti.cc) endif(CUPTI_FOUND) nv_library(dynload_cuda SRCS ${CUDA_SRCS} DEPS dynamic_loader) +if (NOT WIN32) cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc) +endif(NOT WIN32) if (WITH_MKLML) cc_library(dynload_mklml SRCS mklml.cc DEPS dynamic_loader mklml) endif() diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 93bf7c13516..4fbfa6354ab 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -13,8 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/platform/dynload/dynamic_loader.h" -#include - #include #include // NOLINT #include @@ -23,6 +21,7 @@ limitations under the License. */ #include "glog/logging.h" #include "paddle/fluid/platform/dynload/cupti_lib_path.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/port.h" DEFINE_string(cudnn_dir, "", "Specify path for loading libcudnn.so. For instance, " diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index c27552b4190..70e442da506 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -117,7 +117,12 @@ struct EOFException : public std::exception { // always forces branch prediction of true. // This generates faster binary code. __builtin_expect is since C++11. // For more details, please check https://stackoverflow.com/a/43870188/724872. +#if !defined(_WIN32) #define UNLIKELY(condition) __builtin_expect(static_cast(condition), 0) +#else +// there is no equivalent intrinsics in msvc. +#define UNLIKELY(condition) (condition == 0) +#endif template inline typename std::enable_if::type throw_on_error( @@ -230,6 +235,7 @@ inline void throw_on_error(T e) { throw_on_error(e, ""); } +#if !defined(_WIN32) #define PADDLE_THROW(...) \ do { \ throw ::paddle::platform::EnforceNotMet( \ @@ -248,15 +254,27 @@ inline void throw_on_error(T e) { __FILE__, __LINE__); \ } \ } while (false) -#else -#define PADDLE_ENFORCE(...) ::paddle::platform::throw_on_error(__VA_ARGS__); -#endif #define PADDLE_THROW_EOF() \ do { \ throw ::paddle::platform::EOFException("There is no next data.", __FILE__, \ __LINE__); \ } while (false) + +#else +#define PADDLE_ENFORCE(...) ::paddle::platform::throw_on_error(__VA_ARGS__); +#endif +#else // !_WIN32 +#define GLOG_NO_ABBREVIATED_SEVERITIES + // disable enforce, caused by the varardic macro exception error +#define PADDLE_THROW(x) \ + do { \ + throw std::make_exception_ptr(std::runtime_error("Windows disable the enforce.")); \ + } while (false) + +#define PADDLE_ENFORCE(x) x +#endif // !_WIN32 + /* * Some enforce helpers here, usage: * int a = 1; diff --git a/paddle/fluid/platform/float16.h b/paddle/fluid/platform/float16.h index efb021c838e..ee16fc66e4a 100644 --- a/paddle/fluid/platform/float16.h +++ b/paddle/fluid/platform/float16.h @@ -56,7 +56,11 @@ limitations under the License. */ #include #endif // PADDLE_ARM +#if !defined(_WIN32) #define PADDLE_ALIGN(x) __attribute__((aligned(x))) +#else +#define PADDLE_ALIGN(x) /*do nothing*/ +#endif namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/macros.h b/paddle/fluid/platform/macros.h index 4cc04b09051..78775a2bb17 100644 --- a/paddle/fluid/platform/macros.h +++ b/paddle/fluid/platform/macros.h @@ -14,6 +14,8 @@ limitations under the License. */ #pragma once +#define GLOG_NO_ABBREVIATED_SEVERITIES + // Disable the copy and assignment operator for a class. #ifndef DISABLE_COPY_AND_ASSIGN #define DISABLE_COPY_AND_ASSIGN(classname) \ diff --git a/paddle/fluid/platform/port.h b/paddle/fluid/platform/port.h index 6aabfd3d693..3caad45e764 100644 --- a/paddle/fluid/platform/port.h +++ b/paddle/fluid/platform/port.h @@ -7,6 +7,7 @@ #include // for dladdr #include // for backtrace #else +#define NOMINMAX // windows min(), max() macro will mess std::min,max #include #include namespace { diff --git a/paddle/fluid/platform/profiler.h b/paddle/fluid/platform/profiler.h index c99d9c807d1..2cf26a1fe05 100644 --- a/paddle/fluid/platform/profiler.h +++ b/paddle/fluid/platform/profiler.h @@ -69,6 +69,7 @@ void PushEvent(const std::string& name, const DeviceContext* dev_ctx); void PopEvent(const std::string& name, const DeviceContext* dev_ctx); +#if !defined(_WIN32) struct RecordEvent { RecordEvent(const std::string& name, const DeviceContext* dev_ctx); @@ -94,6 +95,16 @@ struct RecordBlock { std::string name_; uint64_t start_ns_; }; +#else +// Our profiler deeply coupled in many operators. +// use fake object to avoid large modifies these files. +struct RecordEvent { + RecordEvent(const std::string& name, const DeviceContext* dev_ctx) {} +}; +struct RecordBlock { + explicit RecordBlock(int block_id) {} +}; +#endif // Return the event list of all threads. Assumed the returned value calls // event_lists, event_lists[i][j] represents the j-th Event of i-th thread. diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index d6a14b3305c..ab5b27e4c40 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -1,17 +1,18 @@ -set(PYBIND_DEPS pybind python proto_desc memory executor prune profiler feed_fetch_method - ) +set(PYBIND_DEPS pybind python proto_desc memory executor prune feed_fetch_method) +set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc) if(NOT WIN32) -list(APPEND PYBIND_DEPS parallel_executor) +list(APPEND PYBIND_DEPS parallel_executor profiler) +list(APPEND PYBIND_SRCS recordio.cc) endif() if(WITH_PYTHON) if(WITH_AMD_GPU) hip_library(paddle_pybind SHARED - SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc + SRCS ${PYBIND_SRCS} DEPS ${PYBIND_DEPS} ${GLOB_OP_LIB}) else() cc_library(paddle_pybind SHARED - SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc + SRCS ${PYBIND_SRCS} DEPS ${PYBIND_DEPS} ${GLOB_OP_LIB}) if(NOT APPLE AND NOT ANDROID AND NOT WIN32) diff --git a/paddle/fluid/recordio/CMakeLists.txt b/paddle/fluid/recordio/CMakeLists.txt index 92e97a6c85d..f401c25dbfc 100644 --- a/paddle/fluid/recordio/CMakeLists.txt +++ b/paddle/fluid/recordio/CMakeLists.txt @@ -1,4 +1,5 @@ # internal library. +message("why it hurts!") cc_library(header SRCS header.cc) cc_test(header_test SRCS header_test.cc DEPS header) cc_library(chunk SRCS chunk.cc DEPS snappystream snappy header zlib) -- GitLab