提交 1286767d 编写于 作者: C chenjianping

support building on windows

上级 d90e1215
......@@ -12,6 +12,7 @@ else()
endif()
set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
......@@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
if(Python3_FOUND)
set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}")
set(PYTHON_LIBRARIES "${Python3_LIBRARIES}")
if (WIN32)
if (Python3_DIR)
message("Python3_DIR set already: " ${Python3_DIR})
else()
string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN)
string(LENGTH "libpythonxx.a" Python3_NAME_LEN)
math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN})
string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR)
message("Python3_DIR: " ${Python3_DIR})
endif()
link_directories(${Python3_DIR})
endif()
else()
find_python_package(py_inc py_lib)
set(PYTHON_INCLUDE_DIRS "${py_inc}")
......
@echo off
@title mindspore_build
SET BASEPATH=%CD%
IF NOT EXIST %BASEPATH%/build (
md "build"
)
cd %BASEPATH%/build
SET BUILD_PATH=%CD%
IF NOT EXIST %BUILD_PATH%/mindspore (
md "mindspore"
)
cd %CD%/mindspore
cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -G "CodeBlocks - MinGW Makefiles" ../..
IF NOT %errorlevel% == 0 (
goto run_fail
)
cmake --build . --target all -- -j6
IF NOT %errorlevel% == 0 (
goto run_fail
)
cd %BASEPATH%
goto run_eof
:run_fail
cd %BASEPATH%
echo "build fail."
:run_eof
set(flatbuffers_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(flatbuffers_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
if (WIN32)
set(flatbuffers_USE_STATIC_LIBS ON)
endif()
mindspore_add_pkg(flatbuffers
VER 1.11.0
LIBS flatbuffers
......
set(onednn_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(onednn_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
mindspore_add_pkg(onednn
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
mindspore_add_pkg(onednn
VER 1.1.1
LIBS dnnl mkldnn
HEAD_ONLY ./
RELEASE on
URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip
MD5 ecaab9ed549643067699c80e5cea1c23)
else()
mindspore_add_pkg(onednn
VER 1.1.2
LIBS dnnl mkldnn
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF)
endif()
include_directories(${onednn_INC})
add_library(mindspore::dnnl ALIAS onednn::dnnl)
add_library(mindspore::mkldnn ALIAS onednn::mkldnn)
......@@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl")
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
else()
set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2")
set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
endif()
mindspore_add_pkg(opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
include_directories(${opencv_INC}/opencv4)
add_library(mindspore::opencv_core ALIAS opencv::opencv_core)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs)
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc)
if (WIN32)
mindspore_add_pkg(opencv
VER 4.2.0
LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a
LIB_PATH x64/mingw/lib
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
else()
mindspore_add_pkg(opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=${tiff_INC}
-DTIFF_LIBRARY=${tiff_LIB})
endif()
if (WIN32)
include_directories(${opencv_INC})
add_library(mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a)
add_library(mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a)
else()
include_directories(${opencv_INC}/opencv4)
add_library(mindspore::opencv_core ALIAS opencv::opencv_core)
add_library(mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs)
add_library(mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc)
endif()
......@@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc")
list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h")
list(APPEND ${py_var} "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
if (WIN32)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi.bak -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
else()
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}.pb.h"
"${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/${rel_path}"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND protobuf::protoc -I${file_dir} --python_out=${CMAKE_BINARY_DIR}/${rel_path} ${abs_file}
COMMAND perl -pi -e "s/import (.+_pb2.*)/from . import \\1/" "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py"
COMMAND cp "${CMAKE_BINARY_DIR}/${rel_path}/${file_name}_pb2.py" "${PROJECT_SOURCE_DIR}/mindspore/train/"
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endif()
endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} ${${py_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)
......
if (WIN32)
mindspore_add_pkg(sqlite-head
VER 3.31.1
HEAD_ONLY ./
URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip
MD5 2b7bfcdd97dc281903a9aee966213fe4)
include_directories(${sqlite-head_INC})
mindspore_add_pkg(sqlite
VER 3.31.1
LIBS sqlite3
LIB_PATH ./
HEAD_ONLY ./
RELEASE ON
URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip
MD5 662c9d2b05467d590ba5c0443e7fd6bd)
set(sqlite_USE_STATIC_LIBS ON)
set(sqlite_CXXFLAGS)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
else ()
set(sqlite_USE_STATIC_LIBS ON)
set(sqlite_CXXFLAGS)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(sqlite_CFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
else()
set(sqlite_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2")
endif()
set(sqlite_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
mindspore_add_pkg(sqlite
VER 3.31.1
LIBS sqlite3
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
MD5 5f4e7b4016c15f4fb5855615279819da
PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sqlite/sqlite.patch001
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1)
include_directories(${sqlite_INC})
add_library(mindspore::sqlite ALIAS sqlite::sqlite3)
\ No newline at end of file
include_directories(${sqlite_INC})
endif ()
add_library(mindspore::sqlite ALIAS sqlite::sqlite3)
set(SECURE_CXX_FLAGS "")
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
if (WIN32)
set(SECURE_CXX_FLAGS "-fstack-protector-all")
else()
set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif()
set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
......
......@@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
option(ENABLE_AKG "enable akg" OFF)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
if (WIN32)
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all")
else()
set(OPTION_CXX_FLAGS "${OPTION_CXX_FLAGS} -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
endif()
endif()
if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
......@@ -106,4 +110,4 @@ endif()
if(ENABLE_DUMP_E2E)
add_compile_definitions(ENABLE_DUMP_E2E)
endif()
\ No newline at end of file
endif()
......@@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
endfunction()
function(__find_pkg_then_add_target pkg_name pkg_exe)
function(__find_pkg_then_add_target pkg_name pkg_exe lib_path)
unset(${pkg_name}_LIBS)
......@@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
set(_LIB_TYPE STATIC)
endif ()
set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/${lib_path} NO_DEFAULT_PATH)
if(NOT ${_LIB_NAME}_LIB)
return()
endif()
add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include"
IMPORTED_LOCATION ${${_LIB_NAME}_LIB}
)
if (WIN32 AND ${_LIB_TYPE} STREQUAL "SHARED")
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_IMPLIB_RELEASE ${${_LIB_NAME}_LIB})
else()
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES IMPORTED_LOCATION ${${_LIB_NAME}_LIB})
endif()
if (EXISTS ${${pkg_name}_BASE_DIR}/include)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include")
endif ()
list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
message("found ${${_LIB_NAME}_LIB}")
STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
......@@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
function(mindspore_add_pkg pkg_name )
set(options )
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH)
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH)
set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES)
cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )
if (NOT PKG_CMAKE_PATH)
set(PKG_CMAKE_PATH .)
if (NOT PKG_LIB_PATH)
set(PKG_LIB_PATH lib)
endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(__FIND_PKG_NAME ${pkg_name})
string(TOLOWER ${pkg_name} pkg_name)
message("pkg name:${__FIND_PKG_NAME},${pkg_name}")
......@@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
if (${PKG_RELEASE})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
endif ()
return()
endif ()
if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)
if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
if(${pkg_name}_LIBS)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found libs: ${${pkg_name}_LIBS}")
......@@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
if (NOT PKG_RELEASE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
endif ()
elseif (PKG_CMAKE_OPTION)
# in cmake
......@@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
endif()
if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIB_PATH} ${PKG_LIBS})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
if(NOT ${pkg_name}_LIBS)
message(FATAL_ERROR "Can not find pkg: ${pkg_name}")
......
......@@ -5,6 +5,10 @@ if(ENABLE_CPU)
file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc"
)
if (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
add_compile_definitions(BUILDING_DLL)
endif()
endif()
if(ENABLE_GPU)
......@@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel/kash/*.cc"
"device/gpu/distribution/collective_init.cc"
)
if (ENABLE_CPU)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc")
if (WIN32)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc")
endif()
endif()
if (NOT ENABLE_GPU)
list(APPEND MINDSPORE_SRC_LIST "device/gpu/distribution/collective_fake_init.cc")
endif()
file(GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"pre_activate/mem_reuse/*.cc"
)
......@@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
add_library(mindspore_gvar SHARED ${MS_GVAR_SRC_LIST})
add_library(mindspore STATIC ${MINDSPORE_SRC_LIST})
add_dependencies(mindspore GENERATED_OUTPUT_DIR)
if(ENABLE_D)
list(APPEND MINDSPORE_PROTO_LIST ${MINDSPORE_PROTO_AICPU_LIST})
......@@ -315,19 +328,33 @@ if(ENABLE_D)
endif()
target_link_libraries(mindspore securec)
target_link_libraries(mindspore dl)
if (NOT WIN32)
target_link_libraries(mindspore dl)
endif()
target_link_libraries(mindspore mindspore::flatbuffers)
# link protobuf
if (ENABLE_D)
target_link_libraries(mindspore mindspore::protobuf)
endif()
if (WIN32)
target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar)
endif()
# set c_expression building
if (WIN32)
set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST}
pipeline/init.cc
kernel/oplib/oplib.cc
${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
else()
set(PYTHON_MODULE_SOURCE
pipeline/init.cc
kernel/oplib/oplib.cc
${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST}
${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST})
endif()
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
pybind11_add_module(_c_expression ${PYTHON_MODULE_SOURCE})
......@@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
set_target_properties(_c_expression PROPERTIES MACOSX_RPATH ON)
set(ORIGIN_PATH @loader_path)
elseif (CMAKE_SYSTEM_NAME MATCHES "Windows")
set(ORIGIN_PATH $ORIGIN)
else ()
MESSAGE(FATAL_ERROR "other platform: ${CMAKE_SYSTEM_NAME}")
endif ()
......@@ -346,11 +375,22 @@ endif ()
set(ORIGIN_PATH ${ORIGIN_PATH}/lib)
set_target_properties(_c_expression PROPERTIES INSTALL_RPATH ${ORIGIN_PATH})
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
)
if (WIN32)
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
securec
proto_input
mindspore::flatbuffers
mindspore::glog
)
else()
target_link_libraries(_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
mindspore::glog
)
endif()
if(ENABLE_GPU)
execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/third_party/apply_patches.sh
......@@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
add_subdirectory(mindrecord)
add_subdirectory(dataset)
endif()
if (USE_GLOG)
target_link_libraries(_c_expression PRIVATE mindspore::glog)
endif()
\ No newline at end of file
......@@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
######################################################################
################# Link with external libraries ########################
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY})
if (WIN32)
target_link_libraries(_c_dataengine PRIVATE mindspore)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY})
else()
target_link_libraries(_c_dataengine PRIVATE mindspore mindspore_gvar)
target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf ${SECUREC_LIBRARY})
endif()
target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc)
if (ENABLE_GPUQUE)
target_link_libraries(_c_dataengine PRIVATE gpu_queue
${CUDNN_PATH}/lib64/libcudnn.so
${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/libcudart.so
${CUDA_PATH}/lib64/stubs/libcuda.so)
endif ()
......@@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
endif ()
add_dependencies(_c_dataengine _c_mindrecord)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
if (WIN32)
set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a)
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite)
else()
target_link_libraries(_c_dataengine PRIVATE _c_mindrecord)
endif()
if (USE_GLOG)
target_link_libraries(_c_dataengine PRIVATE mindspore::glog)
......
......@@ -19,7 +19,9 @@
#include "dataset/kernels/no_op.h"
#include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/image/center_crop_op.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "dataset/kernels/image/change_mode_op.h"
#endif
#include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h"
......@@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
py::arg("fillG") = RandomCropOp::kDefFillG, py::arg("fillB") = RandomCropOp::kDefFillB);
(void)py::class_<HwcToChwOp, TensorOp, std::shared_ptr<HwcToChwOp>>(*m, "ChannelSwapOp").def(py::init<>());
#if !defined(_WIN32) && !defined(_WIN64)
(void)py::class_<ChangeModeOp, TensorOp, std::shared_ptr<ChangeModeOp>>(
*m, "ChangeModeOp", "Tensor operation to change colors from BGR to RGB")
.def(py::init<>());
#endif
(void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>(
*m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.")
......
......@@ -22,6 +22,10 @@
#include <vector>
#include "./securec.h"
#include "utils/log_adapter.h"
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
......
......@@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
if (!reshuffle_each_epoch_) {
rng_ = std::mt19937_64(shuffle_seed_);
} else {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<int32_t> distribution(0, std::numeric_limits<int32_t>::max());
shuffle_seed_ = distribution(random_device);
rng_ = std::mt19937_64(shuffle_seed_);
......
add_library(kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
if (WIN32)
add_library(kernels-image OBJECT
center_crop_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
else()
add_library(kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
endif()
......@@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(*cinfo->mem->alloc_small)(reinterpret_cast<j_common_ptr>(cinfo), JPOOL_PERMANENT, sizeof(struct jpeg_source_mgr)));
cinfo->src->init_source = JpegInitSource;
cinfo->src->fill_input_buffer = JpegFillInputBuffer;
#if defined(_WIN32) || defined(_WIN64)
cinfo->src->skip_input_data = reinterpret_cast<void (*)(j_decompress_ptr, long)>(JpegSkipInputData);
#else
cinfo->src->skip_input_data = JpegSkipInputData;
#endif
cinfo->src->resync_to_restart = jpeg_resync_to_restart;
cinfo->src->term_source = JpegTermSource;
cinfo->src->bytes_in_buffer = datasize;
......
......@@ -22,6 +22,10 @@
#include <random>
#include <string>
#include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h"
#include "./jerror.h"
#include <opencv2/imgproc/imgproc.hpp>
......
......@@ -27,7 +27,7 @@
namespace mindspore {
namespace dataset {
#ifdef _WIN32
char Path::_separator = '\\';
char Path::separator_ = '\\';
#else
char Path::separator_ = '/';
#endif
......@@ -129,7 +129,11 @@ bool Path::IsDirectory() {
Status Path::CreateDirectory() {
if (!Exists()) {
#if defined(_WIN32) || defined(_WIN64)
int rc = mkdir(common::SafeCStr(path_));
#else
int rc = mkdir(common::SafeCStr(path_), 0700);
#endif
if (rc) {
std::ostringstream oss;
oss << "Unable to create directory " << path_ << ". Errno = " << errno;
......
......@@ -32,7 +32,11 @@ namespace dataset {
uint32_t GetSeed() {
uint32_t seed = GlobalContext::config_manager()->seed();
if (seed == std::mt19937::default_seed) {
#if defined(_WIN32) || defined(_WIN64)
std::random_device random_device;
#else
std::random_device random_device("/dev/urandom");
#endif
std::uniform_int_distribution<uint32_t> distribution(0, std::numeric_limits<uint32_t>::max());
seed = distribution(random_device);
}
......
......@@ -16,7 +16,9 @@
#include "dataset/util/services.h"
#include <limits.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/syscall.h>
#endif
#include <unistd.h>
#include <random>
#include "dataset/util/circular_pool.h"
......@@ -28,6 +30,7 @@ namespace dataset {
std::unique_ptr<Services> Services::instance_ = nullptr;
std::once_flag Services::init_instance_flag_;
#if !defined(_WIN32) && !defined(_WIN64)
std::string Services::GetUserName() {
char user[LOGIN_NAME_MAX];
(void)getlogin_r(user, sizeof(user));
......@@ -41,10 +44,15 @@ std::string Services::GetHostName() {
}
int Services::GetLWP() { return syscall(SYS_gettid); }
#endif
std::string Services::GetUniqueID() {
const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789";
#if defined(_WIN32) || defined(_WIN64)
std::mt19937 gen{std::random_device{}()};
#else
std::mt19937 gen{std::random_device{"/dev/urandom"}()};
#endif
std::uniform_int_distribution<> dist(0, kStr.size() - 1);
char buffer[UNIQUEID_LEN];
for (int i = 0; i < UNIQUEID_LEN; i++) {
......
......@@ -62,11 +62,13 @@ class Services {
std::shared_ptr<MemoryPool> GetServiceMemPool() { return pool_; }
#if !defined(_WIN32) && !defined(_WIN64)
static std::string GetUserName();
static std::string GetHostName();
static int GetLWP();
#endif
static std::string GetUniqueID();
......
......@@ -16,13 +16,16 @@
#include "dataset/util/sig_handler.h"
#include <signal.h>
#include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <ucontext.h>
#endif
#include <unistd.h>
#include "dataset/util/task_manager.h"
namespace mindspore {
namespace dataset {
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
void RegisterHandlers() {
struct sigaction new_int_action;
......@@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
// Wake up the watchdog which is designed as async-signal-safe.
TaskManager::WakeUpWatchDog();
}
#endif
} // namespace dataset
} // namespace mindspore
......@@ -22,12 +22,14 @@
namespace mindspore {
namespace dataset {
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
extern void RegisterHandlers();
// A signal handler for SIGINT. Drives interrupt to watchdog
extern void IntHandler(int sig_num, // The signal that was raised
siginfo_t *sig_info, // The siginfo structure.
void *context); // context info
#endif
} // namespace dataset
} // namespace mindspore
......
......@@ -14,6 +14,9 @@
* limitations under the License.
*/
#include "debug/anf_ir_dump.h"
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <fstream>
#include <map>
#include <memory>
......@@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
return;
}
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, filename.c_str(), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "dir " << filename << " does not exit.";
}
#else
if (nullptr == realpath(filename.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << filename << " does not exit.";
}
#endif
OrderedMap<AnfNodePtr, int32_t> para_map;
std::string path_string = real_path;
......
......@@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
if (path_ptr != nullptr) {
path = path_ptr;
char real_path[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (path.size() > PATH_MAX || _fullpath(real_path, path.c_str(), PATH_MAX) == nullptr) {
MS_LOG(EXCEPTION) << "MS IR Path error, " << path_ptr;
}
#else
if (path.size() > PATH_MAX || nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(EXCEPTION) << "MS IR path error, " << path_ptr;
}
#endif
path = real_path;
}
return path;
......@@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
return;
}
char real_path[PATH_MAX] = {0};
if (nullptr == realpath(file_path.c_str(), real_path)) {
MS_LOG(DEBUG) << "Dir " << file_path << " does not exit.";
char* real_path_ret = nullptr;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret = _fullpath(real_path, file_path.c_str(), PATH_MAX);
#else
real_path_ret = realpath(file_path.c_str(), real_path);
#endif
if (nullptr == real_path_ret) {
MS_LOG(DEBUG) << "dir " << file_path << " does not exit.";
} else {
std::string path_string = real_path;
if (chmod(common::SafeCStr(path_string), S_IRUSR | S_IWUSR) == -1) {
......
......@@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
}
char path[PATH_MAX + 1] = {0x00};
#if defined(_WIN32) || defined(_WIN64)
if (file_name_.size() > PATH_MAX || _fullpath(path, file_name_.c_str(), PATH_MAX) == nullptr) {
return debug_info_ss.str();
}
#else
if (file_name_.size() > PATH_MAX || realpath(file_name_.c_str(), path) == nullptr) {
return debug_info_ss.str();
}
#endif
auto src_path = std::string(path);
std::ifstream file(src_path);
if (!file.is_open()) {
......
......@@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
void *CPUResourceManager::MemMalloc(size_t mem_size) {
void *ptr = malloc(mem_size);
if (ptr != nullptr) {
memset_s(ptr, mem_size, 0, mem_size);
dynamic_mem_[ptr] = mem_size;
return ptr;
} else {
......
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "device/gpu/distribution/collective_fake_init.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace device {
namespace gpu {
void CollectiveFakeInitializer::InitCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
void CollectiveFakeInitializer::FinalizeCollective() { MS_LOG(EXCEPTION) << "build without enable gpu!"; }
} // namespace gpu
} // namespace device
} // namespace mindspore
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
namespace mindspore {
namespace device {
namespace gpu {
class CollectiveFakeInitializer {
public:
CollectiveFakeInitializer() = default;
~CollectiveFakeInitializer() = default;
CollectiveFakeInitializer(CollectiveFakeInitializer const &) = delete;
CollectiveFakeInitializer &operator=(const CollectiveFakeInitializer &) = delete;
static void InitCollective();
static void FinalizeCollective();
};
} // namespace gpu
} // namespace device
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
......@@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
bool ret = false;
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
bool is_task_sink = context_ptr->enable_task_sink();
if (is_task_sink) {
ret = RunTask(graph);
} else {
ret = LaunchKernel(graph);
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Call MS Run Success in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Call MS Run Success in " << cost << " us";
#endif
return ret;
}
......@@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
AddressPtrList kernel_workspaces;
AddressPtrList kernel_outputs;
GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs);
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
auto ret =
kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, reinterpret_cast<uintptr_t>(stream_));
if (!ret) {
......@@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
if (AnfAlgo::GetKernelType(kernel) == TBE_KERNEL && !SyncStream()) {
MS_LOG(EXCEPTION) << "SyncStream failed.";
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(DEBUG) << "d " << kernel->fullname_with_scope() << " in " << cost << " us";
#endif
}
}
return true;
......
......@@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
bool KernelMeta::ReadIndex(const std::string &bin_dir) {
DIR *dir = opendir(bin_dir.c_str());
if (dir == nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto ret = mkdir(bin_dir.c_str());
#else
auto ret = mkdir(bin_dir.c_str(), S_IRWXG | S_IRWXU);
#endif
if (ret != 0) {
MS_LOG(INFO) << "kernel dir not exist[" << bin_dir << "].";
return false;
......@@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
}
filewrite << info << std::endl;
filewrite.close();
#if defined(_WIN32) || defined(_WIN64)
if (nullptr == _fullpath(real_path, path.c_str(), PATH_MAX)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return;
}
#else
if (nullptr == realpath(path.c_str(), real_path)) {
MS_LOG(DEBUG) << "dir " << path << " does not exit.";
return;
}
#endif
MS_LOG(INFO) << "real path is :" << real_path;
if (chmod(real_path, S_IRUSR) == -1) {
MS_LOG(DEBUG) << "modify file:" << real_path << " to read only fail.";
......
......@@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
)
# add link library
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)
if (WIN32)
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf)
else()
target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf)
endif()
if (USE_GLOG)
target_link_libraries(_c_mindrecord PRIVATE mindspore::glog)
......
......@@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
return {FAILED, ""};
}
char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
......@@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully";
}
#endif
std::string s = real_path;
char sep = '/';
size_t i = s.rfind(sep, s.length());
......@@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
return {FAILED, ""};
}
char tmp[PATH_MAX] = {0};
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(tmp, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
}
if (_fullpath(real_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path: " << common::SafeCStr(path) << "check successfully";
}
#else
if (realpath(dirname(&(buf[0])), tmp) == nullptr) {
MS_LOG(ERROR) << "Invalid file path, path: " << buf;
return {FAILED, ""};
......@@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
if (realpath(common::SafeCStr(path), real_path) == nullptr) {
MS_LOG(DEBUG) << "Path: " << path << "check successfully";
}
#endif
std::string s = real_path;
if (s.rfind('/') + 1 <= s.size()) {
return {SUCCESS, s.substr(0, s.rfind('/') + 1)};
......@@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
}
std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type) {
#if defined(_WIN32) || defined(_WIN64)
return {SUCCESS, 100};
#else
uint64_t ll_count = 0;
struct statfs disk_info;
if (statfs(common::SafeCStr(str_dir), &disk_info) == -1) {
......@@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
}
return {SUCCESS, ll_count};
#endif
}
uint32_t GetMaxThreadNum() {
......
......@@ -21,8 +21,10 @@
#include <limits.h>
#include <stdlib.h>
#include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/statfs.h>
#include <sys/wait.h>
#endif
#include <unistd.h>
#include <cassert>
#include <cmath>
......
......@@ -19,7 +19,9 @@
#include <dirent.h>
#include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/prctl.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
......
......@@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus ShardReader::ConsumerByRow(int consumer_id) {
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever
for (;;) {
......@@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
MSRStatus ShardReader::ConsumerByBlock(int consumer_id) {
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto thread_id = kThreadName + std::to_string(consumer_id);
prctl(PR_SET_NAME, common::SafeCStr(thread_id), 0, 0, 0);
#endif
// Loop forever
for (;;) {
......
......@@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
MS_LOG(ERROR) << "Securec func failed";
return FAILED;
}
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(resolved_path, dirname(&(buf[0])), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Invalid file path";
return FAILED;
}
if (_fullpath(resolved_path, common::SafeCStr(path), PATH_MAX) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path;
}
#else
if (realpath(dirname(&(buf[0])), resolved_path) == nullptr) {
MS_LOG(ERROR) << "Invalid file path";
return FAILED;
......@@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
if (realpath(common::SafeCStr(path), resolved_path) == nullptr) {
MS_LOG(DEBUG) << "Path " << resolved_path;
}
#endif
file_paths_.emplace_back(string(resolved_path));
}
......
......@@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
(root->has_flag(ALLREDUCE_FUSION_RUN_ONCE_ONLY))) {
return changes;
}
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time, end_time;
(void)gettimeofday(&start_time, nullptr);
#endif
MS_LOG(INFO) << "Now entering allreduce fusion";
DumpGraph(root, std::string(ALLREDUCE_FUSION_BEGIN));
......@@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
// allreduce fusion only run once
root->flags()[ALLREDUCE_FUSION_RUN_ONCE_ONLY] = true;
res->results()[pipeline::kStepParallelGraph] = root;
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
uint64_t time = 1000000 * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
time += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Now leaving allreduce fusion, used time: " << time << " us";
#endif
return changes;
}
} // namespace parallel
......
......@@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case kNumberTypeInt:
return sizeof(int);
case kNumberTypeUInt:
return sizeof(uint);
return sizeof(unsigned int);
case kNumberTypeFloat:
return sizeof(float);
default:
......
......@@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
auto bc_ptr = res->results()[kBackend].cast<compile::BackendPtr>();
std::vector<PrimitivePtr> cut_list = compile::nonlinear_ops;
if (bc_ptr->name() == kMsConvert) {
cut_list = compile::ms_nonlinear_ops;
cut_list = compile::GetMsNonlinearOps();
}
std::shared_ptr<CompileGraphs> compile = std::make_shared<CompileGraphs>(bc_ptr, cut_list);
res->results()[kOutput] = compile->CompileAndLink(func_graph);
......
......@@ -29,8 +29,11 @@
#include "parallel/context.h"
#include "parallel/device_manager.h"
#include "parallel/costmodel_context.h"
#ifdef ENABLE_GPUQUE
#include "device/gpu/distribution/collective_init.h"
#else
#include "device/gpu/distribution/collective_fake_init.h"
#endif
namespace py = pybind11;
using FuncGraph = mindspore::FuncGraph;
......@@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
(void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib")
.def(py::init())
.def("reg_op", &OpLib::RegOp, "Register op info.");
#ifdef ENABLE_GPUQUE
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective,
"Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::FinalizeCollective,
"Finalize gpu collective communication mode.");
#else
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::InitCollective,
"Init gpu collective communication mode.");
(void)m.def("finalize_gpu_collective", &mindspore::device::gpu::CollectiveFakeInitializer::FinalizeCollective,
"Finalize gpu collective communication mode.");
#endif
}
......@@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
size_t num = 0;
for (const auto &pass : passes) {
if (pass != nullptr) {
#if defined(_WIN32) || defined(_WIN64)
auto start_time = std::chrono::steady_clock::now();
#else
struct timeval start_time {};
struct timeval end_time {};
(void)gettimeofday(&start_time, nullptr);
#endif
if (pass->Run(func_graph)) {
changed = true;
}
#if defined(_WIN32) || defined(_WIN64)
auto end_time = std::chrono::steady_clock::now();
std::chrono::duration<double, std::ratio<1, 1000000>> cost = end_time - start_time;
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost.count() << " us";
#else
(void)gettimeofday(&end_time, nullptr);
const uint64_t kUSecondInSecond = 1000000;
uint64_t cost = kUSecondInSecond * static_cast<uint64_t>(end_time.tv_sec - start_time.tv_sec);
cost += static_cast<uint64_t>(end_time.tv_usec - start_time.tv_usec);
MS_LOG(INFO) << "Run pass hwopt_" + name() + "_" << num << "_" + pass->name() + " in " << cost << " us";
#endif
if (save_graphs) {
auto dump_file_path =
save_graphs_path + "/" + "hwopt_" + name() + "_" + std::to_string(num) + "_" + pass->name() + ".ir";
......
......@@ -26,12 +26,19 @@ namespace mindspore {
#ifdef USE_GLOG
static std::string GetTime() {
#define BUFLEN 80
static char buf[BUFLEN];
#if defined(_WIN32) || defined(_WIN64)
time_t time_seconds = time(0);
struct tm now_time;
localtime_s(&now_time, &time_seconds);
sprintf_s(buf, BUFLEN, "%d-%d-%d %d:%d:%d", now_time.tm_year + 1900, now_time.tm_mon + 1, now_time.tm_mday,
now_time.tm_hour, now_time.tm_min, now_time.tm_sec);
#else
struct timeval cur_time;
(void)gettimeofday(&cur_time, NULL);
struct tm now;
(void)localtime_r(&cur_time.tv_sec, &now);
static char buf[BUFLEN];
(void)strftime(buf, BUFLEN, "%Y-%m-%d-%H:%M:%S", &now); // format date and time
// set micro-second
buf[27] = '\0';
......@@ -44,6 +51,7 @@ static std::string GetTime() {
buf[idx--] = '.';
}
}
#endif
return std::string(buf);
}
......
......@@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
std::vector<PrimitivePtr> nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch,
prim::kPrimMakeTuple};
std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch};
const std::vector<PrimitivePtr>& GetMsNonlinearOps() {
static const std::vector<PrimitivePtr> ms_nonlinear_ops = {prim::kPrimReturn, prim::kPrimPartial, prim::kPrimSwitch};
return ms_nonlinear_ops;
}
CompileGraph::CompileGraph(const BackendPtr& backend, const std::vector<PrimitivePtr>& cut_list)
: backend_(backend), cut_list_(cut_list) {
......
......@@ -42,7 +42,7 @@ extern const char kGeVm[];
// A sub namespace in ME to support compile related definition.
namespace compile {
extern std::vector<PrimitivePtr> nonlinear_ops;
extern std::vector<PrimitivePtr> ms_nonlinear_ops;
const std::vector<PrimitivePtr>& GetMsNonlinearOps();
using VmEvalFunc = std::function<BaseRef(const VectorRef&)>;
using VmEvalFuncPtr = std::shared_ptr<std::function<BaseRef(const VectorRef&)>>;
......
......@@ -19,11 +19,13 @@ import sys
import os
import stat
import time
import fcntl
import logging
from logging.handlers import RotatingFileHandler
import traceback
import threading
import platform
if platform.system() != "Windows":
import fcntl
__all__ = ['get_level', 'get_log_config']
......@@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
# Attain an exclusive lock with bloking mode by `fcntl` module.
with open(self.baseFilename, 'a') as file_pointer:
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX)
if platform.system() != "Windows":
fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX)
if self.backupCount > 0:
self.rolling_rename()
......
SET(CMAKE_BUILD_TYPE "Debug")
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
if (WIN32)
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer")
else()
SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='")
endif()
SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
#add flags
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror")
if (WIN32)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include")
else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror")
endif()
include_directories(./include)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册