Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
magicwindyyd
mindspore
提交
1286767d
M
mindspore
项目概览
magicwindyyd
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
1286767d
编写于
4月 15, 2020
作者:
C
chenjianping
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
support building on windows
上级
d90e1215
变更
48
隐藏空白更改
内联
并排
Showing
48 changed file
with
619 addition
and
133 deletion
+619
-133
CMakeLists.txt
CMakeLists.txt
+13
-0
build.bat
build.bat
+36
-0
cmake/external_libs/flatbuffers.cmake
cmake/external_libs/flatbuffers.cmake
+3
-0
cmake/external_libs/mkl_dnn.cmake
cmake/external_libs/mkl_dnn.cmake
+12
-1
cmake/external_libs/opencv.cmake
cmake/external_libs/opencv.cmake
+63
-26
cmake/external_libs/protobuf.cmake
cmake/external_libs/protobuf.cmake
+29
-15
cmake/external_libs/sqlite.cmake
cmake/external_libs/sqlite.cmake
+29
-12
cmake/mind_expression.cmake
cmake/mind_expression.cmake
+5
-1
cmake/options.cmake
cmake/options.cmake
+6
-2
cmake/utils.cmake
cmake/utils.cmake
+33
-17
mindspore/ccsrc/CMakeLists.txt
mindspore/ccsrc/CMakeLists.txt
+49
-13
mindspore/ccsrc/dataset/CMakeLists.txt
mindspore/ccsrc/dataset/CMakeLists.txt
+14
-4
mindspore/ccsrc/dataset/api/python_bindings.cc
mindspore/ccsrc/dataset/api/python_bindings.cc
+4
-0
mindspore/ccsrc/dataset/core/tensor.h
mindspore/ccsrc/dataset/core/tensor.h
+4
-0
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
+4
-0
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
+46
-22
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
+4
-0
mindspore/ccsrc/dataset/kernels/image/image_utils.h
mindspore/ccsrc/dataset/kernels/image/image_utils.h
+4
-0
mindspore/ccsrc/dataset/util/path.cc
mindspore/ccsrc/dataset/util/path.cc
+5
-1
mindspore/ccsrc/dataset/util/random.cc
mindspore/ccsrc/dataset/util/random.cc
+4
-0
mindspore/ccsrc/dataset/util/services.cc
mindspore/ccsrc/dataset/util/services.cc
+8
-0
mindspore/ccsrc/dataset/util/services.h
mindspore/ccsrc/dataset/util/services.h
+2
-0
mindspore/ccsrc/dataset/util/sig_handler.cc
mindspore/ccsrc/dataset/util/sig_handler.cc
+4
-0
mindspore/ccsrc/dataset/util/sig_handler.h
mindspore/ccsrc/dataset/util/sig_handler.h
+2
-0
mindspore/ccsrc/debug/anf_ir_dump.cc
mindspore/ccsrc/debug/anf_ir_dump.cc
+9
-0
mindspore/ccsrc/debug/anf_ir_utils.cc
mindspore/ccsrc/debug/anf_ir_utils.cc
+14
-2
mindspore/ccsrc/debug/info.cc
mindspore/ccsrc/debug/info.cc
+6
-1
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
+1
-0
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc
...ore/ccsrc/device/gpu/distribution/collective_fake_init.cc
+28
-0
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h
...pore/ccsrc/device/gpu/distribution/collective_fake_init.h
+37
-0
mindspore/ccsrc/device/kernel_runtime.cc
mindspore/ccsrc/device/kernel_runtime.cc
+20
-0
mindspore/ccsrc/kernel/common_utils.cc
mindspore/ccsrc/kernel/common_utils.cc
+11
-0
mindspore/ccsrc/mindrecord/CMakeLists.txt
mindspore/ccsrc/mindrecord/CMakeLists.txt
+5
-1
mindspore/ccsrc/mindrecord/common/shard_utils.cc
mindspore/ccsrc/mindrecord/common/shard_utils.cc
+24
-0
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
+2
-0
mindspore/ccsrc/mindrecord/include/shard_reader.h
mindspore/ccsrc/mindrecord/include/shard_reader.h
+2
-0
mindspore/ccsrc/mindrecord/io/shard_reader.cc
mindspore/ccsrc/mindrecord/io/shard_reader.cc
+4
-0
mindspore/ccsrc/mindrecord/io/shard_writer.cc
mindspore/ccsrc/mindrecord/io/shard_writer.cc
+10
-0
mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
.../ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
+10
-3
mindspore/ccsrc/parallel/step_auto_parallel.cc
mindspore/ccsrc/parallel/step_auto_parallel.cc
+1
-1
mindspore/ccsrc/pipeline/action.cc
mindspore/ccsrc/pipeline/action.cc
+1
-1
mindspore/ccsrc/pipeline/init.cc
mindspore/ccsrc/pipeline/init.cc
+12
-2
mindspore/ccsrc/pre_activate/common/pass_manager.cc
mindspore/ccsrc/pre_activate/common/pass_manager.cc
+10
-0
mindspore/ccsrc/utils/log_adapter.cc
mindspore/ccsrc/utils/log_adapter.cc
+9
-1
mindspore/ccsrc/vm/transform.cc
mindspore/ccsrc/vm/transform.cc
+4
-2
mindspore/ccsrc/vm/transform.h
mindspore/ccsrc/vm/transform.h
+1
-1
mindspore/log.py
mindspore/log.py
+5
-2
third_party/securec/CMakeLists.txt
third_party/securec/CMakeLists.txt
+10
-2
未找到文件。
CMakeLists.txt
浏览文件 @
1286767d
...
...
@@ -12,6 +12,7 @@ else()
endif
()
set
(
CMAKE_CXX_FLAGS_DEBUG
"$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC"
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
...
...
@@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
if
(
Python3_FOUND
)
set
(
PYTHON_INCLUDE_DIRS
"
${
Python3_INCLUDE_DIRS
}
"
)
set
(
PYTHON_LIBRARIES
"
${
Python3_LIBRARIES
}
"
)
if
(
WIN32
)
if
(
Python3_DIR
)
message
(
"Python3_DIR set already: "
${
Python3_DIR
}
)
else
()
string
(
LENGTH
${
PYTHON_LIBRARIES
}
PYTHON_LIBRARIES_LEN
)
string
(
LENGTH
"libpythonxx.a"
Python3_NAME_LEN
)
math
(
EXPR Python3_DIR_LEN
${
PYTHON_LIBRARIES_LEN
}
-
${
Python3_NAME_LEN
}
)
string
(
SUBSTRING
${
Python3_LIBRARIES
}
0
${
Python3_DIR_LEN
}
Python3_DIR
)
message
(
"Python3_DIR: "
${
Python3_DIR
}
)
endif
()
link_directories
(
${
Python3_DIR
}
)
endif
()
else
()
find_python_package
(
py_inc py_lib
)
set
(
PYTHON_INCLUDE_DIRS
"
${
py_inc
}
"
)
...
...
build.bat
0 → 100644
浏览文件 @
1286767d
@echo
off
@title
mindspore_build
SET
BASEPATH
=
%CD%
IF
NOT
EXIST
%BASEPATH%
/build
(
md
"build"
)
cd
%BASEPATH%
/build
SET
BUILD_PATH
=
%CD%
IF
NOT
EXIST
%BUILD_PATH%
/mindspore
(
md
"mindspore"
)
cd
%CD%
/mindspore
cmake
-DCMAKE
_BUILD_TYPE
=
Release
-DENABLE
_CPU
=
ON
-DENABLE
_MINDDATA
=
ON
-DUSE
_GLOG
=
ON
-G
"CodeBlocks - MinGW Makefiles"
../..
IF
NOT
%errorlevel%
==
0
(
goto
run_fail
)
cmake
--build
.
--target
all
--
-j
6
IF
NOT
%errorlevel%
==
0
(
goto
run_fail
)
cd
%BASEPATH%
goto
run_eof
:run
_fail
cd
%BASEPATH%
echo
"build fail."
:run
_eof
cmake/external_libs/flatbuffers.cmake
浏览文件 @
1286767d
set
(
flatbuffers_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
flatbuffers_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
if
(
WIN32
)
set
(
flatbuffers_USE_STATIC_LIBS ON
)
endif
()
mindspore_add_pkg
(
flatbuffers
VER 1.11.0
LIBS flatbuffers
...
...
cmake/external_libs/mkl_dnn.cmake
浏览文件 @
1286767d
set
(
onednn_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
onednn_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
mindspore_add_pkg
(
onednn
if
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
mindspore_add_pkg
(
onednn
VER 1.1.1
LIBS dnnl mkldnn
HEAD_ONLY ./
RELEASE on
URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip
MD5 ecaab9ed549643067699c80e5cea1c23
)
else
()
mindspore_add_pkg
(
onednn
VER 1.1.2
LIBS dnnl mkldnn
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF
)
endif
()
include_directories
(
${
onednn_INC
}
)
add_library
(
mindspore::dnnl ALIAS onednn::dnnl
)
add_library
(
mindspore::mkldnn ALIAS onednn::mkldnn
)
cmake/external_libs/opencv.cmake
浏览文件 @
1286767d
...
...
@@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_LDFLAGS
"-Wl"
)
elseif
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Windows"
)
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
else
()
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
mindspore_add_pkg
(
opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
include_directories
(
${
opencv_INC
}
/opencv4
)
add_library
(
mindspore::opencv_core ALIAS opencv::opencv_core
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs
)
add_library
(
mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc
)
if
(
WIN32
)
mindspore_add_pkg
(
opencv
VER 4.2.0
LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a
LIB_PATH x64/mingw/lib
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
else
()
mindspore_add_pkg
(
opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
endif
()
if
(
WIN32
)
include_directories
(
${
opencv_INC
}
)
add_library
(
mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a
)
add_library
(
mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a
)
else
()
include_directories
(
${
opencv_INC
}
/opencv4
)
add_library
(
mindspore::opencv_core ALIAS opencv::opencv_core
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs
)
add_library
(
mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc
)
endif
()
cmake/external_libs/protobuf.cmake
浏览文件 @
1286767d
...
...
@@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
list
(
APPEND
${
c_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
)
list
(
APPEND
${
h_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
)
list
(
APPEND
${
py_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
)
add_custom_command
(
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND perl -pi -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND cp
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
DEPENDS protobuf::protoc
${
abs_file
}
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
if
(
WIN32
)
add_custom_command
(
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND perl -pi.bak -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND
${
CMAKE_COMMAND
}
-E copy
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
DEPENDS protobuf::protoc
${
abs_file
}
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
else
()
add_custom_command
(
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND perl -pi -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND cp
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
DEPENDS protobuf::protoc
${
abs_file
}
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
endif
()
endforeach
()
set_source_files_properties
(
${${
c_var
}}
${${
h_var
}}
${${
py_var
}}
PROPERTIES GENERATED TRUE
)
set
(
${
c_var
}
${${
c_var
}}
PARENT_SCOPE
)
set
(
${
h_var
}
${${
h_var
}}
PARENT_SCOPE
)
...
...
cmake/external_libs/sqlite.cmake
浏览文件 @
1286767d
if
(
WIN32
)
mindspore_add_pkg
(
sqlite-head
VER 3.31.1
HEAD_ONLY ./
URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip
MD5 2b7bfcdd97dc281903a9aee966213fe4
)
include_directories
(
${
sqlite-head_INC
}
)
mindspore_add_pkg
(
sqlite
VER 3.31.1
LIBS sqlite3
LIB_PATH ./
HEAD_ONLY ./
RELEASE ON
URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip
MD5 662c9d2b05467d590ba5c0443e7fd6bd
)
set
(
sqlite_USE_STATIC_LIBS ON
)
set
(
sqlite_CXXFLAGS
)
if
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Darwin"
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2
"
)
else
(
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
endif
(
)
set
(
sqlite_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
mindspore_add_pkg
(
sqlite
else
(
)
set
(
sqlite_USE_STATIC_LIBS ON
)
set
(
sqlite_CXXFLAGS
)
if
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Darwin
"
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
else
(
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
endif
(
)
set
(
sqlite_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
mindspore_add_pkg
(
sqlite
VER 3.31.1
LIBS sqlite3
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
MD5 5f4e7b4016c15f4fb5855615279819da
PATCHES
${
CMAKE_SOURCE_DIR
}
/third_party/patch/sqlite/sqlite.patch001
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1
)
include_directories
(
${
sqlite_INC
}
)
add_library
(
mindspore::sqlite ALIAS sqlite::sqlite3
)
\ No newline at end of file
include_directories
(
${
sqlite_INC
}
)
endif
()
add_library
(
mindspore::sqlite ALIAS sqlite::sqlite3
)
cmake/mind_expression.cmake
浏览文件 @
1286767d
set
(
SECURE_CXX_FLAGS
""
)
if
(
"
${
CMAKE_CXX_COMPILER_ID
}
"
STREQUAL
"GNU"
)
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
if
(
WIN32
)
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all"
)
else
()
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
endif
()
set
(
_ms_tmp_CMAKE_CXX_FLAGS_F
${
CMAKE_CXX_FLAGS
}
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-fvisibility=hidden"
)
...
...
cmake/options.cmake
浏览文件 @
1286767d
...
...
@@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
option
(
ENABLE_AKG
"enable akg"
OFF
)
if
(
CMAKE_CXX_COMPILER_ID STREQUAL
"GNU"
)
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
if
(
WIN32
)
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all"
)
else
()
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
endif
()
if
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
...
...
@@ -106,4 +110,4 @@ endif()
if
(
ENABLE_DUMP_E2E
)
add_compile_definitions
(
ENABLE_DUMP_E2E
)
endif
()
\ No newline at end of file
endif
()
cmake/utils.cmake
浏览文件 @
1286767d
...
...
@@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
endfunction
()
function
(
__find_pkg_then_add_target pkg_name pkg_exe
)
function
(
__find_pkg_then_add_target pkg_name pkg_exe
lib_path
)
unset
(
${
pkg_name
}
_LIBS
)
...
...
@@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
set
(
_LIB_TYPE STATIC
)
endif
()
set
(
${
_LIB_NAME
}
_LIB
${
_LIB_NAME
}
_LIB-NOTFOUND
)
find_library
(
${
_LIB_NAME
}
_LIB
${
_LIB_SEARCH_NAME
}
PATHS
${${
pkg_name
}
_BASE_DIR
}
/lib NO_DEFAULT_PATH
)
find_library
(
${
_LIB_NAME
}
_LIB
${
_LIB_SEARCH_NAME
}
PATHS
${${
pkg_name
}
_BASE_DIR
}
/
${
lib_path
}
NO_DEFAULT_PATH
)
if
(
NOT
${
_LIB_NAME
}
_LIB
)
return
()
endif
()
add_library
(
${
pkg_name
}
::
${
_LIB_NAME
}
${
_LIB_TYPE
}
IMPORTED GLOBAL
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
"
${${
pkg_name
}
_BASE_DIR
}
/include"
IMPORTED_LOCATION
${${
_LIB_NAME
}
_LIB
}
)
if
(
WIN32 AND
${
_LIB_TYPE
}
STREQUAL
"SHARED"
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES IMPORTED_IMPLIB_RELEASE
${${
_LIB_NAME
}
_LIB
}
)
else
()
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES IMPORTED_LOCATION
${${
_LIB_NAME
}
_LIB
}
)
endif
()
if
(
EXISTS
${${
pkg_name
}
_BASE_DIR
}
/include
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
"
${${
pkg_name
}
_BASE_DIR
}
/include"
)
endif
()
list
(
APPEND
${
pkg_name
}
_LIBS
${
pkg_name
}
::
${
_LIB_NAME
}
)
message
(
"found
${${
_LIB_NAME
}
_LIB
}
"
)
STRING
(
REGEX REPLACE
"(.+)/(.+)"
"
\\
1"
LIBPATH
${${
_LIB_NAME
}
_LIB
}
)
...
...
@@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
function
(
mindspore_add_pkg pkg_name
)
set
(
options
)
set
(
oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH
)
set
(
oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH
RELEASE LIB_PATH
)
set
(
multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES
)
cmake_parse_arguments
(
PKG
"
${
options
}
"
"
${
oneValueArgs
}
"
"
${
multiValueArgs
}
"
${
ARGN
}
)
if
(
NOT PKG_CMAKE_PATH
)
set
(
PKG_CMAKE_PATH .
)
if
(
NOT PKG_LIB_PATH
)
set
(
PKG_LIB_PATH lib
)
endif
()
if
(
NOT PKG_EXE
)
set
(
PKG_EXE 0
)
endif
()
set
(
__FIND_PKG_NAME
${
pkg_name
}
)
string
(
TOLOWER
${
pkg_name
}
pkg_name
)
message
(
"pkg name:
${
__FIND_PKG_NAME
}
,
${
pkg_name
}
"
)
...
...
@@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
add_library
(
${
pkg_name
}
INTERFACE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
if
(
${
PKG_RELEASE
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB_PATH
}
${
PKG_LIBS
}
)
endif
()
return
()
endif
()
if
(
NOT PKG_EXE
)
set
(
PKG_EXE 0
)
endif
()
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
)
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
PARENT_SCOPE
)
if
(
PKG_LIBS
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIBS
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB
_PATH
}
${
PKG_LIB
S
}
)
if
(
${
pkg_name
}
_LIBS
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
message
(
"Found libs:
${${
pkg_name
}
_LIBS
}
"
)
...
...
@@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
file
(
GLOB
${
pkg_name
}
_SOURCE_SUBDIRS
${${
pkg_name
}
_SOURCE_DIR
}
/*
)
file
(
COPY
${${
pkg_name
}
_SOURCE_SUBDIRS
}
DESTINATION
${${
pkg_name
}
_BASE_DIR
}
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
add_library
(
${
pkg_name
}
INTERFACE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
if
(
NOT PKG_RELEASE
)
add_library
(
${
pkg_name
}
INTERFACE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
endif
()
elseif
(
PKG_CMAKE_OPTION
)
# in cmake
...
...
@@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
endif
()
if
(
PKG_LIBS
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIBS
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB
_PATH
}
${
PKG_LIB
S
}
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
if
(
NOT
${
pkg_name
}
_LIBS
)
message
(
FATAL_ERROR
"Can not find pkg:
${
pkg_name
}
"
)
...
...
mindspore/ccsrc/CMakeLists.txt
浏览文件 @
1286767d
...
...
@@ -5,6 +5,10 @@ if(ENABLE_CPU)
file
(
GLOB_RECURSE CPU_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
"device/cpu/*.cc"
)
if
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-Wno-attributes -DHAVE_SNPRINTF"
)
add_compile_definitions
(
BUILDING_DLL
)
endif
()
endif
()
if
(
ENABLE_GPU
)
...
...
@@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel/kash/*.cc"
"device/gpu/distribution/collective_init.cc"
)
if
(
ENABLE_CPU
)
list
(
REMOVE_ITEM MINDSPORE_SRC_LIST
"device/gpu/distribution/collective_init.cc"
)
if
(
WIN32
)
list
(
REMOVE_ITEM MINDSPORE_SRC_LIST
"kernel/kernel_query.cc"
)
endif
()
endif
()
if
(
NOT ENABLE_GPU
)
list
(
APPEND MINDSPORE_SRC_LIST
"device/gpu/distribution/collective_fake_init.cc"
)
endif
()
file
(
GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
"pre_activate/mem_reuse/*.cc"
)
...
...
@@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
add_library
(
mindspore_gvar SHARED
${
MS_GVAR_SRC_LIST
}
)
add_library
(
mindspore STATIC
${
MINDSPORE_SRC_LIST
}
)
add_dependencies
(
mindspore GENERATED_OUTPUT_DIR
)
if
(
ENABLE_D
)
list
(
APPEND MINDSPORE_PROTO_LIST
${
MINDSPORE_PROTO_AICPU_LIST
}
)
...
...
@@ -315,19 +328,33 @@ if(ENABLE_D)
endif
()
target_link_libraries
(
mindspore securec
)
target_link_libraries
(
mindspore dl
)
if
(
NOT WIN32
)
target_link_libraries
(
mindspore dl
)
endif
()
target_link_libraries
(
mindspore mindspore::flatbuffers
)
# link protobuf
if
(
ENABLE_D
)
target_link_libraries
(
mindspore mindspore::protobuf
)
endif
()
if
(
WIN32
)
target_link_libraries
(
mindspore
${
PYTHON_LIBRARIES
}
mindspore_gvar
)
endif
()
# set c_expression building
if
(
WIN32
)
set
(
PYTHON_MODULE_SOURCE
${
MS_GVAR_SRC_LIST
}
pipeline/init.cc
kernel/oplib/oplib.cc
${
MINDSPORE_SRC_LIST
}
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
else
()
set
(
PYTHON_MODULE_SOURCE
pipeline/init.cc
kernel/oplib/oplib.cc
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
endif
()
set
(
CMAKE_BUILD_WITH_INSTALL_RPATH TRUE
)
pybind11_add_module
(
_c_expression
${
PYTHON_MODULE_SOURCE
}
)
...
...
@@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
elseif
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
set_target_properties
(
_c_expression PROPERTIES MACOSX_RPATH ON
)
set
(
ORIGIN_PATH @loader_path
)
elseif
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
set
(
ORIGIN_PATH $ORIGIN
)
else
()
MESSAGE
(
FATAL_ERROR
"other platform:
${
CMAKE_SYSTEM_NAME
}
"
)
endif
()
...
...
@@ -346,11 +375,22 @@ endif ()
set
(
ORIGIN_PATH
${
ORIGIN_PATH
}
/lib
)
set_target_properties
(
_c_expression PROPERTIES INSTALL_RPATH
${
ORIGIN_PATH
}
)
target_link_libraries
(
_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
)
if
(
WIN32
)
target_link_libraries
(
_c_expression PRIVATE
mindspore::pybind11_module
securec
proto_input
mindspore::flatbuffers
mindspore::glog
)
else
()
target_link_libraries
(
_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
mindspore::glog
)
endif
()
if
(
ENABLE_GPU
)
execute_process
(
COMMAND bash
${
CMAKE_SOURCE_DIR
}
/third_party/apply_patches.sh
...
...
@@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
add_subdirectory
(
mindrecord
)
add_subdirectory
(
dataset
)
endif
()
if
(
USE_GLOG
)
target_link_libraries
(
_c_expression PRIVATE mindspore::glog
)
endif
()
\ No newline at end of file
mindspore/ccsrc/dataset/CMakeLists.txt
浏览文件 @
1286767d
...
...
@@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
######################################################################
################# Link with external libraries ########################
target_link_libraries
(
_c_dataengine PRIVATE mindspore mindspore_gvar
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf
${
SECUREC_LIBRARY
}
)
if
(
WIN32
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module
${
PYTHON_LIBRARIES
}
mindspore::protobuf
${
SECUREC_LIBRARY
}
)
else
()
target_link_libraries
(
_c_dataengine PRIVATE mindspore mindspore_gvar
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf
${
SECUREC_LIBRARY
}
)
endif
()
target_link_libraries
(
_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc
)
if
(
ENABLE_GPUQUE
)
target_link_libraries
(
_c_dataengine PRIVATE gpu_queue
${
CUDNN_PATH
}
/lib64/libcudnn.so
${
CUDA_PATH
}
/lib64/libcudart.so
${
CUDA_PATH
}
/lib64/libcudart.so
${
CUDA_PATH
}
/lib64/stubs/libcuda.so
)
endif
()
...
...
@@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
endif
()
add_dependencies
(
_c_dataengine _c_mindrecord
)
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
)
if
(
WIN32
)
set
(
MINDRECORD_LINK_OBJECT
${
CMAKE_BINARY_DIR
}
/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a
)
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
${
MINDRECORD_LINK_OBJECT
}
mindspore::sqlite
)
else
()
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
)
endif
()
if
(
USE_GLOG
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::glog
)
...
...
mindspore/ccsrc/dataset/api/python_bindings.cc
浏览文件 @
1286767d
...
...
@@ -19,7 +19,9 @@
#include "dataset/kernels/no_op.h"
#include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/image/center_crop_op.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "dataset/kernels/image/change_mode_op.h"
#endif
#include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h"
...
...
@@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
py
::
arg
(
"fillG"
)
=
RandomCropOp
::
kDefFillG
,
py
::
arg
(
"fillB"
)
=
RandomCropOp
::
kDefFillB
);
(
void
)
py
::
class_
<
HwcToChwOp
,
TensorOp
,
std
::
shared_ptr
<
HwcToChwOp
>>
(
*
m
,
"ChannelSwapOp"
).
def
(
py
::
init
<>
());
#if !defined(_WIN32) && !defined(_WIN64)
(
void
)
py
::
class_
<
ChangeModeOp
,
TensorOp
,
std
::
shared_ptr
<
ChangeModeOp
>>
(
*
m
,
"ChangeModeOp"
,
"Tensor operation to change colors from BGR to RGB"
)
.
def
(
py
::
init
<>
());
#endif
(
void
)
py
::
class_
<
OneHotOp
,
TensorOp
,
std
::
shared_ptr
<
OneHotOp
>>
(
*
m
,
"OneHotOp"
,
"Tensor operation to apply one hot encoding. Takes number of classes."
)
...
...
mindspore/ccsrc/dataset/core/tensor.h
浏览文件 @
1286767d
...
...
@@ -22,6 +22,10 @@
#include <vector>
#include "./securec.h"
#include "utils/log_adapter.h"
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
...
...
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
浏览文件 @
1286767d
...
...
@@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
if
(
!
reshuffle_each_epoch_
)
{
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
}
else
{
#if defined(_WIN32) || defined(_WIN64)
std
::
random_device
random_device
;
#else
std
::
random_device
random_device
(
"/dev/urandom"
);
#endif
std
::
uniform_int_distribution
<
int32_t
>
distribution
(
0
,
std
::
numeric_limits
<
int32_t
>::
max
());
shuffle_seed_
=
distribution
(
random_device
);
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
...
...
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
浏览文件 @
1286767d
add_library
(
kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
if
(
WIN32
)
add_library
(
kernels-image OBJECT
center_crop_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
else
()
add_library
(
kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
endif
()
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
浏览文件 @
1286767d
...
...
@@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(
*
cinfo
->
mem
->
alloc_small
)(
reinterpret_cast
<
j_common_ptr
>
(
cinfo
),
JPOOL_PERMANENT
,
sizeof
(
struct
jpeg_source_mgr
)));
cinfo
->
src
->
init_source
=
JpegInitSource
;
cinfo
->
src
->
fill_input_buffer
=
JpegFillInputBuffer
;
#if defined(_WIN32) || defined(_WIN64)
cinfo
->
src
->
skip_input_data
=
reinterpret_cast
<
void
(
*
)(
j_decompress_ptr
,
long
)
>
(
JpegSkipInputData
);
#else
cinfo
->
src
->
skip_input_data
=
JpegSkipInputData
;
#endif
cinfo
->
src
->
resync_to_restart
=
jpeg_resync_to_restart
;
cinfo
->
src
->
term_source
=
JpegTermSource
;
cinfo
->
src
->
bytes_in_buffer
=
datasize
;
...
...
mindspore/ccsrc/dataset/kernels/image/image_utils.h
浏览文件 @
1286767d
...
...
@@ -22,6 +22,10 @@
#include <random>
#include <string>
#include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h"
#include "./jerror.h"
#include <opencv2/imgproc/imgproc.hpp>
...
...
mindspore/ccsrc/dataset/util/path.cc
浏览文件 @
1286767d
...
...
@@ -27,7 +27,7 @@
namespace
mindspore
{
namespace
dataset
{
#ifdef _WIN32
char
Path
::
_separator
=
'\\'
;
char
Path
::
separator_
=
'\\'
;
#else
char
Path
::
separator_
=
'/'
;
#endif
...
...
@@ -129,7 +129,11 @@ bool Path::IsDirectory() {
Status
Path
::
CreateDirectory
()
{
if
(
!
Exists
())
{
#if defined(_WIN32) || defined(_WIN64)
int
rc
=
mkdir
(
common
::
SafeCStr
(
path_
));
#else
int
rc
=
mkdir
(
common
::
SafeCStr
(
path_
),
0700
);
#endif
if
(
rc
)
{
std
::
ostringstream
oss
;
oss
<<
"Unable to create directory "
<<
path_
<<
". Errno = "
<<
errno
;
...
...
mindspore/ccsrc/dataset/util/random.cc
浏览文件 @
1286767d
...
...
@@ -32,7 +32,11 @@ namespace dataset {
uint32_t
GetSeed
()
{
uint32_t
seed
=
GlobalContext
::
config_manager
()
->
seed
();
if
(
seed
==
std
::
mt19937
::
default_seed
)
{
#if defined(_WIN32) || defined(_WIN64)
std
::
random_device
random_device
;
#else
std
::
random_device
random_device
(
"/dev/urandom"
);
#endif
std
::
uniform_int_distribution
<
uint32_t
>
distribution
(
0
,
std
::
numeric_limits
<
uint32_t
>::
max
());
seed
=
distribution
(
random_device
);
}
...
...
mindspore/ccsrc/dataset/util/services.cc
浏览文件 @
1286767d
...
...
@@ -16,7 +16,9 @@
#include "dataset/util/services.h"
#include <limits.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/syscall.h>
#endif
#include <unistd.h>
#include <random>
#include "dataset/util/circular_pool.h"
...
...
@@ -28,6 +30,7 @@ namespace dataset {
std
::
unique_ptr
<
Services
>
Services
::
instance_
=
nullptr
;
std
::
once_flag
Services
::
init_instance_flag_
;
#if !defined(_WIN32) && !defined(_WIN64)
std
::
string
Services
::
GetUserName
()
{
char
user
[
LOGIN_NAME_MAX
];
(
void
)
getlogin_r
(
user
,
sizeof
(
user
));
...
...
@@ -41,10 +44,15 @@ std::string Services::GetHostName() {
}
int
Services
::
GetLWP
()
{
return
syscall
(
SYS_gettid
);
}
#endif
std
::
string
Services
::
GetUniqueID
()
{
const
std
::
string
kStr
=
"abcdefghijklmnopqrstuvwxyz0123456789"
;
#if defined(_WIN32) || defined(_WIN64)
std
::
mt19937
gen
{
std
::
random_device
{}()};
#else
std
::
mt19937
gen
{
std
::
random_device
{
"/dev/urandom"
}()};
#endif
std
::
uniform_int_distribution
<>
dist
(
0
,
kStr
.
size
()
-
1
);
char
buffer
[
UNIQUEID_LEN
];
for
(
int
i
=
0
;
i
<
UNIQUEID_LEN
;
i
++
)
{
...
...
mindspore/ccsrc/dataset/util/services.h
浏览文件 @
1286767d
...
...
@@ -62,11 +62,13 @@ class Services {
std
::
shared_ptr
<
MemoryPool
>
GetServiceMemPool
()
{
return
pool_
;
}
#if !defined(_WIN32) && !defined(_WIN64)
static
std
::
string
GetUserName
();
static
std
::
string
GetHostName
();
static
int
GetLWP
();
#endif
static
std
::
string
GetUniqueID
();
...
...
mindspore/ccsrc/dataset/util/sig_handler.cc
浏览文件 @
1286767d
...
...
@@ -16,13 +16,16 @@
#include "dataset/util/sig_handler.h"
#include <signal.h>
#include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <ucontext.h>
#endif
#include <unistd.h>
#include "dataset/util/task_manager.h"
namespace
mindspore
{
namespace
dataset
{
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
void
RegisterHandlers
()
{
struct
sigaction
new_int_action
;
...
...
@@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
// Wake up the watchdog which is designed as async-signal-safe.
TaskManager
::
WakeUpWatchDog
();
}
#endif
}
// namespace dataset
}
// namespace mindspore
mindspore/ccsrc/dataset/util/sig_handler.h
浏览文件 @
1286767d
...
...
@@ -22,12 +22,14 @@
namespace
mindspore
{
namespace
dataset
{
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
extern
void
RegisterHandlers
();
// A signal handler for SIGINT. Drives interrupt to watchdog
extern
void
IntHandler
(
int
sig_num
,
// The signal that was raised
siginfo_t
*
sig_info
,
// The siginfo structure.
void
*
context
);
// context info
#endif
}
// namespace dataset
}
// namespace mindspore
...
...
mindspore/ccsrc/debug/anf_ir_dump.cc
浏览文件 @
1286767d
...
...
@@ -14,6 +14,9 @@
* limitations under the License.
*/
#include "debug/anf_ir_dump.h"
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <fstream>
#include <map>
#include <memory>
...
...
@@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
return
;
}
char
real_path
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
real_path
,
filename
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
filename
<<
" does not exit."
;
}
#else
if
(
nullptr
==
realpath
(
filename
.
c_str
(),
real_path
))
{
MS_LOG
(
DEBUG
)
<<
"Dir "
<<
filename
<<
" does not exit."
;
}
#endif
OrderedMap
<
AnfNodePtr
,
int32_t
>
para_map
;
std
::
string
path_string
=
real_path
;
...
...
mindspore/ccsrc/debug/anf_ir_utils.cc
浏览文件 @
1286767d
...
...
@@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
if
(
path_ptr
!=
nullptr
)
{
path
=
path_ptr
;
char
real_path
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
path
.
size
()
>
PATH_MAX
||
_fullpath
(
real_path
,
path
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
EXCEPTION
)
<<
"MS IR Path error, "
<<
path_ptr
;
}
#else
if
(
path
.
size
()
>
PATH_MAX
||
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
MS_LOG
(
EXCEPTION
)
<<
"MS IR path error, "
<<
path_ptr
;
}
#endif
path
=
real_path
;
}
return
path
;
...
...
@@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
return
;
}
char
real_path
[
PATH_MAX
]
=
{
0
};
if
(
nullptr
==
realpath
(
file_path
.
c_str
(),
real_path
))
{
MS_LOG
(
DEBUG
)
<<
"Dir "
<<
file_path
<<
" does not exit."
;
char
*
real_path_ret
=
nullptr
;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret
=
_fullpath
(
real_path
,
file_path
.
c_str
(),
PATH_MAX
);
#else
real_path_ret
=
realpath
(
file_path
.
c_str
(),
real_path
);
#endif
if
(
nullptr
==
real_path_ret
)
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
file_path
<<
" does not exit."
;
}
else
{
std
::
string
path_string
=
real_path
;
if
(
chmod
(
common
::
SafeCStr
(
path_string
),
S_IRUSR
|
S_IWUSR
)
==
-
1
)
{
...
...
mindspore/ccsrc/debug/info.cc
浏览文件 @
1286767d
...
...
@@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
}
char
path
[
PATH_MAX
+
1
]
=
{
0x00
};
#if defined(_WIN32) || defined(_WIN64)
if
(
file_name_
.
size
()
>
PATH_MAX
||
_fullpath
(
path
,
file_name_
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
return
debug_info_ss
.
str
();
}
#else
if
(
file_name_
.
size
()
>
PATH_MAX
||
realpath
(
file_name_
.
c_str
(),
path
)
==
nullptr
)
{
return
debug_info_ss
.
str
();
}
#endif
auto
src_path
=
std
::
string
(
path
);
std
::
ifstream
file
(
src_path
);
if
(
!
file
.
is_open
())
{
...
...
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
浏览文件 @
1286767d
...
...
@@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
void
*
CPUResourceManager
::
MemMalloc
(
size_t
mem_size
)
{
void
*
ptr
=
malloc
(
mem_size
);
if
(
ptr
!=
nullptr
)
{
memset_s
(
ptr
,
mem_size
,
0
,
mem_size
);
dynamic_mem_
[
ptr
]
=
mem_size
;
return
ptr
;
}
else
{
...
...
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc
0 → 100644
浏览文件 @
1286767d
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "device/gpu/distribution/collective_fake_init.h"
#include "utils/log_adapter.h"
namespace
mindspore
{
namespace
device
{
namespace
gpu
{
void
CollectiveFakeInitializer
::
InitCollective
()
{
MS_LOG
(
EXCEPTION
)
<<
"build without enable gpu!"
;
}
void
CollectiveFakeInitializer
::
FinalizeCollective
()
{
MS_LOG
(
EXCEPTION
)
<<
"build without enable gpu!"
;
}
}
// namespace gpu
}
// namespace device
}
// namespace mindspore
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h
0 → 100644
浏览文件 @
1286767d
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
namespace
mindspore
{
namespace
device
{
namespace
gpu
{
class
CollectiveFakeInitializer
{
public:
CollectiveFakeInitializer
()
=
default
;
~
CollectiveFakeInitializer
()
=
default
;
CollectiveFakeInitializer
(
CollectiveFakeInitializer
const
&
)
=
delete
;
CollectiveFakeInitializer
&
operator
=
(
const
CollectiveFakeInitializer
&
)
=
delete
;
static
void
InitCollective
();
static
void
FinalizeCollective
();
};
}
// namespace gpu
}
// namespace device
}
// namespace mindspore
#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
mindspore/ccsrc/device/kernel_runtime.cc
浏览文件 @
1286767d
...
...
@@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
bool
ret
=
false
;
auto
context_ptr
=
MsContext
::
GetInstance
();
MS_EXCEPTION_IF_NULL
(
context_ptr
);
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
bool
is_task_sink
=
context_ptr
->
enable_task_sink
();
if
(
is_task_sink
)
{
ret
=
RunTask
(
graph
);
}
else
{
ret
=
LaunchKernel
(
graph
);
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Call MS Run Success in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Call MS Run Success in "
<<
cost
<<
" us"
;
#endif
return
ret
;
}
...
...
@@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
AddressPtrList
kernel_workspaces
;
AddressPtrList
kernel_outputs
;
GenLaunchArgs
(
*
kernel_mod
,
kernel
,
&
kernel_inputs
,
&
kernel_workspaces
,
&
kernel_outputs
);
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
auto
ret
=
kernel_mod
->
Launch
(
kernel_inputs
,
kernel_workspaces
,
kernel_outputs
,
reinterpret_cast
<
uintptr_t
>
(
stream_
));
if
(
!
ret
)
{
...
...
@@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
if
(
AnfAlgo
::
GetKernelType
(
kernel
)
==
TBE_KERNEL
&&
!
SyncStream
())
{
MS_LOG
(
EXCEPTION
)
<<
"SyncStream failed."
;
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
DEBUG
)
<<
"d "
<<
kernel
->
fullname_with_scope
()
<<
" in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
DEBUG
)
<<
"d "
<<
kernel
->
fullname_with_scope
()
<<
" in "
<<
cost
<<
" us"
;
#endif
}
}
return
true
;
...
...
mindspore/ccsrc/kernel/common_utils.cc
浏览文件 @
1286767d
...
...
@@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
bool
KernelMeta
::
ReadIndex
(
const
std
::
string
&
bin_dir
)
{
DIR
*
dir
=
opendir
(
bin_dir
.
c_str
());
if
(
dir
==
nullptr
)
{
#if defined(_WIN32) || defined(_WIN64)
auto
ret
=
mkdir
(
bin_dir
.
c_str
());
#else
auto
ret
=
mkdir
(
bin_dir
.
c_str
(),
S_IRWXG
|
S_IRWXU
);
#endif
if
(
ret
!=
0
)
{
MS_LOG
(
INFO
)
<<
"kernel dir not exist["
<<
bin_dir
<<
"]."
;
return
false
;
...
...
@@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
}
filewrite
<<
info
<<
std
::
endl
;
filewrite
.
close
();
#if defined(_WIN32) || defined(_WIN64)
if
(
nullptr
==
_fullpath
(
real_path
,
path
.
c_str
(),
PATH_MAX
))
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
path
<<
" does not exit."
;
return
;
}
#else
if
(
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
path
<<
" does not exit."
;
return
;
}
#endif
MS_LOG
(
INFO
)
<<
"real path is :"
<<
real_path
;
if
(
chmod
(
real_path
,
S_IRUSR
)
==
-
1
)
{
MS_LOG
(
DEBUG
)
<<
"modify file:"
<<
real_path
<<
" to read only fail."
;
...
...
mindspore/ccsrc/mindrecord/CMakeLists.txt
浏览文件 @
1286767d
...
...
@@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
)
# add link library
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite
${
PYTHON_LIB
}
${
SECUREC_LIBRARY
}
mindspore mindspore_gvar mindspore::protobuf
)
if
(
WIN32
)
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf
)
else
()
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite
${
PYTHON_LIB
}
${
SECUREC_LIBRARY
}
mindspore mindspore_gvar mindspore::protobuf
)
endif
()
if
(
USE_GLOG
)
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::glog
)
...
...
mindspore/ccsrc/mindrecord/common/shard_utils.cc
浏览文件 @
1286767d
...
...
@@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
return
{
FAILED
,
""
};
}
char
tmp
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
tmp
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
}
if
(
_fullpath
(
real_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
common
::
SafeCStr
(
path
)
<<
"check successfully"
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
...
...
@@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
}
#endif
std
::
string
s
=
real_path
;
char
sep
=
'/'
;
size_t
i
=
s
.
rfind
(
sep
,
s
.
length
());
...
...
@@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
return
{
FAILED
,
""
};
}
char
tmp
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
tmp
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
}
if
(
_fullpath
(
real_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
common
::
SafeCStr
(
path
)
<<
"check successfully"
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
...
...
@@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
}
#endif
std
::
string
s
=
real_path
;
if
(
s
.
rfind
(
'/'
)
+
1
<=
s
.
size
())
{
return
{
SUCCESS
,
s
.
substr
(
0
,
s
.
rfind
(
'/'
)
+
1
)};
...
...
@@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
}
std
::
pair
<
MSRStatus
,
uint64_t
>
GetDiskSize
(
const
std
::
string
&
str_dir
,
const
DiskSizeType
&
disk_type
)
{
#if defined(_WIN32) || defined(_WIN64)
return
{
SUCCESS
,
100
};
#else
uint64_t
ll_count
=
0
;
struct
statfs
disk_info
;
if
(
statfs
(
common
::
SafeCStr
(
str_dir
),
&
disk_info
)
==
-
1
)
{
...
...
@@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
}
return
{
SUCCESS
,
ll_count
};
#endif
}
uint32_t
GetMaxThreadNum
()
{
...
...
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
浏览文件 @
1286767d
...
...
@@ -21,8 +21,10 @@
#include <limits.h>
#include <stdlib.h>
#include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/statfs.h>
#include <sys/wait.h>
#endif
#include <unistd.h>
#include <cassert>
#include <cmath>
...
...
mindspore/ccsrc/mindrecord/include/shard_reader.h
浏览文件 @
1286767d
...
...
@@ -19,7 +19,9 @@
#include <dirent.h>
#include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/prctl.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
...
...
mindspore/ccsrc/mindrecord/io/shard_reader.cc
浏览文件 @
1286767d
...
...
@@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus
ShardReader
::
ConsumerByRow
(
int
consumer_id
)
{
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
#endif
// Loop forever
for
(;;)
{
...
...
@@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
MSRStatus
ShardReader
::
ConsumerByBlock
(
int
consumer_id
)
{
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
#endif
// Loop forever
for
(;;)
{
...
...
mindspore/ccsrc/mindrecord/io/shard_writer.cc
浏览文件 @
1286767d
...
...
@@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
MS_LOG
(
ERROR
)
<<
"Securec func failed"
;
return
FAILED
;
}
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
resolved_path
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path"
;
return
FAILED
;
}
if
(
_fullpath
(
resolved_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path "
<<
resolved_path
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
resolved_path
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path"
;
return
FAILED
;
...
...
@@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
if
(
realpath
(
common
::
SafeCStr
(
path
),
resolved_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path "
<<
resolved_path
;
}
#endif
file_paths_
.
emplace_back
(
string
(
resolved_path
));
}
...
...
mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
浏览文件 @
1286767d
...
...
@@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
(
root
->
has_flag
(
ALLREDUCE_FUSION_RUN_ONCE_ONLY
)))
{
return
changes
;
}
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
MS_LOG
(
INFO
)
<<
"Now entering allreduce fusion"
;
DumpGraph
(
root
,
std
::
string
(
ALLREDUCE_FUSION_BEGIN
));
...
...
@@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
// allreduce fusion only run once
root
->
flags
()[
ALLREDUCE_FUSION_RUN_ONCE_ONLY
]
=
true
;
res
->
results
()[
pipeline
::
kStepParallelGraph
]
=
root
;
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Now leaving allreduce fusion, used time: "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
uint64_t
time
=
1000000
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
time
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Now leaving allreduce fusion, used time: "
<<
time
<<
" us"
;
#endif
return
changes
;
}
}
// namespace parallel
...
...
mindspore/ccsrc/parallel/step_auto_parallel.cc
浏览文件 @
1286767d
...
...
@@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case
kNumberTypeInt
:
return
sizeof
(
int
);
case
kNumberTypeUInt
:
return
sizeof
(
uint
);
return
sizeof
(
u
nsigned
int
);
case
kNumberTypeFloat
:
return
sizeof
(
float
);
default:
...
...
mindspore/ccsrc/pipeline/action.cc
浏览文件 @
1286767d
...
...
@@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
auto
bc_ptr
=
res
->
results
()[
kBackend
].
cast
<
compile
::
BackendPtr
>
();
std
::
vector
<
PrimitivePtr
>
cut_list
=
compile
::
nonlinear_ops
;
if
(
bc_ptr
->
name
()
==
kMsConvert
)
{
cut_list
=
compile
::
ms_nonlinear_ops
;
cut_list
=
compile
::
GetMsNonlinearOps
()
;
}
std
::
shared_ptr
<
CompileGraphs
>
compile
=
std
::
make_shared
<
CompileGraphs
>
(
bc_ptr
,
cut_list
);
res
->
results
()[
kOutput
]
=
compile
->
CompileAndLink
(
func_graph
);
...
...
mindspore/ccsrc/pipeline/init.cc
浏览文件 @
1286767d
...
...
@@ -29,8 +29,11 @@
#include "parallel/context.h"
#include "parallel/device_manager.h"
#include "parallel/costmodel_context.h"
#ifdef ENABLE_GPUQUE
#include "device/gpu/distribution/collective_init.h"
#else
#include "device/gpu/distribution/collective_fake_init.h"
#endif
namespace
py
=
pybind11
;
using
FuncGraph
=
mindspore
::
FuncGraph
;
...
...
@@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
(
void
)
py
::
class_
<
OpLib
,
std
::
shared_ptr
<
OpLib
>>
(
m
,
"Oplib"
)
.
def
(
py
::
init
())
.
def
(
"reg_op"
,
&
OpLib
::
RegOp
,
"Register op info."
);
#ifdef ENABLE_GPUQUE
(
void
)
m
.
def
(
"init_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
InitCollective
,
"Init gpu collective communication mode."
);
(
void
)
m
.
def
(
"finalize_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
FinalizeCollective
,
"Finalize gpu collective communication mode."
);
#else
(
void
)
m
.
def
(
"init_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveFakeInitializer
::
InitCollective
,
"Init gpu collective communication mode."
);
(
void
)
m
.
def
(
"finalize_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveFakeInitializer
::
FinalizeCollective
,
"Finalize gpu collective communication mode."
);
#endif
}
mindspore/ccsrc/pre_activate/common/pass_manager.cc
浏览文件 @
1286767d
...
...
@@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
size_t
num
=
0
;
for
(
const
auto
&
pass
:
passes
)
{
if
(
pass
!=
nullptr
)
{
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
{};
struct
timeval
end_time
{};
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
if
(
pass
->
Run
(
func_graph
))
{
changed
=
true
;
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Run pass hwopt_"
+
name
()
+
"_"
<<
num
<<
"_"
+
pass
->
name
()
+
" in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Run pass hwopt_"
+
name
()
+
"_"
<<
num
<<
"_"
+
pass
->
name
()
+
" in "
<<
cost
<<
" us"
;
#endif
if
(
save_graphs
)
{
auto
dump_file_path
=
save_graphs_path
+
"/"
+
"hwopt_"
+
name
()
+
"_"
+
std
::
to_string
(
num
)
+
"_"
+
pass
->
name
()
+
".ir"
;
...
...
mindspore/ccsrc/utils/log_adapter.cc
浏览文件 @
1286767d
...
...
@@ -26,12 +26,19 @@ namespace mindspore {
#ifdef USE_GLOG
static
std
::
string
GetTime
()
{
#define BUFLEN 80
static
char
buf
[
BUFLEN
];
#if defined(_WIN32) || defined(_WIN64)
time_t
time_seconds
=
time
(
0
);
struct
tm
now_time
;
localtime_s
(
&
now_time
,
&
time_seconds
);
sprintf_s
(
buf
,
BUFLEN
,
"%d-%d-%d %d:%d:%d"
,
now_time
.
tm_year
+
1900
,
now_time
.
tm_mon
+
1
,
now_time
.
tm_mday
,
now_time
.
tm_hour
,
now_time
.
tm_min
,
now_time
.
tm_sec
);
#else
struct
timeval
cur_time
;
(
void
)
gettimeofday
(
&
cur_time
,
NULL
);
struct
tm
now
;
(
void
)
localtime_r
(
&
cur_time
.
tv_sec
,
&
now
);
static
char
buf
[
BUFLEN
];
(
void
)
strftime
(
buf
,
BUFLEN
,
"%Y-%m-%d-%H:%M:%S"
,
&
now
);
// format date and time
// set micro-second
buf
[
27
]
=
'\0'
;
...
...
@@ -44,6 +51,7 @@ static std::string GetTime() {
buf
[
idx
--
]
=
'.'
;
}
}
#endif
return
std
::
string
(
buf
);
}
...
...
mindspore/ccsrc/vm/transform.cc
浏览文件 @
1286767d
...
...
@@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
,
prim
::
kPrimMakeTuple
};
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
};
const
std
::
vector
<
PrimitivePtr
>&
GetMsNonlinearOps
()
{
static
const
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
};
return
ms_nonlinear_ops
;
}
CompileGraph
::
CompileGraph
(
const
BackendPtr
&
backend
,
const
std
::
vector
<
PrimitivePtr
>&
cut_list
)
:
backend_
(
backend
),
cut_list_
(
cut_list
)
{
...
...
mindspore/ccsrc/vm/transform.h
浏览文件 @
1286767d
...
...
@@ -42,7 +42,7 @@ extern const char kGeVm[];
// A sub namespace in ME to support compile related definition.
namespace
compile
{
extern
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
;
extern
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
;
const
std
::
vector
<
PrimitivePtr
>&
GetMsNonlinearOps
()
;
using
VmEvalFunc
=
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>
;
using
VmEvalFuncPtr
=
std
::
shared_ptr
<
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>>
;
...
...
mindspore/log.py
浏览文件 @
1286767d
...
...
@@ -19,11 +19,13 @@ import sys
import
os
import
stat
import
time
import
fcntl
import
logging
from
logging.handlers
import
RotatingFileHandler
import
traceback
import
threading
import
platform
if
platform
.
system
()
!=
"Windows"
:
import
fcntl
__all__
=
[
'get_level'
,
'get_log_config'
]
...
...
@@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
# Attain an exclusive lock with bloking mode by `fcntl` module.
with
open
(
self
.
baseFilename
,
'a'
)
as
file_pointer
:
fcntl
.
lockf
(
file_pointer
.
fileno
(),
fcntl
.
LOCK_EX
)
if
platform
.
system
()
!=
"Windows"
:
fcntl
.
lockf
(
file_pointer
.
fileno
(),
fcntl
.
LOCK_EX
)
if
self
.
backupCount
>
0
:
self
.
rolling_rename
()
...
...
third_party/securec/CMakeLists.txt
浏览文件 @
1286767d
SET
(
CMAKE_BUILD_TYPE
"Debug"
)
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='"
)
if
(
WIN32
)
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer"
)
else
()
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='"
)
endif
()
SET
(
CMAKE_C_FLAGS_RELEASE
"$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations"
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
#add flags
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include -Werror"
)
if
(
WIN32
)
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include"
)
else
()
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include -Werror"
)
endif
()
include_directories
(
./include
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录