Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
magicwindyyd
mindspore
提交
1286767d
M
mindspore
项目概览
magicwindyyd
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
1286767d
编写于
4月 15, 2020
作者:
C
chenjianping
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
support building on windows
上级
d90e1215
变更
48
隐藏空白更改
内联
并排
Showing
48 changed file
with
619 addition
and
133 deletion
+619
-133
CMakeLists.txt
CMakeLists.txt
+13
-0
build.bat
build.bat
+36
-0
cmake/external_libs/flatbuffers.cmake
cmake/external_libs/flatbuffers.cmake
+3
-0
cmake/external_libs/mkl_dnn.cmake
cmake/external_libs/mkl_dnn.cmake
+12
-1
cmake/external_libs/opencv.cmake
cmake/external_libs/opencv.cmake
+63
-26
cmake/external_libs/protobuf.cmake
cmake/external_libs/protobuf.cmake
+29
-15
cmake/external_libs/sqlite.cmake
cmake/external_libs/sqlite.cmake
+29
-12
cmake/mind_expression.cmake
cmake/mind_expression.cmake
+5
-1
cmake/options.cmake
cmake/options.cmake
+6
-2
cmake/utils.cmake
cmake/utils.cmake
+33
-17
mindspore/ccsrc/CMakeLists.txt
mindspore/ccsrc/CMakeLists.txt
+49
-13
mindspore/ccsrc/dataset/CMakeLists.txt
mindspore/ccsrc/dataset/CMakeLists.txt
+14
-4
mindspore/ccsrc/dataset/api/python_bindings.cc
mindspore/ccsrc/dataset/api/python_bindings.cc
+4
-0
mindspore/ccsrc/dataset/core/tensor.h
mindspore/ccsrc/dataset/core/tensor.h
+4
-0
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
+4
-0
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
+46
-22
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
+4
-0
mindspore/ccsrc/dataset/kernels/image/image_utils.h
mindspore/ccsrc/dataset/kernels/image/image_utils.h
+4
-0
mindspore/ccsrc/dataset/util/path.cc
mindspore/ccsrc/dataset/util/path.cc
+5
-1
mindspore/ccsrc/dataset/util/random.cc
mindspore/ccsrc/dataset/util/random.cc
+4
-0
mindspore/ccsrc/dataset/util/services.cc
mindspore/ccsrc/dataset/util/services.cc
+8
-0
mindspore/ccsrc/dataset/util/services.h
mindspore/ccsrc/dataset/util/services.h
+2
-0
mindspore/ccsrc/dataset/util/sig_handler.cc
mindspore/ccsrc/dataset/util/sig_handler.cc
+4
-0
mindspore/ccsrc/dataset/util/sig_handler.h
mindspore/ccsrc/dataset/util/sig_handler.h
+2
-0
mindspore/ccsrc/debug/anf_ir_dump.cc
mindspore/ccsrc/debug/anf_ir_dump.cc
+9
-0
mindspore/ccsrc/debug/anf_ir_utils.cc
mindspore/ccsrc/debug/anf_ir_utils.cc
+14
-2
mindspore/ccsrc/debug/info.cc
mindspore/ccsrc/debug/info.cc
+6
-1
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
+1
-0
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc
...ore/ccsrc/device/gpu/distribution/collective_fake_init.cc
+28
-0
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h
...pore/ccsrc/device/gpu/distribution/collective_fake_init.h
+37
-0
mindspore/ccsrc/device/kernel_runtime.cc
mindspore/ccsrc/device/kernel_runtime.cc
+20
-0
mindspore/ccsrc/kernel/common_utils.cc
mindspore/ccsrc/kernel/common_utils.cc
+11
-0
mindspore/ccsrc/mindrecord/CMakeLists.txt
mindspore/ccsrc/mindrecord/CMakeLists.txt
+5
-1
mindspore/ccsrc/mindrecord/common/shard_utils.cc
mindspore/ccsrc/mindrecord/common/shard_utils.cc
+24
-0
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
+2
-0
mindspore/ccsrc/mindrecord/include/shard_reader.h
mindspore/ccsrc/mindrecord/include/shard_reader.h
+2
-0
mindspore/ccsrc/mindrecord/io/shard_reader.cc
mindspore/ccsrc/mindrecord/io/shard_reader.cc
+4
-0
mindspore/ccsrc/mindrecord/io/shard_writer.cc
mindspore/ccsrc/mindrecord/io/shard_writer.cc
+10
-0
mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
.../ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
+10
-3
mindspore/ccsrc/parallel/step_auto_parallel.cc
mindspore/ccsrc/parallel/step_auto_parallel.cc
+1
-1
mindspore/ccsrc/pipeline/action.cc
mindspore/ccsrc/pipeline/action.cc
+1
-1
mindspore/ccsrc/pipeline/init.cc
mindspore/ccsrc/pipeline/init.cc
+12
-2
mindspore/ccsrc/pre_activate/common/pass_manager.cc
mindspore/ccsrc/pre_activate/common/pass_manager.cc
+10
-0
mindspore/ccsrc/utils/log_adapter.cc
mindspore/ccsrc/utils/log_adapter.cc
+9
-1
mindspore/ccsrc/vm/transform.cc
mindspore/ccsrc/vm/transform.cc
+4
-2
mindspore/ccsrc/vm/transform.h
mindspore/ccsrc/vm/transform.h
+1
-1
mindspore/log.py
mindspore/log.py
+5
-2
third_party/securec/CMakeLists.txt
third_party/securec/CMakeLists.txt
+10
-2
未找到文件。
CMakeLists.txt
浏览文件 @
1286767d
...
@@ -12,6 +12,7 @@ else()
...
@@ -12,6 +12,7 @@ else()
endif
()
endif
()
set
(
CMAKE_CXX_FLAGS_DEBUG
"$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp"
)
set
(
CMAKE_CXX_FLAGS_DEBUG
"$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)=' -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-I/usr/local/include -std=c++17 -Werror -Wall -Wno-deprecated-declarations -fPIC"
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
...
@@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
...
@@ -29,6 +30,18 @@ find_package(Python3 3.7 COMPONENTS Interpreter Development)
if
(
Python3_FOUND
)
if
(
Python3_FOUND
)
set
(
PYTHON_INCLUDE_DIRS
"
${
Python3_INCLUDE_DIRS
}
"
)
set
(
PYTHON_INCLUDE_DIRS
"
${
Python3_INCLUDE_DIRS
}
"
)
set
(
PYTHON_LIBRARIES
"
${
Python3_LIBRARIES
}
"
)
set
(
PYTHON_LIBRARIES
"
${
Python3_LIBRARIES
}
"
)
if
(
WIN32
)
if
(
Python3_DIR
)
message
(
"Python3_DIR set already: "
${
Python3_DIR
}
)
else
()
string
(
LENGTH
${
PYTHON_LIBRARIES
}
PYTHON_LIBRARIES_LEN
)
string
(
LENGTH
"libpythonxx.a"
Python3_NAME_LEN
)
math
(
EXPR Python3_DIR_LEN
${
PYTHON_LIBRARIES_LEN
}
-
${
Python3_NAME_LEN
}
)
string
(
SUBSTRING
${
Python3_LIBRARIES
}
0
${
Python3_DIR_LEN
}
Python3_DIR
)
message
(
"Python3_DIR: "
${
Python3_DIR
}
)
endif
()
link_directories
(
${
Python3_DIR
}
)
endif
()
else
()
else
()
find_python_package
(
py_inc py_lib
)
find_python_package
(
py_inc py_lib
)
set
(
PYTHON_INCLUDE_DIRS
"
${
py_inc
}
"
)
set
(
PYTHON_INCLUDE_DIRS
"
${
py_inc
}
"
)
...
...
build.bat
0 → 100644
浏览文件 @
1286767d
@echo
off
@title
mindspore_build
SET
BASEPATH
=
%CD%
IF
NOT
EXIST
%BASEPATH%
/build
(
md
"build"
)
cd
%BASEPATH%
/build
SET
BUILD_PATH
=
%CD%
IF
NOT
EXIST
%BUILD_PATH%
/mindspore
(
md
"mindspore"
)
cd
%CD%
/mindspore
cmake
-DCMAKE
_BUILD_TYPE
=
Release
-DENABLE
_CPU
=
ON
-DENABLE
_MINDDATA
=
ON
-DUSE
_GLOG
=
ON
-G
"CodeBlocks - MinGW Makefiles"
../..
IF
NOT
%errorlevel%
==
0
(
goto
run_fail
)
cmake
--build
.
--target
all
--
-j
6
IF
NOT
%errorlevel%
==
0
(
goto
run_fail
)
cd
%BASEPATH%
goto
run_eof
:run
_fail
cd
%BASEPATH%
echo
"build fail."
:run
_eof
cmake/external_libs/flatbuffers.cmake
浏览文件 @
1286767d
set
(
flatbuffers_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
flatbuffers_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
flatbuffers_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
flatbuffers_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
if
(
WIN32
)
set
(
flatbuffers_USE_STATIC_LIBS ON
)
endif
()
mindspore_add_pkg
(
flatbuffers
mindspore_add_pkg
(
flatbuffers
VER 1.11.0
VER 1.11.0
LIBS flatbuffers
LIBS flatbuffers
...
...
cmake/external_libs/mkl_dnn.cmake
浏览文件 @
1286767d
set
(
onednn_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
onednn_CXXFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
onednn_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
set
(
onednn_CFLAGS
"-D_FORTIFY_SOURCE=2 -O2"
)
mindspore_add_pkg
(
onednn
if
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
mindspore_add_pkg
(
onednn
VER 1.1.1
LIBS dnnl mkldnn
HEAD_ONLY ./
RELEASE on
URL https://github.com/oneapi-src/oneDNN/releases/download/v1.1.1/dnnl_win_1.1.1_cpu_vcomp.zip
MD5 ecaab9ed549643067699c80e5cea1c23
)
else
()
mindspore_add_pkg
(
onednn
VER 1.1.2
VER 1.1.2
LIBS dnnl mkldnn
LIBS dnnl mkldnn
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
URL https://github.com/oneapi-src/oneDNN/archive/v1.1.2.tar.gz
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
MD5 ab40d52230f3ad1d7a6f06ce0f6bc17a
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF
)
CMAKE_OPTION -DDNNL_ARCH_OPT_FLAGS='' -DDNNL_CPU_RUNTIME='SEQ' -DDNNL_BUILD_EXAMPLES=OFF -DDNNL_BUILD_TESTS=OFF
)
endif
()
include_directories
(
${
onednn_INC
}
)
include_directories
(
${
onednn_INC
}
)
add_library
(
mindspore::dnnl ALIAS onednn::dnnl
)
add_library
(
mindspore::dnnl ALIAS onednn::dnnl
)
add_library
(
mindspore::mkldnn ALIAS onednn::mkldnn
)
add_library
(
mindspore::mkldnn ALIAS onednn::mkldnn
)
cmake/external_libs/opencv.cmake
浏览文件 @
1286767d
...
@@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
...
@@ -2,35 +2,72 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_LDFLAGS
"-Wl"
)
set
(
opencv_LDFLAGS
"-Wl"
)
elseif
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Windows"
)
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
else
()
else
()
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CXXFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2"
)
set
(
opencv_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
set
(
opencv_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
endif
()
mindspore_add_pkg
(
opencv
if
(
WIN32
)
VER 4.2.0
mindspore_add_pkg
(
opencv
LIBS opencv_core opencv_imgcodecs opencv_imgproc
VER 4.2.0
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
LIBS libopencv_core420.dll.a libopencv_imgcodecs420.dll.a libopencv_imgproc420.dll.a
MD5 e8cb208ce2723481408b604b480183b6
LIB_PATH x64/mingw/lib
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
-DBUILD_ZLIB=ON
MD5 e8cb208ce2723481408b604b480183b6
-DBUILD_JPEG=ON
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_PNG=ON
-DBUILD_ZLIB=ON
-DBUILD_OPENEXR=ON
-DBUILD_JPEG=ON
-DBUILD_TESTS=OFF
-DBUILD_PNG=ON
-DBUILD_PERF_TESTS=OFF
-DBUILD_OPENEXR=ON
-DBUILD_opencv_apps=OFF
-DBUILD_TESTS=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_python3=OFF
-DBUILD_opencv_apps=OFF
-DWITH_FFMPEG=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DWITH_TIFF=ON
-DBUILD_opencv_python3=OFF
-DBUILD_TIFF=OFF
-DWITH_FFMPEG=OFF
-DWITH_JASPER=OFF
-DWITH_TIFF=ON
-DBUILD_JASPER=OFF
-DBUILD_TIFF=OFF
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
-DWITH_JASPER=OFF
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
-DBUILD_JASPER=OFF
include_directories
(
${
opencv_INC
}
/opencv4
)
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
add_library
(
mindspore::opencv_core ALIAS opencv::opencv_core
)
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs
)
else
()
add_library
(
mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc
)
mindspore_add_pkg
(
opencv
VER 4.2.0
LIBS opencv_core opencv_imgcodecs opencv_imgproc
URL https://github.com/opencv/opencv/archive/4.2.0.tar.gz
MD5 e8cb208ce2723481408b604b480183b6
CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DWITH_PROTOBUF=OFF -DWITH_WEBP=OFF -DWITH_IPP=OFF -DWITH_ADE=OFF
-DBUILD_ZLIB=ON
-DBUILD_JPEG=ON
-DBUILD_PNG=ON
-DBUILD_OPENEXR=ON
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DBUILD_opencv_apps=OFF
-DCMAKE_SKIP_RPATH=TRUE
-DBUILD_opencv_python3=OFF
-DWITH_FFMPEG=OFF
-DWITH_TIFF=ON
-DBUILD_TIFF=OFF
-DWITH_JASPER=OFF
-DBUILD_JASPER=OFF
-DTIFF_INCLUDE_DIR=
${
tiff_INC
}
-DTIFF_LIBRARY=
${
tiff_LIB
}
)
endif
()
if
(
WIN32
)
include_directories
(
${
opencv_INC
}
)
add_library
(
mindspore::opencv_core ALIAS opencv::libopencv_core420.dll.a
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::libopencv_imgcodecs420.dll.a
)
add_library
(
mindspore::opencv_imgproc ALIAS opencv::libopencv_imgproc420.dll.a
)
else
()
include_directories
(
${
opencv_INC
}
/opencv4
)
add_library
(
mindspore::opencv_core ALIAS opencv::opencv_core
)
add_library
(
mindspore::opencv_imgcodecs ALIAS opencv::opencv_imgcodecs
)
add_library
(
mindspore::opencv_imgproc ALIAS opencv::opencv_imgproc
)
endif
()
cmake/external_libs/protobuf.cmake
浏览文件 @
1286767d
...
@@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
...
@@ -77,22 +77,36 @@ function(ms_protobuf_generate_py c_var h_var py_var)
list
(
APPEND
${
c_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
)
list
(
APPEND
${
c_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
)
list
(
APPEND
${
h_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
)
list
(
APPEND
${
h_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
)
list
(
APPEND
${
py_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
)
list
(
APPEND
${
py_var
}
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
)
if
(
WIN32
)
add_custom_command
(
add_custom_command
(
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND perl -pi -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND perl -pi.bak -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND cp
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
COMMAND
${
CMAKE_COMMAND
}
-E copy
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
DEPENDS protobuf::protoc
${
abs_file
}
DEPENDS protobuf::protoc
${
abs_file
}
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
else
()
add_custom_command
(
OUTPUT
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.cc"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
.pb.h"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
WORKING_DIRECTORY
${
PROJECT_SOURCE_DIR
}
COMMAND
${
CMAKE_COMMAND
}
-E make_directory
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
"
COMMAND protobuf::protoc -I
${
file_dir
}
--cpp_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND protobuf::protoc -I
${
file_dir
}
--python_out=
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
${
abs_file
}
COMMAND perl -pi -e
"s/import (.+_pb2.*)/from . import
\\
1/"
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
COMMAND cp
"
${
CMAKE_BINARY_DIR
}
/
${
rel_path
}
/
${
file_name
}
_pb2.py"
"
${
PROJECT_SOURCE_DIR
}
/mindspore/train/"
DEPENDS protobuf::protoc
${
abs_file
}
COMMENT
"Running C++ protocol buffer compiler on
${
file
}
"
VERBATIM
)
endif
()
endforeach
()
endforeach
()
set_source_files_properties
(
${${
c_var
}}
${${
h_var
}}
${${
py_var
}}
PROPERTIES GENERATED TRUE
)
set_source_files_properties
(
${${
c_var
}}
${${
h_var
}}
${${
py_var
}}
PROPERTIES GENERATED TRUE
)
set
(
${
c_var
}
${${
c_var
}}
PARENT_SCOPE
)
set
(
${
c_var
}
${${
c_var
}}
PARENT_SCOPE
)
set
(
${
h_var
}
${${
h_var
}}
PARENT_SCOPE
)
set
(
${
h_var
}
${${
h_var
}}
PARENT_SCOPE
)
...
...
cmake/external_libs/sqlite.cmake
浏览文件 @
1286767d
if
(
WIN32
)
mindspore_add_pkg
(
sqlite-head
VER 3.31.1
HEAD_ONLY ./
URL https://sqlite.org/2020/sqlite-amalgamation-3310100.zip
MD5 2b7bfcdd97dc281903a9aee966213fe4
)
include_directories
(
${
sqlite-head_INC
}
)
mindspore_add_pkg
(
sqlite
VER 3.31.1
LIBS sqlite3
LIB_PATH ./
HEAD_ONLY ./
RELEASE ON
URL https://sqlite.org/2020/sqlite-dll-win64-x64-3310100.zip
MD5 662c9d2b05467d590ba5c0443e7fd6bd
)
set
(
sqlite_USE_STATIC_LIBS ON
)
else
(
)
set
(
sqlite_CXXFLAGS
)
set
(
sqlite_USE_STATIC_LIBS ON
)
if
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Darwin"
)
set
(
sqlite_CXXFLAGS
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2
"
)
if
(
${
CMAKE_SYSTEM_NAME
}
MATCHES
"Darwin
"
)
else
(
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
else
(
)
endif
(
)
set
(
sqlite_CFLAGS
"-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -D_FORTIFY_SOURCE=2 -O2"
)
set
(
sqlite_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
(
)
set
(
sqlite_LDFLAGS
"-Wl,-z,relro,-z,now,-z,noexecstack"
)
mindspore_add_pkg
(
sqlite
mindspore_add_pkg
(
sqlite
VER 3.31.1
VER 3.31.1
LIBS sqlite3
LIBS sqlite3
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
URL https://github.com/sqlite/sqlite/archive/version-3.31.1.tar.gz
MD5 5f4e7b4016c15f4fb5855615279819da
MD5 5f4e7b4016c15f4fb5855615279819da
PATCHES
${
CMAKE_SOURCE_DIR
}
/third_party/patch/sqlite/sqlite.patch001
PATCHES
${
CMAKE_SOURCE_DIR
}
/third_party/patch/sqlite/sqlite.patch001
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1
)
CONFIGURE_COMMAND ./configure --enable-shared=no --disable-tcl --disable-editline --enable-json1
)
include_directories
(
${
sqlite_INC
}
)
include_directories
(
${
sqlite_INC
}
)
add_library
(
mindspore::sqlite ALIAS sqlite::sqlite3
)
endif
()
\ No newline at end of file
add_library
(
mindspore::sqlite ALIAS sqlite::sqlite3
)
cmake/mind_expression.cmake
浏览文件 @
1286767d
set
(
SECURE_CXX_FLAGS
""
)
set
(
SECURE_CXX_FLAGS
""
)
if
(
"
${
CMAKE_CXX_COMPILER_ID
}
"
STREQUAL
"GNU"
)
if
(
"
${
CMAKE_CXX_COMPILER_ID
}
"
STREQUAL
"GNU"
)
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
if
(
WIN32
)
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all"
)
else
()
set
(
SECURE_CXX_FLAGS
"-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
endif
()
endif
()
set
(
_ms_tmp_CMAKE_CXX_FLAGS_F
${
CMAKE_CXX_FLAGS
}
)
set
(
_ms_tmp_CMAKE_CXX_FLAGS_F
${
CMAKE_CXX_FLAGS
}
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-fvisibility=hidden"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-fvisibility=hidden"
)
...
...
cmake/options.cmake
浏览文件 @
1286767d
...
@@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
...
@@ -19,7 +19,11 @@ option(ENABLE_MPI "enable mpi" OFF)
option
(
ENABLE_AKG
"enable akg"
OFF
)
option
(
ENABLE_AKG
"enable akg"
OFF
)
if
(
CMAKE_CXX_COMPILER_ID STREQUAL
"GNU"
)
if
(
CMAKE_CXX_COMPILER_ID STREQUAL
"GNU"
)
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
if
(
WIN32
)
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all"
)
else
()
set
(
OPTION_CXX_FLAGS
"
${
OPTION_CXX_FLAGS
}
-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack"
)
endif
()
endif
()
endif
()
if
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
if
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
...
@@ -106,4 +110,4 @@ endif()
...
@@ -106,4 +110,4 @@ endif()
if
(
ENABLE_DUMP_E2E
)
if
(
ENABLE_DUMP_E2E
)
add_compile_definitions
(
ENABLE_DUMP_E2E
)
add_compile_definitions
(
ENABLE_DUMP_E2E
)
endif
()
endif
()
\ No newline at end of file
cmake/utils.cmake
浏览文件 @
1286767d
...
@@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
...
@@ -103,7 +103,7 @@ function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)
endfunction
()
endfunction
()
function
(
__find_pkg_then_add_target pkg_name pkg_exe
)
function
(
__find_pkg_then_add_target pkg_name pkg_exe
lib_path
)
unset
(
${
pkg_name
}
_LIBS
)
unset
(
${
pkg_name
}
_LIBS
)
...
@@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
...
@@ -129,15 +129,24 @@ function(__find_pkg_then_add_target pkg_name pkg_exe)
set
(
_LIB_TYPE STATIC
)
set
(
_LIB_TYPE STATIC
)
endif
()
endif
()
set
(
${
_LIB_NAME
}
_LIB
${
_LIB_NAME
}
_LIB-NOTFOUND
)
set
(
${
_LIB_NAME
}
_LIB
${
_LIB_NAME
}
_LIB-NOTFOUND
)
find_library
(
${
_LIB_NAME
}
_LIB
${
_LIB_SEARCH_NAME
}
PATHS
${${
pkg_name
}
_BASE_DIR
}
/lib NO_DEFAULT_PATH
)
find_library
(
${
_LIB_NAME
}
_LIB
${
_LIB_SEARCH_NAME
}
PATHS
${${
pkg_name
}
_BASE_DIR
}
/
${
lib_path
}
NO_DEFAULT_PATH
)
if
(
NOT
${
_LIB_NAME
}
_LIB
)
if
(
NOT
${
_LIB_NAME
}
_LIB
)
return
()
return
()
endif
()
endif
()
add_library
(
${
pkg_name
}
::
${
_LIB_NAME
}
${
_LIB_TYPE
}
IMPORTED GLOBAL
)
add_library
(
${
pkg_name
}
::
${
_LIB_NAME
}
${
_LIB_TYPE
}
IMPORTED GLOBAL
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES
if
(
WIN32 AND
${
_LIB_TYPE
}
STREQUAL
"SHARED"
)
INTERFACE_INCLUDE_DIRECTORIES
"
${${
pkg_name
}
_BASE_DIR
}
/include"
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES IMPORTED_IMPLIB_RELEASE
${${
_LIB_NAME
}
_LIB
}
)
IMPORTED_LOCATION
${${
_LIB_NAME
}
_LIB
}
else
()
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES IMPORTED_LOCATION
${${
_LIB_NAME
}
_LIB
}
)
endif
()
if
(
EXISTS
${${
pkg_name
}
_BASE_DIR
}
/include
)
set_target_properties
(
${
pkg_name
}
::
${
_LIB_NAME
}
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES
"
${${
pkg_name
}
_BASE_DIR
}
/include"
)
endif
()
list
(
APPEND
${
pkg_name
}
_LIBS
${
pkg_name
}
::
${
_LIB_NAME
}
)
list
(
APPEND
${
pkg_name
}
_LIBS
${
pkg_name
}
::
${
_LIB_NAME
}
)
message
(
"found
${${
_LIB_NAME
}
_LIB
}
"
)
message
(
"found
${${
_LIB_NAME
}
_LIB
}
"
)
STRING
(
REGEX REPLACE
"(.+)/(.+)"
"
\\
1"
LIBPATH
${${
_LIB_NAME
}
_LIB
}
)
STRING
(
REGEX REPLACE
"(.+)/(.+)"
"
\\
1"
LIBPATH
${${
_LIB_NAME
}
_LIB
}
)
...
@@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
...
@@ -192,12 +201,18 @@ set(MS_FIND_NO_DEFAULT_PATH ${MS_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)
function
(
mindspore_add_pkg pkg_name
)
function
(
mindspore_add_pkg pkg_name
)
set
(
options
)
set
(
options
)
set
(
oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH
)
set
(
oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH
RELEASE LIB_PATH
)
set
(
multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES
)
set
(
multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES
)
cmake_parse_arguments
(
PKG
"
${
options
}
"
"
${
oneValueArgs
}
"
"
${
multiValueArgs
}
"
${
ARGN
}
)
cmake_parse_arguments
(
PKG
"
${
options
}
"
"
${
oneValueArgs
}
"
"
${
multiValueArgs
}
"
${
ARGN
}
)
if
(
NOT PKG_CMAKE_PATH
)
set
(
PKG_CMAKE_PATH .
)
if
(
NOT PKG_LIB_PATH
)
set
(
PKG_LIB_PATH lib
)
endif
()
endif
()
if
(
NOT PKG_EXE
)
set
(
PKG_EXE 0
)
endif
()
set
(
__FIND_PKG_NAME
${
pkg_name
}
)
set
(
__FIND_PKG_NAME
${
pkg_name
}
)
string
(
TOLOWER
${
pkg_name
}
pkg_name
)
string
(
TOLOWER
${
pkg_name
}
pkg_name
)
message
(
"pkg name:
${
__FIND_PKG_NAME
}
,
${
pkg_name
}
"
)
message
(
"pkg name:
${
__FIND_PKG_NAME
}
,
${
pkg_name
}
"
)
...
@@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
...
@@ -225,18 +240,17 @@ function(mindspore_add_pkg pkg_name )
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
add_library
(
${
pkg_name
}
INTERFACE
)
add_library
(
${
pkg_name
}
INTERFACE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
if
(
${
PKG_RELEASE
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB_PATH
}
${
PKG_LIBS
}
)
endif
()
return
()
return
()
endif
()
endif
()
if
(
NOT PKG_EXE
)
set
(
PKG_EXE 0
)
endif
()
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
)
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
)
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
PARENT_SCOPE
)
set
(
${
__FIND_PKG_NAME
}
_ROOT
${${
pkg_name
}
_BASE_DIR
}
PARENT_SCOPE
)
if
(
PKG_LIBS
)
if
(
PKG_LIBS
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIBS
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB
_PATH
}
${
PKG_LIB
S
}
)
if
(
${
pkg_name
}
_LIBS
)
if
(
${
pkg_name
}
_LIBS
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
message
(
"Found libs:
${${
pkg_name
}
_LIBS
}
"
)
message
(
"Found libs:
${${
pkg_name
}
_LIBS
}
"
)
...
@@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
...
@@ -283,8 +297,10 @@ function(mindspore_add_pkg pkg_name )
file
(
GLOB
${
pkg_name
}
_SOURCE_SUBDIRS
${${
pkg_name
}
_SOURCE_DIR
}
/*
)
file
(
GLOB
${
pkg_name
}
_SOURCE_SUBDIRS
${${
pkg_name
}
_SOURCE_DIR
}
/*
)
file
(
COPY
${${
pkg_name
}
_SOURCE_SUBDIRS
}
DESTINATION
${${
pkg_name
}
_BASE_DIR
}
)
file
(
COPY
${${
pkg_name
}
_SOURCE_SUBDIRS
}
DESTINATION
${${
pkg_name
}
_BASE_DIR
}
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/
${
PKG_HEAD_ONLY
}
PARENT_SCOPE
)
add_library
(
${
pkg_name
}
INTERFACE
)
if
(
NOT PKG_RELEASE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
add_library
(
${
pkg_name
}
INTERFACE
)
target_include_directories
(
${
pkg_name
}
INTERFACE
${${
pkg_name
}
_INC
}
)
endif
()
elseif
(
PKG_CMAKE_OPTION
)
elseif
(
PKG_CMAKE_OPTION
)
# in cmake
# in cmake
...
@@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
...
@@ -355,7 +371,7 @@ function(mindspore_add_pkg pkg_name )
endif
()
endif
()
if
(
PKG_LIBS
)
if
(
PKG_LIBS
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIBS
}
)
__find_pkg_then_add_target
(
${
pkg_name
}
${
PKG_EXE
}
${
PKG_LIB
_PATH
}
${
PKG_LIB
S
}
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
set
(
${
pkg_name
}
_INC
${${
pkg_name
}
_BASE_DIR
}
/include PARENT_SCOPE
)
if
(
NOT
${
pkg_name
}
_LIBS
)
if
(
NOT
${
pkg_name
}
_LIBS
)
message
(
FATAL_ERROR
"Can not find pkg:
${
pkg_name
}
"
)
message
(
FATAL_ERROR
"Can not find pkg:
${
pkg_name
}
"
)
...
...
mindspore/ccsrc/CMakeLists.txt
浏览文件 @
1286767d
...
@@ -5,6 +5,10 @@ if(ENABLE_CPU)
...
@@ -5,6 +5,10 @@ if(ENABLE_CPU)
file
(
GLOB_RECURSE CPU_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
file
(
GLOB_RECURSE CPU_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
"device/cpu/*.cc"
"device/cpu/*.cc"
)
)
if
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-Wno-attributes -DHAVE_SNPRINTF"
)
add_compile_definitions
(
BUILDING_DLL
)
endif
()
endif
()
endif
()
if
(
ENABLE_GPU
)
if
(
ENABLE_GPU
)
...
@@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
...
@@ -150,7 +154,15 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"kernel/kash/*.cc"
"kernel/kash/*.cc"
"device/gpu/distribution/collective_init.cc"
"device/gpu/distribution/collective_init.cc"
)
)
if
(
ENABLE_CPU
)
list
(
REMOVE_ITEM MINDSPORE_SRC_LIST
"device/gpu/distribution/collective_init.cc"
)
if
(
WIN32
)
list
(
REMOVE_ITEM MINDSPORE_SRC_LIST
"kernel/kernel_query.cc"
)
endif
()
endif
()
if
(
NOT ENABLE_GPU
)
list
(
APPEND MINDSPORE_SRC_LIST
"device/gpu/distribution/collective_fake_init.cc"
)
endif
()
file
(
GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
file
(
GLOB_RECURSE MEM_REUSE_SRC_LIST RELATIVE
${
CMAKE_CURRENT_SOURCE_DIR
}
"pre_activate/mem_reuse/*.cc"
"pre_activate/mem_reuse/*.cc"
)
)
...
@@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
...
@@ -248,6 +260,7 @@ file(GLOB_RECURSE MS_GVAR_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
add_library
(
mindspore_gvar SHARED
${
MS_GVAR_SRC_LIST
}
)
add_library
(
mindspore_gvar SHARED
${
MS_GVAR_SRC_LIST
}
)
add_library
(
mindspore STATIC
${
MINDSPORE_SRC_LIST
}
)
add_library
(
mindspore STATIC
${
MINDSPORE_SRC_LIST
}
)
add_dependencies
(
mindspore GENERATED_OUTPUT_DIR
)
if
(
ENABLE_D
)
if
(
ENABLE_D
)
list
(
APPEND MINDSPORE_PROTO_LIST
${
MINDSPORE_PROTO_AICPU_LIST
}
)
list
(
APPEND MINDSPORE_PROTO_LIST
${
MINDSPORE_PROTO_AICPU_LIST
}
)
...
@@ -315,19 +328,33 @@ if(ENABLE_D)
...
@@ -315,19 +328,33 @@ if(ENABLE_D)
endif
()
endif
()
target_link_libraries
(
mindspore securec
)
target_link_libraries
(
mindspore securec
)
target_link_libraries
(
mindspore dl
)
if
(
NOT WIN32
)
target_link_libraries
(
mindspore dl
)
endif
()
target_link_libraries
(
mindspore mindspore::flatbuffers
)
target_link_libraries
(
mindspore mindspore::flatbuffers
)
# link protobuf
# link protobuf
if
(
ENABLE_D
)
if
(
ENABLE_D
)
target_link_libraries
(
mindspore mindspore::protobuf
)
target_link_libraries
(
mindspore mindspore::protobuf
)
endif
()
endif
()
if
(
WIN32
)
target_link_libraries
(
mindspore
${
PYTHON_LIBRARIES
}
mindspore_gvar
)
endif
()
# set c_expression building
# set c_expression building
if
(
WIN32
)
set
(
PYTHON_MODULE_SOURCE
${
MS_GVAR_SRC_LIST
}
pipeline/init.cc
kernel/oplib/oplib.cc
${
MINDSPORE_SRC_LIST
}
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
else
()
set
(
PYTHON_MODULE_SOURCE
set
(
PYTHON_MODULE_SOURCE
pipeline/init.cc
pipeline/init.cc
kernel/oplib/oplib.cc
kernel/oplib/oplib.cc
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
MS_STEPS_SRC_LIST
}
${
MS_CCE_SRC_LIST
}
${
MS_AICPU_SRC_LIST
}
${
MS_TASKINFO_LIST
}
${
MS_RT_SRC_LIST
}
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
${
GPU_NCCL_LIST
}
${
MS_HCCL_SRC_LIST
}
${
MS_PREDICT_SRC_LIST
}
${
CPU_SRC_LIST
}
${
MEM_REUSE_SRC_LIST
}
${
GPU_KERNEL_SRC_LIST
}
)
endif
()
set
(
CMAKE_BUILD_WITH_INSTALL_RPATH TRUE
)
set
(
CMAKE_BUILD_WITH_INSTALL_RPATH TRUE
)
pybind11_add_module
(
_c_expression
${
PYTHON_MODULE_SOURCE
}
)
pybind11_add_module
(
_c_expression
${
PYTHON_MODULE_SOURCE
}
)
...
@@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
...
@@ -339,6 +366,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
elseif
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
elseif
(
CMAKE_SYSTEM_NAME MATCHES
"Darwin"
)
set_target_properties
(
_c_expression PROPERTIES MACOSX_RPATH ON
)
set_target_properties
(
_c_expression PROPERTIES MACOSX_RPATH ON
)
set
(
ORIGIN_PATH @loader_path
)
set
(
ORIGIN_PATH @loader_path
)
elseif
(
CMAKE_SYSTEM_NAME MATCHES
"Windows"
)
set
(
ORIGIN_PATH $ORIGIN
)
else
()
else
()
MESSAGE
(
FATAL_ERROR
"other platform:
${
CMAKE_SYSTEM_NAME
}
"
)
MESSAGE
(
FATAL_ERROR
"other platform:
${
CMAKE_SYSTEM_NAME
}
"
)
endif
()
endif
()
...
@@ -346,11 +375,22 @@ endif ()
...
@@ -346,11 +375,22 @@ endif ()
set
(
ORIGIN_PATH
${
ORIGIN_PATH
}
/lib
)
set
(
ORIGIN_PATH
${
ORIGIN_PATH
}
/lib
)
set_target_properties
(
_c_expression PROPERTIES INSTALL_RPATH
${
ORIGIN_PATH
}
)
set_target_properties
(
_c_expression PROPERTIES INSTALL_RPATH
${
ORIGIN_PATH
}
)
target_link_libraries
(
_c_expression PRIVATE
if
(
WIN32
)
mindspore::pybind11_module
target_link_libraries
(
_c_expression PRIVATE
mindspore
mindspore::pybind11_module
mindspore_gvar
securec
)
proto_input
mindspore::flatbuffers
mindspore::glog
)
else
()
target_link_libraries
(
_c_expression PRIVATE
mindspore::pybind11_module
mindspore
mindspore_gvar
mindspore::glog
)
endif
()
if
(
ENABLE_GPU
)
if
(
ENABLE_GPU
)
execute_process
(
COMMAND bash
${
CMAKE_SOURCE_DIR
}
/third_party/apply_patches.sh
execute_process
(
COMMAND bash
${
CMAKE_SOURCE_DIR
}
/third_party/apply_patches.sh
...
@@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
...
@@ -493,7 +533,3 @@ if(ENABLE_MINDDATA)
add_subdirectory
(
mindrecord
)
add_subdirectory
(
mindrecord
)
add_subdirectory
(
dataset
)
add_subdirectory
(
dataset
)
endif
()
endif
()
if
(
USE_GLOG
)
target_link_libraries
(
_c_expression PRIVATE mindspore::glog
)
endif
()
\ No newline at end of file
mindspore/ccsrc/dataset/CMakeLists.txt
浏览文件 @
1286767d
...
@@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
...
@@ -80,14 +80,19 @@ set_target_properties(_c_dataengine PROPERTIES
######################################################################
######################################################################
################# Link with external libraries ########################
################# Link with external libraries ########################
target_link_libraries
(
_c_dataengine PRIVATE mindspore mindspore_gvar
)
if
(
WIN32
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf
${
SECUREC_LIBRARY
}
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module
${
PYTHON_LIBRARIES
}
mindspore::protobuf
${
SECUREC_LIBRARY
}
)
else
()
target_link_libraries
(
_c_dataengine PRIVATE mindspore mindspore_gvar
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::pybind11_module -ldl mindspore::protobuf
${
SECUREC_LIBRARY
}
)
endif
()
target_link_libraries
(
_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
target_link_libraries
(
_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::opencv_core mindspore::opencv_imgcodecs
mindspore::opencv_imgproc
)
mindspore::opencv_imgproc
)
if
(
ENABLE_GPUQUE
)
if
(
ENABLE_GPUQUE
)
target_link_libraries
(
_c_dataengine PRIVATE gpu_queue
target_link_libraries
(
_c_dataengine PRIVATE gpu_queue
${
CUDNN_PATH
}
/lib64/libcudnn.so
${
CUDNN_PATH
}
/lib64/libcudnn.so
${
CUDA_PATH
}
/lib64/libcudart.so
${
CUDA_PATH
}
/lib64/libcudart.so
${
CUDA_PATH
}
/lib64/stubs/libcuda.so
)
${
CUDA_PATH
}
/lib64/stubs/libcuda.so
)
endif
()
endif
()
...
@@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
...
@@ -96,7 +101,12 @@ if (ENABLE_TDTQUE)
endif
()
endif
()
add_dependencies
(
_c_dataengine _c_mindrecord
)
add_dependencies
(
_c_dataengine _c_mindrecord
)
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
)
if
(
WIN32
)
set
(
MINDRECORD_LINK_OBJECT
${
CMAKE_BINARY_DIR
}
/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a
)
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
${
MINDRECORD_LINK_OBJECT
}
mindspore::sqlite
)
else
()
target_link_libraries
(
_c_dataengine PRIVATE _c_mindrecord
)
endif
()
if
(
USE_GLOG
)
if
(
USE_GLOG
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::glog
)
target_link_libraries
(
_c_dataengine PRIVATE mindspore::glog
)
...
...
mindspore/ccsrc/dataset/api/python_bindings.cc
浏览文件 @
1286767d
...
@@ -19,7 +19,9 @@
...
@@ -19,7 +19,9 @@
#include "dataset/kernels/no_op.h"
#include "dataset/kernels/no_op.h"
#include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/data/one_hot_op.h"
#include "dataset/kernels/image/center_crop_op.h"
#include "dataset/kernels/image/center_crop_op.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "dataset/kernels/image/change_mode_op.h"
#include "dataset/kernels/image/change_mode_op.h"
#endif
#include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/cut_out_op.h"
#include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/decode_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h"
#include "dataset/kernels/image/distort_bounding_box_crop_op.h"
...
@@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
...
@@ -279,9 +281,11 @@ void bindTensorOps2(py::module *m) {
py
::
arg
(
"fillG"
)
=
RandomCropOp
::
kDefFillG
,
py
::
arg
(
"fillB"
)
=
RandomCropOp
::
kDefFillB
);
py
::
arg
(
"fillG"
)
=
RandomCropOp
::
kDefFillG
,
py
::
arg
(
"fillB"
)
=
RandomCropOp
::
kDefFillB
);
(
void
)
py
::
class_
<
HwcToChwOp
,
TensorOp
,
std
::
shared_ptr
<
HwcToChwOp
>>
(
*
m
,
"ChannelSwapOp"
).
def
(
py
::
init
<>
());
(
void
)
py
::
class_
<
HwcToChwOp
,
TensorOp
,
std
::
shared_ptr
<
HwcToChwOp
>>
(
*
m
,
"ChannelSwapOp"
).
def
(
py
::
init
<>
());
#if !defined(_WIN32) && !defined(_WIN64)
(
void
)
py
::
class_
<
ChangeModeOp
,
TensorOp
,
std
::
shared_ptr
<
ChangeModeOp
>>
(
(
void
)
py
::
class_
<
ChangeModeOp
,
TensorOp
,
std
::
shared_ptr
<
ChangeModeOp
>>
(
*
m
,
"ChangeModeOp"
,
"Tensor operation to change colors from BGR to RGB"
)
*
m
,
"ChangeModeOp"
,
"Tensor operation to change colors from BGR to RGB"
)
.
def
(
py
::
init
<>
());
.
def
(
py
::
init
<>
());
#endif
(
void
)
py
::
class_
<
OneHotOp
,
TensorOp
,
std
::
shared_ptr
<
OneHotOp
>>
(
(
void
)
py
::
class_
<
OneHotOp
,
TensorOp
,
std
::
shared_ptr
<
OneHotOp
>>
(
*
m
,
"OneHotOp"
,
"Tensor operation to apply one hot encoding. Takes number of classes."
)
*
m
,
"OneHotOp"
,
"Tensor operation to apply one hot encoding. Takes number of classes."
)
...
...
mindspore/ccsrc/dataset/core/tensor.h
浏览文件 @
1286767d
...
@@ -22,6 +22,10 @@
...
@@ -22,6 +22,10 @@
#include <vector>
#include <vector>
#include "./securec.h"
#include "./securec.h"
#include "utils/log_adapter.h"
#include "utils/log_adapter.h"
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "pybind11/numpy.h"
#include "pybind11/numpy.h"
#include "pybind11/pybind11.h"
#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
#include "pybind11/stl.h"
...
...
mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc
浏览文件 @
1286767d
...
@@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
...
@@ -85,7 +85,11 @@ Status ShuffleOp::SelfReset() {
if
(
!
reshuffle_each_epoch_
)
{
if
(
!
reshuffle_each_epoch_
)
{
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
}
else
{
}
else
{
#if defined(_WIN32) || defined(_WIN64)
std
::
random_device
random_device
;
#else
std
::
random_device
random_device
(
"/dev/urandom"
);
std
::
random_device
random_device
(
"/dev/urandom"
);
#endif
std
::
uniform_int_distribution
<
int32_t
>
distribution
(
0
,
std
::
numeric_limits
<
int32_t
>::
max
());
std
::
uniform_int_distribution
<
int32_t
>
distribution
(
0
,
std
::
numeric_limits
<
int32_t
>::
max
());
shuffle_seed_
=
distribution
(
random_device
);
shuffle_seed_
=
distribution
(
random_device
);
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
rng_
=
std
::
mt19937_64
(
shuffle_seed_
);
...
...
mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt
浏览文件 @
1286767d
add_library
(
kernels-image OBJECT
if
(
WIN32
)
center_crop_op.cc
add_library
(
kernels-image OBJECT
change_mode_op.cc
center_crop_op.cc
cut_out_op.cc
cut_out_op.cc
decode_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
hwc_to_chw_op.cc
image_utils.cc
image_utils.cc
normalize_op.cc
normalize_op.cc
pad_op.cc
pad_op.cc
random_color_adjust_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_resize_op.cc
random_rotation_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
random_vertical_flip_op.cc
rescale_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_bilinear_op.cc
resize_op.cc
resize_op.cc
)
)
else
()
add_library
(
kernels-image OBJECT
center_crop_op.cc
change_mode_op.cc
cut_out_op.cc
decode_op.cc
distort_bounding_box_crop_op.cc
hwc_to_chw_op.cc
image_utils.cc
normalize_op.cc
pad_op.cc
random_color_adjust_op.cc
random_crop_decode_resize_op.cc
random_crop_and_resize_op.cc
random_crop_op.cc
random_horizontal_flip_op.cc
random_resize_op.cc
random_rotation_op.cc
random_vertical_flip_op.cc
rescale_op.cc
resize_bilinear_op.cc
resize_op.cc
)
endif
()
mindspore/ccsrc/dataset/kernels/image/image_utils.cc
浏览文件 @
1286767d
...
@@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
...
@@ -186,7 +186,11 @@ void JpegSetSource(j_decompress_ptr cinfo, const void *data, int64_t datasize) {
(
*
cinfo
->
mem
->
alloc_small
)(
reinterpret_cast
<
j_common_ptr
>
(
cinfo
),
JPOOL_PERMANENT
,
sizeof
(
struct
jpeg_source_mgr
)));
(
*
cinfo
->
mem
->
alloc_small
)(
reinterpret_cast
<
j_common_ptr
>
(
cinfo
),
JPOOL_PERMANENT
,
sizeof
(
struct
jpeg_source_mgr
)));
cinfo
->
src
->
init_source
=
JpegInitSource
;
cinfo
->
src
->
init_source
=
JpegInitSource
;
cinfo
->
src
->
fill_input_buffer
=
JpegFillInputBuffer
;
cinfo
->
src
->
fill_input_buffer
=
JpegFillInputBuffer
;
#if defined(_WIN32) || defined(_WIN64)
cinfo
->
src
->
skip_input_data
=
reinterpret_cast
<
void
(
*
)(
j_decompress_ptr
,
long
)
>
(
JpegSkipInputData
);
#else
cinfo
->
src
->
skip_input_data
=
JpegSkipInputData
;
cinfo
->
src
->
skip_input_data
=
JpegSkipInputData
;
#endif
cinfo
->
src
->
resync_to_restart
=
jpeg_resync_to_restart
;
cinfo
->
src
->
resync_to_restart
=
jpeg_resync_to_restart
;
cinfo
->
src
->
term_source
=
JpegTermSource
;
cinfo
->
src
->
term_source
=
JpegTermSource
;
cinfo
->
src
->
bytes_in_buffer
=
datasize
;
cinfo
->
src
->
bytes_in_buffer
=
datasize
;
...
...
mindspore/ccsrc/dataset/kernels/image/image_utils.h
浏览文件 @
1286767d
...
@@ -22,6 +22,10 @@
...
@@ -22,6 +22,10 @@
#include <random>
#include <random>
#include <string>
#include <string>
#include <vector>
#include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h"
#include "./jpeglib.h"
#include "./jerror.h"
#include "./jerror.h"
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/imgproc/imgproc.hpp>
...
...
mindspore/ccsrc/dataset/util/path.cc
浏览文件 @
1286767d
...
@@ -27,7 +27,7 @@
...
@@ -27,7 +27,7 @@
namespace
mindspore
{
namespace
mindspore
{
namespace
dataset
{
namespace
dataset
{
#ifdef _WIN32
#ifdef _WIN32
char
Path
::
_separator
=
'\\'
;
char
Path
::
separator_
=
'\\'
;
#else
#else
char
Path
::
separator_
=
'/'
;
char
Path
::
separator_
=
'/'
;
#endif
#endif
...
@@ -129,7 +129,11 @@ bool Path::IsDirectory() {
...
@@ -129,7 +129,11 @@ bool Path::IsDirectory() {
Status
Path
::
CreateDirectory
()
{
Status
Path
::
CreateDirectory
()
{
if
(
!
Exists
())
{
if
(
!
Exists
())
{
#if defined(_WIN32) || defined(_WIN64)
int
rc
=
mkdir
(
common
::
SafeCStr
(
path_
));
#else
int
rc
=
mkdir
(
common
::
SafeCStr
(
path_
),
0700
);
int
rc
=
mkdir
(
common
::
SafeCStr
(
path_
),
0700
);
#endif
if
(
rc
)
{
if
(
rc
)
{
std
::
ostringstream
oss
;
std
::
ostringstream
oss
;
oss
<<
"Unable to create directory "
<<
path_
<<
". Errno = "
<<
errno
;
oss
<<
"Unable to create directory "
<<
path_
<<
". Errno = "
<<
errno
;
...
...
mindspore/ccsrc/dataset/util/random.cc
浏览文件 @
1286767d
...
@@ -32,7 +32,11 @@ namespace dataset {
...
@@ -32,7 +32,11 @@ namespace dataset {
uint32_t
GetSeed
()
{
uint32_t
GetSeed
()
{
uint32_t
seed
=
GlobalContext
::
config_manager
()
->
seed
();
uint32_t
seed
=
GlobalContext
::
config_manager
()
->
seed
();
if
(
seed
==
std
::
mt19937
::
default_seed
)
{
if
(
seed
==
std
::
mt19937
::
default_seed
)
{
#if defined(_WIN32) || defined(_WIN64)
std
::
random_device
random_device
;
#else
std
::
random_device
random_device
(
"/dev/urandom"
);
std
::
random_device
random_device
(
"/dev/urandom"
);
#endif
std
::
uniform_int_distribution
<
uint32_t
>
distribution
(
0
,
std
::
numeric_limits
<
uint32_t
>::
max
());
std
::
uniform_int_distribution
<
uint32_t
>
distribution
(
0
,
std
::
numeric_limits
<
uint32_t
>::
max
());
seed
=
distribution
(
random_device
);
seed
=
distribution
(
random_device
);
}
}
...
...
mindspore/ccsrc/dataset/util/services.cc
浏览文件 @
1286767d
...
@@ -16,7 +16,9 @@
...
@@ -16,7 +16,9 @@
#include "dataset/util/services.h"
#include "dataset/util/services.h"
#include <limits.h>
#include <limits.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/syscall.h>
#include <sys/syscall.h>
#endif
#include <unistd.h>
#include <unistd.h>
#include <random>
#include <random>
#include "dataset/util/circular_pool.h"
#include "dataset/util/circular_pool.h"
...
@@ -28,6 +30,7 @@ namespace dataset {
...
@@ -28,6 +30,7 @@ namespace dataset {
std
::
unique_ptr
<
Services
>
Services
::
instance_
=
nullptr
;
std
::
unique_ptr
<
Services
>
Services
::
instance_
=
nullptr
;
std
::
once_flag
Services
::
init_instance_flag_
;
std
::
once_flag
Services
::
init_instance_flag_
;
#if !defined(_WIN32) && !defined(_WIN64)
std
::
string
Services
::
GetUserName
()
{
std
::
string
Services
::
GetUserName
()
{
char
user
[
LOGIN_NAME_MAX
];
char
user
[
LOGIN_NAME_MAX
];
(
void
)
getlogin_r
(
user
,
sizeof
(
user
));
(
void
)
getlogin_r
(
user
,
sizeof
(
user
));
...
@@ -41,10 +44,15 @@ std::string Services::GetHostName() {
...
@@ -41,10 +44,15 @@ std::string Services::GetHostName() {
}
}
int
Services
::
GetLWP
()
{
return
syscall
(
SYS_gettid
);
}
int
Services
::
GetLWP
()
{
return
syscall
(
SYS_gettid
);
}
#endif
std
::
string
Services
::
GetUniqueID
()
{
std
::
string
Services
::
GetUniqueID
()
{
const
std
::
string
kStr
=
"abcdefghijklmnopqrstuvwxyz0123456789"
;
const
std
::
string
kStr
=
"abcdefghijklmnopqrstuvwxyz0123456789"
;
#if defined(_WIN32) || defined(_WIN64)
std
::
mt19937
gen
{
std
::
random_device
{}()};
#else
std
::
mt19937
gen
{
std
::
random_device
{
"/dev/urandom"
}()};
std
::
mt19937
gen
{
std
::
random_device
{
"/dev/urandom"
}()};
#endif
std
::
uniform_int_distribution
<>
dist
(
0
,
kStr
.
size
()
-
1
);
std
::
uniform_int_distribution
<>
dist
(
0
,
kStr
.
size
()
-
1
);
char
buffer
[
UNIQUEID_LEN
];
char
buffer
[
UNIQUEID_LEN
];
for
(
int
i
=
0
;
i
<
UNIQUEID_LEN
;
i
++
)
{
for
(
int
i
=
0
;
i
<
UNIQUEID_LEN
;
i
++
)
{
...
...
mindspore/ccsrc/dataset/util/services.h
浏览文件 @
1286767d
...
@@ -62,11 +62,13 @@ class Services {
...
@@ -62,11 +62,13 @@ class Services {
std
::
shared_ptr
<
MemoryPool
>
GetServiceMemPool
()
{
return
pool_
;
}
std
::
shared_ptr
<
MemoryPool
>
GetServiceMemPool
()
{
return
pool_
;
}
#if !defined(_WIN32) && !defined(_WIN64)
static
std
::
string
GetUserName
();
static
std
::
string
GetUserName
();
static
std
::
string
GetHostName
();
static
std
::
string
GetHostName
();
static
int
GetLWP
();
static
int
GetLWP
();
#endif
static
std
::
string
GetUniqueID
();
static
std
::
string
GetUniqueID
();
...
...
mindspore/ccsrc/dataset/util/sig_handler.cc
浏览文件 @
1286767d
...
@@ -16,13 +16,16 @@
...
@@ -16,13 +16,16 @@
#include "dataset/util/sig_handler.h"
#include "dataset/util/sig_handler.h"
#include <signal.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/types.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <ucontext.h>
#include <ucontext.h>
#endif
#include <unistd.h>
#include <unistd.h>
#include "dataset/util/task_manager.h"
#include "dataset/util/task_manager.h"
namespace
mindspore
{
namespace
mindspore
{
namespace
dataset
{
namespace
dataset
{
// Register the custom signal handlers
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
void
RegisterHandlers
()
{
void
RegisterHandlers
()
{
struct
sigaction
new_int_action
;
struct
sigaction
new_int_action
;
...
@@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
...
@@ -40,5 +43,6 @@ extern void IntHandler(int sig_num, // The signal that was raised
// Wake up the watchdog which is designed as async-signal-safe.
// Wake up the watchdog which is designed as async-signal-safe.
TaskManager
::
WakeUpWatchDog
();
TaskManager
::
WakeUpWatchDog
();
}
}
#endif
}
// namespace dataset
}
// namespace dataset
}
// namespace mindspore
}
// namespace mindspore
mindspore/ccsrc/dataset/util/sig_handler.h
浏览文件 @
1286767d
...
@@ -22,12 +22,14 @@
...
@@ -22,12 +22,14 @@
namespace
mindspore
{
namespace
mindspore
{
namespace
dataset
{
namespace
dataset
{
// Register the custom signal handlers
// Register the custom signal handlers
#if !defined(_WIN32) && !defined(_WIN64)
extern
void
RegisterHandlers
();
extern
void
RegisterHandlers
();
// A signal handler for SIGINT. Drives interrupt to watchdog
// A signal handler for SIGINT. Drives interrupt to watchdog
extern
void
IntHandler
(
int
sig_num
,
// The signal that was raised
extern
void
IntHandler
(
int
sig_num
,
// The signal that was raised
siginfo_t
*
sig_info
,
// The siginfo structure.
siginfo_t
*
sig_info
,
// The siginfo structure.
void
*
context
);
// context info
void
*
context
);
// context info
#endif
}
// namespace dataset
}
// namespace dataset
}
// namespace mindspore
}
// namespace mindspore
...
...
mindspore/ccsrc/debug/anf_ir_dump.cc
浏览文件 @
1286767d
...
@@ -14,6 +14,9 @@
...
@@ -14,6 +14,9 @@
* limitations under the License.
* limitations under the License.
*/
*/
#include "debug/anf_ir_dump.h"
#include "debug/anf_ir_dump.h"
#if defined(_WIN32) || defined(_WIN64)
#include <stdlib.h>
#endif
#include <fstream>
#include <fstream>
#include <map>
#include <map>
#include <memory>
#include <memory>
...
@@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
...
@@ -434,9 +437,15 @@ void DumpIR(const std::string &filename, const FuncGraphPtr &graph, bool dump_fu
return
;
return
;
}
}
char
real_path
[
PATH_MAX
]
=
{
0
};
char
real_path
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
real_path
,
filename
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
filename
<<
" does not exit."
;
}
#else
if
(
nullptr
==
realpath
(
filename
.
c_str
(),
real_path
))
{
if
(
nullptr
==
realpath
(
filename
.
c_str
(),
real_path
))
{
MS_LOG
(
DEBUG
)
<<
"Dir "
<<
filename
<<
" does not exit."
;
MS_LOG
(
DEBUG
)
<<
"Dir "
<<
filename
<<
" does not exit."
;
}
}
#endif
OrderedMap
<
AnfNodePtr
,
int32_t
>
para_map
;
OrderedMap
<
AnfNodePtr
,
int32_t
>
para_map
;
std
::
string
path_string
=
real_path
;
std
::
string
path_string
=
real_path
;
...
...
mindspore/ccsrc/debug/anf_ir_utils.cc
浏览文件 @
1286767d
...
@@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
...
@@ -48,9 +48,15 @@ std::string GetMsIrPath(void) {
if
(
path_ptr
!=
nullptr
)
{
if
(
path_ptr
!=
nullptr
)
{
path
=
path_ptr
;
path
=
path_ptr
;
char
real_path
[
PATH_MAX
]
=
{
0
};
char
real_path
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
path
.
size
()
>
PATH_MAX
||
_fullpath
(
real_path
,
path
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
EXCEPTION
)
<<
"MS IR Path error, "
<<
path_ptr
;
}
#else
if
(
path
.
size
()
>
PATH_MAX
||
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
if
(
path
.
size
()
>
PATH_MAX
||
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
MS_LOG
(
EXCEPTION
)
<<
"MS IR path error, "
<<
path_ptr
;
MS_LOG
(
EXCEPTION
)
<<
"MS IR path error, "
<<
path_ptr
;
}
}
#endif
path
=
real_path
;
path
=
real_path
;
}
}
return
path
;
return
path
;
...
@@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
...
@@ -2247,8 +2253,14 @@ void DumpIRProto(const FuncGraphPtr& func_graph, const std::string& suffix) {
return
;
return
;
}
}
char
real_path
[
PATH_MAX
]
=
{
0
};
char
real_path
[
PATH_MAX
]
=
{
0
};
if
(
nullptr
==
realpath
(
file_path
.
c_str
(),
real_path
))
{
char
*
real_path_ret
=
nullptr
;
MS_LOG
(
DEBUG
)
<<
"Dir "
<<
file_path
<<
" does not exit."
;
#if defined(_WIN32) || defined(_WIN64)
real_path_ret
=
_fullpath
(
real_path
,
file_path
.
c_str
(),
PATH_MAX
);
#else
real_path_ret
=
realpath
(
file_path
.
c_str
(),
real_path
);
#endif
if
(
nullptr
==
real_path_ret
)
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
file_path
<<
" does not exit."
;
}
else
{
}
else
{
std
::
string
path_string
=
real_path
;
std
::
string
path_string
=
real_path
;
if
(
chmod
(
common
::
SafeCStr
(
path_string
),
S_IRUSR
|
S_IWUSR
)
==
-
1
)
{
if
(
chmod
(
common
::
SafeCStr
(
path_string
),
S_IRUSR
|
S_IWUSR
)
==
-
1
)
{
...
...
mindspore/ccsrc/debug/info.cc
浏览文件 @
1286767d
...
@@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
...
@@ -53,10 +53,15 @@ std::string Location::ToString(SourceLineTip tip) {
}
}
char
path
[
PATH_MAX
+
1
]
=
{
0x00
};
char
path
[
PATH_MAX
+
1
]
=
{
0x00
};
#if defined(_WIN32) || defined(_WIN64)
if
(
file_name_
.
size
()
>
PATH_MAX
||
_fullpath
(
path
,
file_name_
.
c_str
(),
PATH_MAX
)
==
nullptr
)
{
return
debug_info_ss
.
str
();
}
#else
if
(
file_name_
.
size
()
>
PATH_MAX
||
realpath
(
file_name_
.
c_str
(),
path
)
==
nullptr
)
{
if
(
file_name_
.
size
()
>
PATH_MAX
||
realpath
(
file_name_
.
c_str
(),
path
)
==
nullptr
)
{
return
debug_info_ss
.
str
();
return
debug_info_ss
.
str
();
}
}
#endif
auto
src_path
=
std
::
string
(
path
);
auto
src_path
=
std
::
string
(
path
);
std
::
ifstream
file
(
src_path
);
std
::
ifstream
file
(
src_path
);
if
(
!
file
.
is_open
())
{
if
(
!
file
.
is_open
())
{
...
...
mindspore/ccsrc/device/cpu/cpu_resource_manager.cc
浏览文件 @
1286767d
...
@@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
...
@@ -60,6 +60,7 @@ void CPUResourceManager::MemMalloc(const session::KernelGraph *graph) {
void
*
CPUResourceManager
::
MemMalloc
(
size_t
mem_size
)
{
void
*
CPUResourceManager
::
MemMalloc
(
size_t
mem_size
)
{
void
*
ptr
=
malloc
(
mem_size
);
void
*
ptr
=
malloc
(
mem_size
);
if
(
ptr
!=
nullptr
)
{
if
(
ptr
!=
nullptr
)
{
memset_s
(
ptr
,
mem_size
,
0
,
mem_size
);
dynamic_mem_
[
ptr
]
=
mem_size
;
dynamic_mem_
[
ptr
]
=
mem_size
;
return
ptr
;
return
ptr
;
}
else
{
}
else
{
...
...
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.cc
0 → 100644
浏览文件 @
1286767d
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "device/gpu/distribution/collective_fake_init.h"
#include "utils/log_adapter.h"
namespace
mindspore
{
namespace
device
{
namespace
gpu
{
void
CollectiveFakeInitializer
::
InitCollective
()
{
MS_LOG
(
EXCEPTION
)
<<
"build without enable gpu!"
;
}
void
CollectiveFakeInitializer
::
FinalizeCollective
()
{
MS_LOG
(
EXCEPTION
)
<<
"build without enable gpu!"
;
}
}
// namespace gpu
}
// namespace device
}
// namespace mindspore
mindspore/ccsrc/device/gpu/distribution/collective_fake_init.h
0 → 100644
浏览文件 @
1286767d
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
#define MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
namespace
mindspore
{
namespace
device
{
namespace
gpu
{
class
CollectiveFakeInitializer
{
public:
CollectiveFakeInitializer
()
=
default
;
~
CollectiveFakeInitializer
()
=
default
;
CollectiveFakeInitializer
(
CollectiveFakeInitializer
const
&
)
=
delete
;
CollectiveFakeInitializer
&
operator
=
(
const
CollectiveFakeInitializer
&
)
=
delete
;
static
void
InitCollective
();
static
void
FinalizeCollective
();
};
}
// namespace gpu
}
// namespace device
}
// namespace mindspore
#endif // MINDSPORE_CCSRC_DEVICE_GPU_DISTRIBUTION_COLLECTIVE_FAKE_INIT_H_
mindspore/ccsrc/device/kernel_runtime.cc
浏览文件 @
1286767d
...
@@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
...
@@ -44,19 +44,29 @@ bool KernelRuntime::Run(session::KernelGraph *graph) {
bool
ret
=
false
;
bool
ret
=
false
;
auto
context_ptr
=
MsContext
::
GetInstance
();
auto
context_ptr
=
MsContext
::
GetInstance
();
MS_EXCEPTION_IF_NULL
(
context_ptr
);
MS_EXCEPTION_IF_NULL
(
context_ptr
);
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
bool
is_task_sink
=
context_ptr
->
enable_task_sink
();
bool
is_task_sink
=
context_ptr
->
enable_task_sink
();
if
(
is_task_sink
)
{
if
(
is_task_sink
)
{
ret
=
RunTask
(
graph
);
ret
=
RunTask
(
graph
);
}
else
{
}
else
{
ret
=
LaunchKernel
(
graph
);
ret
=
LaunchKernel
(
graph
);
}
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Call MS Run Success in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Call MS Run Success in "
<<
cost
<<
" us"
;
MS_LOG
(
INFO
)
<<
"Call MS Run Success in "
<<
cost
<<
" us"
;
#endif
return
ret
;
return
ret
;
}
}
...
@@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
...
@@ -561,8 +571,12 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
AddressPtrList
kernel_workspaces
;
AddressPtrList
kernel_workspaces
;
AddressPtrList
kernel_outputs
;
AddressPtrList
kernel_outputs
;
GenLaunchArgs
(
*
kernel_mod
,
kernel
,
&
kernel_inputs
,
&
kernel_workspaces
,
&
kernel_outputs
);
GenLaunchArgs
(
*
kernel_mod
,
kernel
,
&
kernel_inputs
,
&
kernel_workspaces
,
&
kernel_outputs
);
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
auto
ret
=
auto
ret
=
kernel_mod
->
Launch
(
kernel_inputs
,
kernel_workspaces
,
kernel_outputs
,
reinterpret_cast
<
uintptr_t
>
(
stream_
));
kernel_mod
->
Launch
(
kernel_inputs
,
kernel_workspaces
,
kernel_outputs
,
reinterpret_cast
<
uintptr_t
>
(
stream_
));
if
(
!
ret
)
{
if
(
!
ret
)
{
...
@@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
...
@@ -572,11 +586,17 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) {
if
(
AnfAlgo
::
GetKernelType
(
kernel
)
==
TBE_KERNEL
&&
!
SyncStream
())
{
if
(
AnfAlgo
::
GetKernelType
(
kernel
)
==
TBE_KERNEL
&&
!
SyncStream
())
{
MS_LOG
(
EXCEPTION
)
<<
"SyncStream failed."
;
MS_LOG
(
EXCEPTION
)
<<
"SyncStream failed."
;
}
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
DEBUG
)
<<
"d "
<<
kernel
->
fullname_with_scope
()
<<
" in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
DEBUG
)
<<
"d "
<<
kernel
->
fullname_with_scope
()
<<
" in "
<<
cost
<<
" us"
;
MS_LOG
(
DEBUG
)
<<
"d "
<<
kernel
->
fullname_with_scope
()
<<
" in "
<<
cost
<<
" us"
;
#endif
}
}
}
}
return
true
;
return
true
;
...
...
mindspore/ccsrc/kernel/common_utils.cc
浏览文件 @
1286767d
...
@@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
...
@@ -117,7 +117,11 @@ bool IsAtomicNode(const CNodePtr &kernel_node) {
bool
KernelMeta
::
ReadIndex
(
const
std
::
string
&
bin_dir
)
{
bool
KernelMeta
::
ReadIndex
(
const
std
::
string
&
bin_dir
)
{
DIR
*
dir
=
opendir
(
bin_dir
.
c_str
());
DIR
*
dir
=
opendir
(
bin_dir
.
c_str
());
if
(
dir
==
nullptr
)
{
if
(
dir
==
nullptr
)
{
#if defined(_WIN32) || defined(_WIN64)
auto
ret
=
mkdir
(
bin_dir
.
c_str
());
#else
auto
ret
=
mkdir
(
bin_dir
.
c_str
(),
S_IRWXG
|
S_IRWXU
);
auto
ret
=
mkdir
(
bin_dir
.
c_str
(),
S_IRWXG
|
S_IRWXU
);
#endif
if
(
ret
!=
0
)
{
if
(
ret
!=
0
)
{
MS_LOG
(
INFO
)
<<
"kernel dir not exist["
<<
bin_dir
<<
"]."
;
MS_LOG
(
INFO
)
<<
"kernel dir not exist["
<<
bin_dir
<<
"]."
;
return
false
;
return
false
;
...
@@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
...
@@ -500,10 +504,17 @@ void SaveJsonInfo(const std::string &json_name, const std::string &info) {
}
}
filewrite
<<
info
<<
std
::
endl
;
filewrite
<<
info
<<
std
::
endl
;
filewrite
.
close
();
filewrite
.
close
();
#if defined(_WIN32) || defined(_WIN64)
if
(
nullptr
==
_fullpath
(
real_path
,
path
.
c_str
(),
PATH_MAX
))
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
path
<<
" does not exit."
;
return
;
}
#else
if
(
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
if
(
nullptr
==
realpath
(
path
.
c_str
(),
real_path
))
{
MS_LOG
(
DEBUG
)
<<
"dir "
<<
path
<<
" does not exit."
;
MS_LOG
(
DEBUG
)
<<
"dir "
<<
path
<<
" does not exit."
;
return
;
return
;
}
}
#endif
MS_LOG
(
INFO
)
<<
"real path is :"
<<
real_path
;
MS_LOG
(
INFO
)
<<
"real path is :"
<<
real_path
;
if
(
chmod
(
real_path
,
S_IRUSR
)
==
-
1
)
{
if
(
chmod
(
real_path
,
S_IRUSR
)
==
-
1
)
{
MS_LOG
(
DEBUG
)
<<
"modify file:"
<<
real_path
<<
" to read only fail."
;
MS_LOG
(
DEBUG
)
<<
"modify file:"
<<
real_path
<<
" to read only fail."
;
...
...
mindspore/ccsrc/mindrecord/CMakeLists.txt
浏览文件 @
1286767d
...
@@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
...
@@ -26,7 +26,11 @@ set_target_properties(_c_mindrecord PROPERTIES
)
)
# add link library
# add link library
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite
${
PYTHON_LIB
}
${
SECUREC_LIBRARY
}
mindspore mindspore_gvar mindspore::protobuf
)
if
(
WIN32
)
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf
)
else
()
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::sqlite
${
PYTHON_LIB
}
${
SECUREC_LIBRARY
}
mindspore mindspore_gvar mindspore::protobuf
)
endif
()
if
(
USE_GLOG
)
if
(
USE_GLOG
)
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::glog
)
target_link_libraries
(
_c_mindrecord PRIVATE mindspore::glog
)
...
...
mindspore/ccsrc/mindrecord/common/shard_utils.cc
浏览文件 @
1286767d
...
@@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
...
@@ -65,6 +65,15 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
return
{
FAILED
,
""
};
return
{
FAILED
,
""
};
}
}
char
tmp
[
PATH_MAX
]
=
{
0
};
char
tmp
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
tmp
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
}
if
(
_fullpath
(
real_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
common
::
SafeCStr
(
path
)
<<
"check successfully"
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
return
{
FAILED
,
""
};
...
@@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
...
@@ -72,6 +81,7 @@ std::pair<MSRStatus, std::string> GetFileName(const std::string &path) {
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
}
}
#endif
std
::
string
s
=
real_path
;
std
::
string
s
=
real_path
;
char
sep
=
'/'
;
char
sep
=
'/'
;
size_t
i
=
s
.
rfind
(
sep
,
s
.
length
());
size_t
i
=
s
.
rfind
(
sep
,
s
.
length
());
...
@@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
...
@@ -91,6 +101,15 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
return
{
FAILED
,
""
};
return
{
FAILED
,
""
};
}
}
char
tmp
[
PATH_MAX
]
=
{
0
};
char
tmp
[
PATH_MAX
]
=
{
0
};
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
tmp
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
}
if
(
_fullpath
(
real_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
common
::
SafeCStr
(
path
)
<<
"check successfully"
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
tmp
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
MS_LOG
(
ERROR
)
<<
"Invalid file path, path: "
<<
buf
;
return
{
FAILED
,
""
};
return
{
FAILED
,
""
};
...
@@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
...
@@ -98,6 +117,7 @@ std::pair<MSRStatus, std::string> GetParentDir(const std::string &path) {
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
if
(
realpath
(
common
::
SafeCStr
(
path
),
real_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
MS_LOG
(
DEBUG
)
<<
"Path: "
<<
path
<<
"check successfully"
;
}
}
#endif
std
::
string
s
=
real_path
;
std
::
string
s
=
real_path
;
if
(
s
.
rfind
(
'/'
)
+
1
<=
s
.
size
())
{
if
(
s
.
rfind
(
'/'
)
+
1
<=
s
.
size
())
{
return
{
SUCCESS
,
s
.
substr
(
0
,
s
.
rfind
(
'/'
)
+
1
)};
return
{
SUCCESS
,
s
.
substr
(
0
,
s
.
rfind
(
'/'
)
+
1
)};
...
@@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
...
@@ -144,6 +164,9 @@ bool IsLegalFile(const std::string &path) {
}
}
std
::
pair
<
MSRStatus
,
uint64_t
>
GetDiskSize
(
const
std
::
string
&
str_dir
,
const
DiskSizeType
&
disk_type
)
{
std
::
pair
<
MSRStatus
,
uint64_t
>
GetDiskSize
(
const
std
::
string
&
str_dir
,
const
DiskSizeType
&
disk_type
)
{
#if defined(_WIN32) || defined(_WIN64)
return
{
SUCCESS
,
100
};
#else
uint64_t
ll_count
=
0
;
uint64_t
ll_count
=
0
;
struct
statfs
disk_info
;
struct
statfs
disk_info
;
if
(
statfs
(
common
::
SafeCStr
(
str_dir
),
&
disk_info
)
==
-
1
)
{
if
(
statfs
(
common
::
SafeCStr
(
str_dir
),
&
disk_info
)
==
-
1
)
{
...
@@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
...
@@ -166,6 +189,7 @@ std::pair<MSRStatus, uint64_t> GetDiskSize(const std::string &str_dir, const Dis
}
}
return
{
SUCCESS
,
ll_count
};
return
{
SUCCESS
,
ll_count
};
#endif
}
}
uint32_t
GetMaxThreadNum
()
{
uint32_t
GetMaxThreadNum
()
{
...
...
mindspore/ccsrc/mindrecord/include/common/shard_utils.h
浏览文件 @
1286767d
...
@@ -21,8 +21,10 @@
...
@@ -21,8 +21,10 @@
#include <limits.h>
#include <limits.h>
#include <stdlib.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/stat.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/statfs.h>
#include <sys/statfs.h>
#include <sys/wait.h>
#include <sys/wait.h>
#endif
#include <unistd.h>
#include <unistd.h>
#include <cassert>
#include <cassert>
#include <cmath>
#include <cmath>
...
...
mindspore/ccsrc/mindrecord/include/shard_reader.h
浏览文件 @
1286767d
...
@@ -19,7 +19,9 @@
...
@@ -19,7 +19,9 @@
#include <dirent.h>
#include <dirent.h>
#include <signal.h>
#include <signal.h>
#if !defined(_WIN32) && !defined(_WIN64)
#include <sys/prctl.h>
#include <sys/prctl.h>
#endif
#include <sys/stat.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/types.h>
#include <unistd.h>
#include <unistd.h>
...
...
mindspore/ccsrc/mindrecord/io/shard_reader.cc
浏览文件 @
1286767d
...
@@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
...
@@ -987,8 +987,10 @@ TASK_RETURN_CONTENT ShardReader::ConsumerOneTask(int task_id, uint32_t consumer_
MSRStatus
ShardReader
::
ConsumerByRow
(
int
consumer_id
)
{
MSRStatus
ShardReader
::
ConsumerByRow
(
int
consumer_id
)
{
// Set thread name
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
#endif
// Loop forever
// Loop forever
for
(;;)
{
for
(;;)
{
...
@@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
...
@@ -1040,8 +1042,10 @@ MSRStatus ShardReader::ReadBlob(const int &shard_id, const uint64_t &page_offset
MSRStatus
ShardReader
::
ConsumerByBlock
(
int
consumer_id
)
{
MSRStatus
ShardReader
::
ConsumerByBlock
(
int
consumer_id
)
{
// Set thread name
// Set thread name
#if !defined(_WIN32) && !defined(_WIN64)
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
auto
thread_id
=
kThreadName
+
std
::
to_string
(
consumer_id
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
prctl
(
PR_SET_NAME
,
common
::
SafeCStr
(
thread_id
),
0
,
0
,
0
);
#endif
// Loop forever
// Loop forever
for
(;;)
{
for
(;;)
{
...
...
mindspore/ccsrc/mindrecord/io/shard_writer.cc
浏览文件 @
1286767d
...
@@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
...
@@ -63,6 +63,15 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
MS_LOG
(
ERROR
)
<<
"Securec func failed"
;
MS_LOG
(
ERROR
)
<<
"Securec func failed"
;
return
FAILED
;
return
FAILED
;
}
}
#if defined(_WIN32) || defined(_WIN64)
if
(
_fullpath
(
resolved_path
,
dirname
(
&
(
buf
[
0
])),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path"
;
return
FAILED
;
}
if
(
_fullpath
(
resolved_path
,
common
::
SafeCStr
(
path
),
PATH_MAX
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path "
<<
resolved_path
;
}
#else
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
resolved_path
)
==
nullptr
)
{
if
(
realpath
(
dirname
(
&
(
buf
[
0
])),
resolved_path
)
==
nullptr
)
{
MS_LOG
(
ERROR
)
<<
"Invalid file path"
;
MS_LOG
(
ERROR
)
<<
"Invalid file path"
;
return
FAILED
;
return
FAILED
;
...
@@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
...
@@ -70,6 +79,7 @@ MSRStatus ShardWriter::Open(const std::vector<std::string> &paths, bool append)
if
(
realpath
(
common
::
SafeCStr
(
path
),
resolved_path
)
==
nullptr
)
{
if
(
realpath
(
common
::
SafeCStr
(
path
),
resolved_path
)
==
nullptr
)
{
MS_LOG
(
DEBUG
)
<<
"Path "
<<
resolved_path
;
MS_LOG
(
DEBUG
)
<<
"Path "
<<
resolved_path
;
}
}
#endif
file_paths_
.
emplace_back
(
string
(
resolved_path
));
file_paths_
.
emplace_back
(
string
(
resolved_path
));
}
}
...
...
mindspore/ccsrc/parallel/allreduce_fusion/step_allreduce_fusion.cc
浏览文件 @
1286767d
...
@@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
...
@@ -38,10 +38,12 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
(
root
->
has_flag
(
ALLREDUCE_FUSION_RUN_ONCE_ONLY
)))
{
(
root
->
has_flag
(
ALLREDUCE_FUSION_RUN_ONCE_ONLY
)))
{
return
changes
;
return
changes
;
}
}
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
,
end_time
;
struct
timeval
start_time
,
end_time
;
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
MS_LOG
(
INFO
)
<<
"Now entering allreduce fusion"
;
MS_LOG
(
INFO
)
<<
"Now entering allreduce fusion"
;
DumpGraph
(
root
,
std
::
string
(
ALLREDUCE_FUSION_BEGIN
));
DumpGraph
(
root
,
std
::
string
(
ALLREDUCE_FUSION_BEGIN
));
...
@@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
...
@@ -63,11 +65,16 @@ bool StepAllreduceFusion(const FuncGraphPtr &root, const opt::OptimizerPtr &opti
// allreduce fusion only run once
// allreduce fusion only run once
root
->
flags
()[
ALLREDUCE_FUSION_RUN_ONCE_ONLY
]
=
true
;
root
->
flags
()[
ALLREDUCE_FUSION_RUN_ONCE_ONLY
]
=
true
;
res
->
results
()[
pipeline
::
kStepParallelGraph
]
=
root
;
res
->
results
()[
pipeline
::
kStepParallelGraph
]
=
root
;
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Now leaving allreduce fusion, used time: "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
uint64_t
time
=
1000000
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
uint64_t
time
=
1000000
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
time
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
time
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Now leaving allreduce fusion, used time: "
<<
time
<<
" us"
;
MS_LOG
(
INFO
)
<<
"Now leaving allreduce fusion, used time: "
<<
time
<<
" us"
;
#endif
return
changes
;
return
changes
;
}
}
}
// namespace parallel
}
// namespace parallel
...
...
mindspore/ccsrc/parallel/step_auto_parallel.cc
浏览文件 @
1286767d
...
@@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
...
@@ -229,7 +229,7 @@ size_t GetLengthOfDataType(const TypePtr &type) {
case
kNumberTypeInt
:
case
kNumberTypeInt
:
return
sizeof
(
int
);
return
sizeof
(
int
);
case
kNumberTypeUInt
:
case
kNumberTypeUInt
:
return
sizeof
(
uint
);
return
sizeof
(
u
nsigned
int
);
case
kNumberTypeFloat
:
case
kNumberTypeFloat
:
return
sizeof
(
float
);
return
sizeof
(
float
);
default:
default:
...
...
mindspore/ccsrc/pipeline/action.cc
浏览文件 @
1286767d
...
@@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
...
@@ -264,7 +264,7 @@ bool TaskEmitAction(const ResourcePtr& res) {
auto
bc_ptr
=
res
->
results
()[
kBackend
].
cast
<
compile
::
BackendPtr
>
();
auto
bc_ptr
=
res
->
results
()[
kBackend
].
cast
<
compile
::
BackendPtr
>
();
std
::
vector
<
PrimitivePtr
>
cut_list
=
compile
::
nonlinear_ops
;
std
::
vector
<
PrimitivePtr
>
cut_list
=
compile
::
nonlinear_ops
;
if
(
bc_ptr
->
name
()
==
kMsConvert
)
{
if
(
bc_ptr
->
name
()
==
kMsConvert
)
{
cut_list
=
compile
::
ms_nonlinear_ops
;
cut_list
=
compile
::
GetMsNonlinearOps
()
;
}
}
std
::
shared_ptr
<
CompileGraphs
>
compile
=
std
::
make_shared
<
CompileGraphs
>
(
bc_ptr
,
cut_list
);
std
::
shared_ptr
<
CompileGraphs
>
compile
=
std
::
make_shared
<
CompileGraphs
>
(
bc_ptr
,
cut_list
);
res
->
results
()[
kOutput
]
=
compile
->
CompileAndLink
(
func_graph
);
res
->
results
()[
kOutput
]
=
compile
->
CompileAndLink
(
func_graph
);
...
...
mindspore/ccsrc/pipeline/init.cc
浏览文件 @
1286767d
...
@@ -29,8 +29,11 @@
...
@@ -29,8 +29,11 @@
#include "parallel/context.h"
#include "parallel/context.h"
#include "parallel/device_manager.h"
#include "parallel/device_manager.h"
#include "parallel/costmodel_context.h"
#include "parallel/costmodel_context.h"
#ifdef ENABLE_GPUQUE
#include "device/gpu/distribution/collective_init.h"
#include "device/gpu/distribution/collective_init.h"
#else
#include "device/gpu/distribution/collective_fake_init.h"
#endif
namespace
py
=
pybind11
;
namespace
py
=
pybind11
;
using
FuncGraph
=
mindspore
::
FuncGraph
;
using
FuncGraph
=
mindspore
::
FuncGraph
;
...
@@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
...
@@ -297,9 +300,16 @@ PYBIND11_MODULE(_c_expression, m) {
(
void
)
py
::
class_
<
OpLib
,
std
::
shared_ptr
<
OpLib
>>
(
m
,
"Oplib"
)
(
void
)
py
::
class_
<
OpLib
,
std
::
shared_ptr
<
OpLib
>>
(
m
,
"Oplib"
)
.
def
(
py
::
init
())
.
def
(
py
::
init
())
.
def
(
"reg_op"
,
&
OpLib
::
RegOp
,
"Register op info."
);
.
def
(
"reg_op"
,
&
OpLib
::
RegOp
,
"Register op info."
);
#ifdef ENABLE_GPUQUE
(
void
)
m
.
def
(
"init_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
InitCollective
,
(
void
)
m
.
def
(
"init_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
InitCollective
,
"Init gpu collective communication mode."
);
"Init gpu collective communication mode."
);
(
void
)
m
.
def
(
"finalize_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
FinalizeCollective
,
(
void
)
m
.
def
(
"finalize_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveInitializer
::
FinalizeCollective
,
"Finalize gpu collective communication mode."
);
"Finalize gpu collective communication mode."
);
#else
(
void
)
m
.
def
(
"init_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveFakeInitializer
::
InitCollective
,
"Init gpu collective communication mode."
);
(
void
)
m
.
def
(
"finalize_gpu_collective"
,
&
mindspore
::
device
::
gpu
::
CollectiveFakeInitializer
::
FinalizeCollective
,
"Finalize gpu collective communication mode."
);
#endif
}
}
mindspore/ccsrc/pre_activate/common/pass_manager.cc
浏览文件 @
1286767d
...
@@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
...
@@ -52,17 +52,27 @@ bool PassManager::Run(const FuncGraphPtr &func_graph, const std::vector<PassPtr>
size_t
num
=
0
;
size_t
num
=
0
;
for
(
const
auto
&
pass
:
passes
)
{
for
(
const
auto
&
pass
:
passes
)
{
if
(
pass
!=
nullptr
)
{
if
(
pass
!=
nullptr
)
{
#if defined(_WIN32) || defined(_WIN64)
auto
start_time
=
std
::
chrono
::
steady_clock
::
now
();
#else
struct
timeval
start_time
{};
struct
timeval
start_time
{};
struct
timeval
end_time
{};
struct
timeval
end_time
{};
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
(
void
)
gettimeofday
(
&
start_time
,
nullptr
);
#endif
if
(
pass
->
Run
(
func_graph
))
{
if
(
pass
->
Run
(
func_graph
))
{
changed
=
true
;
changed
=
true
;
}
}
#if defined(_WIN32) || defined(_WIN64)
auto
end_time
=
std
::
chrono
::
steady_clock
::
now
();
std
::
chrono
::
duration
<
double
,
std
::
ratio
<
1
,
1000000
>>
cost
=
end_time
-
start_time
;
MS_LOG
(
INFO
)
<<
"Run pass hwopt_"
+
name
()
+
"_"
<<
num
<<
"_"
+
pass
->
name
()
+
" in "
<<
cost
.
count
()
<<
" us"
;
#else
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
(
void
)
gettimeofday
(
&
end_time
,
nullptr
);
const
uint64_t
kUSecondInSecond
=
1000000
;
const
uint64_t
kUSecondInSecond
=
1000000
;
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
uint64_t
cost
=
kUSecondInSecond
*
static_cast
<
uint64_t
>
(
end_time
.
tv_sec
-
start_time
.
tv_sec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
cost
+=
static_cast
<
uint64_t
>
(
end_time
.
tv_usec
-
start_time
.
tv_usec
);
MS_LOG
(
INFO
)
<<
"Run pass hwopt_"
+
name
()
+
"_"
<<
num
<<
"_"
+
pass
->
name
()
+
" in "
<<
cost
<<
" us"
;
MS_LOG
(
INFO
)
<<
"Run pass hwopt_"
+
name
()
+
"_"
<<
num
<<
"_"
+
pass
->
name
()
+
" in "
<<
cost
<<
" us"
;
#endif
if
(
save_graphs
)
{
if
(
save_graphs
)
{
auto
dump_file_path
=
auto
dump_file_path
=
save_graphs_path
+
"/"
+
"hwopt_"
+
name
()
+
"_"
+
std
::
to_string
(
num
)
+
"_"
+
pass
->
name
()
+
".ir"
;
save_graphs_path
+
"/"
+
"hwopt_"
+
name
()
+
"_"
+
std
::
to_string
(
num
)
+
"_"
+
pass
->
name
()
+
".ir"
;
...
...
mindspore/ccsrc/utils/log_adapter.cc
浏览文件 @
1286767d
...
@@ -26,12 +26,19 @@ namespace mindspore {
...
@@ -26,12 +26,19 @@ namespace mindspore {
#ifdef USE_GLOG
#ifdef USE_GLOG
static
std
::
string
GetTime
()
{
static
std
::
string
GetTime
()
{
#define BUFLEN 80
#define BUFLEN 80
static
char
buf
[
BUFLEN
];
#if defined(_WIN32) || defined(_WIN64)
time_t
time_seconds
=
time
(
0
);
struct
tm
now_time
;
localtime_s
(
&
now_time
,
&
time_seconds
);
sprintf_s
(
buf
,
BUFLEN
,
"%d-%d-%d %d:%d:%d"
,
now_time
.
tm_year
+
1900
,
now_time
.
tm_mon
+
1
,
now_time
.
tm_mday
,
now_time
.
tm_hour
,
now_time
.
tm_min
,
now_time
.
tm_sec
);
#else
struct
timeval
cur_time
;
struct
timeval
cur_time
;
(
void
)
gettimeofday
(
&
cur_time
,
NULL
);
(
void
)
gettimeofday
(
&
cur_time
,
NULL
);
struct
tm
now
;
struct
tm
now
;
(
void
)
localtime_r
(
&
cur_time
.
tv_sec
,
&
now
);
(
void
)
localtime_r
(
&
cur_time
.
tv_sec
,
&
now
);
static
char
buf
[
BUFLEN
];
(
void
)
strftime
(
buf
,
BUFLEN
,
"%Y-%m-%d-%H:%M:%S"
,
&
now
);
// format date and time
(
void
)
strftime
(
buf
,
BUFLEN
,
"%Y-%m-%d-%H:%M:%S"
,
&
now
);
// format date and time
// set micro-second
// set micro-second
buf
[
27
]
=
'\0'
;
buf
[
27
]
=
'\0'
;
...
@@ -44,6 +51,7 @@ static std::string GetTime() {
...
@@ -44,6 +51,7 @@ static std::string GetTime() {
buf
[
idx
--
]
=
'.'
;
buf
[
idx
--
]
=
'.'
;
}
}
}
}
#endif
return
std
::
string
(
buf
);
return
std
::
string
(
buf
);
}
}
...
...
mindspore/ccsrc/vm/transform.cc
浏览文件 @
1286767d
...
@@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
...
@@ -41,8 +41,10 @@ using TypedPrimitiveAbstractClosurePtr = std::shared_ptr<abstract::TypedPrimitiv
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
,
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
,
prim
::
kPrimMakeTuple
};
prim
::
kPrimMakeTuple
};
const
std
::
vector
<
PrimitivePtr
>&
GetMsNonlinearOps
()
{
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
};
static
const
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
=
{
prim
::
kPrimReturn
,
prim
::
kPrimPartial
,
prim
::
kPrimSwitch
};
return
ms_nonlinear_ops
;
}
CompileGraph
::
CompileGraph
(
const
BackendPtr
&
backend
,
const
std
::
vector
<
PrimitivePtr
>&
cut_list
)
CompileGraph
::
CompileGraph
(
const
BackendPtr
&
backend
,
const
std
::
vector
<
PrimitivePtr
>&
cut_list
)
:
backend_
(
backend
),
cut_list_
(
cut_list
)
{
:
backend_
(
backend
),
cut_list_
(
cut_list
)
{
...
...
mindspore/ccsrc/vm/transform.h
浏览文件 @
1286767d
...
@@ -42,7 +42,7 @@ extern const char kGeVm[];
...
@@ -42,7 +42,7 @@ extern const char kGeVm[];
// A sub namespace in ME to support compile related definition.
// A sub namespace in ME to support compile related definition.
namespace
compile
{
namespace
compile
{
extern
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
;
extern
std
::
vector
<
PrimitivePtr
>
nonlinear_ops
;
extern
std
::
vector
<
PrimitivePtr
>
ms_nonlinear_ops
;
const
std
::
vector
<
PrimitivePtr
>&
GetMsNonlinearOps
()
;
using
VmEvalFunc
=
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>
;
using
VmEvalFunc
=
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>
;
using
VmEvalFuncPtr
=
std
::
shared_ptr
<
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>>
;
using
VmEvalFuncPtr
=
std
::
shared_ptr
<
std
::
function
<
BaseRef
(
const
VectorRef
&
)
>>
;
...
...
mindspore/log.py
浏览文件 @
1286767d
...
@@ -19,11 +19,13 @@ import sys
...
@@ -19,11 +19,13 @@ import sys
import
os
import
os
import
stat
import
stat
import
time
import
time
import
fcntl
import
logging
import
logging
from
logging.handlers
import
RotatingFileHandler
from
logging.handlers
import
RotatingFileHandler
import
traceback
import
traceback
import
threading
import
threading
import
platform
if
platform
.
system
()
!=
"Windows"
:
import
fcntl
__all__
=
[
'get_level'
,
'get_log_config'
]
__all__
=
[
'get_level'
,
'get_log_config'
]
...
@@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
...
@@ -90,7 +92,8 @@ class _MultiCompatibleRotatingFileHandler(RotatingFileHandler):
# Attain an exclusive lock with bloking mode by `fcntl` module.
# Attain an exclusive lock with bloking mode by `fcntl` module.
with
open
(
self
.
baseFilename
,
'a'
)
as
file_pointer
:
with
open
(
self
.
baseFilename
,
'a'
)
as
file_pointer
:
fcntl
.
lockf
(
file_pointer
.
fileno
(),
fcntl
.
LOCK_EX
)
if
platform
.
system
()
!=
"Windows"
:
fcntl
.
lockf
(
file_pointer
.
fileno
(),
fcntl
.
LOCK_EX
)
if
self
.
backupCount
>
0
:
if
self
.
backupCount
>
0
:
self
.
rolling_rename
()
self
.
rolling_rename
()
...
...
third_party/securec/CMakeLists.txt
浏览文件 @
1286767d
SET
(
CMAKE_BUILD_TYPE
"Debug"
)
SET
(
CMAKE_BUILD_TYPE
"Debug"
)
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='"
)
if
(
WIN32
)
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer"
)
else
()
SET
(
CMAKE_C_FLAGS_DEBUG
"$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='"
)
endif
()
SET
(
CMAKE_C_FLAGS_RELEASE
"$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations"
)
SET
(
CMAKE_C_FLAGS_RELEASE
"$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations"
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
set
(
CMAKE_EXPORT_COMPILE_COMMANDS ON
)
#add flags
#add flags
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include -Werror"
)
if
(
WIN32
)
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include"
)
else
()
set
(
CMAKE_C_FLAGS
"
${
CMAKE_C_FLAGS
}
-I/usr/local/include -Werror"
)
endif
()
include_directories
(
./include
)
include_directories
(
./include
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录