Skip to content

O
oneflow

Oneflow-Inc / oneflow
上一次同步 2 年多

通知 13
Star 2733
Fork 0
O
oneflow

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 b3fb9acf 编写于 作者: J Jin Yi 提交者： Jinhui Yuan
浏览文件
操作

feat: enhance cmake download & options (#1281) 

* feat: enhance cmake download & options

* feat(tools/): add share libs build scripts

* fix: add cmake options

* feat: add 3rd party download

* chore: updat README

* fix: fix protobuf & cmake repo

* fix: fix options name

* chore: merge 3rd_party.cmake & third_party.cmake

* chore: revert pre cmake URL fix

* chore: update ExternalProject check

* fix: fix typo & missing download

* fix: fix download url

* chore: update readme

* chore: fix typo

* fix: fix bugs

* fix: fix bugs

* fix: fix pre

* print all third party libs

* refine readme

* DOWNLOAD_THIRD_PARTY -> PRECOMPILED_THIRD_PARTY

* refine readme

* minor typo fix


Former-commit-id: d7d1ec98a868c32e3a43658823ae136caa73feb5
上级 71d34a97
隐藏空白更改
内联 并排
Showing with 114 addition and 51 deletion
CMakeLists.txt
浏览文件 @ b3fb9acf
# Minimum CMake required
cmake_minimum_required(VERSION 3.5)
option(BUILD_THIRD_PARTY "Build third party or oneflow" OFF)
option(BUILD_RDMA "" ON)
option(THIRD_PARTY "Build third party or oneflow" OFF)
option(PRECOMPILED_THIRD_PARTY "Donwload precompiled third party instead of building from source" ON)
option(USE_CLANG_FORMAT "" OFF)
option(BUILD_RDMA "" OFF)
option(BUILD_CUDA "" ON)
option(RELEASE_VERSION "" OFF)
option(RELEASE_VERSION "" ON)
# Project
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY)
project(third_party C CXX)
else()
project(oneflow C CXX)
......@@ -78,9 +81,9 @@ include(util)
include(proto2cpp)
include(third_party)
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY)
add_custom_target(prepare_oneflow_third_party ALL DEPENDS ${oneflow_third_party_dependencies})
set(BUILD_THIRD_PARTY OFF CACHE BOOL "" FORCE)
set(THIRD_PARTY OFF CACHE BOOL "" FORCE)
else()
include(oneflow)
endif()
......
README.md
浏览文件 @ b3fb9acf
......@@ -2,9 +2,42 @@
### 1.1 Linux
### Build
Building OneFlow from source requires a `BLAS libary` installed. On CentOS, if you have `Intel MKL` installed, please update the environment variable.
```
export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
```
Or you can install OpenBLAS and other tools through:
```
sudo yum -y install epel-release && sudo yum -y install git gcc-c++ cmake3 openblas-devel kernel-devel-$(uname -r) nasm
```
#### create `build` directory:
```
cd oneflow && mkdir build && cd build
```
#### build third party from source
```
cmake -DTHIRD_PARTY=ON -DPRECOMPILED_THIRD_PARTY=OFF .. && make -j
```
If you do not want to compile the third party code from source, you can also download the pre-compiled third party dependencies from oneflow.org
#### download pre-compiled third party
```
cmake -DTHIRD_PARTY=ON .. && make -j
```
#### build oneflow
```
export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
mkdir build && cd build
cmake -DBUILD_THIRD_PARTY=ON .. && make
cmake -DBUILD_THIRD_PARTY=OFF .. && make
cmake -DTHIRD_PARTY=OFF .. && make -j
```
cmake/oneflow.cmake
浏览文件 @ b3fb9acf
......@@ -125,7 +125,9 @@ include_directories(${PROJECT_BINARY_DIR})
oneflow_add_library(of_ccobj ${of_all_obj_cc})
target_link_libraries(of_ccobj ${oneflow_third_party_libs})
add_dependencies(of_ccobj of_protoobj)
add_dependencies(of_ccobj of_format)
if (USE_CLANG_FORMAT)
add_dependencies(of_ccobj of_format)
endif()
if(APPLE)
set(of_libs -Wl,-force_load of_ccobj of_protoobj)
......
cmake/third_party.cmake
浏览文件 @ b3fb9acf
......@@ -12,6 +12,25 @@ include(libjpeg-turbo)
include(opencv)
include(eigen)
if (THIRD_PARTY AND PRECOMPILED_THIRD_PARTY)
if (BUILD_CUDA)
set(THIRD_PARTY_URL http://download.oneflow.org/third_party_with_cuda.tgz)
else()
set(THIRD_PARTY_URL http://download.oneflow.org/third_party_without_cuda.tgz)
endif()
if (NOT WIN32)
message(STATUS "downloading third_partys ...")
file(DOWNLOAD ${THIRD_PARTY_URL} ${PROJECT_BINARY_DIR}/third_party.tgz SHOW_PROGRESS)
execute_process(
COMMAND ${CMAKE_COMMAND} -E tar zxvf ${PROJECT_BINARY_DIR}/third_party.tgz
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
)
else()
message(STATUS "win32 not support yet.")
endif()
endif()
if (BUILD_CUDA)
set(CUDA_SEPARABLE_COMPILATION ON)
find_package(CUDA REQUIRED)
......@@ -59,8 +78,6 @@ set(oneflow_third_party_libs
${OPENCV_STATIC_LIBRARIES}
)
message(STATUS "oneflow_third_party_libs: " ${oneflow_third_party_libs})
if(WIN32)
# static gflags lib requires "PathMatchSpecA" defined in "ShLwApi.Lib"
list(APPEND oneflow_third_party_libs "ShLwApi.Lib")
......@@ -70,24 +87,29 @@ endif()
set(oneflow_third_party_dependencies
zlib_copy_headers_to_destination
zlib_copy_libs_to_destination
gflags_copy_headers_to_destination
gflags_copy_libs_to_destination
glog_copy_headers_to_destination
glog_copy_libs_to_destination
googletest_copy_headers_to_destination
googletest_copy_libs_to_destination
googlemock_copy_headers_to_destination
googlemock_copy_libs_to_destination
protobuf_copy_headers_to_destination
protobuf_copy_libs_to_destination
protobuf_copy_binary_to_destination
grpc_copy_headers_to_destination
grpc_copy_libs_to_destination
opencv_copy_headers_to_destination
opencv_copy_libs_to_destination
eigen
)
if (NOT PRECOMPILED_THIRD_PARTY)
list(APPEND oneflow_third_party_dependencies
gflags_copy_headers_to_destination
gflags_copy_libs_to_destination
glog_copy_headers_to_destination
glog_copy_libs_to_destination
googletest_copy_headers_to_destination
googletest_copy_libs_to_destination
googlemock_copy_headers_to_destination
googlemock_copy_libs_to_destination
grpc_copy_headers_to_destination
grpc_copy_libs_to_destination
opencv_copy_headers_to_destination
opencv_copy_libs_to_destination
eigen
)
endif()
include_directories(
${ZLIB_INCLUDE_DIR}
${GFLAGS_INCLUDE_DIR}
......@@ -119,3 +141,5 @@ if (BUILD_CUDA)
${NCCL_INCLUDE_DIR}
)
endif()
message(STATUS "oneflow_third_party_libs: " ${oneflow_third_party_libs})
cmake/third_party/cub.cmake
浏览文件 @ b3fb9acf
......@@ -6,7 +6,8 @@ set(CUB_BUILD_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub/cub)
set(CUB_URL https://github.com/NVlabs/cub.git)
set(CUB_TAG c3cceac115c072fb63df1836ff46d8c60d9eb304)
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(cub
PREFIX cub
......@@ -24,4 +25,4 @@ add_custom_target(cub_copy_headers_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_directory ${CUB_BUILD_INCLUDE} ${CUB_INCLUDE_DIR}/cub
DEPENDS cub_create_header_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY)
cmake/third_party/eigen.cmake
浏览文件 @ b3fb9acf
......@@ -8,7 +8,7 @@ set(eigen_TAG e9e95489a0b241412e31f0525e85b2fab386c786)
add_definitions(-DEIGEN_NO_AUTOMATIC_RESIZING -DEIGEN_NO_MALLOC -DEIGEN_USE_GPU)
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(eigen
PREFIX eigen
......@@ -26,4 +26,4 @@ ExternalProject_Add(eigen
)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/gflags.cmake
浏览文件 @ b3fb9acf
......@@ -24,7 +24,7 @@ foreach(LIBRARY_NAME ${GFLAGS_LIBRARY_NAMES})
list(APPEND GFLAGS_BUILD_STATIC_LIBRARIES ${GFLAGS_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(gflags
PREFIX gflags
......@@ -57,4 +57,4 @@ add_custom_target(gflags_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GFLAGS_BUILD_STATIC_LIBRARIES} ${GFLAGS_LIBRARY_DIR}
DEPENDS gflags_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/glog.cmake
浏览文件 @ b3fb9acf
......@@ -31,7 +31,7 @@ set (GLOG_PUBLIC_H
${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/src/glog/log_severity.h
)
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(glog
DEPENDS gflags_copy_headers_to_destination gflags_copy_libs_to_destination
......@@ -74,4 +74,4 @@ add_custom_target(glog_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GLOG_BUILD_STATIC_LIBRARIES} ${GLOG_LIBRARY_DIR}
DEPENDS glog_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/googletest.cmake
浏览文件 @ b3fb9acf
......@@ -39,7 +39,7 @@ foreach(LIBRARY_NAME ${GOOGLEMOCK_LIBRARY_NAMES})
list(APPEND GOOGLEMOCK_BUILD_STATIC_LIBRARIES ${GOOGLEMOCK_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(googletest
PREFIX googletest
......@@ -88,4 +88,4 @@ add_custom_target(googlemock_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GOOGLEMOCK_BUILD_STATIC_LIBRARIES} ${GOOGLEMOCK_LIBRARY_DIR}
DEPENDS googlemock_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/grpc.cmake
浏览文件 @ b3fb9acf
......@@ -26,7 +26,7 @@ foreach(LIBRARY_NAME ${GRPC_LIBRARY_NAMES})
list(APPEND GRPC_BUILD_STATIC_LIBRARIES ${GRPC_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(grpc
PREFIX grpc
......@@ -64,4 +64,4 @@ add_custom_target(grpc_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GRPC_BUILD_STATIC_LIBRARIES} ${GRPC_LIBRARY_DIR}
DEPENDS grpc_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/libjpeg-turbo.cmake
浏览文件 @ b3fb9acf
......@@ -46,7 +46,7 @@ set(LIBJPEG_HEADERS
"${LIBJPEG_BUILD_LIBRARY_DIR}/turbojpeg.h"
)
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(libjpeg-turbo
PREFIX libjpeg-turbo
......@@ -83,4 +83,4 @@ add_custom_target(libjpeg_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${LIBJPEG_BUILD_STATIC_LIBRARIES} ${LIBJPEG_LIBRARY_DIR}
DEPENDS libjpeg_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/nccl.cmake
浏览文件 @ b3fb9acf
......@@ -27,7 +27,7 @@ set(NCCL_HEADERS
"${NCCL_BUILD_DIR}/include/nccl.h"
)
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(nccl
PREFIX nccl
......@@ -62,4 +62,4 @@ add_custom_target(nccl_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${NCCL_BUILD_STATIC_LIBRARIES} ${NCCL_LIBRARY_DIR}
DEPENDS nccl_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY)
cmake/third_party/opencv.cmake
浏览文件 @ b3fb9acf
......@@ -5,7 +5,7 @@ set(OPENCV_LIBRARY_DIR ${THIRD_PARTY_DIR}/opencv/lib)
set(OPENCV_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build/install)
set(OPENCV_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/src)
set(OPENCV_URL https://github.com/Oneflow-Inc/opencv.git)
set(OPENCV_URL https://github.com/opencv/opencv.git)
set(OPENCV_TAG 51cef2651e91003e6a6760f496719dbb325cfc61)
if(WIN32)
......@@ -29,7 +29,7 @@ foreach(LIBRARY_NAME ${OPENCV_3RDPARTY_LIBRARY_NAMES})
endforeach()
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
ExternalProject_Add(opencv
DEPENDS libjpeg_copy_headers_to_destination libjpeg_copy_libs_to_destination
......@@ -124,4 +124,4 @@ add_custom_target(opencv_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${OPENCV_BUILD_STATIC_LIBRARIES} ${OPENCV_LIBRARY_DIR}
DEPENDS opencv_create_library_dir)
endif(BUILD_THIRD_PARTY)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
cmake/third_party/protobuf.cmake
浏览文件 @ b3fb9acf
......@@ -5,7 +5,7 @@ set(PROTOBUF_LIBRARY_DIR ${THIRD_PARTY_DIR}/protobuf/lib)
set(PROTOBUF_BINARY_DIR ${THIRD_PARTY_DIR}/protobuf/bin)
set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
set(PROTOBUF_URL https://github.com/mrry/protobuf.git) # Includes MSVC fix.
set(PROTOBUF_URL https://github.com/protocolbuffers/protobuf.git)
set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
if(WIN32)
......@@ -31,7 +31,7 @@ endforeach()
set(PROTOBUF_BUILD_PROTOC_EXECUTABLE ${PROTOBUF_BUILD_LIBRARY_DIR}/${PROTOC_EXECUTABLE_NAME})
set(PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_BINARY_DIR}/${PROTOC_EXECUTABLE_NAME})
if (BUILD_THIRD_PARTY)
if (THIRD_PARTY)
ExternalProject_Add(protobuf
PREFIX protobuf
......@@ -83,4 +83,4 @@ add_custom_target(protobuf_copy_binary_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${PROTOBUF_BUILD_PROTOC_EXECUTABLE} ${PROTOBUF_BINARY_DIR}
DEPENDS protobuf_create_binary_dir)
endif(BUILD_THIRD_PARTY)
\ No newline at end of file
endif(THIRD_PARTY)
cmake/third_party/zlib.cmake
浏览文件 @ b3fb9acf
......@@ -3,8 +3,8 @@ include (ExternalProject)
set(ZLIB_INCLUDE_DIR ${THIRD_PARTY_DIR}/zlib/include)
set(ZLIB_LIBRARY_DIR ${THIRD_PARTY_DIR}/zlib/lib)
set(ZLIB_URL https://github.com/madler/zlib)
set(ZLIB_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/zlib/install)
set(ZLIB_URL https://github.com/madler/zlib)
set(ZLIB_TAG 50893291621658f355bc5b4d450a8d06a563053d)
if(WIN32)
......@@ -28,7 +28,7 @@ set(ZLIB_HEADERS
"${ZLIB_INSTALL}/include/zlib.h"
)
if(BUILD_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(zlib
PREFIX zlib
......@@ -66,4 +66,4 @@ add_custom_target(zlib_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ZLIB_BUILD_STATIC_LIBRARIES} ${ZLIB_LIBRARY_DIR}
DEPENDS zlib_create_library_dir)
endif(BUILD_THIRD_PARTY)
\ No newline at end of file
endif(THIRD_PARTY)
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或