提交 b4258477 编写于 作者: J Jin Yi 提交者: Jinhui Yuan

Dev jinyi offline build (#1476)

* chore: remove pre compiler funcs

* chore: add submoudles

* fix: fix project build URL from git_url -> submodule_dir_url

* fix: fix submodule commit id

* fix: fix .gitmodules

* chore: mv third_party dir

* chore: remove test-driver(glog#188) link in glog submodule

* fix: update glog from: da816ea70645e463aa04f9564544939fa327d5a7 ==> to: 4f3e18bf26cdb794fc66cec348f57b5838a0c929

* chore: update README.md


Former-commit-id: 8cc052f38cfd53c40186dc487df41b0c1f4a7189
上级 5d034a39
[submodule "cub"]
path = build/third_party/cub/src/cub
url = https://github.com/NVlabs/cub.git
[submodule "eigen"]
path = build/third_party/eigen/src/eigen
url = https://github.com/eigenteam/eigen-git-mirror
[submodule "gflags"]
path = build/third_party/gflags/src/gflags
url = https://github.com/gflags/gflags
[submodule "glog"]
path = build/third_party/glog/src/glog
url = https://github.com/google/glog
[submodule "googletest"]
path = build/third_party/googletest/src/googletest
url = https://github.com/google/googletest
[submodule "grpc"]
path = build/third_party/grpc/src/grpc
url = https://github.com/yuanms2/grpc
[submodule "libjpeg-turbo"]
path = build/third_party/libjpeg-turbo/src/libjpeg-turbo
url = https://github.com/libjpeg-turbo/libjpeg-turbo
[submodule "nccl"]
path = build/third_party/nccl/src/nccl
url = https://github.com/NVIDIA/nccl
[submodule "opencv"]
path = build/third_party/opencv/src/opencv
url = https://github.com/opencv/opencv
[submodule "protobuf"]
path = build/third_party/protobuf/src/protobuf
url = https://github.com/protocolbuffers/protobuf
[submodule "zlib"]
path = build/third_party/zlib/src/zlib
url = https://github.com/madler/zlib
......@@ -2,7 +2,6 @@
cmake_minimum_required(VERSION 3.5)
option(THIRD_PARTY "Build third party or oneflow" OFF)
option(PRECOMPILED_THIRD_PARTY "Donwload precompiled third party instead of building from source" ON)
option(USE_CLANG_FORMAT "" OFF)
option(BUILD_RDMA "" OFF)
......
......@@ -16,24 +16,25 @@ Or you can install OpenBLAS and other tools through:
sudo yum -y install epel-release && sudo yum -y install git gcc-c++ cmake3 openblas-devel kernel-devel-$(uname -r) nasm
```
#### create `build` directory:
#### clone source code
> note: with `--recursive` flag to clone third_party submodules
```
cd oneflow && mkdir build && cd build
git clone https://github.com/Oneflow-Inc/oneflow --recursive
```
#### build third party from source
or you can just clone source code and submodules step by step
```
cmake -DTHIRD_PARTY=ON -DPRECOMPILED_THIRD_PARTY=OFF .. && make -j
git clone https://github.com/Oneflow-Inc/oneflow
git submodule update --init --recursive
```
If you do not want to compile the third party code from source, you can also download the pre-compiled third party dependencies from oneflow.org
#### download pre-compiled third party
#### build third party from source
```
cmake -DTHIRD_PARTY=ON .. && make -j
cmake -DTHIRD_PARTY=ON .. && make -j
```
#### build oneflow
......
Subproject commit c3cceac115c072fb63df1836ff46d8c60d9eb304
Subproject commit e9e95489a0b241412e31f0525e85b2fab386c786
Subproject commit 9314597d4b742ed6f95665241345e590a0f5759b
Subproject commit 4f3e18bf26cdb794fc66cec348f57b5838a0c929
Subproject commit ec44c6c1675c25b9827aacd08c02433cccde7780
Subproject commit e0db46e140405f0f94f03c9a55b302e39a514c48
Subproject commit 3041cf67ffdc7230e377802cba0e5c325d6d01c6
Subproject commit f93fe9bfd94884cec2ba711897222e0df5569a53
Subproject commit 51cef2651e91003e6a6760f496719dbb325cfc61
Subproject commit 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05
Subproject commit 50893291621658f355bc5b4d450a8d06a563053d
......@@ -12,25 +12,6 @@ include(libjpeg-turbo)
include(opencv)
include(eigen)
if (THIRD_PARTY AND PRECOMPILED_THIRD_PARTY)
if (BUILD_CUDA)
set(THIRD_PARTY_URL http://download.oneflow.org/third_party_with_cuda.tgz)
else()
set(THIRD_PARTY_URL http://download.oneflow.org/third_party_without_cuda.tgz)
endif()
if (NOT WIN32)
message(STATUS "downloading third_partys ...")
file(DOWNLOAD ${THIRD_PARTY_URL} ${PROJECT_BINARY_DIR}/third_party.tgz SHOW_PROGRESS)
execute_process(
COMMAND ${CMAKE_COMMAND} -E tar zxvf ${PROJECT_BINARY_DIR}/third_party.tgz
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
)
else()
message(STATUS "win32 not support yet.")
endif()
endif()
if (BUILD_CUDA)
set(CUDA_SEPARABLE_COMPILATION ON)
find_package(CUDA REQUIRED)
......@@ -90,26 +71,21 @@ set(oneflow_third_party_dependencies
protobuf_copy_headers_to_destination
protobuf_copy_libs_to_destination
protobuf_copy_binary_to_destination
gflags_copy_headers_to_destination
gflags_copy_libs_to_destination
glog_copy_headers_to_destination
glog_copy_libs_to_destination
googletest_copy_headers_to_destination
googletest_copy_libs_to_destination
googlemock_copy_headers_to_destination
googlemock_copy_libs_to_destination
grpc_copy_headers_to_destination
grpc_copy_libs_to_destination
opencv_copy_headers_to_destination
opencv_copy_libs_to_destination
eigen
)
if (NOT PRECOMPILED_THIRD_PARTY)
list(APPEND oneflow_third_party_dependencies
gflags_copy_headers_to_destination
gflags_copy_libs_to_destination
glog_copy_headers_to_destination
glog_copy_libs_to_destination
googletest_copy_headers_to_destination
googletest_copy_libs_to_destination
googlemock_copy_headers_to_destination
googlemock_copy_libs_to_destination
grpc_copy_headers_to_destination
grpc_copy_libs_to_destination
opencv_copy_headers_to_destination
opencv_copy_libs_to_destination
eigen
)
endif()
include_directories(
${ZLIB_INCLUDE_DIR}
${GFLAGS_INCLUDE_DIR}
......
......@@ -3,16 +3,13 @@ include (ExternalProject)
set(CUB_INCLUDE_DIR ${THIRD_PARTY_DIR}/cub/include)
set(CUB_BUILD_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub/cub)
set(CUB_URL https://github.com/NVlabs/cub.git)
set(CUB_TAG c3cceac115c072fb63df1836ff46d8c60d9eb304)
set(CUB_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/cub/src/cub)
if(THIRD_PARTY)
ExternalProject_Add(cub
PREFIX cub
GIT_REPOSITORY ${CUB_URL}
GIT_TAG ${CUB_TAG}
URL ${CUB_URL}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
......
......@@ -3,17 +3,15 @@ include (ExternalProject)
set(EIGEN_INCLUDE_DIR ${THIRD_PARTY_DIR}/eigen/include/eigen3)
set(EIGEN_INSTALL_DIR ${THIRD_PARTY_DIR}/eigen)
set(eigen_URL https://github.com/eigenteam/eigen-git-mirror)
set(eigen_TAG e9e95489a0b241412e31f0525e85b2fab386c786)
set(EIGEN_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/eigen/src/eigen)
add_definitions(-DEIGEN_NO_AUTOMATIC_RESIZING -DEIGEN_NO_MALLOC -DEIGEN_USE_GPU)
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if (THIRD_PARTY)
ExternalProject_Add(eigen
PREFIX eigen
GIT_REPOSITORY ${eigen_URL}
GIT_TAG ${eigen_TAG}
URL ${EIGEN_URL}
UPDATE_COMMAND ""
INSTALL_DIR "${EIGEN_INSTALL_DIR}"
CMAKE_CACHE_ARGS
......@@ -26,4 +24,4 @@ ExternalProject_Add(eigen
)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -5,8 +5,7 @@ set(GFLAGS_LIBRARY_DIR ${THIRD_PARTY_DIR}/gflags/lib)
set(gflags_HEADERS_DIR ${CMAKE_CURRENT_BINARY_DIR}/gflags/src/gflags/include)
set(gflags_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/gflags/src/gflags/lib)
set(gflags_URL https://github.com/gflags/gflags.git)
set(gflags_TAG 9314597d4b742ed6f95665241345e590a0f5759b)
set(gflags_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/src/gflags)
if(WIN32)
set(GFLAGS_BUILD_LIBRARY_DIR ${gflags_LIB_DIR}/${CMAKE_BUILD_TYPE})
......@@ -24,12 +23,11 @@ foreach(LIBRARY_NAME ${GFLAGS_LIBRARY_NAMES})
list(APPEND GFLAGS_BUILD_STATIC_LIBRARIES ${GFLAGS_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if (THIRD_PARTY)
ExternalProject_Add(gflags
PREFIX gflags
GIT_REPOSITORY ${gflags_URL}
GIT_TAG ${gflags_TAG}
URL ${gflags_URL}
UPDATE_COMMAND ""
BUILD_IN_SOURCE 1
INSTALL_COMMAND ""
......@@ -57,4 +55,4 @@ add_custom_target(gflags_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GFLAGS_BUILD_STATIC_LIBRARIES} ${GFLAGS_LIBRARY_DIR}
DEPENDS gflags_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -3,8 +3,7 @@ include (ExternalProject)
set(GLOG_INCLUDE_DIR ${THIRD_PARTY_DIR}/glog/include)
set(GLOG_LIBRARY_DIR ${THIRD_PARTY_DIR}/glog/lib)
set(glog_URL https://github.com/google/glog.git)
set(glog_TAG da816ea70645e463aa04f9564544939fa327d5a7)
set(glog_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/glog/src/glog)
if(WIN32)
set(GLOG_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/${CMAKE_BUILD_TYPE})
......@@ -31,13 +30,12 @@ set (GLOG_PUBLIC_H
${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/src/glog/log_severity.h
)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(glog
DEPENDS gflags_copy_headers_to_destination gflags_copy_libs_to_destination
PREFIX glog
GIT_REPOSITORY ${glog_URL}
GIT_TAG ${glog_TAG}
URL ${glog_URL}
UPDATE_COMMAND ""
BUILD_IN_SOURCE 1
INSTALL_COMMAND ""
......@@ -74,4 +72,4 @@ add_custom_target(glog_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GLOG_BUILD_STATIC_LIBRARIES} ${GLOG_LIBRARY_DIR}
DEPENDS glog_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -9,8 +9,7 @@ set(GOOGLEMOCK_LIBRARY_DIR ${THIRD_PARTY_DIR}/googlemock/lib)
set(googletest_SRC_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/include)
set(googlemock_SRC_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googlemock/include)
set(googletest_URL https://github.com/google/googletest.git)
set(googletest_TAG ec44c6c1675c25b9827aacd08c02433cccde7780)
set(googletest_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/googletest/src/googletest)
if(WIN32)
set(GOOGLETEST_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googlemock/gtest/${CMAKE_BUILD_TYPE})
......@@ -39,12 +38,11 @@ foreach(LIBRARY_NAME ${GOOGLEMOCK_LIBRARY_NAMES})
list(APPEND GOOGLEMOCK_BUILD_STATIC_LIBRARIES ${GOOGLEMOCK_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(googletest
PREFIX googletest
GIT_REPOSITORY ${googletest_URL}
GIT_TAG ${googletest_TAG}
URL ${googletest_URL}
UPDATE_COMMAND ""
BUILD_IN_SOURCE 1
INSTALL_COMMAND ""
......@@ -88,4 +86,4 @@ add_custom_target(googlemock_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GOOGLEMOCK_BUILD_STATIC_LIBRARIES} ${GOOGLEMOCK_LIBRARY_DIR}
DEPENDS googlemock_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -4,8 +4,7 @@ set(GRPC_INCLUDE_DIR ${THIRD_PARTY_DIR}/grpc/include)
set(GRPC_LIBRARY_DIR ${THIRD_PARTY_DIR}/grpc/lib)
set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
set(GRPC_URL https://github.com/yuanms2/grpc.git)
set(GRPC_TAG e0db46e140405f0f94f03c9a55b302e39a514c48)
set(GRPC_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/grpc/src/grpc)
if(WIN32)
set(GRPC_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/${CMAKE_BUILD_TYPE})
......@@ -26,13 +25,12 @@ foreach(LIBRARY_NAME ${GRPC_LIBRARY_NAMES})
list(APPEND GRPC_BUILD_STATIC_LIBRARIES ${GRPC_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
endforeach()
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(grpc
PREFIX grpc
DEPENDS protobuf zlib
GIT_REPOSITORY ${GRPC_URL}
GIT_TAG ${GRPC_TAG}
URL ${GRPC_URL}
UPDATE_COMMAND ""
BUILD_IN_SOURCE 1
INSTALL_COMMAND ""
......@@ -64,4 +62,4 @@ add_custom_target(grpc_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GRPC_BUILD_STATIC_LIBRARIES} ${GRPC_LIBRARY_DIR}
DEPENDS grpc_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -3,8 +3,7 @@ include (ExternalProject)
set(LIBJPEG_INCLUDE_DIR ${THIRD_PARTY_DIR}/libjpeg-turbo/include)
set(LIBJPEG_LIBRARY_DIR ${THIRD_PARTY_DIR}/libjpeg-turbo/lib)
set(LIBJPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
set(LIBJPEG_TAG 3041cf67ffdc7230e377802cba0e5c325d6d01c6)
set(LIBJPEG_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/libjpeg-turbo/src/libjpeg-turbo)
if(WIN32)
elseif(APPLE AND ("${CMAKE_GENERATOR}" STREQUAL "Xcode"))
......@@ -46,12 +45,11 @@ set(LIBJPEG_HEADERS
"${LIBJPEG_BUILD_LIBRARY_DIR}/turbojpeg.h"
)
if(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if(THIRD_PARTY)
ExternalProject_Add(libjpeg-turbo
PREFIX libjpeg-turbo
GIT_REPOSITORY ${LIBJPEG_URL}
GIT_TAG ${LIBJPEG_TAG}
URL ${LIBJPEG_URL}
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_IN_SOURCE 1
......@@ -83,4 +81,4 @@ add_custom_target(libjpeg_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${LIBJPEG_BUILD_STATIC_LIBRARIES} ${LIBJPEG_LIBRARY_DIR}
DEPENDS libjpeg_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -3,8 +3,7 @@ include (ExternalProject)
set(NCCL_INCLUDE_DIR ${THIRD_PARTY_DIR}/nccl/include)
set(NCCL_LIBRARY_DIR ${THIRD_PARTY_DIR}/nccl/lib)
set(NCCL_URL https://github.com/NVIDIA/nccl.git)
set(NCCL_TAG f93fe9bfd94884cec2ba711897222e0df5569a53)
set(NCCL_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/nccl/src/nccl)
set(NCCL_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/nccl/src/nccl/build)
if(WIN32)
......@@ -31,8 +30,7 @@ if(THIRD_PARTY)
ExternalProject_Add(nccl
PREFIX nccl
GIT_REPOSITORY ${NCCL_URL}
GIT_TAG ${NCCL_TAG}
URL ${NCCL_URL}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
......
......@@ -5,8 +5,7 @@ set(OPENCV_LIBRARY_DIR ${THIRD_PARTY_DIR}/opencv/lib)
set(OPENCV_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build/install)
set(OPENCV_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/src)
set(OPENCV_URL https://github.com/opencv/opencv.git)
set(OPENCV_TAG 51cef2651e91003e6a6760f496719dbb325cfc61)
set(OPENCV_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/opencv/src/opencv)
if(WIN32)
elseif(APPLE AND ("${CMAKE_GENERATOR}" STREQUAL "Xcode"))
......@@ -29,13 +28,12 @@ foreach(LIBRARY_NAME ${OPENCV_3RDPARTY_LIBRARY_NAMES})
endforeach()
if (THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
if (THIRD_PARTY)
ExternalProject_Add(opencv
DEPENDS libjpeg_copy_headers_to_destination libjpeg_copy_libs_to_destination
PREFIX opencv
GIT_REPOSITORY ${OPENCV_URL}
GIT_TAG ${OPENCV_TAG}
URL ${OPENCV_URL}
UPDATE_COMMAND ""
PATCH_COMMAND cmake -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build
BUILD_IN_SOURCE 0
......@@ -124,4 +122,4 @@ add_custom_target(opencv_copy_libs_to_destination
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${OPENCV_BUILD_STATIC_LIBRARIES} ${OPENCV_LIBRARY_DIR}
DEPENDS opencv_create_library_dir)
endif(THIRD_PARTY AND NOT PRECOMPILED_THIRD_PARTY)
endif(THIRD_PARTY)
......@@ -5,8 +5,7 @@ set(PROTOBUF_LIBRARY_DIR ${THIRD_PARTY_DIR}/protobuf/lib)
set(PROTOBUF_BINARY_DIR ${THIRD_PARTY_DIR}/protobuf/bin)
set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
set(PROTOBUF_URL https://github.com/protocolbuffers/protobuf.git)
set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
set(PROTOBUF_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/protobuf/src/protobuf)
if(WIN32)
set(PROTOBUF_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/${CMAKE_BUILD_TYPE})
......@@ -36,8 +35,7 @@ if (THIRD_PARTY)
ExternalProject_Add(protobuf
PREFIX protobuf
DEPENDS zlib
GIT_REPOSITORY ${PROTOBUF_URL}
GIT_TAG ${PROTOBUF_TAG}
URL ${PROTOBUF_URL}
UPDATE_COMMAND ""
BUILD_IN_SOURCE 1
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf
......
......@@ -4,8 +4,7 @@ set(ZLIB_INCLUDE_DIR ${THIRD_PARTY_DIR}/zlib/include)
set(ZLIB_LIBRARY_DIR ${THIRD_PARTY_DIR}/zlib/lib)
set(ZLIB_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/zlib/install)
set(ZLIB_URL https://github.com/madler/zlib)
set(ZLIB_TAG 50893291621658f355bc5b4d450a8d06a563053d)
set(ZLIB_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib/src/zlib)
if(WIN32)
set(ZLIB_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib)
......@@ -32,8 +31,7 @@ if(THIRD_PARTY)
ExternalProject_Add(zlib
PREFIX zlib
GIT_REPOSITORY ${ZLIB_URL}
GIT_TAG ${ZLIB_TAG}
URL ${ZLIB_URL}
UPDATE_COMMAND ""
INSTALL_DIR ${ZLIB_INSTALL}
BUILD_IN_SOURCE 1
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册