Merge branch 'master' of https://github.com/Oneflow-Inc/oneflow into...

Merge branch 'master' of https://github.com/Oneflow-Inc/oneflow into dev_build_docker_img_on_hosted_azure_env


Former-commit-id: 280dda1a01f47545db68775333b76b56011dcfda

.github/workflows/release.yml

@@ -43,22 +43,25 @@ jobs:
         $HOME/ossutil64 cp --update -r $ONEFLOW_CI_TMP_DIR/wheelhouse oss://oneflow-static/staging/${dir}
   pack_src:
     name: Pack source code
-    runs-on: [self-hosted, linux]
+    runs-on: ubuntu-latest
     steps:
-    - name: Fix permissions
-      run: |
-        export ONEFLOW_CI_TMP_DIR=$HOME/ci-tmp-xla-102
-        docker run --rm -v $PWD:/p -w /p busybox chmod -R o+w .
-        docker run --rm -v $ONEFLOW_CI_TMP_DIR/:$HOME/ci-tmp/ busybox rm -rf $HOME/ci-tmp/wheelhouse
     - uses: actions/checkout@v2
     - name: Setup submodule
-      uses: ./.github/actions/setup
+      run: |
+        auth_header="$(git config --local --get http.https://github.com/.extraheader)"
+        git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --recursive
     - name: Pack OneFlow source code
       if: github.ref == 'refs/heads/master'
+      env:
+        OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+        OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
       run: |
         git reset --hard
         git clean -f
         cd ..
         zip -r oneflow-src.zip oneflow
+        curl http://gosspublic.alicdn.com/ossutil/1.6.19/ossutil64 -o $HOME/ossutil64
+        chmod 755 $HOME/ossutil64
+        $HOME/ossutil64 config -e oss-cn-beijing.aliyuncs.com -i ${OSS_ACCESS_KEY_ID} -k ${OSS_ACCESS_KEY_SECRET}  -L EN -c $HOME/.ossutilconfig
         $HOME/ossutil64 cp --update oneflow-src.zip oss://oneflow-public/oneflow-src.zip
         rm oneflow-src.zip

.github/workflows/test.yml

@@ -150,7 +150,7 @@ jobs:
       uses: ./.github/actions/setup
     - name: Build OneFlow
       run: |
-        export ONEFLOW_CI_EXTRA_ONEFLOW_CMAKE_ARGS="-DWITH_XLA=ON"
+        export ONEFLOW_CI_EXTRA_ONEFLOW_CMAKE_ARGS="-DWITH_XLA=ON -DTHIRD_PARTY_MIRROR=aliyun"
         export ONEFLOW_CI_TMP_DIR=$HOME/ci-tmp-xla
         export ONEFLOW_CI_CUDA_VERSION=10.1
         export ONEFLOW_CI_PACKAGE_SUFFIX="_cu101_xla"

CMakeLists.txt

@@ -16,6 +16,7 @@ option(WITH_XLA "Option to build with XLA" OFF)
 option(WITH_TENSORRT "Option to build with TensorRT" OFF)
 option(FOR_CI "" OFF)
 option(BUILD_GIT_VERSION "" ON)
+set(THIRD_PARTY_MIRROR "" CACHE STRING "")
 
 if (CMAKE_BUILD_TYPE MATCHES Debug)
   set(CUDNN_STATIC OFF CACHE BOOL "")

README.md

@@ -138,6 +138,7 @@
     ```
 
     - For pure CPU build, please add this CMake flag `-DBUILD_CUDA=OFF`.
+    - If you are in China, please add this CMake flag `-DTHIRD_PARTY_MIRROR=aliyun` to speed up the downloading procedure for some dependency tar files. 
 
 ### Troubleshooting
 

cmake/oneflow.cmake

 include(python)
-
 # main cpp
 # TODO(tsai): skip for now, fail to link when building CPU only
 if (BUILD_CUDA)

cmake/pybind11.cmake

 include(FetchContent)
+set(PYBIND11_TAR_URL https://github.com/Oneflow-Inc/pybind11/archive/1534e17.zip)
+use_mirror(VARIABLE PYBIND11_TAR_URL URL ${PYBIND11_TAR_URL})
+set(PYBIND11_URL_HASH c49f9edab467699dd3d8be9224fc9cf3)
 
 FetchContent_Declare(
     pybind11
-    URL https://github.com/Oneflow-Inc/pybind11/archive/1534e17.zip
+    URL ${PYBIND11_TAR_URL} 
+    URL_HASH MD5=${PYBIND11_URL_HASH}
 )
 FetchContent_GetProperties(pybind11)
 if(NOT pybind11_POPULATED)

cmake/third_party/absl.cmake

@@ -2,7 +2,8 @@ include(ExternalProject)
 include(GNUInstallDirs)
 
 SET(ABSL_PROJECT absl)
-
+SET(ABSL_TAR_URL https://github.com/Oneflow-Inc/abseil-cpp/archive/d0.tar.gz)
+use_mirror(VARIABLE ABSL_TAR_URL URL ${ABSL_TAR_URL})
 SET(ABSL_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/absl/src/absl)
 SET(ABSL_INSTALL ${THIRD_PARTY_DIR}/absl)
 
@@ -29,7 +30,7 @@ endforeach()
 if(THIRD_PARTY)
   ExternalProject_Add(${ABSL_PROJECT}
     PREFIX absl
-    URL https://github.com/Oneflow-Inc/abseil-cpp/archive/d0.tar.gz
+    URL ${ABSL_TAR_URL} 
     URL_MD5 20126998c9b17e5f7a93711972f03f79
     UPDATE_COMMAND ""
     CMAKE_ARGS

cmake/third_party/cares.cmake

 include(ExternalProject)
 set(CARES_TAR_URL https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz)
+use_mirror(VARIABLE CARES_TAR_URL URL ${CARES_TAR_URL})
 set(CARES_URL_HASH d2391da274653f7643270623e822dff7)
 set(CARES_INSTALL ${THIRD_PARTY_DIR}/cares)
 SET(CARES_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cares/src/cares)

cmake/third_party/cocoapi.cmake

@@ -4,6 +4,8 @@ set(COCOAPI_INCLUDE_DIR ${THIRD_PARTY_DIR}/cocoapi/include)
 set(COCOAPI_LIBRARY_DIR ${THIRD_PARTY_DIR}/cocoapi/lib)
 
 set(COCOAPI_URL https://github.com/Oneflow-Inc/cocoapi/archive/ed842bf.tar.gz)
+use_mirror(VARIABLE COCOAPI_URL URL ${COCOAPI_URL})
+set(COCOAPI_URL_HASH e7e0504231e5614ffaa34f081773f7f1)
 set(COCOAPI_BASE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cocoapi/src/cocoapi)
 set(COCOAPI_LIBRARY_NAME libcocoapi_static.a)
 
@@ -19,6 +21,7 @@ if(THIRD_PARTY)
 ExternalProject_Add(cocoapi
     PREFIX cocoapi
     URL ${COCOAPI_URL}
+    URL_HASH MD5=${COCOAPI_URL_HASH}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""
     BUILD_IN_SOURCE 1

cmake/third_party/eigen.cmake

@@ -6,6 +6,7 @@ set(EIGEN_INSTALL_DIR ${THIRD_PARTY_DIR}/eigen)
 if(WITH_XLA)
   #set(EIGEN_URL "https://storage.googleapis.com/mirror.tensorflow.org/gitlab.com/libeigen/eigen/-/archive/386d809bde475c65b7940f290efe80e6a05878c4/eigen-386d809bde475c65b7940f290efe80e6a05878c4.tar.gz")
   set(EIGEN_URL "https://gitlab.com/libeigen/eigen/-/archive/386d809bde475c65b7940f290efe80e6a05878c4/eigen-386d809bde475c65b7940f290efe80e6a05878c4.tar.gz")
+  use_mirror(VARIABLE EIGEN_URL URL ${EIGEN_URL})
 else()
   set(EIGEN_URL ${THIRD_PARTY_SUBMODULE_DIR}/eigen/src/eigen)
 endif()

cmake/third_party/grpc.cmake

@@ -5,6 +5,7 @@ set(GRPC_LIBRARY_DIR ${THIRD_PARTY_DIR}/grpc/lib)
 
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 SET(GRPC_TAR_URL https://github.com/grpc/grpc/archive/v1.27.3.tar.gz)
+use_mirror(VARIABLE GRPC_TAR_URL URL ${GRPC_TAR_URL})
 set(GRPC_URL_HASH 0c6c3fc8682d4262dd0e5e6fabe1a7e2)
 SET(GRPC_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc)
 

cmake/third_party/half.cmake

@@ -3,8 +3,9 @@ include (ExternalProject)
 set(HALF_INCLUDE_DIR ${THIRD_PARTY_DIR}/half/include)
 
 set(HALF_URL https://github.com/Oneflow-Inc/half/archive/v2.1.0-fix-cuda-raise.zip)
+use_mirror(VARIABLE HALF_URL URL ${HALF_URL})
 set(HALF_BASE_DIR ${CMAKE_CURRENT_BINARY_DIR}/half/src/half)
-
+set(HALF_URL_HASH 30b0dc289729f9e85ddf6995f2e6968f)
 set(HALF_HEADERS
     "${HALF_BASE_DIR}/include/half.hpp"
 )
@@ -14,6 +15,7 @@ if(THIRD_PARTY)
 ExternalProject_Add(half
     PREFIX half
     URL ${HALF_URL}
+    URL_HASH MD5=${HALF_URL_HASH}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""

cmake/third_party/json.cmake

 include(ExternalProject)
 
 SET(JSON_URL https://github.com/nlohmann/json/releases/download/v3.7.3/include.zip)
-set(JSON_BASE_DIR ${CMAKE_CURRENT_BINARY_DIR}/json/src/json)
+use_mirror(VARIABLE JSON_URL URL ${JSON_URL})
+SET(JSON_BASE_DIR ${CMAKE_CURRENT_BINARY_DIR}/json/src/json)
 SET(JSON_INSTALL_DIR ${THIRD_PARTY_DIR}/json)
 SET(JSON_INCLUDE_DIR ${JSON_INSTALL_DIR}/include CACHE PATH "" FORCE)
-
+SET(JSON_URL_HASH fb96f95cdf609143e998db401ca4f324)
 SET(JSON_HEADERS
     "${JSON_BASE_DIR}/single_include/nlohmann/json.hpp"
 )
@@ -13,6 +14,7 @@ if(THIRD_PARTY)
     ExternalProject_Add(json
         PREFIX json
         URL ${JSON_URL}
+        URL_HASH MD5=${JSON_URL_HASH}
         UPDATE_COMMAND ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/third_party/openssl.cmake

@@ -4,7 +4,8 @@ set(OPENSSL_INSTALL ${THIRD_PARTY_DIR}/openssl)
 set(OPENSSL_INCLUDE_DIR ${THIRD_PARTY_DIR}/openssl/include)
 set(OPENSSL_LIBRARY_DIR ${THIRD_PARTY_DIR}/openssl/lib)
 
-SET(OPENSSL_TAR_URL https://github.com/openssl/openssl/archive/OpenSSL_1_1_1g.tar.gz)
+set(OPENSSL_TAR_URL https://github.com/openssl/openssl/archive/OpenSSL_1_1_1g.tar.gz)
+use_mirror(VARIABLE OPENSSL_TAR_URL URL ${OPENSSL_TAR_URL})
 set(OPENSSL_URL_HASH dd32f35dd5d543c571bc9ebb90ebe54e)
 set(OPENSSL_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/openssl)
 

cmake/third_party/protobuf.cmake

@@ -7,6 +7,7 @@ set(PROTOBUF_BINARY_DIR ${THIRD_PARTY_DIR}/protobuf/bin)
 set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
 if(WITH_XLA)
   set(PROTOBUF_URL "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip")
+  use_mirror(VARIABLE PROTOBUF_URL URL ${PROTOBUF_URL})
 else()
   set(PROTOBUF_URL ${THIRD_PARTY_SUBMODULE_DIR}/protobuf/src/protobuf)
 endif()

cmake/third_party/tensorflow.cmake

@@ -67,6 +67,7 @@ link_directories(${TENSORFLOW_INSTALL_DIR}/lib)
 
 if(NOT XRT_TF_URL)
   set(XRT_TF_URL https://github.com/Oneflow-Inc/tensorflow/archive/1f_dep_v2.3.0r4.zip)
+  use_mirror(VARIABLE XRT_TF_URL URL ${XRT_TF_URL})
 endif()
 if (THIRD_PARTY)
   ExternalProject_Add(${TENSORFLOW_PROJECT}

cmake/util.cmake

@@ -72,3 +72,34 @@ function(add_copy_headers_target)
     COMMAND ${CMAKE_COMMAND} -E copy_if_different "${PARSED_ARGS_SRC}/${header_file}" "${PARSED_ARGS_DST}/${header_file}")
   endforeach()
 endfunction()
+
+function(use_mirror)
+  cmake_parse_arguments(
+    PARSED_ARGS
+    ""
+    "VARIABLE;URL"
+    ""
+    ${ARGN}
+  )
+  if(NOT PARSED_ARGS_VARIABLE)
+    message(FATAL_ERROR "VARIABLE required")
+  endif(NOT PARSED_ARGS_VARIABLE)
+  if(NOT PARSED_ARGS_URL)
+    message(FATAL_ERROR "url required")
+  endif(NOT PARSED_ARGS_URL)
+  if(DEFINED THIRD_PARTY_MIRROR)
+    if(THIRD_PARTY_MIRROR STREQUAL "aliyun")
+      execute_process( 
+        COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/package_mirror.py -u ${PARSED_ARGS_URL} 
+        OUTPUT_VARIABLE temp_url
+        RESULT_VARIABLE ret_code)
+      if (NOT (ret_code EQUAL "0"))
+        message(FATAL_ERROR "Fail to execute the script package_mirror.py.")
+      else()
+        set(${PARSED_ARGS_VARIABLE} ${temp_url} PARENT_SCOPE)
+      endif()
+    elseif(NOT THIRD_PARTY_MIRROR STREQUAL "")
+      message(FATAL_ERROR "Invalid key for third party mirror.")
+    endif()
+  endif()
+endfunction()

tools/package_mirror.py

@@ -2,13 +2,18 @@ import glob
 import argparse
 import os
 import re
-import oss2
 from urllib.parse import urlparse
 import requests
 import hashlib
 import base64
 import tempfile
 
+try:
+    import oss2
+except:
+    pass
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument("-i", "--src_path", type=str, required=False)
 parser.add_argument("-u", "--url", type=str, required=False)
@@ -44,9 +49,10 @@ def convert_url_to_oss_key(url):
     return os.path.join("third_party_mirror", parsed.scheme, parsed.netloc, path)
 
 
-# https://oneflow-static.oss-cn-beijing.aliyuncs.com/...
 def convert_url_to_oss_https_url(url):
-    raise NotImplemented
+    key = convert_url_to_oss_key(url)
+    prefix = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/"
+    return os.path.join(prefix, key)
 
 
 def should_be_mirrored(url: str):
@@ -106,5 +112,5 @@ if __name__ == "__main__":
     if args.src_path != None:
         upload_to_aliyun(args.src_path)
     if args.url != None:
-        oss_url = convert_url_to_oss_https(args.url)
+        oss_url = convert_url_to_oss_https_url(args.url)
         print(oss_url)