提交 238f6493 编写于 作者: S ShawnXuan

test

上级 fc47d1b0
......@@ -78,16 +78,13 @@ jobs:
matrix: ${{ steps.find-cache.outputs.matrix }}
steps:
- uses: actions/checkout@v2
# if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
- name: Checkout Oneflow-Inc/oneflow
uses: actions/checkout@v2
# if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
with:
repository: Oneflow-Inc/oneflow
ref: ${{ github.event.inputs.of_branch_or_commit }}
path: ${{ env.ONEFLOW_SRC }}
- uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@2a9efceab8d45b725a687e73f870f9b75a15e472
# if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
name: find cache
id: find-cache
with:
......@@ -98,108 +95,100 @@ jobs:
oneflow-src: ${{ env.ONEFLOW_SRC }}
entries: |
${{ github.event.inputs.compute_platform }}
# build-manylinux:
# name: "Build manylinux"
# runs-on: ['self-hosted', 'linux', 'provision']
# needs: [find-build-cache]
# strategy:
# fail-fast: true
# max-parallel: 5
# matrix: ${{ fromJson(needs.find-build-cache.outputs.matrix) }}
# env:
# MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/${{ matrix.entry }}
# WHEELHOUSE_DIR: manylinux-wheelhouse
# OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
# OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
# steps:
# - name: Fix permissions
# run: |
# set -x
# docker run --rm -v $PWD:/p -w /p busybox chown -R $(id -u):$(id -g) .
# - name: Remove leftover cuda-installer.log
# run: |
# docker run --rm -v /tmp:/host/tmp -w /p busybox rm -f /host/tmp/cuda-installer.log
# - uses: actions/checkout@v2
# - name: Checkout Oneflow-Inc/oneflow
# uses: actions/checkout@v2
# with:
# repository: Oneflow-Inc/oneflow
# ref: ${{ github.event.inputs.of_branch_or_commit }}
# path: ${{ env.ONEFLOW_SRC }}
# - name: Set environment variables
# id: set-env
# run: |
# set -x
# current_dir=$PWD
# cd ${{ env.ONEFLOW_SRC }}
# oneflow_branch=`git rev-parse --abbrev-ref HEAD`
# # oneflow_branch=`git branch --show-current`
# # oneflow_branch=`git symbolic-ref --short HEAD`
# oneflow_commit=`git rev-parse HEAD`
# oss_branch_dir=branch/${oneflow_branch}/${{ github.event.inputs.compute_platform }}
# oss_dir=${oss_branch_dir}/${oneflow_commit}
# echo "oss_branch_dir=${oss_branch_dir}" >> $GITHUB_ENV
# echo "oss_dir=${oss_dir}" >> $GITHUB_ENV
# oneflow_wheel_path=http://oneflow-staging.oss-cn-beijing.aliyuncs.com/commit/${oneflow_commit}/${{ github.event.inputs.compute_platform }}
# # echo "ONEFLOW_WHEEL_PATH=${oneflow_wheel_path}" >> $GITHUB_ENV
# echo "::set-output name=ONEFLOW_WHEEL_PATH::${oneflow_wheel_path}"
# cd $current_dir
# set +x
# - uses: Oneflow-Inc/get-oneflow/cache-complete@2a9efceab8d45b725a687e73f870f9b75a15e472
# name: Save cache if successful
# id: save-cache
# timeout-minutes: 5
# with:
# oneflow-src: ${{ env.ONEFLOW_SRC }}
# entry: ${{ matrix.entry }}
# digest-type: build
# mark-as-completed: ${{ contains(matrix.runs-on, 'self-hosted') }}
# # - name: Check digest and fail if cache result not identical to matrix
# # if: ${{ fromJSON(steps.save-cache.outputs.cache-hit) != matrix.cache-hit }}
# # run: |
# # echo "::error file=test.yml,line=204,col=10::steps.save-cache.outputs.cache-hit != matrix.cache-hit"
# # exit 1
# - uses: Oneflow-Inc/get-oneflow@2a9efceab8d45b725a687e73f870f9b75a15e472
# name: Build manylinux ${{ github.event.inputs.compute_platform }}
# id: build-cuda
# if: ${{ matrix.entry =='${{ github.event.inputs.compute_platform }}' && !matrix.cache-hit }}
# with:
# cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/cuda.cmake
# build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc7.sh
# oneflow-src: ${{ env.ONEFLOW_SRC }}
# oneflow-build-env: manylinux
# wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
# clear-wheelhouse-dir: true
# self-hosted: true
# cuda-version: "10.2"
# manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
# docker-run-use-system-http-proxy: false
# docker-run-use-lld: false
# retry-failed-build: true
# python-versions: |
# ${{ github.event.inputs.python_version }}
# - name: Upload wheel
# if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
# uses: ./.github/actions/upload_oss
# with:
# src_path: ${{ env.WHEELHOUSE_DIR }}
# oss_dst_path: oss://oneflow-staging/${{ env.oss_dir }}
# oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
# oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
# - name: Update pip index
# if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
# env:
# OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
# OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
# run: |
# python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# python3 -m pip install oss2 beautifulsoup4 --user
# python3 ${{ env.ONEFLOW_SRC }}/tools/create_pip_index.py --dir_key ${oss_dir} -b oneflow-staging --index_key=${oss_branch_dir}/index.html --index_key=${oss_dir}/index.html
build-oneflow:
name: "Build OneFlow ${{ github.event.inputs.compute_platform }}"
runs-on: ['self-hosted', 'linux', 'provision']
needs: [find-build-cache]
strategy:
fail-fast: true
max-parallel: 5
matrix: ${{ fromJson(needs.find-build-cache.outputs.matrix) }}
env:
MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/${{ matrix.entry }}
WHEELHOUSE_DIR: manylinux-wheelhouse
OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
steps:
- name: Fix permissions
run: |
set -x
docker run --rm -v $PWD:/p -w /p busybox chown -R $(id -u):$(id -g) .
- name: Remove leftover cuda-installer.log
run: |
docker run --rm -v /tmp:/host/tmp -w /p busybox rm -f /host/tmp/cuda-installer.log
- uses: actions/checkout@v2
- name: Checkout Oneflow-Inc/oneflow
uses: actions/checkout@v2
with:
repository: Oneflow-Inc/oneflow
ref: ${{ github.event.inputs.of_branch_or_commit }}
path: ${{ env.ONEFLOW_SRC }}
- name: Set environment variables
id: set-env
run: |
set -x
current_dir=$PWD
cd ${{ env.ONEFLOW_SRC }}
oneflow_branch=`git rev-parse --abbrev-ref HEAD`
# oneflow_branch=`git branch --show-current`
# oneflow_branch=`git symbolic-ref --short HEAD`
oneflow_commit=`git rev-parse HEAD`
oss_branch_dir=branch/${oneflow_branch}/${{ github.event.inputs.compute_platform }}
oss_dir=${oss_branch_dir}/${oneflow_commit}
echo "oss_branch_dir=${oss_branch_dir}" >> $GITHUB_ENV
echo "oss_dir=${oss_dir}" >> $GITHUB_ENV
cd $current_dir
set +x
- uses: Oneflow-Inc/get-oneflow/cache-complete@2a9efceab8d45b725a687e73f870f9b75a15e472
name: Save cache if successful
id: save-cache
timeout-minutes: 5
with:
oneflow-src: ${{ env.ONEFLOW_SRC }}
entry: ${{ matrix.entry }}
digest-type: build
mark-as-completed: ${{ contains(matrix.runs-on, 'self-hosted') }}
- uses: Oneflow-Inc/get-oneflow@2a9efceab8d45b725a687e73f870f9b75a15e472
name: Build manylinux ${{ github.event.inputs.compute_platform }}
id: build-cuda
if: ${{ matrix.entry =='${{ github.event.inputs.compute_platform }}' && !matrix.cache-hit }}
with:
cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/cuda.cmake
build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc7.sh
oneflow-src: ${{ env.ONEFLOW_SRC }}
oneflow-build-env: manylinux
wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
clear-wheelhouse-dir: true
self-hosted: true
cuda-version: "10.2"
manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
docker-run-use-system-http-proxy: false
docker-run-use-lld: false
retry-failed-build: true
python-versions: |
${{ github.event.inputs.python_version }}
- name: Upload wheel
if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
uses: ./.github/actions/upload_oss
with:
src_path: ${{ env.WHEELHOUSE_DIR }}
oss_dst_path: oss://oneflow-staging/${{ env.oss_dir }}
oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
- name: Update pip index
if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
env:
OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
run: |
python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
python3 -m pip install oss2 beautifulsoup4 --user
python3 ${{ env.ONEFLOW_SRC }}/tools/create_pip_index.py --dir_key ${oss_dir} -b oneflow-staging --index_key=${oss_branch_dir}/index.html --index_key=${oss_dir}/index.html
# test:
# name: Test suite
# # needs: [find-test-cache]build-manylinux
# needs: [build-manylinux]
# # needs: [find-test-cache]build-oneflow
# needs: [build-oneflow]
# # runs-on: ['self-hosted', 'linux', 'provision']
# runs-on: ['self-hosted', 'linux', 'x64', 'gpu-8-titan-v']
# env:
......@@ -269,7 +258,7 @@ jobs:
# run: |
# docker exec ${TEST_CONTAINER_NAME} python3 --version
# docker exec ${TEST_CONTAINER_NAME} python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
# docker exec ${TEST_CONTAINER_NAME} python3 -m pip install --find-links=${{ needs.build-manylinux.outputs.ONEFLOW_WHEEL_PATH }} oneflow
# docker exec ${TEST_CONTAINER_NAME} python3 -m pip install --find-links=${{ needs.build-oneflow.outputs.ONEFLOW_WHEEL_PATH }} oneflow
# - name: Test container
# # if: ${{ !fromJson(matrix.cache-hit) && contains(matrix.runs-on, 'self-hosted') }}
# run: |
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册