test

.github/workflows/cnn_e2e.yml

@@ -78,16 +78,13 @@ jobs:
       matrix: ${{ steps.find-cache.outputs.matrix }}
     steps:
       - uses: actions/checkout@v2
-        # if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
       - name: Checkout Oneflow-Inc/oneflow
         uses: actions/checkout@v2
-        # if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
         with:
           repository: Oneflow-Inc/oneflow
           ref: ${{ github.event.inputs.of_branch_or_commit }}
           path: ${{ env.ONEFLOW_SRC }}
       - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@2a9efceab8d45b725a687e73f870f9b75a15e472
-        # if: ${{ needs.find-oss-wheel.outputs.find-wheel-hit != 1 }}
         name: find cache
         id: find-cache
         with:
@@ -98,108 +95,100 @@ jobs:
           oneflow-src: ${{ env.ONEFLOW_SRC }}
           entries: |
             ${{ github.event.inputs.compute_platform }}
-  # build-manylinux:
-  #   name: "Build manylinux"
-  #   runs-on: ['self-hosted', 'linux', 'provision']
-  #   needs: [find-build-cache]
-  #   strategy:
-  #     fail-fast: true
-  #     max-parallel: 5
-  #     matrix: ${{ fromJson(needs.find-build-cache.outputs.matrix) }}
-  #   env:
-  #     MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/${{ matrix.entry }}
-  #     WHEELHOUSE_DIR: manylinux-wheelhouse
-  #     OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
-  #     OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
-  #   steps:
-  #     - name: Fix permissions
-  #       run: |
-  #         set -x
-  #         docker run --rm -v $PWD:/p -w /p busybox chown -R $(id -u):$(id -g) .
-  #     - name: Remove leftover cuda-installer.log
-  #       run: |
-  #         docker run --rm -v /tmp:/host/tmp -w /p busybox rm -f /host/tmp/cuda-installer.log
-  #     - uses: actions/checkout@v2
-  #     - name: Checkout Oneflow-Inc/oneflow
-  #       uses: actions/checkout@v2
-  #       with:
-  #         repository: Oneflow-Inc/oneflow
-  #         ref: ${{ github.event.inputs.of_branch_or_commit }}
-  #         path: ${{ env.ONEFLOW_SRC }}
-  #     - name: Set environment variables
-  #       id: set-env
-  #       run: |
-  #         set -x
-  #         current_dir=$PWD
-  #         cd ${{ env.ONEFLOW_SRC }}
-  #         oneflow_branch=`git rev-parse --abbrev-ref HEAD`
-  #         # oneflow_branch=`git branch --show-current`
-  #         # oneflow_branch=`git symbolic-ref --short HEAD`
-  #         oneflow_commit=`git rev-parse HEAD`
-  #         oss_branch_dir=branch/${oneflow_branch}/${{ github.event.inputs.compute_platform }}
-  #         oss_dir=${oss_branch_dir}/${oneflow_commit}
-  #         echo "oss_branch_dir=${oss_branch_dir}" >> $GITHUB_ENV
-  #         echo "oss_dir=${oss_dir}" >> $GITHUB_ENV
-  #         oneflow_wheel_path=http://oneflow-staging.oss-cn-beijing.aliyuncs.com/commit/${oneflow_commit}/${{ github.event.inputs.compute_platform }}
-  #         # echo "ONEFLOW_WHEEL_PATH=${oneflow_wheel_path}" >> $GITHUB_ENV
-  #         echo "::set-output name=ONEFLOW_WHEEL_PATH::${oneflow_wheel_path}"
-  #         cd $current_dir
-  #         set +x
-  #     - uses: Oneflow-Inc/get-oneflow/cache-complete@2a9efceab8d45b725a687e73f870f9b75a15e472
-  #       name: Save cache if successful
-  #       id: save-cache
-  #       timeout-minutes: 5
-  #       with:
-  #         oneflow-src: ${{ env.ONEFLOW_SRC }}
-  #         entry: ${{ matrix.entry }}
-  #         digest-type: build
-  #         mark-as-completed: ${{ contains(matrix.runs-on, 'self-hosted') }}
-  #     # - name: Check digest and fail if cache result not identical to matrix
-  #     #   if: ${{ fromJSON(steps.save-cache.outputs.cache-hit) != matrix.cache-hit }}
-  #     #   run: |
-  #     #     echo "::error file=test.yml,line=204,col=10::steps.save-cache.outputs.cache-hit != matrix.cache-hit"
-  #     #     exit 1
-  #     - uses: Oneflow-Inc/get-oneflow@2a9efceab8d45b725a687e73f870f9b75a15e472
-  #       name: Build manylinux ${{ github.event.inputs.compute_platform }}
-  #       id: build-cuda
-  #       if: ${{ matrix.entry =='${{ github.event.inputs.compute_platform }}' && !matrix.cache-hit }}
-  #       with:
-  #         cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/cuda.cmake
-  #         build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc7.sh
-  #         oneflow-src: ${{ env.ONEFLOW_SRC }}
-  #         oneflow-build-env: manylinux
-  #         wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
-  #         clear-wheelhouse-dir: true
-  #         self-hosted: true
-  #         cuda-version: "10.2"
-  #         manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
-  #         docker-run-use-system-http-proxy: false
-  #         docker-run-use-lld: false
-  #         retry-failed-build: true
-  #         python-versions: |
-  #           ${{ github.event.inputs.python_version }}
-  #     - name: Upload wheel
-  #       if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
-  #       uses: ./.github/actions/upload_oss
-  #       with:
-  #         src_path: ${{ env.WHEELHOUSE_DIR }}
-  #         oss_dst_path: oss://oneflow-staging/${{ env.oss_dir }}
-  #         oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
-  #         oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
-  #     - name: Update pip index
-  #       if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
-  #       env:
-  #         OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
-  #         OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
-  #       run: |
-  #         python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-  #         python3 -m pip install oss2 beautifulsoup4 --user
-  #         python3 ${{ env.ONEFLOW_SRC }}/tools/create_pip_index.py --dir_key ${oss_dir} -b oneflow-staging --index_key=${oss_branch_dir}/index.html --index_key=${oss_dir}/index.html
+  build-oneflow:
+    name: "Build OneFlow ${{ github.event.inputs.compute_platform }}"
+    runs-on: ['self-hosted', 'linux', 'provision']
+    needs: [find-build-cache]
+    strategy:
+      fail-fast: true
+      max-parallel: 5
+      matrix: ${{ fromJson(needs.find-build-cache.outputs.matrix) }}
+    env:
+      MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/${{ matrix.entry }}
+      WHEELHOUSE_DIR: manylinux-wheelhouse
+      OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+      OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
+    steps:
+      - name: Fix permissions
+        run: |
+          set -x
+          docker run --rm -v $PWD:/p -w /p busybox chown -R $(id -u):$(id -g) .
+      - name: Remove leftover cuda-installer.log
+        run: |
+          docker run --rm -v /tmp:/host/tmp -w /p busybox rm -f /host/tmp/cuda-installer.log
+      - uses: actions/checkout@v2
+      - name: Checkout Oneflow-Inc/oneflow
+        uses: actions/checkout@v2
+        with:
+          repository: Oneflow-Inc/oneflow
+          ref: ${{ github.event.inputs.of_branch_or_commit }}
+          path: ${{ env.ONEFLOW_SRC }}
+      - name: Set environment variables
+        id: set-env
+        run: |
+          set -x
+          current_dir=$PWD
+          cd ${{ env.ONEFLOW_SRC }}
+          oneflow_branch=`git rev-parse --abbrev-ref HEAD`
+          # oneflow_branch=`git branch --show-current`
+          # oneflow_branch=`git symbolic-ref --short HEAD`
+          oneflow_commit=`git rev-parse HEAD`
+          oss_branch_dir=branch/${oneflow_branch}/${{ github.event.inputs.compute_platform }}
+          oss_dir=${oss_branch_dir}/${oneflow_commit}
+          echo "oss_branch_dir=${oss_branch_dir}" >> $GITHUB_ENV
+          echo "oss_dir=${oss_dir}" >> $GITHUB_ENV
+          cd $current_dir
+          set +x
+      - uses: Oneflow-Inc/get-oneflow/cache-complete@2a9efceab8d45b725a687e73f870f9b75a15e472
+        name: Save cache if successful
+        id: save-cache
+        timeout-minutes: 5
+        with:
+          oneflow-src: ${{ env.ONEFLOW_SRC }}
+          entry: ${{ matrix.entry }}
+          digest-type: build
+          mark-as-completed: ${{ contains(matrix.runs-on, 'self-hosted') }}
+      - uses: Oneflow-Inc/get-oneflow@2a9efceab8d45b725a687e73f870f9b75a15e472
+        name: Build manylinux ${{ github.event.inputs.compute_platform }}
+        id: build-cuda
+        if: ${{ matrix.entry =='${{ github.event.inputs.compute_platform }}' && !matrix.cache-hit }}
+        with:
+          cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/cuda.cmake
+          build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc7.sh
+          oneflow-src: ${{ env.ONEFLOW_SRC }}
+          oneflow-build-env: manylinux
+          wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
+          clear-wheelhouse-dir: true
+          self-hosted: true
+          cuda-version: "10.2"
+          manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
+          docker-run-use-system-http-proxy: false
+          docker-run-use-lld: false
+          retry-failed-build: true
+          python-versions: |
+            ${{ github.event.inputs.python_version }}
+      - name: Upload wheel
+        if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
+        uses: ./.github/actions/upload_oss
+        with:
+          src_path: ${{ env.WHEELHOUSE_DIR }}
+          oss_dst_path: oss://oneflow-staging/${{ env.oss_dir }}
+          oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }}
+          oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
+      - name: Update pip index
+        if: ${{ !fromJson(matrix.cache-hit) && steps.build-cuda.outcome == 'success' }}
+        env:
+          OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }}
+          OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }}
+        run: |
+          python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+          python3 -m pip install oss2 beautifulsoup4 --user
+          python3 ${{ env.ONEFLOW_SRC }}/tools/create_pip_index.py --dir_key ${oss_dir} -b oneflow-staging --index_key=${oss_branch_dir}/index.html --index_key=${oss_dir}/index.html
 
   # test:
   #   name: Test suite
-  #   # needs: [find-test-cache]build-manylinux
-  #   needs: [build-manylinux]
+  #   # needs: [find-test-cache]build-oneflow
+  #   needs: [build-oneflow]
   #   # runs-on: ['self-hosted', 'linux', 'provision']
   #   runs-on: ['self-hosted', 'linux', 'x64', 'gpu-8-titan-v']
   #   env:
@@ -269,7 +258,7 @@ jobs:
   #       run: |
   #         docker exec ${TEST_CONTAINER_NAME} python3 --version
   #         docker exec ${TEST_CONTAINER_NAME} python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-  #         docker exec ${TEST_CONTAINER_NAME} python3 -m pip install --find-links=${{ needs.build-manylinux.outputs.ONEFLOW_WHEEL_PATH }} oneflow
+  #         docker exec ${TEST_CONTAINER_NAME} python3 -m pip install --find-links=${{ needs.build-oneflow.outputs.ONEFLOW_WHEEL_PATH }} oneflow
   #     - name: Test container
   #       # if: ${{ !fromJson(matrix.cache-hit) && contains(matrix.runs-on, 'self-hosted') }}
   #       run: |