From 96742c19ae2e4ae048646bdb06331092dd412db2 Mon Sep 17 00:00:00 2001
From: luzzyzhang <luzzyzhang@gmail.com>
Date: Mon, 30 Mar 2020 18:36:19 +0800
Subject: [PATCH] ci: add github action

---
 .github/workflows/ci-cpu.yml | 25 +++++++++++++++++++++++
 ci/cmake.sh                  | 37 ++++++++++++++++++++++++++++++++++
 ci/docker_env/Dockerfile     | 39 ++++++++++++++++++++++++++++++++++++
 ci/run_cpp_test.sh           | 24 ++++++++++++++++++++++
 ci/run_python_test.sh        | 16 +++++++++++++++
 ci/utils.sh                  |  7 +++++++
 6 files changed, 148 insertions(+)
 create mode 100644 .github/workflows/ci-cpu.yml
 create mode 100755 ci/cmake.sh
 create mode 100644 ci/docker_env/Dockerfile
 create mode 100755 ci/run_cpp_test.sh
 create mode 100755 ci/run_python_test.sh
 create mode 100644 ci/utils.sh

diff --git a/.github/workflows/ci-cpu.yml b/.github/workflows/ci-cpu.yml
new file mode 100644
index 00000000..08c03ae6
--- /dev/null
+++ b/.github/workflows/ci-cpu.yml
@@ -0,0 +1,25 @@
+name: CI CPU
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  cpu-test:
+    runs-on: self-hosted
+    container:
+      image: localhost:5000/megengine-ci:latest
+    steps:
+      - name: Checkout MegEngine
+        uses: actions/checkout@v2
+      - name: Checkout submodules
+        run: |
+          ./third_party/prepare.sh
+          ./third_party/install-mkl.sh
+      - name: Build MegEngine
+        run: ./ci/cmake.sh cpu
+      - name: Python test
+        run: ./ci/run_python_test.sh
+      - name: C++ test
+        run: ./ci/run_cpp_test.sh cpu
diff --git a/ci/cmake.sh b/ci/cmake.sh
new file mode 100755
index 00000000..8d8c55bf
--- /dev/null
+++ b/ci/cmake.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+BASEDIR=$(readlink -f "$(dirname "$0")"/..)
+
+source "${BASEDIR}/ci/utils.sh"
+
+if [[ "$1" == cpu ]]; then
+    DMGE_WITH_DISTRIBUTED=OFF
+    DMGE_WITH_CUDA=OFF
+elif [[ "$1" == cuda ]]; then
+    DMGE_WITH_DISTRIBUTED=ON
+    DMGE_WITH_CUDA=ON
+else
+    log "Argument must cpu or cuda"
+    exit 1
+fi
+
+
+function build() {
+    log "Start to build"
+    local build_dir="/tmp/build/${1}"
+    mkdir -p "$build_dir"
+    pushd ${build_dir} >/dev/null
+        cmake -S "${BASEDIR}" -B "${build_dir}" \
+            -DMGE_WITH_DISTRIBUTED=${DMGE_WITH_DISTRIBUTED} \
+            -DMGE_WITH_CUDA=${DMGE_WITH_CUDA} \
+            -DMGE_WITH_TEST=ON \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo
+        make -j$(($(nproc) * 2)) -I ${build_dir}
+        make develop
+    popd >/dev/null
+    log "End build: $(ls ${build_dir})"
+}
+
+build "$@"
diff --git a/ci/docker_env/Dockerfile b/ci/docker_env/Dockerfile
new file mode 100644
index 00000000..8f525f5c
--- /dev/null
+++ b/ci/docker_env/Dockerfile
@@ -0,0 +1,39 @@
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
+RUN rm /etc/apt/sources.list.d/cuda.list
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    autoconf \
+    autotools-dev \
+    automake \
+    clang-6.0 \
+    curl \
+    git-lfs \
+    libtool \
+    libpcre3-dev \
+    llvm-6.0-dev \
+    openssh-client \
+    openssh-server \
+    pkg-config \
+    python-pip \
+    python3-pip \
+    python3-dev \
+    python-numpy \
+    python3-numpy \
+    python3-setuptools \
+    software-properties-common \
+    swig \
+    vim \
+    wget \
+    zlib1g-dev \
+    # GitLab Runner need Git 2.18 or higher to create a local Git repository
+    && add-apt-repository ppa:git-core/ppa -y && apt-get install --no-install-recommends -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN cd /tmp ; wget https://cmake.org/files/v3.14/cmake-3.14.4.tar.gz;tar -xzvf cmake-3.14.4.tar.gz;cd cmake-3.14.4;./configure; make -j32; make install
+
+RUN git lfs install
+
+ENV PATH=${PATH}:/usr/local/cuda/bin \
+  LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib/stubs:/usr/local/cuda/lib64/stubs:/usr/local/cuda-10.1-libs/cudnn-v7.6.0/lib:/usr/local/cuda-10.1-libs/cudnn-v7.6.0/lib64:/usr/local/cuda-10.1-libs/TensorRT-5.1.5.0/lib:/usr/local/cuda-10.1-libs/TensorRT-5.1.5.0/lib64 \
+  LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib/stubs:/usr/local/cuda/lib64/stubs:/usr/local/cuda-10.1-libs/cudnn-v7.6.0/lib:/usr/local/cuda-10.1-libs/cudnn-v7.6.0/lib64:/usr/local/cuda-10.1-libs/TensorRT-5.1.5.0/lib:/usr/local/cuda-10.1-libs/TensorRT-5.1.5.0/lib64:/usr/local/cuda/lib64/stubs/libcuda.so:/tmp/build/cuda/dnn/cuda-stub/libcuda.so.1 \
+  CPATH=${CPATH}:/usr/local/cuda/include:/usr/local/cuda-10.1-libs/cudnn-v7.6.0/include:/usr/local/cuda-10.1-libs/TensorRT-5.1.5.0/include
\ No newline at end of file
diff --git a/ci/run_cpp_test.sh b/ci/run_cpp_test.sh
new file mode 100755
index 00000000..2b4714d6
--- /dev/null
+++ b/ci/run_cpp_test.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+BASEDIR=$(readlink -f "$(dirname "$0")"/..)
+source "${BASEDIR}/ci/utils.sh"
+
+export MGB_TEST_NO_LOG=1
+export MGB_STABLE_RNG=1
+
+function cpp_test {
+    pushd /tmp/build/"${1}" >/dev/null
+        ./dnn/test/megdnn_test
+        ./test/megbrain_test
+    popd >/dev/null
+}
+
+
+if [[ "$1" != "cpu" && "$1" != "cuda" ]] ; then
+    echo "Argument must cpu or cuda"
+    exit 1
+fi
+
+cpp_test "$@"
\ No newline at end of file
diff --git a/ci/run_python_test.sh b/ci/run_python_test.sh
new file mode 100755
index 00000000..6d55ec72
--- /dev/null
+++ b/ci/run_python_test.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+
+BASEDIR=$(readlink -f "$(dirname "$0")"/..)
+
+function python_test() {
+    pushd "${BASEDIR}"/python_module >/dev/null
+        pip3 install -e '.[ci]'
+        export PYTHONPATH=.
+        ./test/run.sh
+    popd >/dev/null
+}
+
+python_test
diff --git a/ci/utils.sh b/ci/utils.sh
new file mode 100644
index 00000000..43686701
--- /dev/null
+++ b/ci/utils.sh
@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+
+function log() {
+    d=$(date +'%Y-%m-%d %H:%M:%S')
+    echo $d" "$1
+}
\ No newline at end of file
-- 
GitLab