!140 CI: set up CI for isulad

Merge pull request !140 from lifeng_isula/ci

CI/Dockerfile

+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2019. All rights reserved.
+# - lcr licensed under the Mulan PSL v1.
+# - You can use this software according to the terms and conditions of the Mulan PSL v1.
+# - You may obtain a copy of Mulan PSL v1 at:
+# -     http://license.coscl.org.cn/MulanPSL
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v1 for more details.
+##- @Description: prepare compile container envrionment
+##- @Author: lifeng
+##- @Create: 2020-01-10
+#######################################################################
+# This file describes the isulad compile container image.
+#
+# Usage:
+#
+# docker build --build-arg http_proxy=YOUR_HTTP_PROXY_IF_NEEDED \
+#		--build-arg https_proxy=YOUR_HTTPS_PROXY_IF_NEEDED \
+#		-t YOUR_IMAGE_NAME -f ./Dockerfile .
+
+
+FROM	centos:7.6.1810
+MAINTAINER LiFeng <lifeng68@huawei.com>
+
+# Install dependency package
+RUN yum clean all && yum swap -y fakesystemd systemd && \
+	yum install -y epel-release \
+	yum update -y && \
+	yum install -y  automake \
+			autoconf \
+			libtool \
+			make \
+			cmake \
+			which \
+			gdb \
+			strace \
+			rpm-build \
+			graphviz \
+			libcap \
+			libcap-devel \
+			libxslt  \
+			docbook2X \
+			libselinux \
+			libselinux-devel \
+			libseccomp \
+			libseccomp-devel \
+			yajl-devel \
+			git \
+			bridge-utils \
+			dnsmasq \
+			libcgroup \
+			rsync \
+			iptables \
+			iproute \
+			net-tools \
+			unzip \
+			tar \
+			wget \
+			gtest \
+			gtest-devel \
+			gmock \
+			gmock-devel \
+			cppcheck \
+			python3 \
+			python3-pip \
+			python \
+			python-pip \
+			device-mapper-devel \
+			libcurl-devel \
+			zlib-devel \
+			glibc-headers \
+			openssl-devel \
+			gcc \
+			gcc-c++ \
+			hostname \
+			sqlite-devel \
+			gpgme \
+			gpgme-devel \
+			expect \
+			systemd-devel \
+			systemd-libs \
+			go \
+			CUnit \
+			CUnit-devel \
+			valgrind \
+			e2fsprogs
+
+RUN pip install xlrd
+
+RUN yum clean all && \
+    (cd /lib/systemd/system/sysinit.target.wants/; for i in *; \
+    do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+    rm -f /lib/systemd/system/multi-user.target.wants/*;\
+    rm -f /etc/systemd/system/*.wants/*;\
+    rm -f /lib/systemd/system/local-fs.target.wants/*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+    rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+    rm -f /lib/systemd/system/basic.target.wants/*;\
+    rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+RUN echo "export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH" >> /etc/bashrc && \
+    echo "export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:$LD_LIBRARY_PATH" >> /etc/bashrc && \
+    echo "/usr/lib" >> /etc/ld.so.conf && \
+    echo "/usr/local/lib" >> /etc/ld.so.conf
+
+	
+# disalbe sslverify
+RUN git config --global http.sslverify false
+
+# Centos has no protobuf, protobuf-devel, grpc, grpc-devel, grpc-plugin
+# and we should install them manually.
+# install protobuf
+RUN set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/protobuf.git && \
+	cd protobuf && \
+	tar -xzvf protobuf-all-3.9.0.tar.gz && \
+	cd protobuf-3.9.0 && \
+	./autogen.sh && \
+	./configure && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+
+# install c-ares
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/c-ares.git && \
+	cd c-ares && \
+	tar -xzvf c-ares-1.15.0.tar.gz && \
+	cd c-ares-1.15.0 && \
+	autoreconf -if && \
+	./configure --enable-shared --disable-dependency-tracking && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+	
+# install grpc
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/grpc.git && \
+	cd grpc && \
+	tar -xzvf grpc-1.22.0.tar.gz && \
+	cd grpc-1.22.0 && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+
+# install libevent
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/libevent.git && \
+	cd libevent && \
+	tar -xzvf libevent-2.1.11-stable.tar.gz && \
+	cd libevent-2.1.11-stable && \
+	./autogen.sh && \
+	./configure && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+
+# install libevhtp
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/libevhtp.git && \
+	cd libevhtp && \
+	tar -xzvf libevhtp-1.2.16.tar.gz && \
+	cd libevhtp-1.2.16 && \
+	patch -p1 -F1 -s < ../0001-support-dynamic-threads.patch && \
+	patch -p1 -F1 -s < ../0002-close-openssl.patch && \
+	rm -rf build && \
+	mkdir build && \
+	cd build && \
+	cmake -D EVHTP_BUILD_SHARED=on -D EVHTP_DISABLE_SSL=on ../ && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+
+# install http-parser
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/http-parser.git && \
+	cd http-parser && \
+	tar -xzvf http-parser-2.9.2.tar.gz && \
+	cd http-parser-2.9.2 && \
+	make -j CFLAGS="-Wno-error" && \
+	make CFLAGS="-Wno-error" install && \
+	ldconfig
+
+# install libwebsockets
+RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH && \
+	set -x && \
+	cd ~ && \
+	git clone https://gitee.com/src-openeuler/libwebsockets.git && \
+	cd libwebsockets && \
+	tar -xzvf libwebsockets-2.4.2.tar.gz && \
+	cd libwebsockets-2.4.2 && \
+	patch -p1 -F1 -s < ../libwebsockets-fix-coredump.patch && \
+	mkdir build && \
+	cd build && \
+	cmake -DLWS_WITH_SSL=0 -DLWS_MAX_SMP=32 -DCMAKE_BUILD_TYPE=Debug ../ && \
+	make -j $(nproc) && \
+	make install && \
+	ldconfig
+
+VOLUME [ "/sys/fs/cgroup" ]
+CMD ["/usr/sbin/init"]

CI/build.sh

+#!/bin/bash
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description: generate cetification
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+set +e
+set -x
+
+ignore_ci=false
+basepath=$(cd `dirname $0`; pwd)
+source $basepath/helper.sh
+TOPDIR=`pwd`
+src_code_dir="$TOPDIR"
+make_script="${TOPDIR}/CI/make-and-install.sh"
+CIDIR="$TOPDIR/CI"
+testcase_script="${src_code_dir}/CI/run-testcases.sh"
+testcase_test="${src_code_dir}/CI/test.sh"
+testcase_data="/tmp/testcases_data"
+LXC_LOCK_DIR_CONTAINER="/run/lxc/lock/mount_lock"
+LXC_LOCK_DIR_HOST="/tmp/lxc_mount_dir"
+KEEP_CONTAINERS_ALIVE_DIR="/tmp/containerslock"
+TESTCASE_ASSIGN="${CIDIR}/testcase_assign"
+BASE_IMAGE=""
+
+rm -rf ${TESTCASE_ASSIGN}_*
+
+# #Run this file will generate default BASE_IMAGE and auto run isulad unit tests
+# #You should cd the root path of isulad, and run:
+# ./CI/build.sh
+
+declare -a modules
+container_nums=0
+
+function usage() {
+    echo "Usage: $0 [options]"
+    echo "Continuous integration (CI) script for isulad/lcr project"
+    echo "Options:"
+    echo "    -m, --module        Execute scripts related to the specified module"
+    echo "    -n, --container-num Multiple containers execute scripts in parallel"
+    echo "    -g, --enable-gcov   Enable gcov for code coverage analysis"
+    echo "    -i, --ignore-ci     Not running testcase"
+    echo "        --rm            Auto remove containers after testcase run success"
+    echo "    -h, --help          Script help information"
+}
+
+function err() {
+    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2
+}
+
+args=`getopt -o m:n:g:i:h --long module:,container-num:,ignore-ci:,help -- "$@"`
+if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
+eval set -- "$args"
+
+while true; do
+    case "$1" in
+        -m|--module)        modules=${2} ; modules=(${modules// / }) ; shift 2 ;;
+        -n|--container-num) container_nums=${2} ; shift 2 ;;
+        -i|--ignore-ci)     ignore_ci=${2} ; shift 2 ;;
+        -h|--help)          usage ; exit 0 ;;
+        --)                 shift ; break ;;
+        *)                  err "invalid parameter" ; exit -1 ;;
+    esac
+done
+
+declare -A scripts
+pwd
+TESTCASE_PATH="./CI/test_cases"
+
+for file in $(find ${TESTCASE_PATH} -not \( -path '.*/data' -prune \) -regextype posix-extended -regex ".*\.(bash)" | grep -v "helpers.bash" | sort)
+do
+    attributes=$(sed -n '3p' $file)
+    if [[ "x$attributes" == "x" ]] || [[ ! "${attributes}" =~ "attributes:" ]];then
+        attributes=$(cat $file | grep "# attributes:")
+        if [[ "x$attributes" == "x" ]] || [[ ! "${attributes}" =~ "attributes:" ]];then
+            continue
+        fi
+    fi
+    attributes=${attributes#*: }
+    attributes=(${attributes// / })
+    if [[ ${#modules[@]} -ne 0 ]]; then
+        intersection=($(comm -12 <(echo ${modules[*]}| tr " " "\n"| sort) <(echo ${attributes[*]} | tr " " "\n"| sort)| sort -g))
+        if [[ ${#intersection[@]} -eq 0 ]]; then
+            continue
+        fi
+    fi
+
+    concurrent=$(sed -n '4p' $file)
+    concurrent=${concurrent#*: }
+
+    spend_time=$(sed -n '5p' $file)
+    spend_time=${spend_time#*: }
+
+    info=(${spend_time} ${concurrent} ${attributes[@]})
+    scripts+=([${file}]=${info[@]})
+done
+
+function check_concurrent() {
+    attr=${scripts[${1}]}
+    attributes=(${attr// / })
+    #disable concurrent run, testcase may fail while using sleep in testcase due to concurrent
+    return 1
+    if [[ "x${attributes[1]}" == "xYES" ]];then
+        return 0
+    fi
+    return 1
+}
+
+declare -A concurrent_scripts
+declare -A non_concurrent_scripts
+
+for script in "${!scripts[@]}"
+do
+    check_concurrent ${script}
+    if [ $? -eq 0 ];then
+        concurrent_scripts+=([${script}]=${scripts[${script}]})
+    else
+        non_concurrent_scripts+=([${script}]=${scripts[${script}]})
+    fi
+done
+
+CONTAINER_INDEX=1
+
+function calculate_non_concurrent_script_tatol_time() {
+    local result=0
+    for script in ${!non_concurrent_scripts[@]}
+    do
+        attr=${non_concurrent_scripts[${script}]}
+        attributes=(${attr// / })
+        spend_time=${attributes[0]}
+        if [[ ${spend_time} == "-" ]]; then
+            spend_time=3
+        fi
+        result=$((result + spend_time))
+    done
+    echo ${result}
+}
+
+function do_testcase_auto_assignment() {
+    local index=1
+    for script in ${!concurrent_scripts[@]}
+    do
+        script_realpath=$(realpath ${script})
+        echo ${script_realpath} >> ${TESTCASE_ASSIGN}_P${CONTAINER_INDEX}
+        index=$((index + 1))
+        if [[ ${index} -eq 50 ]]; then
+            index=1
+            CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+        fi
+    done
+
+    if [[ ${#concurrent_scripts[@]} -ne 0 ]]; then
+        CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+    fi
+    local acc_time=0
+    for script in ${!non_concurrent_scripts[@]}
+    do
+        attr=${non_concurrent_scripts[${script}]}
+        attributes=(${attr// / })
+        spend_time=${attributes[0]}
+        if [[ ${spend_time} == "-" ]]; then
+            spend_time=3
+        fi
+
+        acc_time=$((acc_time + spend_time))
+        if [[ ${acc_time} -ge 600  ]]; then
+            acc_time=0
+            CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+        fi
+        script_realpath=$(realpath ${script})
+        echo ${script_realpath} >> ${TESTCASE_ASSIGN}_S${CONTAINER_INDEX}
+    done
+    if [[ ${#non_concurrent_scripts[@]} -eq 0 ]]; then
+        CONTAINER_INDEX=$((CONTAINER_INDEX - 1))
+    fi
+}
+
+function do_testcase_manual_assignment() {
+    local index=1
+    rm -rf ${TESTCASE_ASSIGN}_*
+    for script in ${!concurrent_scripts[@]}
+    do
+        script_realpath=$(realpath ${script})
+        echo ${script_realpath} >> ${TESTCASE_ASSIGN}_P${CONTAINER_INDEX}
+        index=$((index + 1))
+        if [[ ${index} -eq 50 ]]; then
+            index=1
+            CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+        fi
+    done
+
+    non_concurrent_tatol_time=$(calculate_non_concurrent_script_tatol_time)
+
+    avg_time_per_container=$((non_concurrent_tatol_time / container_nums))
+
+    if [[ ${#concurrent_scripts[@]} -ne 0 ]]; then
+        CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+    fi
+    local acc_time=0
+    for script in ${!non_concurrent_scripts[@]}
+    do
+        attr=${non_concurrent_scripts[${script}]}
+        attributes=(${attr// / })
+        spend_time=${attributes[0]}
+        if [[ ${spend_time} == "-" ]]; then
+            spend_time=3
+        fi
+        acc_time=$((acc_time + spend_time))
+
+        if [[ ${acc_time} -gt ${avg_time_per_container} ]]; then
+            acc_time=0
+            CONTAINER_INDEX=$((CONTAINER_INDEX + 1))
+        fi
+        script_realpath=$(realpath ${script})
+        echo ${script_realpath} >> ${TESTCASE_ASSIGN}_S${CONTAINER_INDEX}
+    done
+    if [[ ${#non_concurrent_scripts[@]} -eq 0 ]]; then
+        CONTAINER_INDEX=$((CONTAINER_INDEX - 1))
+    fi
+}
+
+if [[ ${container_nums} -le 0 ]]; then
+    do_testcase_auto_assignment
+else
+    do_testcase_manual_assignment ${container_nums}
+fi
+
+function echo_success()
+{
+    echo -e "\033[1;32m"$@"\033[0m"
+}
+
+function echo_error()
+{
+    echo -e "\033[1;31m"$@"\033[0m"
+}
+
+DockerFile=./CI/Dockerfile
+ProcsFile=/sys/fs/cgroup/cpuset/docker/cgroup.clone_children
+function make_sure_cgroup()
+{
+    image=`cat $DockerFile | grep FROM | awk '{print $2}'`
+    if [ ! -e $ProcsFile ];then
+        cid=`docker run -d $image`
+        if [ $? -ne 0 ];then
+            echo "Can not run docker container"
+            return 1
+        fi
+        docker rm -f $cid
+    fi
+    procsval=`cat $ProcsFile`
+    if [ $procsval -ne 1 ];then
+        echo "warning: set $ProcsFile to 1"
+        echo 1 > $ProcsFile
+    fi
+}
+
+function make_base_image()
+{
+    BASE_IMAGE=`docker build -q -f ${DockerFile} .`
+}
+
+make_sure_cgroup
+
+make_base_image
+if [ $? -ne 0 ];then
+    exit 0
+fi
+
+#if you want to debug and disable cleanup all resources, create directory by 'mkdir -p $KEEP_CONTAINERS_ALIVE_DIR'
+#remember to remove $KEEP_CONTAINERS_ALIVE_DIR after finished your debug.
+if [ ! -d $KEEP_CONTAINERS_ALIVE_DIR ];then
+    delete_old_resources $time_id "$containers_clear_time" "listcontainers" "remove_container"
+    delete_old_resources $time_id "$containers_clear_time" "listtmpdirs" "remove_tmpdir"
+fi
+
+RES_CODE=0
+mkdir -p $LXC_LOCK_DIR_HOST
+env_ignore_ci=""
+if [ "x$ignore_ci" == "xON" ];then
+    env_ignore_ci="--env IGNORE_CI=ON"
+fi
+
+function exec_script() {
+    set +e
+    local log_path="/tmp/${1}.log"
+    contname="${1}"
+    # keep -i so testcases which read stdin can success
+    docker exec -itd -e TOPDIR=$src_code_dir -e TESTCASE_FLOCK=/tmp/runflag/${CONTAINER_NAME}.flock -e TESTCASE_SCRIPTS_LOG=/tmp/runflag/${CONTAINER_NAME}.scripts.log  -e TESTCASE_RUNFLAG=/tmp/runflag/${CONTAINER_NAME}.runflag -e TESTCASE_CONTNAME=${contname} ${contname} ${testcase_test} run ${2} ${log_path}
+    docker exec ${1} $testcase_test get
+    if [[ $? -ne 0 ]]; then
+        rm -rf ${CIDIR}/${CONTAINER_NAME}.runflag
+        docker exec ${contname} cat ${log_path}
+        echo_error "testcase execute failed in container ${contname}, log: ${log_path}"
+        return 1
+    fi
+    echo_success "Container: ${contname} success"
+    return 0
+}
+
+cptemp=${tmpdir_prefix}/${CONTAINER_NAME}_cptemp
+# container for testing restful and building
+copycontainer=${CONTAINER_NAME}_R1
+tmpdir="${tmpdir_prefix}/${copycontainer}"
+containers+=(${copycontainer})
+mkdir -p ${tmpdir}
+touch $CIDIR/${CONTAINER_NAME}.runflag
+
+docker run -tid -v /sys/fs/cgroup:/sys/fs/cgroup --tmpfs /tmp:exec,mode=777 --tmpfs /run:exec,mode=777 --name ${copycontainer} -v ${cptemp}:${cptemp} $env_gcov $env_ignore_ci -v ${CIDIR}:/tmp/runflag -v /lib/modules:/lib/modules -v $testcases_data_dir:$testcase_data -v $LXC_LOCK_DIR_HOST:$LXC_LOCK_DIR_CONTAINER -v $TOPDIR:$src_code_dir -v ${tmpdir}:/var/lib/isulad  --privileged -e login_username=$login_username -e login_passwd=$login_passwd $BASE_IMAGE
+docker cp ${CIDIR}/testcase_assign_R1 ${copycontainer}:/root
+echo_success "Run container ${copycontainer} success"
+
+# make and install in rest container
+docker exec -e TOPDIR=${src_code_dir} -e BUILDDIR=${cptemp} ${copycontainer} ${make_script}
+if [ $? -ne 0 ];then
+    echo_error "Make and install failed in container ${copycontainer}"
+    rm -rf ${cptemp}
+    exit 1
+fi
+echo_success "Finished build in container ${copycontainer}"
+
+for index in $(seq 1 ${CONTAINER_INDEX})
+do
+    suffix=$(ls ${CIDIR} | grep testcase_assign_ | grep -E "*[S|P]${index}$" | awk -F '_' '{print $NF}')
+    tmpdir="${tmpdir_prefix}/${CONTAINER_NAME}_${suffix}"
+    mkdir -p ${tmpdir}
+    containers+=(${CONTAINER_NAME}_${suffix})
+    docker run -tid -v /sys/fs/cgroup:/sys/fs/cgroup --tmpfs /tmp:exec,mode=777 --tmpfs /run:exec,mode=777 --name ${CONTAINER_NAME}_${suffix} -v ${cptemp}:${cptemp} $env_gcov $env_ignore_ci -v ${CIDIR}:/tmp/runflag -v /lib/modules:/lib/modules -v $testcases_data_dir:$testcase_data -v $LXC_LOCK_DIR_HOST:$LXC_LOCK_DIR_CONTAINER -v $TOPDIR:$src_code_dir -v ${tmpdir}:/var/lib/isulad  --privileged -e login_username=$login_username -e login_passwd=$login_passwd $BASE_IMAGE
+    docker cp ${CIDIR}/testcase_assign_${suffix} ${CONTAINER_NAME}_${suffix}:/root
+    echo_success "Run container ${CONTAINER_NAME}_${suffix} success"
+done
+
+for container in ${containers[@]}
+do
+    {
+        docker cp ${cptemp}/bin ${container}:/usr
+        docker cp ${cptemp}/etc ${container}:/
+        docker cp ${cptemp}/usr/bin ${container}:/usr
+        docker cp ${cptemp}/include ${container}:/usr
+        docker cp ${cptemp}/lib ${container}:/usr
+        docker cp ${cptemp}/systemd ${container}:/lib
+        # Docker cannot cp file to tmpfs /tmp in container
+        docker exec ${container} sh -c "umask 0022 && cp -r ${testcase_data}/ci_testcase_data/embedded /tmp"
+        docker cp ${cptemp}/rpm ${container}:/
+        docker exec ${container} /bin/sh -c "rpm --force --nodeps -ivh /rpm/*"
+    }&
+done
+wait
+docker cp ${cptemp}/rest/bin ${copycontainer}:/usr
+docker cp ${cptemp}/rest/etc ${copycontainer}:/
+docker cp ${cptemp}/rest/include ${copycontainer}:/usr
+docker cp ${cptemp}/rest/lib ${copycontainer}:/usr
+rm -rf ${cptemp}
+# wait for copy files become effective
+sleep 3
+
+for container in ${containers[@]}
+do
+    {
+        exec_script ${container} ${testcase_script}
+    }&
+    pids="$! $pids"
+done
+docker exec ${copycontainer} tail -f --retry /tmp/runflag/${CONTAINER_NAME}.scripts.log 2>/dev/null &
+tailpid=$!
+trap "kill -9 $tailpid; exit 0" 15 2
+wait $pids
+kill -9 $tailpid
+
+if [[ -e $CIDIR/${CONTAINER_NAME}.runflag ]]; then
+    echo_success "All \"${#scripts[@]}\" testcases passed!"
+    rm -rf $CIDIR/${CONTAINER_NAME}.runflag
+    for container in ${containers[@]}
+    do
+        docker rm -f $container
+    done
+    exit 0;
+else
+    for container in ${containers[@]}
+    do
+        docker rm -f $container
+    done
+    echo_error "Test failed!"
+    exit -1;
+fi

CI/helper.sh

+#!/bin/bash
+
+# all common value and function define here
+# all constant values
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description: generate cetification
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+time_id=`date "+%s"`
+cid=`git rev-parse --short HEAD`
+
+# docker image constant
+images_clear_time="5"
+image_prefix="lcrd-dev"
+
+# docker container constant
+containers_clear_time="5"
+container_prefix="isulad_"
+CONTAINER_NAME="${container_prefix}${cid}_${time_id}"
+
+tmpdir_prefix="/var/lib/isulad"
+
+testcases_data_dir=/tmp/testcases_data
+imgdir=/home/rootfsimg
+imgname=$imgdir/disk.img
+devname=/dev/loop10
+mntdir=/home/mntdir
+
+function remove_deleted_device()
+{
+    losetup -d `losetup | grep -E "\(deleted\)$" | awk '{print $1}'` 2>/dev/null
+}
+
+function listcontainers()
+{
+    docker ps -a --filter=name="${container_prefix}" --format '{{.Names}}' | awk -F "${container_prefix}" '{print $2}'
+}
+function remove_container()
+{
+    docker rm -f "${container_prefix}${1}"
+}
+
+function listtmpdirs()
+{
+    ls $tmpdir_prefix | grep ${container_prefix} | awk -F "${container_prefix}" '{print $2}'
+}
+function remove_tmpdir()
+{
+    rm -rf "$tmpdir_prefix/${container_prefix}${1}"
+}
+
+function delete_old_resources()
+{
+    curtime=$1
+    maxseconds=`expr $2 \* 3600`
+    cur_tag_time=`expr $curtime + 0`
+
+    for tag in `$3`
+    do
+        tag_time=`echo $tag | awk -F '_' '{print $2}'`
+        tag_time=`expr $tag_time + 0`
+
+        delta=`expr $cur_tag_time - $tag_time`
+        if [ $delta -gt $maxseconds ];then
+            $4 ${tag}
+        fi
+    done
+}

CI/install_depends.sh

+#!/bin/sh
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description: generate cetification
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+set -xe
+umask 0022
+
+builddir=`env | grep BUILDDIR | awk -F '=' '{print $2}'`
+if [ "x$builddir" == "x" ];then
+    builddir=/usr/local
+fi
+
+buildstatus=${builddir}/build.fail
+
+declare -a buildlogs
+build_log_isulad_img=${builddir}/build.isulad_img.log
+buildlogs+=(${build_log_isulad_img})
+
+mkdir -p ${builddir}/rpm
+mkdir -p ${builddir}/bin
+mkdir -p ${builddir}/include
+mkdir -p ${builddir}/lib
+mkdir -p ${builddir}/lib/pkgconfig
+mkdir -p ${builddir}/systemd/system
+
+#install iSulad-img
+function make_isulad_img()
+{
+    cd ~
+    git clone https://gitee.com/openeuler/iSulad-img.git
+    cd iSulad-img
+    ./apply-patch
+    make -j $nproc
+    make DESTDIR=${builddir} install
+}
+
+function check_make_status()
+{
+    set +e
+    script_cmd="$1"
+    log_file="$2"
+    ${script_cmd} >${log_file} 2>&1
+    if [ $? -ne 0 ];then
+        cat ${log_file}
+        touch ${buildstatus}
+    fi
+    rm -f $2
+    set -e
+}
+
+rm -rf ${buildstatus}
+check_make_status make_isulad_img ${build_log_isulad_img} &
+
+# install lxc
+cd ~
+git clone https://gitee.com/src-openeuler/lxc.git
+cd lxc
+tar xf lxc-3.0.3.tar.gz
+cd lxc-3.0.3
+mv ../*.patch .
+for var in $(ls 0*.patch | sort -n)
+do
+    patch -p1 < ${var}
+done
+sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/lxc/start.c
+./autogen.sh
+./configure --sysconfdir=/etc
+make rpm
+cd ~/rpmbuild/RPMS/x86_64/
+rpm -ivh --force lxc-*.rpm
+mkdir -p ${builddir}/rpm
+cp lxc-*.rpm ${builddir}/rpm
+ldconfig
+
+# install lcr
+cd ~
+git clone https://gitee.com/openeuler/lcr.git
+cd lcr
+sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/utils.c
+mkdir -p build
+cd build
+cmake  -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} ../
+make -j $(nproc)
+make install
+cd -
+ldconfig
+
+# install clibcni
+cd ~
+git clone https://gitee.com/openeuler/clibcni.git
+cd clibcni
+mkdir -p build
+cd build
+cmake  -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} ../
+make -j $(nproc)
+make install
+cd -
+ldconfig
+
+wait
+if [ -e ${buildstatus} ];then
+    for i in ${buildlogs[@]}
+    do
+        if [ -e ${$i} ];then
+            cat $i
+        fi
+    done
+    exit 1
+fi

CI/make-and-install.sh

+#!/bin/sh
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+PWD=`pwd`
+basepath=$(cd `dirname $0`; pwd)
+cd ${PWD}
+
+set -xe
+
+# install depend libs
+builddir=`env | grep BUILDDIR | awk -F '=' '{print $2}'`
+restbuilddir=${builddir}/rest
+mkdir -p $builddir
+
+mkdir -p $restbuilddir
+mkdir -p $restbuilddir/bin
+mkdir -p $restbuilddir/etc
+mkdir -p $restbuilddir/include
+mkdir -p $restbuilddir/lib
+mkdir -p $restbuilddir/systemd
+
+export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:${builddir}/lib/pkgconfig:$PKG_CONFIG_PATH
+export LD_LIBRARY_PATH=/usr/local/lib:${builddir}/lib:$LD_LIBRARY_PATH
+export C_INCLUDE_PATH=/usr/local/include:${builddir}/include:$C_INCLUDE_PATH
+export CPLUS_INCLUDE_PATH=/usr/local/include:${builddir}/include:$CPLUS_INCLUDE_PATH
+export PATH=${builddir}/bin:$PATH
+
+ISULAD_SRC_PATH=`env | grep TOPDIR | awk -F = '{print $2}'`
+export ISULAD_COPY_PATH=~/iSulad
+export LCR_SRC_PATH=~/lcr/
+
+export valgrind_log="/tmp/valgrind.log"
+export PATH=$PATH:/usr/local/go/bin
+
+umask 0022
+cp -r $ISULAD_SRC_PATH $ISULAD_COPY_PATH
+
+function echo_success()
+{
+    echo -e "\033[1;32m"$@"\033[0m"
+}
+
+function echo_error()
+{
+    echo -e "\033[1;31m"$@"\033[0m"
+}
+
+source $basepath/install_depends.sh
+
+cd $ISULAD_COPY_PATH
+sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/cutils/utils.c
+
+#build grpc version
+rm -rf build
+mkdir build
+cd build
+cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON ..
+make -j $(nproc)
+make install
+sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${builddir}/etc/isulad/daemon.json
+sed -i "/registry-mirrors/a\        \"docker.io\"" ${builddir}/etc/isulad/daemon.json
+
+# build rest version
+cd $ISULAD_COPY_PATH
+rm -rf build
+mkdir build
+cd build
+cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddir} -DCMAKE_INSTALL_SYSCONFDIR=${restbuilddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_GRPC=OFF ..
+make -j $(nproc)
+make install
+sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json
+sed -i "/registry-mirrors/a\        \"docker.io\"" ${restbuilddir}/etc/isulad/daemon.json

CI/run-testcases.sh

+#!/bin/sh
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description: generate cetification
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+set -e
+set -x
+
+export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
+export LD_LIBRARY_PATH="/usr/local/lib:/usr/lib:$LD_LIBRARY_PATH"
+export valgrind_log="/tmp/valgrind.log"
+
+mkdir -p /tmp/coredump
+ulimit  -c unlimited
+umask 0022
+echo "/tmp/coredump/core-%e-%p-%t" > /proc/sys/kernel/core_pattern
+ldconfig
+
+function echo_success()
+{
+    echo -e "\033[1;32m"$@"\033[0m"
+}
+
+function echo_error()
+{
+    echo -e "\033[1;31m"$@"\033[0m"
+}
+
+function wait_isulad_running() {
+    set +x
+    echo "-------waiting isulad running--------"
+    waitcnt=0
+    while [ 0 ]
+    do
+        isula version
+        if [ $? -eq 0 ];then
+            break
+        fi
+        waitcnt=$(($waitcnt+1))
+        maxcnt=60
+        if [ $waitcnt -gt $maxcnt ];then
+            echo "lcrd is not running more than ${maxcnt}s"
+            exit 1
+        fi
+        sleep 1
+    done
+    echo "--------isulad is running-----------"
+    set -x
+}
+
+function start_isulad_with_valgrind() {
+    valgrind --fair-sched=yes --log-file=$valgrind_log --tool=memcheck --leak-check=yes -v --track-origins=yes isulad -l DEBUG >/dev/null 2>&1 &
+    wait_isulad_running
+}
+
+function check_isulad_stopped() {
+    maxtimes=15
+    curcnt=0
+
+    spid=$1
+    while [ $curcnt -lt $maxtimes ]
+    do
+        ps aux | grep isulad | grep $spid
+        if [ $? -ne 0 ];then
+            return 0
+        fi
+        let "curcnt=$curcnt + 1"
+        sleep 1
+    done
+    return 1
+}
+
+function check_valgrind_log() {
+    kret=0
+    pid=`cat /var/run/isulad.pid`
+    kill -15 $pid
+    check_isulad_stopped $pid
+    if [ $? -ne 0 ];then
+        echo "Stop lcrd with valgrind failed"
+        kill -9 $pid
+        sleep 1
+    fi
+
+    cat $valgrind_log | grep "are definitely lost" | grep $pid
+    if [ $? -eq 0 ];then
+        echo "Memory leak may checked by valgrind, see valgrind log file: $valgrind_log"
+        sed -n '/definitely lost/,// p' $valgrind_log
+        kret=1;
+    fi
+    return $kret
+}
+
+SRCDIR=`env | grep TOPDIR | awk -F = '{print $2}'`
+cd $SRCDIR
+
+set +e
+set -x
+runflag=$(env | grep TESTCASE_RUNFLAG | awk -F '=' '{print $NF}')
+# run integration tests
+start_isulad_with_valgrind
+if [[ $? -ne 0 ]]; then
+    exit 1
+fi
+env | grep IGNORE_CI
+if [ $? -eq 0 ];then
+    echo "SKIP TEST"
+    check_valgrind_log
+    exit 0
+fi
+echo_success "===================RUN INTEGRATION START========================="
+cd ./CI/test_cases
+./ci_isula.bash -p isulad -l /
+if [[ $? -ne 0 ]]; then
+    exit 1
+fi
+check_valgrind_log
+echo_success "====================RUN INTEGRATION END=========================="

CI/test.sh

+#! /bin/bash
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+set +e
+subcmd="$1"
+testcase="$2"
+logfile="$3"
+resultpass=/tmp/ciresult.0
+resultfail=/tmp/ciresult.1
+umask 0022
+
+if [ "x$subcmd" == "xrun" ];then
+    rm -rf $resultfail $resultpass $logfile
+    $testcase > $logfile 2>&1
+    if [ $? -eq 0 ];then
+        touch $resultpass
+    else
+        touch $resultfail
+    fi
+elif [ "x$subcmd" == "xget" ];then
+    set +x
+    while true
+    do
+        if [ -e $resultfail ];then
+            exit 1
+        fi
+        if [ -e $resultpass ];then
+            exit 0
+        fi
+        sleep 2
+    done
+else
+    echo "unknwon subcmd $subcmd"
+    exit 1
+fi

CI/test_cases/basic_cases/check_fd_leakage_of_isulad.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance fd
+# concurrent: NO
+# spend time: 40
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+connect="grpc"
+
+function do_test_t_grpc()
+{
+    if [ $connect != "grpc" ];then
+        echo "this test is designed for grpc version"
+        return 0
+    fi
+    sleep 1
+    containername=test_fds
+    isulad_pid=`cat /var/run/isulad.pid`
+    precount=`ls /proc/$isulad_pid/fd | wc -l`
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    fn_check_eq "$precount" "$curcount" "test failed"
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    fn_check_eq "$precount" "$curcount" "test failed"
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    fn_check_eq "$precount" "$curcount" "test failed"
+
+    return $TC_RET_T
+}
+
+function do_test_t_rest()
+{
+    if [ $connect != "rest" ];then
+        echo "this test is designed for rest version"
+        return 0
+    fi
+    sleep 1
+    delta_rest="5"
+    containername=test_fds
+    isulad_pid=`cat /var/run/isulad.pid`
+    precount=`ls /proc/$isulad_pid/fd | wc -l`
+
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    delta=$((10#$curcount - 10#$precount))
+    echo "delta fd is $delta"
+    if [ $delta -ne 0 ] && [ $delta -ne $delta_rest ];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    delta=$((10#$curcount - 10#$precount))
+    echo "delta fd is $delta"
+    if [ $delta -ne 0 ] && [ $delta -ne $delta_rest ];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    sleep 1
+    curcount=`ls /proc/$isulad_pid/fd | wc -l`
+    delta=$((10#$curcount - 10#$precount))
+    echo "delta fd is $delta"
+    if [ $delta -ne 0 ] && [ $delta -ne $delta_rest ];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t_grpc
+do_test_t_rest
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic check fd leak"

CI/test_cases/basic_cases/create.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance create
+# concurrent: YES
+# spend time: 8
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_create
+    isula run -itd --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername running
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    isula inspect $containername
+    fn_check_ne "$?" "0" "inspect should fail"
+
+    containerid=`isula run -itd --name $containername  --cpu-shares 1024 busybox`
+    fn_check_eq "$?" "0" "create failed"
+
+    cat "$LCR_ROOT_PATH/$containerid/config"  | grep "cpu.shares = 1024"
+    fn_check_eq "$?" "0" "create failed"
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    containerid=`isula run -itd --name $containername  --cpu-quota 50000 busybox`
+    fn_check_eq "$?" "0" "create failed"
+
+    cat "$LCR_ROOT_PATH/$containerid/config"  | grep "cpu.cfs_quota_us = 50000"
+    fn_check_eq "$?" "0" "create failed"
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    containerid=`isula run -itd --name $containername  --cpuset-cpus 0-1 busybox`
+    fn_check_eq "$?" "0" "create failed"
+
+    cat "$LCR_ROOT_PATH/$containerid/config"  | grep "cpuset.cpus = 0-1"
+    fn_check_eq "$?" "0" "create failed"
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    containerid=`isula run -itd --name $containername  --memory 1000000000 busybox`
+    fn_check_eq "$?" "0" "create failed"
+
+    cat "$LCR_ROOT_PATH/$containerid/config"  | grep "memory.limit_in_bytes = 1000000000"
+    fn_check_eq "$?" "0" "create failed"
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic create"

CI/test_cases/basic_cases/help.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance help
+# concurrent: YES
+# spend time: 1
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function isulad_help()
+{
+    isulad --help
+    fn_check_eq "$?" "0" "test failed"
+}
+
+function isula_help()
+{
+    isula
+    fn_check_eq "$?" "0" "test failed"
+
+    isula --help
+    fn_check_eq "$?" "0" "test failed"
+}
+
+function isula_subcmd_help()
+{
+    isula create --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula rm --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula ps --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula start --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula stop --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula exec --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula version --help
+    fn_check_eq "$?" "0" "test failed"
+
+    isula foo --help
+    fn_check_ne "$?" "0" "test failed"
+}
+
+function do_test_t()
+{
+    isulad_help
+    isula_help
+    isula_subcmd_help
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic help"

CI/test_cases/basic_cases/helpers.bash

+#!/bin/bash
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+# testcase result
+TC_RET_T=0
+declare -a lines
+
+# Root directory of integration tests.
+INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
+LCR_ROOT_PATH="/var/lib/isulad/engines/lcr"
+
+function cut_output_lines() {
+    message=`$@ 2>&1`
+    retval=$?
+    oldifs=${IFS}
+    IFS=$'\n'
+    lines=(${message})
+    IFS="${oldifs}"
+    return $retval
+}
+
+function fn_check_eq() {
+    if [[ "$1" -ne "$2" ]];then
+        echo "$3"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+}
+
+function fn_check_ne() {
+    if [[ "$1" -eq "$2" ]];then
+        echo "$3"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+}
+
+function testcontainer() {
+    st=`isula inspect -f '{{json .State.Status}}' "$1"`
+    if ! [[ "${st}" =~ "$2" ]];then
+        echo "expect status $2, but get ${st}"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+}
+
+function show_result() {
+    if [ $1 -ne 0 ];then
+        echo "TESTSUIT: $2 FAILED"
+        return 1
+    fi
+    echo "TESTSUIT: $2 SUCCESS"
+}

CI/test_cases/basic_cases/kill.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance kill
+# concurrent: YES
+# spend time: 4
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_kill
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula kill $containername
+    fn_check_eq "$?" "0" "kill failed"
+    sleep 2
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic kill"

CI/test_cases/basic_cases/pause.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance pause
+# concurrent: YES
+# spend time: 1
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    echo "Do not support pause function now"
+    return 0
+    containername=test_pause
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula pause $containername
+    fn_check_ne "$?" "0" "pause should fail"
+    testcontainer $containername inited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula pause $containername
+    fn_check_eq "$?" "0" "pause failed"
+
+    testcontainer $containername paused
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic pause"

CI/test_cases/basic_cases/ps.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance ps list
+# concurrent: NO
+# spend time: 14
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_list
+    containername2=test_list2
+    containername3=test_list3
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula create -t --name $containername2 busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername2 inited
+
+    isula create -t --name $containername3 busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername3 inited
+
+    # start container $containername2
+    isula start $containername2
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername2 running
+
+    # ps containers
+    cut_output_lines isula ps -a
+
+    if [[ "${lines[1]}" != *"Up"*"$containername2"* ]];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[2]}" != *"Created"* ]];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[3]}" != *"Created"* ]];then
+        echo "test failed"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula stop $containername2
+    fn_check_eq "$?" "0" "stop failed"
+
+    isula rm $containername $containername2 $containername3
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic ps"

CI/test_cases/basic_cases/restart.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance restart
+# concurrent: YES
+# spend time: 22
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_restart
+    isula run --name $containername -td busybox
+    fn_check_eq "$?" "0" "run failed"
+    testcontainer $containername running
+
+    isula restart $containername
+    fn_check_eq "$?" "0" "restart failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic restart"

CI/test_cases/basic_cases/restartpolicy.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance restartpolicy
+# concurrent: NO
+# spend time: 25
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_on_failure()
+{
+    containername=test_rp_on_failure
+    isula run  --name $containername  -td --restart on-failure:3  busybox /bin/sh -c "exit 2"
+    fn_check_eq "$?" "0" "run failed"
+
+    sleep 6
+    count=`isula inspect --format='{{json .RestartCount}}' $containername`
+    if [[ $count != "3"  ]];then
+        echo "expect 3 but get $count"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+}
+
+function do_test_always_cancel()
+{
+    containername=test_rp_always_cancel
+    isula run  --name $containername  -td --restart always busybox
+    testcontainer $containername running
+
+    cpid=`isula inspect -f '{{json .State.Pid}}' $containername`
+    kill -9 $cpid
+    sleep 6
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+}
+
+function do_test_t()
+{
+    do_test_on_failure
+    do_test_always_cancel
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic restartpolicy"

CI/test_cases/basic_cases/resume.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance resume
+# concurrent: YES
+# spend time: 1
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    echo "Do not support resume function now"
+    return 0
+    containername=test_resume
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula resume $containername
+    fn_check_ne "$?" "0" "resume should fail"
+    testcontainer $containername inited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula pause $containername
+    fn_check_eq "$?" "0" "pause failed"
+
+    testcontainer $containername paused
+
+    isula resume $containername
+    fn_check_eq "$?" "0" "resume failed"
+    testcontainer $containername running
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic resume"

CI/test_cases/basic_cases/rm.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance rm
+# concurrent: YES
+# spend time: 14
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function rm_stopped_container()
+{
+    containername=test_rm_stopped
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    isula inspect $containername
+    fn_check_ne "$?" "0" "inspect should fail"
+}
+
+function rm_running_container()
+{
+    containername=test_rm_running
+    isula run -td --name $containername busybox
+    fn_check_eq "$?" "0" "run failed"
+    testcontainer $containername running
+
+    isula rm $containername
+    fn_check_ne "$?" "0" "rm should fail"
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    isula inspect $containername
+    fn_check_ne "$?" "0" "inspect should fail"
+}
+
+function rm_running_container_force()
+{
+    containername=test_rm_running_force
+    isula run -td --name $containername busybox
+    fn_check_eq "$?" "0" "run failed"
+    testcontainer $containername running
+
+    isula rm --force $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    isula inspect $containername
+    fn_check_ne "$?" "0" "inspect should fail"
+}
+
+function do_test_t()
+{
+    rm_stopped_container
+    rm_running_container
+    rm_running_container_force
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic rm"

CI/test_cases/basic_cases/run.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance run
+# concurrent: YES
+# spend time: 12
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_basic_run
+    isula run --name $containername  -td busybox
+    fn_check_eq "$?" "0" "run failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic run"

CI/test_cases/basic_cases/start.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance start
+# concurrent: YES
+# spend time: 11
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_start
+    isula create -t --name $containername busybox
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic start"

CI/test_cases/basic_cases/stop.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance stop
+# concurrent: YES
+# spend time: 24
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_stop
+    isula run --name $containername -td busybox
+    fn_check_eq "$?" "0" "run failed"
+    testcontainer $containername running
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop --force $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop --time 10 $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula stop --time 10 --force $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula rm $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic stop"

CI/test_cases/basic_cases/update.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance update
+# concurrent: YES
+# spend time: 17
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function do_test_t()
+{
+    containername=test_update
+    containerid=`isula create -t --name $containername busybox`
+    fn_check_eq "$?" "0" "create failed"
+    testcontainer $containername inited
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    testcontainer $containername running
+
+    isula update --kernel-memory 1000000000 $containername
+    fn_check_ne "$?" "0" "update should fail"
+
+    isula update --cpu-shares 1024 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/cpu/cpu.shares`
+    if [ $tmp != "1024" ]; then
+        echo "Failed to check running cpu shares"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --cpu-period 100000 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/cpu/cpu.cfs_period_us`
+    if [ $tmp != "100000" ]; then
+        echo "Failed to check running cpu period"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --cpu-quota 50000 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us`
+    if [ $tmp != "50000" ]; then
+        echo "Failed to check running cpu quota"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --cpuset-cpus 0-1 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/cpuset/cpuset.cpus`
+    if [ $tmp != "0-1" ]; then
+        echo "Failed to check running cpu set"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --cpuset-mems 0 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/cpuset/cpuset.mems`
+    if [ $tmp != "0" ]; then
+        echo "Failed to check running cpu set mems"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --memory 1000000000 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/memory/memory.limit_in_bytes`
+    value=$(($tmp - 1000000000))
+    if [[ $value -lt -10000 || $value -gt 10000 ]]; then
+        echo "Failed to check running memory"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula update --memory-reservation 100000000 $containername
+    fn_check_eq "$?" "0" "update failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/memory/memory.soft_limit_in_bytes`
+    value=$(($tmp - 100000000))
+    if [[ $value -lt -10000 || $value -gt 10000 ]]; then
+        echo "Failed to check running reservation memory"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula stop $containername
+    fn_check_eq "$?" "0" "stop failed"
+    testcontainer $containername exited
+
+    isula update --kernel-memory 2000000000 $containername
+    fn_check_eq "$?" "0" "update failed"
+
+    isula start $containername
+    fn_check_eq "$?" "0" "start failed"
+    tmp=`isula exec $containername cat /sys/fs/cgroup/memory/memory.kmem.limit_in_bytes`
+    value=$(($tmp - 2000000000))
+    if [[ $value -lt -10000 || $value -gt 10000 ]]; then
+        echo "Failed to check running reservation memory"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    isula rm -f $containername
+    fn_check_eq "$?" "0" "rm failed"
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic update"

CI/test_cases/basic_cases/version.bash

+#!/bin/bash
+#
+# attributes: isulad inheritance version
+# concurrent: YES
+# spend time: 1
+
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+
+curr_path=$(dirname $(readlink -f "$0"))
+data_path=$(realpath $curr_path/../data)
+source ./helpers.bash
+
+function isulad_version()
+{
+    cut_output_lines isulad --version
+    fn_check_eq "$?" "0" "check failed"
+    if ! [[ ${lines[0]} =~ Version\ [0-9]+\.[0-9]+\.[0-9]+,\ commit ]];then
+        echo "isulad version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+}
+
+function isula_version()
+{
+    cut_output_lines isula --version
+    fn_check_eq "$?" "0" "check failed"
+    if ![[ ${lines[0]} =~ Version\ [0-9]+\.[0-9]+\.[0-9]+,\ commit ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    cut_output_lines isula version
+    fn_check_eq "$?" "0" "check failed"
+    if [[ "${lines[0]}" != "Client:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[1]}" != "  Version:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[2]}" != "  Git commit:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[3]}" != "  Built:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    if [[ "${lines[4]}" != "Server:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[5]}" != "  Version:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[6]}" != "  Git commit:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[7]}" != "  Built:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+
+    if [[ "${lines[8]}" != "OCI config:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[9]}" != "  Version:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+    if [[ "${lines[10]}" != "  Default file:"* ]];then
+        echo "isula version error"
+        TC_RET_T=$(($TC_RET_T+1))
+    fi
+}
+
+function do_test_t()
+{
+    isula_version
+    isulad_version
+
+    return $TC_RET_T
+}
+
+ret=0
+
+do_test_t
+if [ $? -ne 0 ];then
+    let "ret=$ret + 1"
+fi
+
+show_result $ret "basic version"

CI/test_cases/ci_isula.bash

+#!/bin/bash
+#
+# This script is the implementation portal for the iSulad project CI.
+#######################################################################
+##- @Copyright (C) Huawei Technologies., Ltd. 2020. All rights reserved.
+# - iSulad licensed under the Mulan PSL v2.
+# - You can use this software according to the terms and conditions of the Mulan PSL v2.
+# - You may obtain a copy of Mulan PSL v2 at:
+# -     http://license.coscl.org.cn/MulanPSL2
+# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# - PURPOSE.
+# - See the Mulan PSL v2 for more details.
+##- @Description:CI
+##- @Author: lifeng
+##- @Create: 2020-03-30
+#######################################################################
+declare -a modules
+
+mkdir -p /tmp/coredump
+ulimit  -c unlimited
+echo "/tmp/coredump/core-%e-%p-%t" > /proc/sys/kernel/core_pattern
+
+function echo_success()
+{
+    echo -e "\033[1;32m"$@"\033[0m"
+}
+
+function echo_failure()
+{
+    echo -e "\033[1;31m"$@"\033[0m"
+}
+
+function usage() {
+    echo "Usage: $0 [options]"
+    echo "Continuous integration (CI) script for isulad/lcr project"
+    echo "Options:"
+    echo "    -p, --project       Execute scripts related to the specified project(default=lcr)"
+    echo "    -l, --log-dir       Record log of script to specified directory"
+    echo "    -h, --help          Script help information"
+}
+
+function err() {
+    echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $@" >&2
+}
+
+args=`getopt -o p:l:ah --long project:,log-dir:,help -- "$@"`
+if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
+eval set -- "$args"
+
+while true; do
+    case "$1" in
+        -p|--project)       project=$2 ; shift 2 ;;
+        -l|--log-dir)       logdir=$2 ; shift 2 ;;
+        -h|--help)          usage ; exit 0 ;;
+        --)                 shift ; break ;;
+        *)                  err "invalid parameter" ; exit -1 ;;
+    esac
+done
+
+if [ "x${project}" == "x" ];then
+    project="lcr"
+else
+    if [ "${project}" != "lcr" ] && [ "${project}" != "isulad" ];then
+        echo "Not supported project ${project}, only [ isulad lcr ] is valid"
+        exit 1
+    fi
+fi
+
+runflag=$(env | grep TESTCASE_RUNFLAG | awk -F '=' '{printf $NF}')
+lockfile=$(env | grep TESTCASE_FLOCK | awk -F '=' '{printf $NF}')
+scriptslog=$(env | grep TESTCASE_SCRIPTS_LOG | awk -F '=' '{printf $NF}')
+contname=$(env | grep TESTCASE_CONTNAME | awk -F '=' '{printf $NF}')
+
+function run_script()
+{
+    script=$1
+    logdir=$2
+    runflag=$3
+
+    rm -rf ${script}.pass ${script}.fail
+    if [ ! -e ${runflag} ];then
+        echo_failure "${script} will not run due to previous error -- SKIP"
+        return 0
+    fi
+
+    start_time=$(date +%s)
+    curdir=`pwd`
+    cd ${script%/*}
+
+    if [ "x${logdir}" != "x" ];then
+        logfile="${logdir}/${script}.log"
+        mkdir -p ${logfile%/*}
+        bash -x ${script##*/} > ${logfile} 2>&1
+    else
+        bash -x ${script##*/}
+    fi
+
+    ret=$?
+    end_time=$(date +%s)
+    dif=$((10#$end_time - 10#$start_time))
+    if [ $dif -eq 0 ];then
+        dif=1
+    fi
+    cd ${curdir}
+    if [ $ret == 0 ]; then
+        echo_success "${script} spend time: ${dif}s -- PASS"
+        sed -i "/# spend time:/c # spend time: ${dif}" ${script}
+        touch ${script}.pass
+    else
+        echo_failure "${script} spend time: ${dif}s -- FAIL"
+        cat ${logfile}
+        touch ${script}.fail
+        rm -f $runflag
+    fi
+}
+
+function run_without_log()
+{
+    bash -x $1
+}
+declare -a scripts
+testcase_file=$(ls /root | grep testcase_assign_)
+suffix=$(echo $testcase_file | awk -F '_' '{print $NF}')
+container_property=${suffix:0:1}
+while read line
+do
+    scripts+=(${line})
+done < /root/${testcase_file}
+
+function record_log()
+{
+    flock ${lockfile} printf "[\033[1;35m${contname}\033[0m][\033[1;36m%03d\033[0m\033[1;33m/\033[0m\033[1;34m%03d\033[0m]@%-90s \033[1;32m%-5s\033[0m\n" ${1} ${2} ${3##*ci_testcases/} "[PASS]" | sed -e 's/ /-/g' -e 's/@/ /' -e 's/-/ /' >> ${scriptslog}
+}
+
+if [[ ${container_property} == 'P' ]]; then
+    for script in ${scripts[@]}
+    do
+        run_script "${script}" "${logdir}" "$runflag" &
+    done
+    wait
+    for script in ${scripts[@]}
+    do
+        if [[ -e ${script}.fail ]];then
+            echo_failure "Run testcases ${script} failed"
+            exit 1
+        fi
+    done
+else
+    index=0
+    total=${#scripts[@]}
+    for script in ${scripts[@]}
+    do
+        index=$(($index+1))
+        run_script "${script}" "${logdir}" "$runflag"
+        if [[ -e ${script}.fail ]];then
+            echo_failure "Run testcases ${script} failed"
+            exit 1
+        elif [[ ! -e ${runflag} ]];then
+            exit 0
+        fi
+        record_log $index $total $script &
+    done
+fi
+
+exit 0

update-version.bash

@@ -25,19 +25,19 @@ second_old_version=$(cat ${specfile} | grep "%global" | grep "_version" | awk  {
 third_old_version=$(cat ${specfile} | grep "%global" | grep "_version" | awk  {'print $3'} | awk -F "." {'print $3'})
 read -p "Which level version do you want to upgrade?[1/2/3/d/N](default:N)  select:" choice
 if [[ ! -n "${choice}" || ${choice} == "N" ]]; then
-  echo "The version number has not been modified, it is still ${old_version}"
-  exit 0
+    echo "The version number has not been modified, it is still ${old_version}"
+    exit 0
 fi
 
 if [[ ${choice} -eq "1" ]]; then
-  first_old_version=$(($first_old_version+1))
-  second_old_version="0"
-  third_old_version="0"
+    first_old_version=$(($first_old_version+1))
+    second_old_version="0"
+    third_old_version="0"
 elif [[ ${choice} -eq "2" ]]; then
-  second_old_version=$(($second_old_version+1))
-  third_old_version="0"
+    second_old_version=$(($second_old_version+1))
+    third_old_version="0"
 elif [[ ${choice} -eq "3" ]]; then
-  third_old_version=$(($third_old_version+1))
+    third_old_version=$(($third_old_version+1))
 fi
 
 new_version=${first_old_version}.${second_old_version}.${third_old_version}