Merge branch '3.0' of https://github.com/taosdata/TDengine into refact/tsdb_optimize

packaging/MPtestJenkinsfile

+def sync_source(branch_name) {
+    sh '''
+        hostname
+        env
+        echo ''' + branch_name + '''
+    '''
+    sh '''
+        cd ${TDINTERNAL_ROOT_DIR}
+        git reset --hard
+        git fetch || git fetch
+        git checkout ''' + branch_name + ''' -f
+        git branch
+        git pull || git pull
+        git log | head -n 20
+        cd ${TDENGINE_ROOT_DIR}
+        git reset --hard
+        git fetch || git fetch
+        git checkout ''' + branch_name + ''' -f
+        git branch
+        git pull || git pull
+        git log | head -n 20
+        git submodule update --init --recursive
+    '''
+    return 1
+}
+def run_test() {
+    sh '''
+        cd ${TDENGINE_ROOT_DIR}/packaging
+        
+    '''
+    sh '''
+        export LD_LIBRARY_PATH=${TDINTERNAL_ROOT_DIR}/debug/build/lib
+        ./fulltest.sh
+    '''
+    sh '''
+        cd ${TDENGINE_ROOT_DIR}/tests
+        ./test-all.sh b1fq
+    '''
+}
+def build_run() {
+    sync_source("${BRANCH_NAME}")
+}
+pipeline {
+    agent none
+    parameters {
+        string (
+            name:'version', 
+            defaultValue:'3.0.0.1', 
+            description: 'release version number,eg: 3.0.0.1 or 3.0.0.'
+        )
+        string (
+            name:'baseVersion', 
+            defaultValue:'3.0.0.1', 
+            description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1'
+        )
+    }
+    environment{
+        WORK_DIR = '/var/lib/jenkins/workspace'
+        TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal'
+        TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community'
+        BRANCH_NAME = '3.0'
+
+        TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz"
+        BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-arm64-x64.tar.gz"
+
+        TD_SERVER_ARM_TAR = "TDengine-server-${version}-Linux-arm64.tar.gz"
+        BASE_TD_SERVER_ARM_TAR = "TDengine-server-${baseVersion}-Linux-arm64.tar.gz"
+
+        TD_SERVER_LITE_TAR = "TDengine-server-${version}-Linux-x64-Lite.tar.gz"
+        BASE_TD_SERVER_LITE_TAR = "TDengine-server-${baseVersion}-Linux-x64-Lite.tar.gz"
+
+        TD_CLIENT_TAR = "TDengine-client-${version}-Linux-x64.tar.gz"
+        BASE_TD_CLIENT_TAR = "TDengine-client-${baseVersion}-arm64-x64.tar.gz"
+
+        TD_CLIENT_ARM_TAR = "TDengine-client-${version}-Linux-arm64.tar.gz"
+        BASE_TD_CLIENT_ARM_TAR = "TDengine-client-${baseVersion}-Linux-arm64.tar.gz"
+
+        TD_CLIENT_LITE_TAR = "TDengine-client-${version}-Linux-x64-Lite.tar.gz"
+        BASE_TD_CLIENT_LITE_TAR = "TDengine-client-${baseVersion}-Linux-x64-Lite.tar.gz"
+
+        TD_SERVER_RPM = "TDengine-server-${version}-Linux-x64.rpm"
+        
+        TD_SERVER_DEB = "TDengine-server-${version}-Linux-x64.deb"
+
+        TD_SERVER_EXE = "TDengine-server-${version}-Windows-x64.exe"
+        
+        TD_CLIENT_EXE = "TDengine-client-${version}-Windows-x64.exe"
+
+
+    }
+    stages {
+        stage ('RUN') {
+        stage('get check package scritps'){
+            agent{label 'ubuntu18'}
+            steps {
+                catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                    script{
+                        sync_source("${BRANCH_NAME}")
+                    }
+                    
+                }
+            }
+        }
+            parallel {
+                stage('ubuntu16') {
+                    agent{label " ubuntu16 "}
+                    steps {
+                        timeout(time: 3, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            rmtaos
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                        }
+                    }
+                }
+                stage('ubuntu18') {
+                    agent{label " ubuntu18 "}
+                    steps {
+                        timeout(time: 3, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            rmtaos
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_DEB} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                        }
+                    }
+                }
+                stage('centos7') {
+                    agent{label " centos7_9 "}
+                    steps {
+                        timeout(time: 240, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            rmtaos
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                        }
+                    }
+                }
+                stage('centos8') {
+                    agent{label " centos8_3 "}
+                    steps {
+                        timeout(time: 240, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            rmtaos
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            python3 checkPackageRuning.py  
+                            '''                     
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

packaging/checkPackageRuning.py

+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/
+
+# -*- coding: utf-8 -*-
+import sys , os
+import getopt
+import subprocess
+# from this import d
+import time
+
+# install taospy
+
+out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ")
+print(out)
+if (out == "" ):
+    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git")
+    print("install taos python connector")
+
+
+
+# start taosd prepare 
+os.system("rm -rf /var/lib/taos/*")
+os.system("systemctl restart taosd ")
+
+# wait a moment ,at least 5 seconds
+time.sleep(5)
+
+# prepare data by taosBenchmark 
+
+os.system("taosBenchmark -y -n 100 -t 100")
+
+import taos
+
+conn = taos.connect(host="localhost",
+                                         user="root",
+                                         password="taosdata",
+                                         database="test",
+                                         port=6030,
+                                         config="/etc/taos",  # for windows the default value is C:\TDengine\cfg
+                                         timezone="Asia/Shanghai")  # default your host's timezone
+
+server_version = conn.server_info
+print("server_version", server_version)
+client_version = conn.client_info
+print("client_version", client_version)  # 3.0.0.0
+
+# Execute a sql and get its result set. It's useful for SELECT statement
+result: taos.TaosResult = conn.query("SELECT count(*) from test.meters")
+
+data = result.fetch_all()
+
+if data[0][0] !=10000:
+    print(" taosBenchmark work not as expected ")
+    sys.exit(1)
+else:
+    print(" taosBenchmark work as expected ")
+
+# test taosdump dump out data and dump in data 
+
+# dump out datas
+os.system("taosdump --version")
+os.system("mkdir -p /tmp/dumpdata")
+os.system("rm -rf /tmp/dumpdata/*")
+
+
+
+# dump data out 
+print("taosdump dump out data")
+
+os.system("taosdump -o /tmp/dumpdata -D test -y ")
+
+# drop database of test
+print("drop database test")
+os.system(" taos -s ' drop database test ;' ")
+
+# dump data in 
+print("taosdump dump data in")
+os.system("taosdump -i /tmp/dumpdata  -y ")
+
+result = conn.query("SELECT count(*) from test.meters")
+
+data = result.fetch_all()
+
+if data[0][0] !=10000:
+    print(" taosdump work not as expected ")
+    sys.exit(1)
+else:
+    print(" taosdump work as expected ")
+
+conn.close()
\ No newline at end of file

packaging/testpackage.sh

+#!/bin/sh
+
+# function installPkgAndCheckFile{
+
+echo "Download package"
+
+packgeName=$1
+version=$2
+originPackageName=$3
+originversion=$4
+testFile=$5
+subFile="taos.tar.gz"
+
+if [ ${testFile} = "server" ];then
+    tdPath="TDengine-server-${version}"
+    originTdpPath="TDengine-server-${originversion}"
+    installCmd="install.sh"
+elif [ ${testFile} = "client" ];then
+    tdPath="TDengine-client-${version}"
+    originTdpPath="TDengine-client-${originversion}"
+    installCmd="install_client.sh"    
+elif [ ${testFile} = "tools" ];then
+    tdPath="taosTools-${version}"
+    originTdpPath="taosTools-${originversion}"
+    installCmd="install-taostools.sh"
+fi
+
+echo "Uninstall all components of TDeingne"
+
+if command -v rmtaos ;then
+    echo "uninstall all components of TDeingne:rmtaos"
+    echo " " 
+else 
+    echo "os doesn't include TDengine "
+fi
+
+if command -v rmtaostools ;then
+    echo "uninstall all components of TDeingne:rmtaostools"
+    echo " " 
+else 
+    echo "os doesn't include rmtaostools "
+fi
+
+echo "new workroom path"
+installPath="/usr/local/src/packageTest"
+oriInstallPath="/usr/local/src/packageTest/3.1"
+
+if [ ! -d ${installPath} ] ;then
+    mkdir -p ${installPath}
+else  
+    echo "${installPath} already exists"
+fi
+
+
+if [ ! -d ${oriInstallPath} ] ;then
+    mkdir -p ${oriInstallPath}
+else  
+    echo "${oriInstallPath} already exists"
+fi
+
+echo "decompress installPackage"
+
+cd ${installPath}
+wget https://www.taosdata.com/assets-download/3.0/${packgeName}
+cd  ${oriInstallPath}
+wget https://www.taosdata.com/assets-download/3.0/${originPackageName}
+
+
+if [[ ${packgeName} =~ "deb" ]];then
+    echo "dpkg ${packgeName}" &&  dpkg -i ${packgeName}
+elif [[ ${packgeName} =~ "rpm" ]];then
+    echo "rpm ${packgeName}"  && rpm -ivh ${packgeName}
+elif [[ ${packgeName} =~ "tar" ]];then
+    echo "tar ${packgeName}" && tar -xvf ${packgeName} 
+    cd ${oriInstallPath}
+    echo "tar -xvf ${originPackageName}" && tar -xvf ${originPackageName} 
+    cd ${installPath} 
+    echo "tar -xvf ${packgeName}" && tar -xvf ${packgeName} 
+
+
+    if [ ${testFile} != "tools" ] ;then
+        cd ${installPath}/${tdPath} && tar vxf ${subFile}
+        cd  ${oriInstallPath}/${originTdpPath}  && tar vxf ${subFile}
+    fi
+
+    echo "check installPackage File"
+
+    cd ${installPath} 
+
+    tree ${oriInstallPath}/${originTdpPath} > ${originPackageName}_checkfile
+    tree ${installPath}/${tdPath} > ${packgeName}_checkfile
+
+    diff  ${packgeName}_checkfile  ${originPackageName}_checkfile  > ${installPath}/diffFile.log
+    diffNumbers=`cat ${installPath}/diffFile.log |wc -l `
+    if [ ${diffNumbers} != 0 ];then
+        echo "The number and names of files have changed from the previous installation package"
+        echo `cat ${installPath}/diffFile.log`
+        exit -1
+    fi
+
+    cd ${installPath}/${tdPath}
+    if [ ${testFile} = "server" ];then
+        bash ${installCmd}  -e no  
+    else
+        bash ${installCmd} 
+    fi
+
+fi 
+# }
+
+# installPkgAndCheckFile 
+