Merge branch 'develop' into fix/TS-654

Jenkinsfile

@@ -583,10 +583,15 @@ pipeline {
             timeout(time: 55, unit: 'MINUTES'){       
               pre_test()
               sh '''
+              cd ${WKC}/tests
+              ./test-all.sh develop-test
+              '''
+              sh '''
               date
               cd ${WKC}/tests
               ./test-all.sh b6fq
               date'''
+              
             }
           }
         }
@@ -596,6 +601,10 @@ pipeline {
             timeout(time: 55, unit: 'MINUTES'){       
               pre_test()
               sh '''
+              cd ${WKC}/tests
+              ./test-all.sh system-test
+              '''
+              sh '''
               date
               cd ${WKC}/tests
               ./test-all.sh b7fq

documentation20/cn/11.administrator/docs.md

@@ -557,7 +557,7 @@ KILL STREAM <stream-id>;
 
 TDengine启动后，会自动创建一个监测数据库log，并自动将服务器的CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入该数据库。TDengine还将重要的系统操作（比如登录、创建、删除数据库等）日志以及各种错误报警信息记录下来存放在log库里。系统管理员可以从CLI直接查看这个数据库，也可以在WEB通过图形化界面查看这些监测信息。
 
-这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。
+这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项monitor将其关闭或打开。
 
 ### TDinsight - 使用监控数据库 + Grafana 对 TDengine 进行监控的解决方案
 

packaging/deb/makedeb.sh

@@ -58,7 +58,6 @@ cp ${compile_dir}/../packaging/tools/startPre.sh    ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/set_core.sh    ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb    ${pkg_dir}${install_home_path}/bin
 
-cp ${compile_dir}/build/bin/taosdemo                ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
 
 if [ -f "${compile_dir}/build/bin/taosadapter" ]; then

packaging/release.sh

@@ -518,7 +518,7 @@ if [ "$osType" != "Darwin" ]; then
           cd ${top_dir}/src/kit/taos-tools/packaging/deb
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
-          taos_tools_ver=$(git describe --tags|sed -e 's/ver-//g')
+          taos_tools_ver=$(git describe --tags|sed -e 's/ver-//g'|awk -F '-' '{print $1}')
           ${csudo} ./make-taos-tools-deb.sh ${top_dir} \
               ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
       fi
@@ -541,7 +541,7 @@ if [ "$osType" != "Darwin" ]; then
           cd ${top_dir}/src/kit/taos-tools/packaging/rpm
           [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
 
-          taos_tools_ver=$(git describe --tags|sed -e 's/ver-//g'|sed -e 's/-/_/g')
+          taos_tools_ver=$(git describe --tags|sed -e 's/ver-//g'|awk -F '-' '{print $1}'|sed -e 's/-/_/g')
           ${csudo} ./make-taos-tools-rpm.sh ${top_dir} \
               ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType}
       fi

packaging/rpm/tdengine.spec

@@ -71,7 +71,6 @@ cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 if [ -f %{_compiledir}/build/bin/taosadapter ]; then
     cp %{_compiledir}/build/bin/taosadapter                    %{buildroot}%{homepath}/bin ||:
 fi
-cp %{_compiledir}/build/bin/taosdemo                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taosdef.h              %{buildroot}%{homepath}/include
@@ -196,7 +195,6 @@ if [ $1 -eq 0 ];then
     ${csudo} rm -f ${bin_link_dir}/taos       || :
     ${csudo} rm -f ${bin_link_dir}/taosd      || :
     ${csudo} rm -f ${bin_link_dir}/taosadapter       || :
-    ${csudo} rm -f ${bin_link_dir}/taosdemo   || :
     ${csudo} rm -f ${cfg_link_dir}/*          || :
     ${csudo} rm -f ${inc_link_dir}/taos.h     || :
     ${csudo} rm -f ${inc_link_dir}/taosdef.h     || :

packaging/tools/install.sh

@@ -887,8 +887,8 @@ function update_TDengine() {
     fi
     tar -zxf taos.tar.gz
     install_jemalloc
-    install_avro lib
-    install_avro lib64
+    #install_avro lib
+    #install_avro lib64
 
     echo -e "${GREEN}Start to update TDengine...${NC}"
     # Stop the service if running
@@ -1001,8 +1001,8 @@ function install_TDengine() {
     install_header
     install_lib
     install_jemalloc
-    install_avro lib
-    install_avro lib64
+    #install_avro lib
+    #install_avro lib64
 
     if [ "$pagMode" != "lite" ]; then
       install_connector

packaging/tools/makeclient.sh

@@ -45,8 +45,9 @@ if [ "$osType" != "Darwin" ]; then
     strip ${build_dir}/bin/taos
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
   else
-    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo \
-               ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
+    bin_files="${script_dir}/remove_client.sh \
+        ${script_dir}/set_core.sh \
+        ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
 else

packaging/tools/makepkg.sh

@@ -31,23 +31,37 @@ else
     install_dir="${release_dir}/TDengine-server-${version}"
 fi
 
+if [ -d ${top_dir}/src/kit/taos-tools/packaging/deb ]; then
+    cd ${top_dir}/src/kit/taos-tools/packaging/deb
+    [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0"
+
+    taostools_ver=$(git describe --tags|sed -e 's/ver-//g'|awk -F '-' '{print $1}')
+    taostools_install_dir="${release_dir}/taos-tools-${taostools_ver}"
+
+    cd ${curr_dir}
+else
+    taostools_install_dir="${release_dir}/taos-tools-${version}"
+fi
+
 # Directories and files
 if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taosd
   strip ${build_dir}/bin/taos
   # lite version doesn't include taosadapter,  which will lead to no restful interface
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh ${script_dir}/startPre.sh"
+  taostools_bin_files=""
 else
   bin_files="${build_dir}/bin/taosd \
       ${build_dir}/bin/taos \
       ${build_dir}/bin/taosadapter \
-      ${build_dir}/bin/taosdump \
-      ${build_dir}/bin/taosdemo \
       ${build_dir}/bin/tarbitrator\
       ${script_dir}/remove.sh \
       ${script_dir}/set_core.sh \
       ${script_dir}/startPre.sh \
       ${script_dir}/taosd-dump-cfg.gdb"
+
+  taostools_bin_files=" ${build_dir}/bin/taosdump \
+      ${build_dir}/bin/taosBenchmark"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -78,6 +92,7 @@ mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg
 
+
 if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.toml                 ${install_dir}/cfg || :
 fi
@@ -102,10 +117,29 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taos
 mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || :
 mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || :
 
-if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then
-    mkdir -p ${install_dir}/avro/{lib,lib/pkgconfig}
-    cp ${build_dir}/lib/libavro.* ${install_dir}/avro/lib
-    cp ${build_dir}/lib/pkgconfig/avro-c.pc ${install_dir}/avro/lib/pkgconfig
+if [ -n "${taostools_bin_files}" ]; then
+    mkdir -p ${taostools_install_dir} || echo -e "failed to create ${taostools_install_dir}"
+    mkdir -p ${taostools_install_dir}/bin \
+        && cp ${taostools_bin_files} ${taostools_install_dir}/bin \
+        && chmod a+x ${taostools_install_dir}/bin/* || :
+    [ -f ${taostools_install_dir}/bin/taosBenchmark ] && \
+        ln -sf ${taostools_install_dir}/bin/taosBenchmark \
+        ${taostools_install_dir}/bin/taosdemo
+
+    if [ -f ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh ]; then
+        cp ${top_dir}/src/kit/taos-tools/packaging/tools/install-taostools.sh \
+            ${taostools_install_dir}/ > /dev/null \
+            && chmod a+x {taostools_install_dir}/install-taostools.sh \
+            || echo -e "failed to copy install-taostools.sh"
+    else
+        echo -e "install-taostools.sh not found"
+    fi
+
+    if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then
+        mkdir -p ${taostools_install_dir}/avro/{lib,lib/pkgconfig} || echo -e "failed to create ${taostools_install_dir}/avro"
+        cp ${build_dir}/lib/libavro.* ${taostools_install_dir}/avro/lib
+        cp ${build_dir}/lib/pkgconfig/avro-c.pc ${taostools_install_dir}/avro/lib/pkgconfig
+    fi
 fi
 
 if [ -f ${build_dir}/bin/jemalloc-config ]; then
@@ -235,6 +269,8 @@ cd ${release_dir}
 #  install_dir has been distinguishes  cluster from  edege, so comments this code
 pkg_name=${install_dir}-${osType}-${cpuType}
 
+taostools_pkg_name=${taostools_install_dir}-${osType}-${cpuType}
+
 # if [ "$verMode" == "cluster" ]; then
 #   pkg_name=${install_dir}-${osType}-${cpuType}
 # elif [ "$verMode" == "edge" ]; then
@@ -246,8 +282,10 @@ pkg_name=${install_dir}-${osType}-${cpuType}
 
 if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then
   pkg_name=${install_dir}-${verType}-${osType}-${cpuType}
+  taostools_pkg_name=${taostools_install_dir}-${verType}-${osType}-${cpuType}
 elif [ "$verType" == "stable" ]; then
   pkg_name=${pkg_name}
+  taostools_pkg_name=${taostools_pkg_name}
 else
   echo "unknow verType, nor stabel or beta"
   exit 1
@@ -264,4 +302,13 @@ if [ "$exitcode" != "0" ]; then
     exit $exitcode
 fi
 
+if [ -n "${taostools_bin_files}" ]; then
+    tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" $(basename ${taostools_install_dir}) --remove-files || :
+    exitcode=$?
+    if [ "$exitcode" != "0" ]; then
+        echo "tar ${taostools_pkg_name}.tar.gz error !!!"
+        exit $exitcode
+    fi
+fi
+
 cd ${curr_dir}

src/client/src/tscSQLParser.c

@@ -6898,17 +6898,15 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlN
 
   // todo refactor
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-    if (!tscQueryTags(pQueryInfo)) {  // local handle the super table tag query
-      if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
-        if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) {
-          return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
-        }
+    if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
+      if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) {
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+      }
 
-        // for projection query on super table, all queries are subqueries
-        if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
-            !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) {
-          pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY;
-        }
+      // for projection query on super table, all queries are subqueries
+      if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
+          !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) {
+        pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY;
       }
     }
 
@@ -9226,7 +9224,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
         SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
 
         int32_t f = pExpr->base.functionId;
-        if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE || f == TSDB_FUNC_RATE || f== TSDB_FUNC_DIFF) {
+        if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE || 
+            f == TSDB_FUNC_RATE       || f == TSDB_FUNC_DIFF) {
           return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
         }
       }

src/common/inc/tdataformat.h

@@ -448,13 +448,14 @@ typedef struct {
 #define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
 #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
 #define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r))
+#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t)))
 #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
 #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
 #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
 #define kvRowFree(r) tfree(r)
 #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
 #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r))
-#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r)))
+#define kvRowTKey(r) (*(TKEY *)(kvRowKeys(r)))
 #define kvRowKey(r) tdGetKey(kvRowTKey(r))
 #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r))
 
@@ -652,7 +653,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) {
 #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
 #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
 #define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
-#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
+#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
 
 #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
 #define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))

taos-tools @ 26f90b54

-Subproject commit b76b5a76756a5c6530ba1d418de51fd336ae23b1
+Subproject commit 26f90b549f6e113ef3d275775062e674b0119645

src/tsdb/src/tsdbMemTable.c

@@ -564,7 +564,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
   }
 }
 
-static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); }
+static char *tsdbGetTsTupleKey(const void *data) { return memRowKeys((SMemRow)data); }
 
 static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
   ASSERT(pMemTable->maxTables < maxTables);

tests/develop-test/1-insert/0-sql/basic.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        ret = tdSql.execute('create table tb (ts timestamp, speed int)')
+
+        insertRows = 10
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into tb values (now + %dm, %d)' %
+                (i, i))
+
+        tdLog.info("insert earlier data")
+        tdSql.execute('insert into tb values (now - 5m , 10)')
+        tdSql.execute('insert into tb values (now - 6m , 10)')
+        tdSql.execute('insert into tb values (now - 7m , 10)')
+        tdSql.execute('insert into tb values (now - 8m , 10)')
+
+        tdSql.query("select * from tb")
+        tdSql.checkRows(insertRows + 4)
+
+        # test case for https://jira.taosdata.com:18080/browse/TD-3716:
+        tdSql.error("insert into tb(now, 1)")
+        # test case for TD-10717
+        tdSql.error("insert into tb values(now,1),,(now+1s,1)")
+        tdSql.execute("insert into tb values(now+2s,1),(now+3s,1),(now+4s,1)")
+        tdSql.query("select * from tb")
+        tdSql.checkRows(insertRows + 4 +3)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/develop-test/fulltest.sh

+python3 test.py -f insert/basic.py 
\ No newline at end of file

tests/develop-test/test.py

+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/
+
+# -*- coding: utf-8 -*-
+import sys
+import getopt
+import subprocess
+import time
+from distutils.log import warn as printf
+from fabric2 import Connection
+sys.path.append("../pytest")
+from util.log import *
+from util.dnodes import *
+from util.cases import *
+
+import taos
+
+
+if __name__ == "__main__":
+    
+    fileName = "all"
+    deployPath = ""
+    masterIp = ""
+    testCluster = False
+    valgrind = 0
+    logSql = True
+    stop = 0
+    restart = False
+    windows = 0
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
+    for key, value in opts:
+        if key in ['-h', '--help']:
+            tdLog.printNoPrefix(
+                'A collection of test cases written using Python')
+            tdLog.printNoPrefix('-f Name of test case file written by Python')
+            tdLog.printNoPrefix('-p Deploy Path for Simulator')
+            tdLog.printNoPrefix('-m Master Ip for Simulator')
+            tdLog.printNoPrefix('-l <True:False> logSql Flag')
+            tdLog.printNoPrefix('-s stop All dnodes')
+            tdLog.printNoPrefix('-c Test Cluster Flag')
+            tdLog.printNoPrefix('-g valgrind Test Flag')
+            tdLog.printNoPrefix('-r taosd restart test')
+            tdLog.printNoPrefix('-w taos on windows')
+            sys.exit(0)
+
+        if key in ['-r', '--restart']: 
+            restart = True
+
+        if key in ['-f', '--file']:
+            fileName = value
+
+        if key in ['-p', '--path']:
+            deployPath = value
+
+        if key in ['-m', '--master']:
+            masterIp = value
+
+        if key in ['-l', '--logSql']:
+            if (value.upper() == "TRUE"):
+                logSql = True
+            elif (value.upper() == "FALSE"):
+                logSql = False
+            else:
+                tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
+                sys.exit(0)
+
+        if key in ['-c', '--cluster']:
+            testCluster = True
+
+        if key in ['-g', '--valgrind']:
+            valgrind = 1
+
+        if key in ['-s', '--stop']:
+            stop = 1
+
+        if key in ['-w', '--windows']:
+            windows = 1
+
+    if (stop != 0):
+        if (valgrind == 0):
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
+
+        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+        processID = subprocess.check_output(psCmd, shell=True)
+
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        for port in range(6030, 6041):
+            usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port
+            processID = subprocess.check_output(usePortPID, shell=True)
+
+            if processID:
+                killCmd = "kill -TERM %s" % processID
+                os.system(killCmd)
+            fuserCmd = "fuser -k -n tcp %d" % port
+            os.system(fuserCmd)
+        if valgrind:
+            time.sleep(2)
+
+        tdLog.info('stop All dnodes')
+    
+    if masterIp == "":
+        host = '127.0.0.1'
+    else:
+        host = masterIp
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    if windows:
+        tdCases.logSql(logSql)
+        tdLog.info("Procedures for testing self-deployment")
+        td_clinet = TDSimClient("C:\\TDengine")
+        td_clinet.deploy()
+        remote_conn = Connection("root@%s"%host)
+        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+            remote_conn.run("python3 ./test.py")
+        conn = taos.connect(
+            host="%s"%(host),
+            config=td_clinet.cfgDir)
+        tdCases.runOneWindows(conn, fileName)
+    else:
+        tdDnodes.init(deployPath)
+        tdDnodes.setTestCluster(testCluster)
+        tdDnodes.setValgrind(valgrind)
+        tdDnodes.stopAll()
+        is_test_framework = 0
+        key_word = 'tdCases.addLinux'
+        try:
+            if key_word in open(fileName).read():
+                is_test_framework = 1
+        except:
+            pass
+        if is_test_framework:
+            moduleName = fileName.replace(".py", "").replace("/", ".")
+            uModule = importlib.import_module(moduleName)
+            try:
+                ucase = uModule.TDTestCase()
+                tdDnodes.deploy(1,ucase.updatecfgDict)
+            except :
+                tdDnodes.deploy(1,{})
+        else:
+            pass
+            tdDnodes.deploy(1,{})
+        tdDnodes.start(1)
+
+
+
+        tdCases.logSql(logSql)
+
+        if testCluster:
+            tdLog.info("Procedures for testing cluster")
+            if fileName == "all":
+                tdCases.runAllCluster()
+            else:
+                tdCases.runOneCluster(fileName)
+        else:
+            tdLog.info("Procedures for testing self-deployment")
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            if fileName == "all":
+                tdCases.runAllLinux(conn)
+            else:
+                tdCases.runOneWindows(conn, fileName)
+        if restart:
+            if fileName == "all":
+                tdLog.info("not need to query ")
+            else:    
+                sp = fileName.rsplit(".", 1)
+                if len(sp) == 2 and sp[1] == "py":
+                    tdDnodes.stopAll()
+                    tdDnodes.start(1)
+                    time.sleep(1)            
+                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    tdLog.info("query test after taosd restart")
+                    tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+                else:
+                    tdLog.info("not need to query")
+    conn.close()

tests/pytest/query/querySort.py

@@ -132,6 +132,25 @@ class TDTestCase:
         tdSql.error("select top(tbcol1, 12) from st order by ts, tbcol1")
         tdSql.error("select top(tbcol1, 2) from st1 group by tbcol1 order by tbcol2")
 
+        fun_list = ['avg','count','twa','sum','stddev','leastsquares','min',
+                    'max','first','last','top','bottom','percentile','apercentile',
+                    'last_row','diff','spread','distinct']
+        key = ['tbol','tagcol']
+        for i in range(1,15):
+            for k in key:
+                for j in fun_list:
+                    if j == 'leastsquares':
+                        pick_func=j+'('+ k + str(i) +',1,1)'
+                    elif j == 'top' or j == 'bottom' : continue
+                    elif j == 'percentile' or j == 'apercentile':
+                        pick_func=j+'('+ k + str(i) +',1)'
+                    else:
+                        pick_func=j+'('+ k + str(i) +')'
+                    sql = 'select %s from st group by %s order by %s' % (pick_func , k+str(i), k+str(i))
+                    tdSql.error(sql)
+                    sql = 'select %s from st6 group by %s order by %s ' % (pick_func , k+str(i), k+str(i))
+                    tdSql.error(sql)
+
         tdSql.query("select top(tbcol1, 2) from st1 group by tbcol2 order by tbcol2")
         tdSql.query("select top(tbcol1, 12) from st order by tbcol1, ts")
 

tests/script/general/parser/nestquery.sim

@@ -419,12 +419,14 @@ if $data03 != @20-09-15 00:00:00.000@ then
   return -1
 endi
 
+sql_error select diff(val) from (select c1 val from nest_tb0);
+
 sql select diff(val) from (select ts,c1 val from nest_tb0);
 if $rows != 9999 then
   return -1
 endi
 
-if $data00 != @70-01-01 08:00:00.000@ then
+if $data00 != @20-09-15 00:01:00.000@ then
   return -1
 endi
 

tests/script/general/parser/slimit_query.sim

@@ -31,6 +31,8 @@ $tsu = $tsu + $ts0
 #sql_error select top(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1
 #sql_error select bottom(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1
 
+sql_error select t1 from $stb slimit 5 offset 1;
+
 ### select from stb + group by + slimit offset
 sql select max(c1), min(c2), avg(c3), sum(c4), spread(c5), sum(c6), count(c7), first(c8), last(c9) from $stb group by t1 slimit 5 soffset 0
 if $rows != 5 then

tests/system-test/1-insert/0-sql/basic.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        ret = tdSql.execute('create table tb (ts timestamp, speed int)')
+
+        insertRows = 10
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(0, insertRows):
+            ret = tdSql.execute(
+                'insert into tb values (now + %dm, %d)' %
+                (i, i))
+
+        tdLog.info("insert earlier data")
+        tdSql.execute('insert into tb values (now - 5m , 10)')
+        tdSql.execute('insert into tb values (now - 6m , 10)')
+        tdSql.execute('insert into tb values (now - 7m , 10)')
+        tdSql.execute('insert into tb values (now - 8m , 10)')
+
+        tdSql.query("select * from tb")
+        tdSql.checkRows(insertRows + 4)
+
+        # test case for https://jira.taosdata.com:18080/browse/TD-3716:
+        tdSql.error("insert into tb(now, 1)")
+        # test case for TD-10717
+        tdSql.error("insert into tb values(now,1),,(now+1s,1)")
+        tdSql.execute("insert into tb values(now+2s,1),(now+3s,1),(now+4s,1)")
+        tdSql.query("select * from tb")
+        tdSql.checkRows(insertRows + 4 +3)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/fulltest.sh

+python3 test.py -f insert/basic.py 
\ No newline at end of file

tests/system-test/test.py

+#!/usr/bin/python
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+# install pip
+# pip install src/connector/python/
+
+# -*- coding: utf-8 -*-
+import sys
+import getopt
+import subprocess
+import time
+from distutils.log import warn as printf
+from fabric2 import Connection
+sys.path.append("../pytest")
+from util.log import *
+from util.dnodes import *
+from util.cases import *
+
+import taos
+
+
+if __name__ == "__main__":
+    
+    fileName = "all"
+    deployPath = ""
+    masterIp = ""
+    testCluster = False
+    valgrind = 0
+    logSql = True
+    stop = 0
+    restart = False
+    windows = 0
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
+    for key, value in opts:
+        if key in ['-h', '--help']:
+            tdLog.printNoPrefix(
+                'A collection of test cases written using Python')
+            tdLog.printNoPrefix('-f Name of test case file written by Python')
+            tdLog.printNoPrefix('-p Deploy Path for Simulator')
+            tdLog.printNoPrefix('-m Master Ip for Simulator')
+            tdLog.printNoPrefix('-l <True:False> logSql Flag')
+            tdLog.printNoPrefix('-s stop All dnodes')
+            tdLog.printNoPrefix('-c Test Cluster Flag')
+            tdLog.printNoPrefix('-g valgrind Test Flag')
+            tdLog.printNoPrefix('-r taosd restart test')
+            tdLog.printNoPrefix('-w taos on windows')
+            sys.exit(0)
+
+        if key in ['-r', '--restart']: 
+            restart = True
+
+        if key in ['-f', '--file']:
+            fileName = value
+
+        if key in ['-p', '--path']:
+            deployPath = value
+
+        if key in ['-m', '--master']:
+            masterIp = value
+
+        if key in ['-l', '--logSql']:
+            if (value.upper() == "TRUE"):
+                logSql = True
+            elif (value.upper() == "FALSE"):
+                logSql = False
+            else:
+                tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
+                sys.exit(0)
+
+        if key in ['-c', '--cluster']:
+            testCluster = True
+
+        if key in ['-g', '--valgrind']:
+            valgrind = 1
+
+        if key in ['-s', '--stop']:
+            stop = 1
+
+        if key in ['-w', '--windows']:
+            windows = 1
+
+    if (stop != 0):
+        if (valgrind == 0):
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
+
+        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+        processID = subprocess.check_output(psCmd, shell=True)
+
+        while(processID):
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(psCmd, shell=True)
+
+        for port in range(6030, 6041):
+            usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port
+            processID = subprocess.check_output(usePortPID, shell=True)
+
+            if processID:
+                killCmd = "kill -TERM %s" % processID
+                os.system(killCmd)
+            fuserCmd = "fuser -k -n tcp %d" % port
+            os.system(fuserCmd)
+        if valgrind:
+            time.sleep(2)
+
+        tdLog.info('stop All dnodes')
+    
+    if masterIp == "":
+        host = '127.0.0.1'
+    else:
+        host = masterIp
+
+    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    if windows:
+        tdCases.logSql(logSql)
+        tdLog.info("Procedures for testing self-deployment")
+        td_clinet = TDSimClient("C:\\TDengine")
+        td_clinet.deploy()
+        remote_conn = Connection("root@%s"%host)
+        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+            remote_conn.run("python3 ./test.py")
+        conn = taos.connect(
+            host="%s"%(host),
+            config=td_clinet.cfgDir)
+        tdCases.runOneWindows(conn, fileName)
+    else:
+        tdDnodes.init(deployPath)
+        tdDnodes.setTestCluster(testCluster)
+        tdDnodes.setValgrind(valgrind)
+        tdDnodes.stopAll()
+        is_test_framework = 0
+        key_word = 'tdCases.addLinux'
+        try:
+            if key_word in open(fileName).read():
+                is_test_framework = 1
+        except:
+            pass
+        if is_test_framework:
+            moduleName = fileName.replace(".py", "").replace("/", ".")
+            uModule = importlib.import_module(moduleName)
+            try:
+                ucase = uModule.TDTestCase()
+                tdDnodes.deploy(1,ucase.updatecfgDict)
+            except :
+                tdDnodes.deploy(1,{})
+        else:
+            pass
+            tdDnodes.deploy(1,{})
+        tdDnodes.start(1)
+
+
+
+        tdCases.logSql(logSql)
+
+        if testCluster:
+            tdLog.info("Procedures for testing cluster")
+            if fileName == "all":
+                tdCases.runAllCluster()
+            else:
+                tdCases.runOneCluster(fileName)
+        else:
+            tdLog.info("Procedures for testing self-deployment")
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            if fileName == "all":
+                tdCases.runAllLinux(conn)
+            else:
+                tdCases.runOneWindows(conn, fileName)
+        if restart:
+            if fileName == "all":
+                tdLog.info("not need to query ")
+            else:    
+                sp = fileName.rsplit(".", 1)
+                if len(sp) == 2 and sp[1] == "py":
+                    tdDnodes.stopAll()
+                    tdDnodes.start(1)
+                    time.sleep(1)            
+                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    tdLog.info("query test after taosd restart")
+                    tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
+                else:
+                    tdLog.info("not need to query")
+    conn.close()

tests/test-all.sh

@@ -158,7 +158,13 @@ function runPyCaseOneByOne {
 }
 
 function runPyCaseOneByOnefq() {
-  cd $tests_dir/pytest
+  if [[ $3 =~ system ]] ; then
+    cd $tests_dir/system-test
+  elif [[ $3 =~ develop ]] ; then
+    cd $tests_dir/develop-test
+  else
+    cd $tests_dir/pytest
+  fi
   if [[ $1 =~ full ]] ; then
     start=1
     end=`sed -n '$=' fulltest.sh`
@@ -361,6 +367,12 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]  && [ "$2" != "
   elif [ "$1" == "p4" ]; then
     echo "### run Python_4 test ###"
     runPyCaseOneByOnefq p4 1
+  elif [ "$1" == "system-test" ]; then
+    echo "### run system-test test ###"
+    runPyCaseOneByOnefq full 1 system
+  elif [ "$1" == "develop-test" ]; then
+    echo "### run develop-test test ###"
+    runPyCaseOneByOnefq full 1 develop
   elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
     exit $(($totalFailed + $totalPyFailed))
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then