Merge remote-tracking branch 'origin/feature/m1' into hotfix/TD-6272

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.1.7.1")
+  SET(TD_VER_NUMBER "2.1.7.2")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

documentation20/cn/01.evaluation/docs.md

@@ -2,18 +2,18 @@
 
 ## <a class="anchor" id="intro"></a>TDengine 简介
 
-TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。
+TDengine 是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL 数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。
 
-TDengine的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与Hadoop等典型的大数据平台相比，它具有如下鲜明的特点：
+TDengine 的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine 还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与 Hadoop 等典型的大数据平台相比，它具有如下鲜明的特点：
 
-* __10倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少2万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。
-* __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。
-* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 
-* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, MATLAB随时进行。
-* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。
-* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。
+* __10 倍以上的性能提升__：定义了创新的数据存储结构，单核每秒能处理至少 2 万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快十倍以上。
+* __硬件或云服务成本降至 1/5__：由于超强性能，计算资源不到通用大数据方案的 1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的 1/10。
+* __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成 Kafka/Redis/HBase/Spark/HDFS 等软件，大幅降低应用开发和维护的复杂度成本。 
+* __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过 Shell, Python, R, MATLAB 随时进行。
+* __与第三方工具无缝连接__：不用一行代码，即可与 Telegraf, Grafana, EMQ, HiveMQ, Prometheus, MATLAB, R 等集成。后续将支持 OPC, Hadoop, Spark 等，BI 工具也将无缝连接。
+* __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类标准 SQL，支持 RESTful，支持 Python/Java/C/C++/C#/Go/Node.js, 与 MySQL 相似，零学习成本。
 
-采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。
+采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。
 
 ![TDengine技术生态图](page://images/eco_system.png)
 <center>图 1. TDengine技术生态图</center>
@@ -21,42 +21,47 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的
 
 ## <a class="anchor" id="scenes"></a>TDengine 总体适用场景
 
-作为一个IOT大数据平台，TDengine的典型适用场景是在IOT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。
+作为一个 IOT 大数据平台，TDengine 的典型适用场景是在 IOT 范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如 CRM，ERP 等，不在本文讨论范围内。
 
 
 ### 数据源特点和需求
-从数据源角度，设计人员可以从下面几个角度分析TDengine在目标应用系统里面的适用性。
+
+从数据源角度，设计人员可以从下面几个角度分析 TDengine 在目标应用系统里面的适用性。
 
 |数据源特点和需求|不适用|可能适用|非常适用|简单说明|
 |---|---|---|---|---|
-|总体数据量巨大|  |  | √ |TDengine在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。|
-|数据输入速度偶尔或者持续巨大|   |   | √ | TDengine的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。|
-|数据源数目巨大|   |   | √ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。|
+|总体数据量巨大|  |  | √ | TDengine 在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。|
+|数据输入速度偶尔或者持续巨大|   |   | √ | TDengine 的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。|
+|数据源数目巨大|   |   | √ | TDengine 设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。|
 
 ### 系统架构要求
+
 |系统架构要求|不适用|可能适用|非常适用|简单说明|
 |---|---|---|---|---|
-|要求简单可靠的系统架构|  |  | √ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。|
-|要求容错和高可靠|  |  | √ |TDengine的集群功能，自动提供容错灾备等高可靠功能。|
-|标准化规范|  |  | √ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范。|
+|要求简单可靠的系统架构|  |  | √ | TDengine 的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。|
+|要求容错和高可靠|  |  | √ | TDengine 的集群功能，自动提供容错灾备等高可靠功能。|
+|标准化规范|  |  | √ | TDengine 使用标准的SQL语言提供主要功能，遵守标准化规范。|
 
 ### 系统功能需求
+
 |系统功能需求|不适用|可能适用|非常适用|简单说明|
 |---|---|---|---|---|
-|要求完整的内置数据处理算法|  | √ |  |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。|
-|需要大量的交叉查询处理|  | √ |  |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能。|
+|要求完整的内置数据处理算法|  | √ |  | TDengine 的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。|
+|需要大量的交叉查询处理|  | √ |  |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑 TDengine 和关系型数据系统配合实现系统功能。|
 
 ### 系统性能需求
+
 |系统性能需求|不适用|可能适用|非常适用|简单说明|
 |---|---|---|---|---|
-|要求较大的总体处理能力|  |  | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。|
-|要求高速处理数据  | | | √ |TDengine的专门为IOT优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。|
-|要求快速处理小粒度数据|  |  | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。|
+|要求较大的总体处理能力|  |  | √ | TDengine 的集群功能可以轻松地让多服务器配合达成处理能力的提升。|
+|要求高速处理数据  | | | √ | TDengine 的专门为 IOT 优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。|
+|要求快速处理小粒度数据|  |  | √ |这方面 TDengine 性能可以完全对标关系型和 NoSQL 型数据处理系统。|
 
 ### 系统维护需求
+
 |系统维护需求|不适用|可能适用|非常适用|简单说明|
 |---|---|---|---|---|
-|要求系统可靠运行|  |  | √ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。|
+|要求系统可靠运行|  |  | √ | TDengine 的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。|
 |要求运维学习成本可控|  |  | √ |同上。|
-|要求市场有大量人才储备| √ |  |  |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。|
+|要求市场有大量人才储备| √ |  |  | TDengine 作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。|
 

packaging/tools/make_install.sh

@@ -142,6 +142,7 @@ function install_bin() {
     if [ "$osType" != "Darwin" ]; then
         ${csudo} rm -f ${bin_link_dir}/taosd    || :
         ${csudo} rm -f ${bin_link_dir}/taosdemo || :
+        ${csudo} rm -f ${bin_link_dir}/perfMonitor || :
         ${csudo} rm -f ${bin_link_dir}/taosdump || :
         ${csudo} rm -f ${bin_link_dir}/set_core || :
     fi
@@ -167,6 +168,7 @@ function install_bin() {
         [ -x ${install_main_dir}/bin/taosd ]     && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd   || :
         [ -x ${install_main_dir}/bin/taosdump ]  && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
         [ -x ${install_main_dir}/bin/taosdemo ]  && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
+        [ -x ${install_main_dir}/bin/perfMonitor ]  && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
         [ -x ${install_main_dir}/set_core.sh ]  && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
     fi
 

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.1.7.1'
+version: '2.1.7.2'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.1.7.1
+      - usr/lib/libtaos.so.2.1.7.2
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/client/inc/tscUtil.h

@@ -190,6 +190,7 @@ void    tscFieldInfoClear(SFieldInfo* pFieldInfo);
 void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList);
 
 static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; }
+int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo);
 
 int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize);
 void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid);

src/client/src/tscAsync.c

@@ -168,6 +168,9 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
     } else {
       pRes->code = numOfRows;
     }
+    if (pRes->code == TSDB_CODE_SUCCESS) {
+      pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE;           
+    }
 
     tscAsyncResultOnError(pSql);
     return;

src/client/src/tscGlobalmerge.c

@@ -643,7 +643,7 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
         for(int32_t j = 0; j < numOfExpr; ++j) {
           pCtx[j].pOutput += (pCtx[j].outputBytes * numOfRows);
           if (pCtx[j].functionId == TSDB_FUNC_TOP || pCtx[j].functionId == TSDB_FUNC_BOTTOM) {
-            pCtx[j].ptsOutputBuf = pCtx[0].pOutput;
+            if(j>0) pCtx[j].ptsOutputBuf = pCtx[j-1].pOutput;
           }
         }
 

src/client/src/tscParseInsert.c

@@ -1757,6 +1757,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
       pSql->res.numOfRows = 0;
       code = doPackSendDataBlock(pSql, pInsertParam, pTableMeta, count, pTableDataBlock);
       if (code != TSDB_CODE_SUCCESS) {
+        pParentSql->res.code = code;
         goto _error;
       }
 

src/client/src/tscPrepare.c

@@ -206,6 +206,8 @@ static int normalStmtPrepare(STscStmt* stmt) {
         return code;
       }
       start = i + token.n;
+    } else if (token.type == TK_ILLEGAL) {
+      return invalidOperationMsg(tscGetErrorMsgPayload(&stmt->pSql->cmd), "invalid sql");
     }
 
     i += token.n;

src/client/src/tscSQLParser.c

@@ -422,7 +422,6 @@ int32_t readFromFile(char *name, uint32_t *len, void **buf) {
     return TSDB_CODE_TSC_APP_ERROR;
   }
   close(fd);
-  tfree(*buf);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -888,6 +887,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
 
     case TSDB_SQL_SELECT: {
+      const char * msg1 = "no nested query supported in union clause";
       code = loadAllTableMeta(pSql, pInfo);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
@@ -901,6 +901,10 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
         tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size);
 
+        if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) {
+          return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+        }
+
 //        normalizeSqlNode(pSqlNode); // normalize the column name in each function
         if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) {
           return code;
@@ -2603,13 +2607,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
       // set the first column ts for diff query
       if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
-        colIndex += 1;
         SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0};
         SExprInfo*   pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP,
                                          TSDB_KEYSIZE, getNewResColId(pCmd), TSDB_KEYSIZE, false);
 
         SColumnList ids = createColumnList(1, 0, 0);
-        insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr);
+        insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr);
       }
 
       SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), intermediateResSize, false);
@@ -2882,7 +2885,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
 
         const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX;
         SColumnList   ids = createColumnList(1, index.tableIndex, TS_COLUMN_INDEX);
-        insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP,
+        insertResultField(pQueryInfo, colIndex, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP,
                           aAggs[TSDB_FUNC_TS].name, pExpr);
 
         colIndex += 1;  // the first column is ts
@@ -4884,10 +4887,6 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) {
     tSqlExprDestroy(pCondExpr->pTableCond);
   }
 
-  if (pCondExpr->pTagCond) {
-    tSqlExprDestroy(pCondExpr->pTagCond);
-  }
-
   if (pCondExpr->pColumnCond) {
     tSqlExprDestroy(pCondExpr->pColumnCond);
   }
@@ -6943,9 +6942,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
         s = &pSchema[colIndex];
       }
     }
-  
-    size_t size = tscNumOfExprs(pQueryInfo);
-  
+    
     if (TSDB_COL_IS_TAG(pColIndex->flag)) {
 
       int32_t f = TSDB_FUNC_TAG;
@@ -6953,8 +6950,10 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
         f = TSDB_FUNC_TAGPRJ;
       }
 
+      int32_t pos = tscGetFirstInvisibleFieldPos(pQueryInfo);      
+
       SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex};
-      SExprInfo*   pExpr = tscExprAppend(pQueryInfo, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true);
+      SExprInfo*   pExpr = tscExprInsert(pQueryInfo, pos, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true);
 
       memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName));
       tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName));
@@ -6964,13 +6963,15 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
 
       // NOTE: tag column does not add to source column list
       SColumnList ids = createColumnList(1, 0, pColIndex->colIndex);
-      insertResultField(pQueryInfo, (int32_t)size, &ids, s->bytes, (int8_t)s->type, s->name, pExpr);
+      insertResultField(pQueryInfo, pos, &ids, s->bytes, (int8_t)s->type, s->name, pExpr);
     } else {
       // if this query is "group by" normal column, time window query is not allowed
       if (isTimeWindowQuery(pQueryInfo)) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
 
+      size_t size = tscNumOfExprs(pQueryInfo);
+
       bool hasGroupColumn = false;
       for (int32_t j = 0; j < size; ++j) {
         SExprInfo* pExpr = tscExprGet(pQueryInfo, j);

src/client/src/tscServer.c

@@ -841,6 +841,11 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
     tableSerialize = totalTables * sizeof(STableIdInfo);
   }
 
+  SCond* pCond = &pQueryInfo->tagCond.tbnameCond;
+  if (pCond->len > 0) {
+    srcColListSize += pCond->len;
+  }
+
   return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize +
          exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
 }

src/client/src/tscSubquery.c

@@ -15,8 +15,9 @@
 #define _GNU_SOURCE
 
 #include "os.h"
-
 #include "texpr.h"
+
+#include "tsched.h"
 #include "qTsbuf.h"
 #include "tcompare.h"
 #include "tscLog.h"
@@ -2264,7 +2265,7 @@ void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) {
 
     destroySup(pSup);
     taos_free_result(pSql);
-    parent->res.code = code;
+    parent->res.code = c;
     tscAsyncResultOnError(parent);
     return;
   }
@@ -2425,6 +2426,26 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
   return terrno;
 }
 
+typedef struct SPair {
+  int32_t first;
+  int32_t second;
+} SPair;
+
+static void doSendQueryReqs(SSchedMsg* pSchedMsg) {
+  SSqlObj* pSql = pSchedMsg->ahandle;
+  SPair* p = pSchedMsg->msg;
+
+  for(int32_t i = p->first; i < p->second; ++i) {
+    SSqlObj* pSub = pSql->pSubs[i];
+    SRetrieveSupport* pSupport = pSub->param;
+
+    tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex);
+    tscBuildAndSendRequest(pSub, NULL);
+  }
+
+  tfree(p);
+}
+
 int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
@@ -2547,13 +2568,33 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     doCleanupSubqueries(pSql, i);
     return pRes->code;
   }
-  
-  for(int32_t j = 0; j < pState->numOfSub; ++j) {
-    SSqlObj* pSub = pSql->pSubs[j];
-    SRetrieveSupport* pSupport = pSub->param;
-    
-    tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex);
-    tscBuildAndSendRequest(pSub, NULL);
+
+  // concurrently sent the query requests.
+  const int32_t MAX_REQUEST_PER_TASK = 8;
+
+  int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK;
+  assert(numOfTasks >= 1);
+
+  int32_t num = (pState->numOfSub/numOfTasks) + 1;
+  tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks);
+
+  for(int32_t j = 0; j < numOfTasks; ++j) {
+    SSchedMsg schedMsg = {0};
+    schedMsg.fp      = doSendQueryReqs;
+    schedMsg.ahandle = (void*)pSql;
+
+    schedMsg.thandle = NULL;
+    SPair* p = calloc(1, sizeof(SPair));
+    p->first  = j * num;
+
+    if (j == numOfTasks - 1) {
+      p->second = pState->numOfSub;
+    } else {
+      p->second = (j + 1) * num;
+    }
+
+    schedMsg.msg = p;
+    taosScheduleTask(tscQhandle, &schedMsg);
   }
 
   return TSDB_CODE_SUCCESS;

src/client/src/tscUtil.c

@@ -2093,6 +2093,22 @@ TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) {
   return f;
 }
 
+int32_t tscGetFirstInvisibleFieldPos(SQueryInfo* pQueryInfo) { 
+  if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) {
+    return 0;
+  }
+
+  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+    SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
+    if (!pField->visible) {
+      return i;
+    }    
+  }
+  
+  return pQueryInfo->fieldsInfo.numOfOutput; 
+}
+
+
 SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
   assert(pFieldInfo != NULL);
   pFieldInfo->numOfOutput++;
@@ -3778,6 +3794,8 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) {
     tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self);
 
     if (code && !((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry)) {
+      pParentSql->res.code = code;
+      
       tscAsyncResultOnError(pParentSql);
       return;
     }
@@ -3858,6 +3876,7 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) {
       pNew->signature = pNew;
       pNew->sqlstr    = strdup(pSql->sqlstr);
       pNew->fp        = tscSubqueryCompleteCallback;
+      pNew->fetchFp   = tscSubqueryCompleteCallback;
       pNew->maxRetry  = pSql->maxRetry;
 
       pNew->cmd.resColumnId = TSDB_RES_COL_ID;

src/common/src/tvariant.c

@@ -31,12 +31,12 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
 
   switch (token->type) {
     case TSDB_DATA_TYPE_BOOL: {
-      int32_t k = strncasecmp(token->z, "true", 4);
-      if (k == 0) {
+      if (strncasecmp(token->z, "true", 4) == 0) {
         pVar->i64 = TSDB_TRUE;
-      } else {
-        assert(strncasecmp(token->z, "false", 5) == 0);
+      } else if (strncasecmp(token->z, "false", 5) == 0) {
         pVar->i64 = TSDB_FALSE;
+      } else {
+        return;
       }
 
       break;

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java

@@ -18,7 +18,7 @@ public class RestfulConnection extends AbstractConnection {
     private final String url;
     private final String database;
     private final String token;
-    /******************************************************/
+
     private boolean isClosed;
     private final DatabaseMetaData metadata;
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java

@@ -88,17 +88,24 @@ public class RestfulStatement extends AbstractStatement {
     }
 
     private String getUrl() throws SQLException {
+        String dbname = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_DBNAME);
+        if (dbname == null || dbname.trim().isEmpty()) {
+            dbname = "";
+        } else {
+            dbname = "/" + dbname.toLowerCase();
+        }
         TimestampFormat timestampFormat = TimestampFormat.valueOf(conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).trim().toUpperCase());
         String url;
+
         switch (timestampFormat) {
             case TIMESTAMP:
-                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt" + dbname;
                 break;
             case UTC:
-                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc" + dbname;
                 break;
             default:
-                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql" + dbname;
         }
         return url;
     }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiConnectionWithDifferentDbTest.java

+package com.taosdata.jdbc.cases;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class MultiConnectionWithDifferentDbTest {
+
+    private static String host = "127.0.0.1";
+    private static String db1 = "db1";
+    private static String db2 = "db2";
+
+    private long ts;
+
+    @Test
+    public void test() {
+        List<Thread> threads = IntStream.range(1, 3).mapToObj(i -> new Thread(new Runnable() {
+            @Override
+            public void run() {
+                for (int j = 0; j < 10; j++) {
+                    queryDb();
+                    try {
+                        TimeUnit.SECONDS.sleep(1);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+
+            private void queryDb() {
+                String url = "jdbc:TAOS-RS://" + host + ":6041/db" + i + "?user=root&password=taosdata";
+                try (Connection connection = DriverManager.getConnection(url)) {
+                    Statement stmt = connection.createStatement();
+
+                    ResultSet rs = stmt.executeQuery("select * from weather");
+                    assertNotNull(rs);
+                    rs.next();
+                    long actual = rs.getTimestamp("ts").getTime();
+                    assertEquals(ts, actual);
+
+                    int f1 = rs.getInt("f1");
+                    assertEquals(i, f1);
+
+                    String loc = i == 1 ? "beijing" : "shanghai";
+                    String loc_actual = rs.getString("loc");
+                    assertEquals(loc, loc_actual);
+
+                    stmt.close();
+                } catch (SQLException e) {
+                    e.printStackTrace();
+                }
+            }
+        }, "thread-" + i)).collect(Collectors.toList());
+
+        threads.forEach(Thread::start);
+
+        for (Thread t : threads) {
+            try {
+                t.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+
+    }
+
+    @Before
+    public void before() {
+        ts = System.currentTimeMillis();
+
+        try {
+            Connection conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata");
+
+            Statement stmt = conn.createStatement();
+            stmt.execute("drop database if exists " + db1);
+            stmt.execute("create database if not exists " + db1);
+            stmt.execute("use " + db1);
+            stmt.execute("create table weather(ts timestamp, f1 int) tags(loc nchar(10))");
+            stmt.execute("insert into t1 using weather tags('beijing') values(" + ts + ", 1)");
+
+            stmt.execute("drop database if exists " + db2);
+            stmt.execute("create database if not exists " + db2);
+            stmt.execute("use " + db2);
+            stmt.execute("create table weather(ts timestamp, f1 int) tags(loc nchar(10))");
+            stmt.execute("insert into t1 using weather tags('shanghai') values(" + ts + ", 2)");
+
+            conn.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/DatabaseSpecifiedTest.java

+package com.taosdata.jdbc.rs;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.*;
+
+import static org.junit.Assert.*;
+
+public class DatabaseSpecifiedTest {
+
+    private static String host = "127.0.0.1";
+    private static String dbname = "test_db_spec";
+
+    private Connection connection;
+    private long ts;
+
+    @Test
+    public void test() throws SQLException {
+        // when
+        connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/" + dbname + "?user=root&password=taosdata");
+        try (Statement stmt = connection.createStatement();) {
+            ResultSet rs = stmt.executeQuery("select * from weather");
+
+            //then
+            assertNotNull(rs);
+            rs.next();
+            long now = rs.getTimestamp("ts").getTime();
+            assertEquals(ts, now);
+            int f1 = rs.getInt(2);
+            assertEquals(1, f1);
+            String loc = rs.getString("loc");
+            assertEquals("beijing", loc);
+        }
+        connection.close();
+    }
+
+    @Before
+    public void before() {
+        ts = System.currentTimeMillis();
+        try {
+            Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata");
+            Statement stmt = connection.createStatement();
+
+            stmt.execute("drop database if exists " + dbname);
+            stmt.execute("create database if not exists " + dbname);
+            stmt.execute("use " + dbname);
+            stmt.execute("create table weather(ts timestamp, f1 int) tags(loc nchar(10))");
+            stmt.execute("insert into t1 using weather tags('beijing') values( " + ts + ", 1)");
+
+            stmt.close();
+            connection.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @After
+    public void after() {
+        try {
+            if (connection != null)
+                connection.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+}

src/connector/python/examples/demo.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect(host='127.0.0.1',
                     user='root',
-                    passworkd='taodata',
+                    password='taosdata',
                     database='log')
 cursor = conn.cursor()
 

src/connector/python/taos/cinterface.py

@@ -268,7 +268,7 @@ def _load_taos():
     try:
         return load_func[platform.system()]()
     except:
-        sys.exit('unsupported platform to TDengine connector')
+        raise InterfaceError('unsupported platform or failed to load taos client library')
 
 
 class CTaosInterface(object):

src/inc/taosdef.h

@@ -87,6 +87,8 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DEFAULT_PASS               "taosdata"
 #endif
 
+#define SHELL_MAX_PASSWORD_LEN          20
+
 #define TSDB_TRUE   1
 #define TSDB_FALSE  0
 #define TSDB_OK     0

src/kit/shell/inc/shell.h

@@ -25,7 +25,6 @@
 #define MAX_USERNAME_SIZE      64
 #define MAX_DBNAME_SIZE        64
 #define MAX_IP_SIZE            20
-#define MAX_PASSWORD_SIZE      20
 #define MAX_HISTORY_SIZE       1000
 #define MAX_COMMAND_SIZE       1048586
 #define HISTORY_FILE           ".taos_history"
@@ -56,6 +55,8 @@ typedef struct SShellArguments {
   int   abort;
   int   port;
   int   pktLen;
+  int   pktNum;
+  char* pktType;
   char* netTestRole;
 } SShellArguments;
 

src/kit/shell/src/shellDarwin.c

@@ -66,7 +66,7 @@ void printHelp() {
 
 char      DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
                              "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
-char g_password[MAX_PASSWORD_SIZE];
+char g_password[SHELL_MAX_PASSWORD_LEN];
 
 void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
   wordexp_t full_path;
@@ -81,19 +81,25 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
       }
     }
       // for password
-    else if (strncmp(argv[i], "-p", 2) == 0) {
+    else if ((strncmp(argv[i], "-p", 2) == 0)
+            || (strncmp(argv[i], "--password", 10) == 0)) {
         strcpy(tsOsName, "Darwin");
         printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info());
-        if (strlen(argv[i]) == 2) {
+        if ((strlen(argv[i]) == 2)
+                  || (strncmp(argv[i], "--password", 10) == 0)) {
             printf("Enter password: ");
+            taosSetConsoleEcho(false);
             if (scanf("%s", g_password) > 1) {
                 fprintf(stderr, "password read error\n");
             }
+            taosSetConsoleEcho(true);
             getchar();
         } else {
-            tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE);
+            tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
         }
         arguments->password = g_password;
+        strcpy(argv[i], "");
+        argc -= 1;
     }
     // for management port
     else if (strcmp(argv[i], "-P") == 0) {

src/kit/shell/src/shellLinux.c

@@ -47,9 +47,11 @@ static struct argp_option options[] = {
   {"thread",     'T', "THREADNUM",  0,                   "Number of threads when using multi-thread to import data."},
   {"check",      'k', "CHECK",      0,                   "Check tables."},
   {"database",   'd', "DATABASE",   0,                   "Database to use when connecting to the server."},
-  {"timezone",   't', "TIMEZONE",   0,                   "Time zone of the shell, default is local."},
-  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."},
+  {"timezone",   'z', "TIMEZONE",   0,                   "Time zone of the shell, default is local."},
+  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speen|fqdn."},
   {"pktlen",     'l', "PKTLEN",     0,                   "Packet length used for net test, default is 1000 bytes."},
+  {"pktnum",     'N', "PKTNUM",     0,                   "Packet numbers used for net test, default is 100."},
+  {"pkttype",    'S', "PKTTYPE",    0,                   "Packet type used for net test, default is TCP."},
   {0}};
 
 static error_t parse_opt(int key, char *arg, struct argp_state *state) {
@@ -74,7 +76,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       }
 
       break;
-    case 't':
+    case 'z':
       arguments->timezone = arg;
       break;
     case 'u':
@@ -106,7 +108,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       arguments->is_raw_time = true;
       break;
     case 'f':
-      if (wordexp(arg, &full_path, 0) != 0) {
+      if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) {
         fprintf(stderr, "Invalid path %s\n", arg);
         return -1;
       }
@@ -146,6 +148,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
         return -1;
       }
       break;
+    case 'N':
+      if (arg) {
+        arguments->pktNum = atoi(arg);
+      } else {
+        fprintf(stderr, "Invalid packet number\n");
+        return -1;
+      }
+      break;
+    case 'S':
+      arguments->pktType = arg;
+      break;
     case OPT_ABORT:
       arguments->abort = 1;
       break;
@@ -160,22 +173,29 @@ static struct argp argp = {options, parse_opt, args_doc, doc};
 
 char      LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
                              "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
-char g_password[MAX_PASSWORD_SIZE];
+char g_password[SHELL_MAX_PASSWORD_LEN];
 
-static void parse_password(
+static void parse_args(
         int argc, char *argv[], SShellArguments *arguments) {
     for (int i = 1; i < argc; i++) {
-        if (strncmp(argv[i], "-p", 2) == 0) {
+        if ((strncmp(argv[i], "-p", 2) == 0)
+              || (strncmp(argv[i], "--password", 10) == 0)) {
             strcpy(tsOsName, "Linux");
             printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info());
-            if (strlen(argv[i]) == 2) {
+            if ((strlen(argv[i]) == 2)
+                  || (strncmp(argv[i], "--password", 10) == 0)) {
                 printf("Enter password: ");
+                taosSetConsoleEcho(false);
                 if (scanf("%20s", g_password) > 1) {
                     fprintf(stderr, "password reading error\n");
                 }
-                getchar();
+                taosSetConsoleEcho(true);
+                if (EOF == getchar()) {
+                    fprintf(stderr, "getchar() return EOF\n");
+                }
             } else {
-                tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE);
+                tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
+                strcpy(argv[i], "-p");
             }
             arguments->password = g_password;
             arguments->is_use_passwd = true;
@@ -190,7 +210,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
   argp_program_version = verType;
 
   if (argc > 1) {
-    parse_password(argc, argv, arguments);
+    parse_args(argc, argv, arguments);
   }
 
   argp_parse(&argp, argc, argv, 0, 0, arguments);

src/kit/shell/src/shellMain.c

@@ -85,6 +85,8 @@ SShellArguments args = {
   .threadNum = 5,
   .commands = NULL,
   .pktLen = 1000,
+  .pktNum = 100,
+  .pktType = "TCP",
   .netTestRole = NULL
 };
 
@@ -118,7 +120,7 @@ int main(int argc, char* argv[]) {
       printf("Failed to init taos");
       exit(EXIT_FAILURE);
     }
-    taosNetTest(args.netTestRole, args.host, args.port, args.pktLen);
+    taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType);
     exit(0);
   }
 

src/kit/shell/src/shellWindows.c

@@ -55,16 +55,20 @@ void printHelp() {
   printf("%s%s\n", indent, "-t");
   printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
   printf("%s%s\n", indent, "-n");
-  printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync.");
+  printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn.");
   printf("%s%s\n", indent, "-l");
   printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
+  printf("%s%s\n", indent, "-N");
+  printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100.");
+  printf("%s%s\n", indent, "-S");
+  printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP.");
   printf("%s%s\n", indent, "-V");
   printf("%s%s%s\n", indent, indent, "Print program version.");
 
   exit(EXIT_SUCCESS);
 }
 
-char g_password[MAX_PASSWORD_SIZE];
+char g_password[SHELL_MAX_PASSWORD_LEN];
 
 void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
   for (int i = 1; i < argc; i++) {
@@ -78,20 +82,26 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
       }
     }
     // for password
-    else if (strncmp(argv[i], "-p", 2) == 0) {
+    else if ((strncmp(argv[i], "-p", 2) == 0)
+            || (strncmp(argv[i], "--password", 10) == 0)) {
         arguments->is_use_passwd = true;
         strcpy(tsOsName, "Windows");
         printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info());
-        if (strlen(argv[i]) == 2) {
+        if ((strlen(argv[i]) == 2)
+                  || (strncmp(argv[i], "--password", 10) == 0)) {
             printf("Enter password: ");
+            taosSetConsoleEcho(false);
             if (scanf("%s", g_password) > 1) {
                 fprintf(stderr, "password read error!\n");
             }
+            taosSetConsoleEcho(true);
             getchar();
         } else {
-            tstrncpy(g_password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE);
+            tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
         }
         arguments->password = g_password;
+        strcpy(argv[i], "");
+        argc -= 1;
     }
     // for management port
     else if (strcmp(argv[i], "-P") == 0) {

src/kit/taosdump/taosdump.c

@@ -62,6 +62,20 @@ typedef struct {
 #define errorPrint(fmt, ...) \
     do { fprintf(stderr, "\033[31m"); fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); fprintf(stderr, "\033[0m"); } while(0)
 
+static bool isStringNumber(char *input)
+{
+    int len = strlen(input);
+    if (0 == len) {
+        return false;
+    }
+
+    for (int i = 0; i < len; i++) {
+        if (!isdigit(input[i]))
+            return false;
+    }
+
+    return true;
+}
 
 // -------------------------- SHOW DATABASE INTERFACE-----------------------
 enum _show_db_index {
@@ -243,19 +257,15 @@ static struct argp_option options[] = {
     {"table-batch", 't', "TABLE_BATCH", 0,  "Number of table dumpout into one output file. Default is 1.",  3},
     {"thread_num",  'T', "THREAD_NUM",  0,  "Number of thread for dump in file. Default is 5.", 3},
     {"debug",   'g', 0, 0,  "Print debug info.",    8},
-    {"verbose", 'b', 0, 0,  "Print verbose debug info.", 9},
-    {"performanceprint", 'm', 0, 0,  "Print performance debug info.", 10},
     {0}
 };
 
-#define MAX_PASSWORD_SIZE   20
-
 /* Used by main to communicate with parse_opt. */
 typedef struct arguments {
     // connection option
     char    *host;
     char    *user;
-    char    password[MAX_PASSWORD_SIZE];
+    char    password[SHELL_MAX_PASSWORD_LEN];
     uint16_t port;
     char     cversion[12];
     uint16_t mysqlFlag;
@@ -432,7 +442,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             break;
             // dump unit option
         case 'A':
-            g_args.all_databases = true;
             break;
         case 'D':
             g_args.databases = true;
@@ -477,6 +486,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
             g_args.table_batch = atoi(arg);
             break;
         case 'T':
+            if (!isStringNumber(arg)) {
+                errorPrint("%s", "\n\t-T need a number following!\n");
+                exit(EXIT_FAILURE);
+            }
             g_args.thread_num = atoi(arg);
             break;
         case OPT_ABORT:
@@ -555,11 +568,14 @@ static void parse_precision_first(
     }
 }
 
-static void parse_password(
+static void parse_args(
         int argc, char *argv[], SArguments *arguments) {
+
     for (int i = 1; i < argc; i++) {
-        if (strncmp(argv[i], "-p", 2) == 0) {
-            if (strlen(argv[i]) == 2) {
+        if ((strncmp(argv[i], "-p", 2) == 0)
+              || (strncmp(argv[i], "--password", 10) == 0)) {
+            if ((strlen(argv[i]) == 2)
+                  || (strncmp(argv[i], "--password", 10) == 0)) {
                 printf("Enter password: ");
                 taosSetConsoleEcho(false);
                 if(scanf("%20s", arguments->password) > 1) {
@@ -567,10 +583,22 @@ static void parse_password(
                 }
                 taosSetConsoleEcho(true);
             } else {
-                tstrncpy(arguments->password, (char *)(argv[i] + 2), MAX_PASSWORD_SIZE);
+                tstrncpy(arguments->password, (char *)(argv[i] + 2),
+                        SHELL_MAX_PASSWORD_LEN);
+                strcpy(argv[i], "-p");
             }
-            argv[i] = "";
+        } else if (strcmp(argv[i], "-gg") == 0) {
+            arguments->verbose_print = true;
+            strcpy(argv[i], "");
+        } else if (strcmp(argv[i], "-PP") == 0) {
+            arguments->performance_print = true;
+            strcpy(argv[i], "");
+        } else if (strcmp(argv[i], "-A") == 0) {
+            g_args.all_databases = true;
+        } else {
+            continue;
         }
+
     }
 }
 
@@ -639,7 +667,7 @@ int main(int argc, char *argv[]) {
     if (argc > 1) {
         parse_precision_first(argc, argv, &g_args);
         parse_timestamp(argc, argv, &g_args);
-        parse_password(argc, argv, &g_args);
+        parse_args(argc, argv, &g_args);
     }
 
     argp_parse(&argp, argc, argv, 0, 0, &g_args);

src/os/src/linux/linuxEnv.c

@@ -50,14 +50,20 @@ void osInit() {
 char* taosGetCmdlineByPID(int pid) {
   static char cmdline[1024];
   sprintf(cmdline, "/proc/%d/cmdline", pid);
-  FILE* f = fopen(cmdline, "r");
-  if (f) {
-    size_t size;
-    size = fread(cmdline, sizeof(char), 1024, f);
-    if (size > 0) {
-      if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0';
-    }
-    fclose(f);
+
+  int fd = open(cmdline, O_RDONLY);
+  if (fd >= 0) {
+    int n = read(fd, cmdline, sizeof(cmdline) - 1);
+    if (n < 0) n = 0;
+
+    if (n > 0 && cmdline[n - 1] == '\n') --n;
+
+    cmdline[n] = 0;
+
+    close(fd);
+  } else {
+    cmdline[0] = 0;
   }
+
   return cmdline;
 }

src/os/src/linux/osSystem.c

@@ -63,12 +63,12 @@ int taosSetConsoleEcho(bool on)
     }
 
     if (on)
-        term.c_lflag|=ECHOFLAGS;
+        term.c_lflag |= ECHOFLAGS;
     else
-        term.c_lflag &=~ECHOFLAGS;
+        term.c_lflag &= ~ECHOFLAGS;
 
-    err = tcsetattr(STDIN_FILENO,TCSAFLUSH,&term);
-    if (err == -1 && err == EINTR) {
+    err = tcsetattr(STDIN_FILENO, TCSAFLUSH, &term);
+    if (err == -1 || err == EINTR) {
         perror("Cannot set the attribution of the terminal");
         return -1;
     }

src/plugins/monitor/src/monMain.c

@@ -34,7 +34,7 @@
 #define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }}
 
 #define SQL_LENGTH     1030
-#define LOG_LEN_STR    100
+#define LOG_LEN_STR    512
 #define IP_LEN_STR     TSDB_EP_LEN
 #define CHECK_INTERVAL 1000
 

src/query/inc/qExecutor.h

@@ -597,6 +597,8 @@ bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilter
 void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
 
 SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
+void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput);
+
 void* destroyOutputBuf(SSDataBlock* pBlock);
 void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
 

src/query/src/qAggMain.c

@@ -4089,7 +4089,7 @@ static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockD
   } else {
     pDist->maxRows = pSrc->maxRows;
     pDist->minRows = pSrc->minRows;
-    
+
     int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS;
     if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
       ++maxSteps;
@@ -4223,7 +4223,7 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) {
     taosArrayDestroy(pDist->dataBlockInfos);
     pDist->dataBlockInfos = NULL;
   }
-  
+
   // cannot set the numOfIteratedElems again since it is set during previous iteration
   pResInfo->numOfRes  = 1;
   pResInfo->hasResult = DATA_SET_FLAG;

src/query/src/qExecutor.c

@@ -3616,7 +3616,7 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i
     // set the timestamp output buffer for top/bottom/diff query
     int32_t fid = pCtx[i].functionId;
     if (fid == TSDB_FUNC_TOP || fid == TSDB_FUNC_BOTTOM || fid == TSDB_FUNC_DIFF || fid == TSDB_FUNC_DERIVATIVE) {
-      pCtx[i].ptsOutputBuf = pCtx[0].pOutput;
+      if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
     }
   }
 
@@ -3651,7 +3651,37 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf
     // re-estabilish output buffer pointer.
     int32_t functionId = pBInfo->pCtx[i].functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
-      pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput;
+      if(i>0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput;
+    }
+  }
+}
+
+void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) {
+  bool    needCopyTs = false;
+  int32_t tsNum = 0;
+  char *src = NULL;
+  for (int32_t i = 0; i < numOfOutput; i++) {
+    int32_t functionId = pCtx[i].functionId;
+    if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
+      needCopyTs = true;
+      if (i > 0  && pCtx[i-1].functionId == TSDB_FUNC_TS_DUMMY){
+        SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data
+        src = pColRes->pData;
+      }
+    }else if(functionId == TSDB_FUNC_TS_DUMMY) {
+      tsNum++;
+    }
+  }
+
+  if (!needCopyTs) return;
+  if (tsNum < 2) return;
+  if (src == NULL) return;
+
+  for (int32_t i = 0; i < numOfOutput; i++) {
+    int32_t functionId = pCtx[i].functionId;
+    if(functionId == TSDB_FUNC_TS_DUMMY) {
+      SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i);
+      memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows);
     }
   }
 }
@@ -3851,7 +3881,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe
     }
 
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
-      pCtx[i].ptsOutputBuf = pCtx[0].pOutput;
+      if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
     }
 
     if (!pResInfo->initialized) {
@@ -3912,7 +3942,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF
 
     int32_t functionId = pCtx[i].functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
-      pCtx[i].ptsOutputBuf = pCtx[0].pOutput;
+      if(i>0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput;
     }
 
     /*
@@ -5698,6 +5728,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
 
     pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
     if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) {
+      copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
       clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput);
       return pRes;
     }
@@ -5723,8 +5754,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
     if (*newgroup) {
       if (pRes->info.rows > 0) {
         pProjectInfo->existDataBlock = pBlock;
-        clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput);
-        return pInfo->pRes;
+        break;
       } else { // init output buffer for a new group data
         for (int32_t j = 0; j < pOperator->numOfOutput; ++j) {
           aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]);
@@ -5754,7 +5784,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
       break;
     }
   }
-
+  copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput);
   clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput);
   return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL;
 }
@@ -7419,10 +7449,12 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
   pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput);
   pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols);
   pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen);
+
   pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset);
   pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen);
   pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks);
   pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder);
+
   pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags);
   pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen);
   pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput);

src/query/src/qTsbuf.c

@@ -375,6 +375,16 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
   sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
 
   if (decomp) {
+    if (pBlock->numOfElem * TSDB_KEYSIZE > pTSBuf->tsData.allocSize) {
+      pTSBuf->tsData.rawBuf = realloc(pTSBuf->tsData.rawBuf, pBlock->numOfElem * TSDB_KEYSIZE);
+      pTSBuf->tsData.allocSize = pBlock->numOfElem * TSDB_KEYSIZE;
+    }
+
+    if (pBlock->numOfElem * TSDB_KEYSIZE > pTSBuf->bufSize) {
+      pTSBuf->assistBuf = realloc(pTSBuf->assistBuf, pBlock->numOfElem * TSDB_KEYSIZE);
+      pTSBuf->bufSize = pBlock->numOfElem * TSDB_KEYSIZE;
+    }    
+    
     pTSBuf->tsData.len =
         tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
                               pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
@@ -471,7 +481,7 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, i
   
   // the size of raw data exceeds the size of the default prepared buffer, so
   // during getBufBlock, the output buffer needs to be large enough.
-  if (ptsData->len >= ptsData->threshold) {
+  if (ptsData->len >= ptsData->threshold - TSDB_KEYSIZE) {
     writeDataToDisk(pTSBuf);
     shrinkBuffer(ptsData);
   }
@@ -603,6 +613,10 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex
     expandBuffer(&pTSBuf->tsData, (int32_t)s);
   }
   
+  if (s > pTSBuf->bufSize) {
+    pTSBuf->assistBuf = realloc(pTSBuf->assistBuf, s);
+    pTSBuf->bufSize = (int32_t)s;
+  }    
   pTSBuf->tsData.len =
       tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
                             pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);

src/tsdb/src/tsdbRead.c

@@ -1572,7 +1572,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
   int32_t numOfColsOfRow1 = 0;
 
   if (pSchema1 == NULL) {
-    pSchema1 = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row1));
+    pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
   }
   if(isRow1DataRow) {
     numOfColsOfRow1 = schemaNCols(pSchema1);
@@ -1584,7 +1584,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
   if(row2) {
     isRow2DataRow = isDataRow(row2);
     if (pSchema2 == NULL) {
-      pSchema2 = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row2));
+      pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
     }
     if(isRow2DataRow) {
       numOfColsOfRow2 = schemaNCols(pSchema2);
@@ -2460,7 +2460,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
-        (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+    (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
       tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle,
                 pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId);

src/util/inc/tnettest.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-void taosNetTest(char *role, char *host, int port, int pkgLen);
+void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
 
 #ifdef __cplusplus
 }

src/util/src/tnettest.c

@@ -27,6 +27,10 @@
 #include "syncMsg.h"
 
 #define MAX_PKG_LEN (64 * 1000)
+#define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024)
+#define MIN_SPEED_PKG_LEN 1024
+#define MAX_SPEED_PKG_NUM 10000
+#define MIN_SPEED_PKG_NUM 1
 #define BUFFER_SIZE (MAX_PKG_LEN + 1024)
 
 extern int32_t tsRpcMaxUdpSize;
@@ -466,6 +470,7 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) {
       sendpkgLen = pkgLen;
     }
 
+    tsRpcForceTcp = 1;
     int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL);
     if (ret < 0) {
       printf("failed to test TCP port:%d\n", port);
@@ -479,6 +484,7 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) {
       sendpkgLen = pkgLen;
     }
 
+    tsRpcForceTcp = 0;
     ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL);
     if (ret < 0) {
       printf("failed to test UDP port:%d\n", port);
@@ -542,12 +548,110 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) {
   }
 }
 
-void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) {
+static void taosNetTestFqdn(char *host) {
+  int code = 0;
+  uint64_t startTime = taosGetTimestampUs();
+  uint32_t ip = taosGetIpv4FromFqdn(host);
+  if (ip == 0xffffffff) {
+    uError("failed to get IP address from %s since %s", host, strerror(errno));
+    code = -1;
+  }
+  uint64_t endTime = taosGetTimestampUs();
+  uint64_t el = endTime - startTime;
+  printf("check convert fqdn spend, status: %d\tcost: %" PRIu64 " us\n", code, el);
+  return;
+}
+
+static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen,
+                              int32_t pkgNum, char *pkgType) {
+  // record config
+  int32_t compressTmp = tsCompressMsgSize;
+  int32_t maxUdpSize  = tsRpcMaxUdpSize;
+  int32_t forceTcp  = tsRpcForceTcp;
+
+  if (0 == strcmp("tcp", pkgType)){
+    tsRpcForceTcp = 1;
+    tsRpcMaxUdpSize = 0;            // force tcp
+  } else {
+    tsRpcForceTcp = 0;
+    tsRpcMaxUdpSize = INT_MAX;
+  }
+  tsCompressMsgSize = -1;
+
+  SRpcEpSet epSet;
+  SRpcMsg   reqMsg;
+  SRpcMsg   rspMsg;
+  void *    pRpcConn;
+  char secretEncrypt[32] = {0};
+  char    spi = 0;
+  pRpcConn = taosNetInitRpc(secretEncrypt, spi);
+  if (NULL == pRpcConn) {
+    uError("failed to init client rpc");
+    return;
+  }
+  printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n\n", host, port, pkgLen, pkgNum, pkgType);
+  int32_t totalSucc = 0;
+  uint64_t startT = taosGetTimestampUs();
+  for (int32_t i = 1; i <= pkgNum; i++) {
+    uint64_t startTime = taosGetTimestampUs();
+
+    memset(&epSet, 0, sizeof(SRpcEpSet));
+    epSet.inUse = 0;
+    epSet.numOfEps = 1;
+    epSet.port[0] = port;
+    strcpy(epSet.fqdn[0], host);
+
+    reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST;
+    reqMsg.pCont = rpcMallocCont(pkgLen);
+    reqMsg.contLen = pkgLen;
+    reqMsg.code = 0;
+    reqMsg.handle = NULL;   // rpc handle returned to app
+    reqMsg.ahandle = NULL;  // app handle set by client
+    strcpy(reqMsg.pCont, "nettest speed");
+
+    rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg);
+
+    int code = 0;
+    if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) {
+      uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code));
+      code = -1;
+    }else{
+      totalSucc ++;
+    }
+
+    rpcFreeCont(rspMsg.pCont);
+
+    uint64_t endTime = taosGetTimestampUs();
+    uint64_t el = endTime - startTime;
+    printf("progress:%5d/%d\tstatus:%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0);
+  }
+  int64_t endT = taosGetTimestampUs();
+  uint64_t elT = endT - startT;
+  printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc);
+
+  rpcClose(pRpcConn);
+
+  // return config
+  tsCompressMsgSize = compressTmp;
+  tsRpcMaxUdpSize = maxUdpSize;
+  tsRpcForceTcp = forceTcp;
+  return;
+}
+
+void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen,
+                 int32_t pkgNum, char *pkgType) {
   tscEmbedded = 1;
   if (host == NULL) host = tsLocalFqdn;
   if (port == 0) port = tsServerPort;
-  if (pkgLen <= 10) pkgLen = 1000;
-  if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN;
+  if (0 == strcmp("speed", role)){
+    if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN;
+    if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN;
+    if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM;
+    if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM;
+  }else{
+    if (pkgLen <= 10) pkgLen = 1000;
+    if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN;
+  }
 
   if (0 == strcmp("client", role)) {
     taosNetTestClient(host, port, pkgLen);
@@ -560,6 +664,12 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) {
     taosNetCheckSync(host, port);
   } else if (0 == strcmp("startup", role)) {
     taosNetTestStartup(host, port);
+  } else if (0 == strcmp("speed", role)) {
+    tscEmbedded = 0;
+    char type[10] = {0};
+    taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType));
+  }else if (0 == strcmp("fqdn", role)) {
+    taosNetTestFqdn(host);
   } else {
     taosNetTestStartup(host, port);
   }

tests/nettest/FQDNnettest.sh

+#!/bin/bash
+
+taos -n fqdn

tests/nettest/TCPUDP.sh

+#!/bin/bash
+
+for N in -1 0 1 10000 10001
+do
+	for l in 1023 1024 1073741824 1073741825 
+	do
+		for S in udp tcp 
+		do
+			taos -n speed -h BCC-2 -P 6030 -N $N -l $l -S $S 2>&1 | tee -a result.txt
+		done
+	done
+done

tests/pytest/alter/alter_table.py

@@ -102,6 +102,20 @@ class TDTestCase:
                 print("check2: i=%d colIdx=%d" % (i, colIdx))
                 tdSql.checkData(0, i, self.rowNum * (colIdx - i + 3))
 
+    def alter_table_255_times(self):   # add case for TD-6207
+        for i in range(255):
+            tdLog.info("alter table st add column cb%d int"%i)
+            tdSql.execute("alter table st add column cb%d int"%i)
+            tdSql.execute("insert into t0 (ts,c1) values(now,1)")
+            tdSql.execute("reset query cache")
+            tdSql.query("select * from st")
+        tdSql.execute("create table mt(ts timestamp, i int)")
+        tdSql.execute("insert into mt values(now,11)")
+        tdSql.query("select * from mt")
+        tdDnodes.stop(1)        
+        tdDnodes.start(1) 
+        tdSql.query("describe db.st")
+
     def run(self):
         # Setup params
         db = "db"
@@ -131,12 +145,14 @@ class TDTestCase:
             tdSql.checkData(0, i, self.rowNum * (size - i))
 
 
-        tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float)")
-        tdSql.execute("create table t0 using st tags(null)")
+        tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float,t2 int,t3 double)")
+        tdSql.execute("create table t0 using st tags(null,1,2.3)")
         tdSql.execute("alter table t0 set tag t1=2.1")
 
         tdSql.query("show tables")
         tdSql.checkRows(2)
+        self.alter_table_255_times()
+
         
     def stop(self):
         tdSql.close()

tests/pytest/fulltest.sh

@@ -255,7 +255,7 @@ python3 ./test.py -f query/queryTsisNull.py
 python3 ./test.py -f query/subqueryFilter.py
 python3 ./test.py -f query/nestedQuery/queryInterval.py
 python3 ./test.py -f query/queryStateWindow.py
-python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py
+# python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py
 python3 ./test.py -f query/nestquery_last_row.py
 python3 ./test.py -f query/queryCnameDisplay.py
 python3 ./test.py -f query/operator_cost.py

tests/pytest/functions/function_bottom.py

@@ -104,6 +104,21 @@ class TDTestCase:
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 1)
         tdSql.checkData(1, 1, 2)
+
+        tdSql.query("select ts,bottom(col1, 2),ts from test1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
+        tdSql.checkData(1, 0, "2018-09-17 09:00:00.001")
+        tdSql.checkData(1, 3, "2018-09-17 09:00:00.001")
+
+
+        tdSql.query("select ts,bottom(col1, 2),ts from test group by tbname")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
+        tdSql.checkData(1, 0, "2018-09-17 09:00:00.001")
+        tdSql.checkData(1, 3, "2018-09-17 09:00:00.001")
                    
         #TD-2457 bottom + interval + order by
         tdSql.error('select top(col2,1) from test interval(1y) order by col2;')

tests/pytest/functions/function_derivative.py

@@ -54,6 +54,28 @@ class TDTestCase:
                 tdSql.query("select derivative(col, 10s, 0) from stb group by tbname")
                 tdSql.checkRows(10)
 
+                tdSql.query("select ts,derivative(col, 10s, 1),ts from stb group by tbname")
+                tdSql.checkRows(4)
+                tdSql.checkData(0, 0, "2018-09-17 09:00:10.000")
+                tdSql.checkData(0, 1, "2018-09-17 09:00:10.000")
+                tdSql.checkData(0, 3, "2018-09-17 09:00:10.000")
+                tdSql.checkData(3, 0, "2018-09-17 09:01:20.000")
+                tdSql.checkData(3, 1, "2018-09-17 09:01:20.000")
+                tdSql.checkData(3, 3, "2018-09-17 09:01:20.000")
+
+                tdSql.query("select ts,derivative(col, 10s, 1),ts from tb1")
+                tdSql.checkRows(2)
+                tdSql.checkData(0, 0, "2018-09-17 09:00:10.000")
+                tdSql.checkData(0, 1, "2018-09-17 09:00:10.000")
+                tdSql.checkData(0, 3, "2018-09-17 09:00:10.000")
+                tdSql.checkData(1, 0, "2018-09-17 09:00:20.009")
+                tdSql.checkData(1, 1, "2018-09-17 09:00:20.009")
+                tdSql.checkData(1, 3, "2018-09-17 09:00:20.009")
+
+                tdSql.query("select ts from(select ts,derivative(col, 10s, 0) from stb group by tbname)")
+
+                tdSql.checkData(0, 0, "2018-09-17 09:00:10.000")
+
                 tdSql.error("select derivative(col, 10s, 0) from tb1 group by tbname")
 
                 tdSql.query("select derivative(col, 10s, 1) from tb1")

tests/pytest/functions/function_diff.py

@@ -94,6 +94,23 @@ class TDTestCase:
         tdSql.error("select diff(col13) from test")
         tdSql.error("select diff(col14) from test")
 
+        tdSql.query("select ts,diff(col1),ts from test1")
+        tdSql.checkRows(10)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 3, "2018-09-17 09:00:00.000")
+        tdSql.checkData(9, 0, "2018-09-17 09:00:00.009")
+        tdSql.checkData(9, 1, "2018-09-17 09:00:00.009")
+        tdSql.checkData(9, 3, "2018-09-17 09:00:00.009")
+
+        tdSql.query("select ts,diff(col1),ts from test group by tbname")
+        tdSql.checkRows(10)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.000")
+        tdSql.checkData(0, 3, "2018-09-17 09:00:00.000")
+        tdSql.checkData(9, 0, "2018-09-17 09:00:00.009")
+        tdSql.checkData(9, 1, "2018-09-17 09:00:00.009")
+        tdSql.checkData(9, 3, "2018-09-17 09:00:00.009")
 
         tdSql.query("select diff(col1) from test1")
         tdSql.checkRows(10)

tests/pytest/functions/function_top.py

@@ -117,6 +117,21 @@ class TDTestCase:
         tdSql.checkRows(2)
         tdSql.checkData(0, 1, 8.1)
         tdSql.checkData(1, 1, 9.1)
+
+        tdSql.query("select ts,top(col1, 2),ts from test1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.008")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.008")
+        tdSql.checkData(1, 0, "2018-09-17 09:00:00.009")
+        tdSql.checkData(1, 3, "2018-09-17 09:00:00.009")
+
+
+        tdSql.query("select ts,top(col1, 2),ts from test group by tbname")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, "2018-09-17 09:00:00.008")
+        tdSql.checkData(0, 1, "2018-09-17 09:00:00.008")
+        tdSql.checkData(1, 0, "2018-09-17 09:00:00.009")
+        tdSql.checkData(1, 3, "2018-09-17 09:00:00.009")
         
         #TD-2563 top + super_table + interval 
         tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") 

tests/pytest/functions/queryTestCases.py

@@ -884,6 +884,204 @@ class TDTestCase:
 
         pass
 
+    def td6068(self):
+        tdLog.printNoPrefix("==========TD-6068==========")
+        tdSql.execute("drop database if exists db")
+        tdSql.execute("create database  if not exists db keep 3650")
+        tdSql.execute("use db")
+
+        tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 float, c3 timestamp, c4 binary(16), c5 double, c6 bool) tags(t1 int)")
+
+        for i in range(100):
+            sql  = f"create table db.t{i} using db.stb1 tags({i})"
+            tdSql.execute(sql)
+            tdSql.execute(f"insert into db.t{i} values (now-10h, {i}, {i+random.random()}, now-10h, 'a_{i}', '{i-random.random()}', True)")
+            tdSql.execute(f"insert into db.t{i} values (now-9h, {i+random.randint(1,10)}, {i+random.random()}, now-9h, 'a_{i}', '{i-random.random()}', FALSE )")
+            tdSql.execute(f"insert into db.t{i} values (now-8h, {i+random.randint(1,10)}, {i+random.random()}, now-8h, 'b_{i}', '{i-random.random()}', True)")
+            tdSql.execute(f"insert into db.t{i} values (now-7h, {i+random.randint(1,10)}, {i+random.random()}, now-7h, 'b_{i}', '{i-random.random()}', FALSE )")
+            tdSql.execute(f"insert into db.t{i} values (now-6h, {i+random.randint(1,10)}, {i+random.random()}, now-6h, 'c_{i}', '{i-random.random()}', True)")
+            tdSql.execute(f"insert into db.t{i} values (now-5h, {i+random.randint(1,10)}, {i+random.random()}, now-5h, 'c_{i}', '{i-random.random()}', FALSE )")
+            tdSql.execute(f"insert into db.t{i} (ts)values (now-4h)")
+            tdSql.execute(f"insert into db.t{i} (ts)values (now-11h)")
+            tdSql.execute(f"insert into db.t{i} (ts)values (now-450m)")
+
+        tdSql.query("select ts as t,derivative(c1, 10m, 0) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, derivative(c1, 1h, 0) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.checkCols(4)
+        tdSql.query("select ts as t, derivative(c1, 1s, 0) from t1")
+        tdSql.query("select ts as t, derivative(c1, 1d, 0) from t1")
+        tdSql.error("select ts as t, derivative(c1, 1h, 0) from stb1")
+        tdSql.query("select ts as t, derivative(c2, 1h, 0) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, derivative(c3, 1h, 0) from t1")
+        tdSql.error("select ts as t, derivative(c4, 1h, 0) from t1")
+        tdSql.query("select ts as t, derivative(c5, 1h, 0) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, derivative(c6, 1h, 0) from t1")
+        tdSql.error("select ts as t, derivative(t1, 1h, 0) from t1")
+
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, diff(c1) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.checkCols(4)
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.error("select ts as t, diff(c1) from stb1")
+        tdSql.query("select ts as t, diff(c2) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, diff(c3) from t1")
+        tdSql.error("select ts as t, diff(c4) from t1")
+        tdSql.query("select ts as t, diff(c5) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, diff(c6) from t1")
+        tdSql.error("select ts as t, diff(t1) from t1")
+        tdSql.error("select ts as t, diff(c1, c2) from t1")
+
+        tdSql.error("select ts as t, bottom(c1, 0) from t1")
+        tdSql.query("select ts as t, bottom(c1, 5) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, bottom(c1, 5) from stb1")
+        tdSql.checkRows(5)
+        tdSql.query("select ts as t, bottom(c1, 5) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.query("select ts as t, bottom(c1, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.query("select ts as t, bottom(c2, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, bottom(c3, 5) from t1")
+        tdSql.error("select ts as t, bottom(c4, 5) from t1")
+        tdSql.query("select ts as t, bottom(c5, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, bottom(c6, 5) from t1")
+        tdSql.error("select ts as t, bottom(c5, 8) as b from t1 order by b")
+        tdSql.error("select ts as t, bottom(t1, 1) from t1")
+        tdSql.error("select ts as t, bottom(t1, 1) from stb1")
+        tdSql.error("select ts as t, bottom(t1, 3) from stb1 order by c3")
+        tdSql.error("select ts as t, bottom(t1, 3) from t1 order by c3")
+
+
+        tdSql.error("select ts as t, top(c1, 0) from t1")
+        tdSql.query("select ts as t, top(c1, 5) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, top(c1, 5) from stb1")
+        tdSql.checkRows(5)
+        tdSql.query("select ts as t, top(c1, 5) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.query("select ts as t, top(c1, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.query("select ts as t, top(c2, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, top(c3, 5) from t1")
+        tdSql.error("select ts as t, top(c4, 5) from t1")
+        tdSql.query("select ts as t, top(c5, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, top(c6, 5) from t1")
+        tdSql.error("select ts as t, top(c5, 8) as b from t1 order by b")
+        tdSql.error("select ts as t, top(t1, 1) from t1")
+        tdSql.error("select ts as t, top(t1, 1) from stb1")
+        tdSql.error("select ts as t, top(t1, 3) from stb1 order by c3")
+        tdSql.error("select ts as t, top(t1, 3) from t1 order by c3")
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, diff(c1) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.checkCols(4)
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.query("select ts as t, diff(c1) from t1")
+        tdSql.error("select ts as t, diff(c1) from stb1")
+        tdSql.query("select ts as t, diff(c2) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, diff(c3) from t1")
+        tdSql.error("select ts as t, diff(c4) from t1")
+        tdSql.query("select ts as t, diff(c5) from t1")
+        tdSql.checkRows(5)
+        tdSql.error("select ts as t, diff(c6) from t1")
+        tdSql.error("select ts as t, diff(t1) from t1")
+        tdSql.error("select ts as t, diff(c1, c2) from t1")
+
+        tdSql.error("select ts as t, bottom(c1, 0) from t1")
+        tdSql.query("select ts as t, bottom(c1, 5) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, bottom(c1, 5) from stb1")
+        tdSql.checkRows(5)
+        tdSql.query("select ts as t, bottom(c1, 5) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.query("select ts as t, bottom(c1, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.query("select ts as t, bottom(c2, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, bottom(c3, 5) from t1")
+        tdSql.error("select ts as t, bottom(c4, 5) from t1")
+        tdSql.query("select ts as t, bottom(c5, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, bottom(c6, 5) from t1")
+        tdSql.error("select ts as t, bottom(c5, 8) as b from t1 order by b")
+        tdSql.error("select ts as t, bottom(t1, 1) from t1")
+        tdSql.error("select ts as t, bottom(t1, 1) from stb1")
+        tdSql.error("select ts as t, bottom(t1, 3) from stb1 order by c3")
+        tdSql.error("select ts as t, bottom(t1, 3) from t1 order by c3")
+
+
+        tdSql.error("select ts as t, top(c1, 0) from t1")
+        tdSql.query("select ts as t, top(c1, 5) from t1")
+        tdSql.checkRows(5)
+        tdSql.checkCols(3)
+        for i in range(5):
+            data=tdSql.getData(i, 0)
+            tdSql.checkData(i, 1, data)
+        tdSql.query("select ts as t, top(c1, 5) from stb1")
+        tdSql.checkRows(5)
+        tdSql.query("select ts as t, top(c1, 5) from stb1 group by tbname")
+        tdSql.checkRows(500)
+        tdSql.query("select ts as t, top(c1, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.query("select ts as t, top(c2, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, top(c3, 5) from t1")
+        tdSql.error("select ts as t, top(c4, 5) from t1")
+        tdSql.query("select ts as t, top(c5, 8) from t1")
+        tdSql.checkRows(6)
+        tdSql.error("select ts as t, top(c6, 5) from t1")
+        tdSql.error("select ts as t, top(c5, 8) as b from t1 order by b")
+        tdSql.error("select ts as t, top(t1, 1) from t1")
+        tdSql.error("select ts as t, top(t1, 1) from stb1")
+        tdSql.error("select ts as t, top(t1, 3) from stb1 order by c3")
+        tdSql.error("select ts as t, top(t1, 3) from t1 order by c3")
+
+        pass
+
+
     def run(self):
 
         # master branch
@@ -891,8 +1089,9 @@ class TDTestCase:
         # self.td4082()
         # self.td4288()
         # self.td4724()
-        self.td5798()
+        # self.td5798()
         # self.td5935()
+        self.td6068()
 
         # develop branch
         # self.td4097()

tests/pytest/insert/insertFromCSVPerformance.py

@@ -28,7 +28,7 @@ class insertFromCSVPerformace:
         self.tbName = tbName
         self.branchName = branchName
         self.type = buildType
-        self.ts = 1500074556514
+        self.ts = 1500000000000
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -46,13 +46,20 @@ class insertFromCSVPerformace:
             config = self.config)
 
     def writeCSV(self):
-        with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
+        tsset = set()
+        rows = 0
+        with open('test4.csv','w', encoding='utf-8', newline='') as csvFile:
             writer = csv.writer(csvFile, dialect='excel')
-            for i in range(1000000):
-                newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000)
-                d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
-                dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
-                writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
+            while True:
+                newTimestamp = self.ts + random.randint(1, 10) * 10000000000 + random.randint(1, 10) * 1000000000 + random.randint(1, 10) * 100000000 + random.randint(1, 10) * 10000000 + random.randint(1, 10) * 1000000 + random.randint(1, 10) * 100000 + random.randint(1, 10) * 10000 + random.randint(1, 10) * 1000 + random.randint(1, 10) * 100 + random.randint(1, 10) * 10 + random.randint(1, 10)
+                if newTimestamp not in tsset:
+                    tsset.add(newTimestamp)
+                    d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
+                    dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
+                    writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
+                    rows += 1
+                    if rows == 2000000:
+                        break
     
     def removCSVHeader(self):
         data = pd.read_csv("ordered.csv")
@@ -71,7 +78,9 @@ class insertFromCSVPerformace:
             cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
             startTime = time.time()
             cursor.execute("insert into t1 file 'outoforder.csv'")
-            totalTime += time.time() - startTime                      
+            totalTime += time.time() - startTime 
+            time.sleep(1)
+                                 
         out_of_order_time = (float) (totalTime / 10)
         print("Out of Order - Insert time: %f" % out_of_order_time)                      
         
@@ -81,7 +90,8 @@ class insertFromCSVPerformace:
             cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
             startTime = time.time()
             cursor.execute("insert into t2 file 'ordered.csv'")
-            totalTime += time.time() - startTime            
+            totalTime += time.time() - startTime
+            time.sleep(1)          
 
         in_order_time = (float) (totalTime / 10)
         print("In order - Insert time: %f" % in_order_time)

tests/pytest/query/filterNoKeyword.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+        self.ts = 1537146000000
+
+    def run(self):
+        tdSql.prepare()
+
+        print("======= Verify filter for bool, nchar and binary type =========")
+        tdLog.debug(
+            "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))")
+        tdSql.execute(
+            "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))")
+
+        tdSql.execute("create table st1 using st tags(true, 'table1', '水表')")
+        for i in range(1, 6):
+            tdSql.execute(
+                "insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f)" %
+                (self.ts + i, i %
+                 2, i, i,
+                 i, i, i, i, 1.0, 1.0))
+
+        # =============Data type keywords cannot be used in filter====================
+        # timestamp
+        tdSql.error("select * from st where timestamp = 1629417600")
+
+        # bool
+        tdSql.error("select * from st where bool = false")
+
+        #binary
+        tdSql.error("select * from st where binary = 'taosdata'")
+
+        # nchar
+        tdSql.error("select * from st where nchar = '涛思数据'")
+
+        # tinyint
+        tdSql.error("select * from st where tinyint = 127")
+
+        # smallint
+        tdSql.error("select * from st where smallint = 32767")
+
+        # int
+        tdSql.error("select * from st where INTEGER = 2147483647")
+        tdSql.error("select * from st where int = 2147483647")
+
+        # bigint
+        tdSql.error("select * from st where bigint = 2147483647")
+
+        # float
+        tdSql.error("select * from st where float = 3.4E38")
+
+        # double
+        tdSql.error("select * from st where double = 1.7E308")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/query.py

-###################################################################
-#           Copyright (c) 2016 by TAOS Technologies, Inc.
-#                     All rights reserved.
-#
-#  This file is proprietary and confidential to TAOS Technologies.
-#  No part of this file may be reproduced, stored, transmitted,
-#  disclosed or used in any form or by any means other than as
-#  expressly provided by the written permission from Jianhui Tao
-#
-###################################################################
-
-# -*- coding: utf-8 -*-
-
-import sys
-import taos
-from util.log import tdLog
-from util.cases import tdCases
-from util.sql import tdSql
-from util.dnodes import tdDnodes
-
-class TDTestCase:
-    def init(self, conn, logSql):
-        tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor(), logSql)
-
-        self.ts = 1538548685000
-
-    def run(self):
-        tdSql.prepare()
-
-        print("==============step1")
-        tdSql.execute(
-            "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
-        tdSql.execute(
-            'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
-        tdSql.execute(
-            'CREATE TABLE if not exists dev_002 using st tags("dev_02")')
-
-        print("==============step2")
-
-        tdSql.execute(
-            """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1),
-            ('2020-05-13 10:00:00.001', 1)
-             dev_002 VALUES('2020-05-13 10:00:00.001', 1)""")
-
-        tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'")
-        tdSql.checkRows(1)
-
-        tdSql.query("select tbname, dev from dev_001") 
-        tdSql.checkRows(1)
-        tdSql.checkData(0, 0, 'dev_001')
-        tdSql.checkData(0, 1, 'dev_01')
-
-        tdSql.query("select tbname, dev, tagtype from dev_001") 
-        tdSql.checkRows(2)
-        tdSql.checkData(0, 0, 'dev_001')
-        tdSql.checkData(0, 1, 'dev_01')
-        tdSql.checkData(0, 2, 1)
-        tdSql.checkData(1, 0, 'dev_001')
-        tdSql.checkData(1, 1, 'dev_01')
-        tdSql.checkData(1, 2, 1)
-
-        ## test case for https://jira.taosdata.com:18080/browse/TD-2488
-        tdSql.execute("create table m1(ts timestamp, k int) tags(a int)")
-        tdSql.execute("create table t1 using m1 tags(1)")
-        tdSql.execute("create table t2 using m1 tags(2)")
-        tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)")
-        tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)")
-        tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)")
-        
-        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
-        tdSql.checkRows(1)
-        tdSql.checkData(0, 0, 1)
-
-        tdDnodes.stop(1)
-        tdDnodes.start(1)
-
-        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
-        tdSql.checkRows(1)
-        tdSql.checkData(0, 0, 1)
-
-        ## test case for https://jira.taosdata.com:18080/browse/TD-1930
-        tdSql.execute("create table tb(ts timestamp, c1 int, c2 binary(10), c3 nchar(10), c4 float, c5 bool)")
-        for i in range(10):
-            tdSql.execute("insert into tb values(%d, %d, 'binary%d', 'nchar%d', %f, %d)" % (self.ts + i, i, i, i, i + 0.1, i % 2))
-        
-        tdSql.error("select * from tb where c2 = binary2")
-        tdSql.error("select * from tb where c3 = nchar2")
-
-        tdSql.query("select * from tb where c2 = 'binary2' ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c3 = 'nchar2' ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c1 = '2' ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c1 = 2 ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c4 = '0.1' ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c4 = 0.1 ")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb where c5 = true ")
-        tdSql.checkRows(5)
-
-        tdSql.query("select * from tb where c5 = 'true' ")
-        tdSql.checkRows(5)
-
-        # For jira: https://jira.taosdata.com:18080/browse/TD-2850
-        tdSql.execute("create database 'Test' ")
-        tdSql.execute("use 'Test' ")
-        tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)")
-        tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)")
-        tdSql.query("select * from tb")
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from tb0")
-        tdSql.checkRows(1)
-        
-
-    def stop(self):
-        tdSql.close()
-        tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1538548685000
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))")
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags("dev_01")')
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags("dev_02")')
+
+        print("==============step2")
+
+        tdSql.execute(
+            """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1),
+            ('2020-05-13 10:00:00.001', 1)
+             dev_002 VALUES('2020-05-13 10:00:00.001', 1)""")
+
+        tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'")
+        tdSql.checkRows(1)
+
+        tdSql.query("select tbname, dev from dev_001")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'dev_001')
+        tdSql.checkData(0, 1, 'dev_01')
+
+        tdSql.query("select tbname, dev, tagtype from dev_001")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, 'dev_001')
+        tdSql.checkData(0, 1, 'dev_01')
+        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(1, 0, 'dev_001')
+        tdSql.checkData(1, 1, 'dev_01')
+        tdSql.checkData(1, 2, 1)
+
+        ## test case for https://jira.taosdata.com:18080/browse/TD-2488
+        tdSql.execute("create table m1(ts timestamp, k int) tags(a int)")
+        tdSql.execute("create table t1 using m1 tags(1)")
+        tdSql.execute("create table t2 using m1 tags(2)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)")
+        tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)")
+
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        ## test case for https://jira.taosdata.com:18080/browse/TD-1930
+        tdSql.execute("create table tb(ts timestamp, c1 int, c2 binary(10), c3 nchar(10), c4 float, c5 bool)")
+        for i in range(10):
+            tdSql.execute(
+                "insert into tb values(%d, %d, 'binary%d', 'nchar%d', %f, %d)" % (self.ts + i, i, i, i, i + 0.1, i % 2))
+
+        tdSql.error("select * from tb where c2 = binary2")
+        tdSql.error("select * from tb where c3 = nchar2")
+
+        tdSql.query("select * from tb where c2 = 'binary2' ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c3 = 'nchar2' ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c1 = '2' ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c1 = 2 ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c4 = '0.1' ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c4 = 0.1 ")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb where c5 = true ")
+        tdSql.checkRows(5)
+
+        tdSql.query("select * from tb where c5 = 'true' ")
+        tdSql.checkRows(5)
+
+        # For jira: https://jira.taosdata.com:18080/browse/TD-2850
+        tdSql.execute("create database 'Test' ")
+        tdSql.execute("use 'Test' ")
+        tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)")
+        tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)")
+        tdSql.query("select * from tb")
+        tdSql.checkRows(1)
+
+        # For jira:https://jira.taosdata.com:18080/browse/TD-6314
+        tdSql.execute("use db")
+        tdSql.execute("create stable stb_001(ts timestamp,v int) tags(c0 int)")
+        tdSql.query("select _block_dist() from stb_001")
+        tdSql.checkRows(1)
+
+        tdSql.query("select * from tb0")
+        tdSql.checkRows(1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/queryPerformance.py

@@ -17,6 +17,7 @@ import os
 import taos
 import time
 import argparse
+import json
 
 
 class taosdemoQueryPerformace:
@@ -48,7 +49,7 @@ class taosdemoQueryPerformace:
         cursor2 = self.conn2.cursor()
         cursor2.execute("create database if not exists %s" % self.dbName)
         cursor2.execute("use %s" % self.dbName)
-        cursor2.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50), branch binary(50), type binary(20)) tags(query_id int, query_sql binary(300))" % self.stbName)
+        cursor2.execute("create table if not exists %s(ts timestamp, query_time_avg float, query_time_max float, query_time_min float, commit_id binary(50), branch binary(50), type binary(20)) tags(query_id int, query_sql binary(300))" % self.stbName)
 
         sql = "select count(*) from test.meters"
         tableid = 1
@@ -74,7 +75,7 @@ class taosdemoQueryPerformace:
         tableid = 6
         cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
-        sql = "select * from meters"
+        sql = "select * from meters limit 10000"
         tableid = 7
         cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
@@ -87,37 +88,96 @@ class taosdemoQueryPerformace:
         cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
 
         cursor2.close()
+    
+    def generateQueryJson(self):
+        
+        sqls = []
+        cursor2 = self.conn2.cursor()
+        cursor2.execute("select query_id, query_sql from %s.%s" % (self.dbName, self.stbName))
+        i = 0
+        for data in cursor2:
+            sql = {
+                "sql": data[1],
+                "result_mode": "onlyformat",
+                "result_file": "./query_sql_res%d.txt" % i
+            }
+            sqls.append(sql)
+            i += 1
+
+        query_data = {
+            "filetype": "query",
+            "cfgdir": "/etc/perf",
+            "host": "127.0.0.1",
+            "port": 6030,
+            "user": "root",
+            "password": "taosdata",
+            "databases": "test",
+            "specified_table_query": {
+                "query_times": 100,
+                "concurrent": 1,
+                "sqls": sqls
+            }
+        }        
+
+        query_json_file = f"/tmp/query.json"
+
+        with open(query_json_file, 'w') as f:
+            json.dump(query_data, f)
+        return query_json_file
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosdemo" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+    
+    def getCMDOutput(self, cmd):
+        cmd = os.popen(cmd)
+        output = cmd.read()
+        cmd.close()
+        return output
 
     def query(self): 
-        cursor = self.conn.cursor() 
-        print("==================== query performance ====================")
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            print("taosdemo not found!")
+            sys.exit(1)
+            
+        binPath = buildPath + "/build/bin/"
+        os.system(
+            "%sperfMonitor -f %s > query_res.txt" %
+            (binPath, self.generateQueryJson()))
 
+        cursor = self.conn2.cursor() 
+        print("==================== query performance ====================")
         cursor.execute("use %s" % self.dbName)
-        cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName)      
+        cursor.execute("select tbname, query_sql from %s" % self.stbName)      
 
+        i = 0
         for data in cursor:
             table_name = data[0]
-            query_id = data[1]
-            sql = data[2]            
+            sql = data[1]
+
+            self.avgDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $2}'" % (i + 1))
+            self.maxDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $5}'" % (i + 1))
+            self.minDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $8}'" % (i + 1))
+            i += 1
+            
+            print("query time for: %s %f seconds" % (sql, float(self.avgDelay)))
+            c = self.conn2.cursor()
+            c.execute("insert into %s.%s values(now, %f, %f, %f, '%s', '%s', '%s')" % (self.dbName, table_name, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.commitID, self.branch, self.type))            
             
-            totalTime = 0            
-            cursor2 = self.conn.cursor()
-            cursor2.execute("use test")
-            for i in range(100):       
-                if(self.clearCache == True):
-                    # root permission is required
-                    os.system("echo 3 > /proc/sys/vm/drop_caches")                
-                
-                startTime = time.time()      
-                cursor2.execute(sql)
-                totalTime += time.time() - startTime
-            cursor2.close()
-            print("query time for: %s %f seconds" % (sql, totalTime / 100))
-                        
-            cursor3 = self.conn2.cursor()
-            cursor3.execute("insert into %s.%s values(now, %f, '%s', '%s', '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID, self.branch, self.type))
-
-        cursor3.close()
+        c.close()    
         cursor.close()
 
 if __name__ == '__main__':
@@ -174,4 +234,4 @@ if __name__ == '__main__':
     args = parser.parse_args()
     perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix, args.git_branch, args.build_type)
     perftest.createPerfTables()
-    perftest.query()
+    perftest.query()
\ No newline at end of file

tests/pytest/tools/taosdemoPerformance.py

@@ -49,24 +49,18 @@ class taosdemoPerformace:
     def generateJson(self):
         db = {
             "name": "%s" % self.insertDB,
-            "drop": "yes",
-            "replica": 1
+            "drop": "yes"
         }
 
         stb = {
             "name": "meters",
-            "child_table_exists": "no",
             "childtable_count": self.numOfTables,
             "childtable_prefix": "stb_",
-            "auto_create_table": "no",
-            "data_source": "rand",
             "batch_create_tbl_num": 10,
-            "insert_mode": "taosc",
+            "insert_mode": "rand",
             "insert_rows": self.numOfRows,
-            "interlace_rows": 0,
-            "max_sql_len": 1024000,
-            "disorder_ratio": 0,
-            "disorder_range": 1000,
+            "batch_rows": 1000000,
+            "max_sql_len": 1048576,
             "timestamp_step": 1,
             "start_timestamp": "2020-10-01 00:00:00.000",
             "sample_format": "csv",
@@ -100,11 +94,8 @@ class taosdemoPerformace:
             "user": "root",
             "password": "taosdata",
             "thread_count": 10,
-            "thread_count_create_tbl": 10,
+            "thread_count_create_tbl": 4,
             "result_file": "./insert_res.txt",
-            "confirm_parameter_prompt": "no",
-            "insert_interval": 0,
-            "num_of_records_per_req": 30000,
             "databases": [db]
         }
 
@@ -145,7 +136,7 @@ class taosdemoPerformace:
         binPath = buildPath + "/build/bin/"
 
         os.system(
-            "%staosdemo -f %s > /dev/null 2>&1" %
+            "%sperfMonitor -f %s > /dev/null 2>&1" %
             (binPath, self.generateJson()))
         self.createTableTime = self.getCMDOutput(
             "grep 'Spent' insert_res.txt | awk 'NR==1{print $2}'")

tests/script/api/makefile

@@ -6,7 +6,8 @@ TARGET=exe
 LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt
 CFLAGS = -O0 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \
 	-Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \
-	-Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99
+	-Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \
+	-fsanitize=address
 
 all: $(TARGET)
 
@@ -14,8 +15,10 @@ exe:
 	gcc $(CFLAGS) ./batchprepare.c  -o $(ROOT)batchprepare  $(LFLAGS)
 	gcc $(CFLAGS) ./stmtBatchTest.c -o $(ROOT)stmtBatchTest $(LFLAGS)
 	gcc $(CFLAGS) ./stmtTest.c -o $(ROOT)stmtTest $(LFLAGS)
+	gcc $(CFLAGS) ./stmt_function.c -o $(ROOT)stmt_function $(LFLAGS)
 
 clean:
 	rm $(ROOT)batchprepare
 	rm $(ROOT)stmtBatchTest
 	rm $(ROOT)stmtTest
+	rm $(ROOT)stmt_function