Merge branch 'develop' into queryPercentile

src/client/inc/tscLocalMerge.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "qextbuffer.h"
-#include "qfill.h"
+#include "qExtbuffer.h"
+#include "qFill.h"
 #include "taosmsg.h"
 #include "tlosertree.h"
 #include "tsclient.h"

src/client/inc/tscUtil.h

@@ -23,11 +23,11 @@ extern "C" {
 /*
  * @date   2018/09/30
  */
-#include "os.h"
-#include "tbuffer.h"
 #include "exception.h"
-#include "qextbuffer.h"
+#include "os.h"
+#include "qExtbuffer.h"
 #include "taosdef.h"
+#include "tbuffer.h"
 #include "tscLocalMerge.h"
 #include "tsclient.h"
 

src/client/inc/tsclient.h

@@ -31,8 +31,8 @@ extern "C" {
 #include "tutil.h"
 
 #include "qExecutor.h"
+#include "qTsbuf.h"
 #include "qsqlparser.h"
-#include "qtsbuf.h"
 #include "tcmdtype.h"
 
 // forward declaration
@@ -52,12 +52,20 @@ typedef struct STableComInfo {
   int32_t rowSize;
 } STableComInfo;
 
+typedef struct SCMCorVgroupInfo {
+  int32_t version;
+  int8_t inUse;
+  int8_t  numOfEps;
+  SEpAddr epAddr[TSDB_MAX_REPLICA];
+} SCMCorVgroupInfo;
+
 typedef struct STableMeta {
   STableComInfo tableInfo;
   uint8_t       tableType;
   int16_t       sversion;
   int16_t       tversion;
-  SCMVgroupInfo vgroupInfo;
+  SCMVgroupInfo  vgroupInfo;
+  SCMCorVgroupInfo  corVgroupInfo;
   int32_t       sid;       // the index of one table in a virtual node
   uint64_t      uid;       // unique id of a table
   SSchema       schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
@@ -456,7 +464,8 @@ extern void *    tscQhandle;
 extern int       tscKeepConn[];
 extern int       tsInsertHeadSize;
 extern int       tscNumOfThreads;
-extern SRpcEpSet tscMgmtEpSet;
+  
+extern SRpcCorEpSet tscMgmtEpSet;
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
 

src/client/src/tscFunctionImpl.c

@@ -14,15 +14,15 @@
  */
 
 #include "os.h"
-#include "qextbuffer.h"
-#include "qfill.h"
-#include "qhistogram.h"
-#include "qpercentile.h"
-#include "qsyntaxtreefunction.h"
-#include "qtsbuf.h"
+#include "qAst.h"
+#include "qExtbuffer.h"
+#include "qFill.h"
+#include "qHistogram.h"
+#include "qPercentile.h"
+#include "qSyntaxtreefunction.h"
+#include "qTsbuf.h"
 #include "taosdef.h"
 #include "taosmsg.h"
-#include "qast.h"
 #include "tscLog.h"
 #include "tscSubquery.h"
 #include "tscompression.h"
@@ -74,7 +74,7 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) {                  \
 void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {}
 void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {}
 
-void doFinalizer(SQLFunctionCtx *pCtx) { resetResultInfo(GET_RES_INFO(pCtx)); }
+void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); }
 
 typedef struct tValuePair {
   tVariant v;
@@ -330,12 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   return TSDB_CODE_SUCCESS;
 }
 
-/**
- * the numOfRes should be kept, since it may be used later
- * and allow the ResultInfo to be re initialized
- */
-void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; }
-
 void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) {
   assert(pResInfo->interResultBuf == NULL);
   

src/client/src/tscLocal.c

@@ -16,14 +16,14 @@
 #include "os.h"
 #include "taosmsg.h"
 
-#include "tcache.h"
-#include "tscUtil.h"
-#include "tsclient.h"
+#include "qExtbuffer.h"
 #include "taosdef.h"
+#include "tcache.h"
+#include "tname.h"
 #include "tscLog.h"
-#include "qextbuffer.h"
+#include "tscUtil.h"
 #include "tschemautil.h"
-#include "tname.h"
+#include "tsclient.h"
 
 static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength);
 

src/client/src/tscSQLParser.c

@@ -18,9 +18,9 @@
 #define _DEFAULT_SOURCE
 
 #include "os.h"
+#include "qAst.h"
 #include "taos.h"
 #include "taosmsg.h"
-#include "qast.h"
 #include "tcompare.h"
 #include "tname.h"
 #include "tscLog.h"

src/client/src/tscSchemaUtil.c

@@ -140,7 +140,15 @@ struct SSchema tscGetTbnameColumnSchema() {
   strcpy(s.name, TSQL_TBNAME_L);
   return s;
 }
-
+static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) {
+  corVgroupInfo->version = 0;
+  corVgroupInfo->inUse = 0;
+  corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
+  for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) {
+    strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN);
+    corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port;
+  }
+}
 STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
   assert(pTableMetaMsg != NULL);
   
@@ -157,6 +165,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   pTableMeta->sid = pTableMetaMsg->sid;
   pTableMeta->uid = pTableMetaMsg->uid;
   pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
+
+  tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
+
   pTableMeta->sversion = pTableMetaMsg->sversion;
   pTableMeta->tversion = pTableMetaMsg->tversion;
   

src/client/src/tscServer.c

@@ -26,10 +26,11 @@
 #include "ttime.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "tlockfree.h"
 
 #define TSC_MGMT_VNODE 999
 
-SRpcEpSet  tscMgmtEpSet;
+SRpcCorEpSet  tscMgmtEpSet;
 SRpcEpSet  tscDnodeEpSet;
 
 int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
@@ -51,37 +52,78 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
     pEpSet->numOfEps = 0;
     return;
   }
-  
+
   pEpSet->numOfEps = pVgroupInfo->numOfEps;
   for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
     strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn);
     pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
   }
 }
-
-void tscPrintMgmtEp() {
-  if (tscMgmtEpSet.numOfEps <= 0) {
-    tscError("invalid mnode EP list:%d", tscMgmtEpSet.numOfEps);
-  } else {
-    for (int i = 0; i < tscMgmtEpSet.numOfEps; ++i) {
-      tscDebug("mnode index:%d %s:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]);
-    }
+static void tscDumpMgmtEpSet(SRpcEpSet *epSet) {
+  taosCorBeginRead(&tscMgmtEpSet.version);
+  *epSet = tscMgmtEpSet.epSet;
+  taosCorEndRead(&tscMgmtEpSet.version);
+}  
+static void tscEpSetHtons(SRpcEpSet *s) {
+   for (int32_t i = 0; i < s->numOfEps; i++) {
+      s->port[i] = htons(s->port[i]);    
+   }
+} 
+bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) {
+   if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) {
+     return false;
+   } 
+   for (int32_t i = 0; i < s1->numOfEps; i++) {
+     if (s1->port[i] != s2->port[i] 
+        || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0)
+        return false;
+   }
+   return true;
+}
+void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) {
+  // no need to update if equal
+  taosCorBeginWrite(&tscMgmtEpSet.version);
+  tscMgmtEpSet.epSet = *pEpSet;
+  taosCorEndWrite(&tscMgmtEpSet.version);
+}
+static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) {
+  if (pVgroupInfo == NULL) { return;}
+  taosCorBeginRead(&pVgroupInfo->version);
+  int8_t inUse = pVgroupInfo->inUse;
+  pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; 
+  pEpSet->numOfEps = pVgroupInfo->numOfEps;  
+  for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
+    strncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN);
+    pEpSet->port[i] = pVgroupInfo->epAddr[i].port;
   }
+  taosCorEndRead(&pVgroupInfo->version);
 }
 
-void tscSetMgmtEpSet(SRpcEpSet *pEpSet) {
-  tscMgmtEpSet.numOfEps = pEpSet->numOfEps;
-  tscMgmtEpSet.inUse = pEpSet->inUse;
-  for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) {
-    tscMgmtEpSet.port[i] = htons(pEpSet->port[i]);
+static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) {
+  SSqlCmd *pCmd = &pObj->cmd;
+  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
+  if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;}
+  SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo;
+
+  taosCorBeginWrite(&pVgroupInfo->version);
+  //TODO(dengyihao), dont care vgid 
+  pVgroupInfo->inUse = pEpSet->inUse;
+  pVgroupInfo->numOfEps = pEpSet->numOfEps;
+  for (int32_t i = 0; pVgroupInfo->numOfEps; i++) {
+    strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN);
+    pVgroupInfo->epAddr[i].port = pEpSet->port[i];
   }
+  taosCorEndWrite(&pVgroupInfo->version);
 }
-
-void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet) {
-  tscMgmtEpSet = *pEpSet;
-  tscDebug("mnode EP list is changed for ufp is called, numOfEps:%d inUse:%d", tscMgmtEpSet.numOfEps, tscMgmtEpSet.inUse);
-  for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) {
-    tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]);
+void tscPrintMgmtEp() {
+  SRpcEpSet dump;
+  tscDumpMgmtEpSet(&dump);
+  if (dump.numOfEps <= 0) {
+    tscError("invalid mnode EP list:%d", dump.numOfEps);
+  } else {
+    for (int i = 0; i < dump.numOfEps; ++i) {
+      tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]);
+    }
   }
 }
 
@@ -95,7 +137,9 @@ void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet) {
 UNUSED_FUNC
 static int32_t tscGetMgmtConnMaxRetryTimes() {
   int32_t factor = 2;
-  return tscMgmtEpSet.numOfEps * factor;
+  SRpcEpSet dump;
+  tscDumpMgmtEpSet(&dump);
+  return dump.numOfEps * factor;
 }
 
 void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
@@ -111,9 +155,11 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
 
   if (code == 0) {
     SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp;
-    SRpcEpSet *      pEpSet = &pRsp->epSet;
-    if (pEpSet->numOfEps > 0) 
-      tscSetMgmtEpSet(pEpSet);
+    SRpcEpSet *      epSet = &pRsp->epSet;
+    if (epSet->numOfEps > 0) {
+      tscEpSetHtons(epSet);
+      tscUpdateMgmtEpSet(epSet);
+    } 
 
     pSql->pTscObj->connId = htonl(pRsp->connId);
 
@@ -185,7 +231,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
 
   // set the mgmt ip list
   if (pSql->cmd.command >= TSDB_SQL_MGMT) {
-    pSql->epSet = tscMgmtEpSet;
+    tscDumpMgmtEpSet(&pSql->epSet);
   }
 
   memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
@@ -236,58 +282,43 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  if (pCmd->command < TSDB_SQL_MGMT) {
-    if (pEpSet) pSql->epSet = *pEpSet;
-  } else {
-    if (pEpSet) tscMgmtEpSet = *pEpSet;
+  if (pEpSet) { 
+    //SRpcEpSet dump; 
+    tscEpSetHtons(pEpSet); 
+    if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
+      if(pCmd->command < TSDB_SQL_MGMT)  { 
+        tscUpdateVgroupInfo(pSql, pEpSet); 
+      } else {
+        tscUpdateMgmtEpSet(pEpSet);
+    }
+    }
   }
 
-  if (rpcMsg->pCont == NULL) {
-    rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-  } else {
-    STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
-    // if (rpcMsg->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-    //   if (pCmd->command == TSDB_SQL_CONNECT) {
-    //     rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-    //     rpcFreeCont(rpcMsg->pCont);
-    //     return;
-    //   }
-
-    //   if (pCmd->command == TSDB_SQL_HB) {
-    //     rpcMsg->code = TSDB_CODE_RPC_NOT_READY;
-    //     rpcFreeCont(rpcMsg->pCont);
-    //     return;
-    //   }
-
-    //   if (pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_DESCRIBE_TABLE ||
-    //       pCmd->command == TSDB_SQL_STABLEVGROUP || pCmd->command == TSDB_SQL_SHOW ||
-    //       pCmd->command == TSDB_SQL_RETRIEVE) {
-    //     // get table meta/vgroup query will not retry, do nothing
-    //   }
-    // }
-
-    if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_INSERT ||
-         pCmd->command == TSDB_SQL_UPDATE_TAGS_VAL) &&
-        (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID ||
-         rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) {
-      tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
-      // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
-      if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
-        pSql->cmd.submitSchema = 1;
-      }
+  STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
 
-      pSql->res.code = rpcMsg->code;  // keep the previous error code
-      if (pSql->retry > pSql->maxRetry) {
-        tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
-      } else {
-        rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name);
-
-        // if there is an error occurring, proceed to the following error handling procedure.
-        // todo add test cases
-        if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
-          rpcFreeCont(rpcMsg->pCont);
-          return;
-        }
+  int32_t cmd = pCmd->command;
+  if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) &&
+      (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID ||
+       rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID ||
+       rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL ||
+       rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) {
+    tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry);
+
+    // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
+    if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+      pSql->cmd.submitSchema = 1;
+    }
+
+    pSql->res.code = rpcMsg->code;  // keep the previous error code
+    if (pSql->retry > pSql->maxRetry) {
+      tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
+    } else {
+      rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name);
+
+      // if there is an error occurring, proceed to the following error handling procedure.
+      if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+        rpcFreeCont(rpcMsg->pCont);
+        return;
       }
     }
   }
@@ -421,7 +452,8 @@ int tscProcessSql(SSqlObj *pSql) {
       return pSql->res.code;
     }
   } else if (pCmd->command < TSDB_SQL_LOCAL) {
-    pSql->epSet = tscMgmtEpSet;
+
+    //pSql->epSet = tscMgmtEpSet;
   } else {  // local handler
     return (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
@@ -525,8 +557,8 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   // pSql->cmd.payloadLen is set during copying data into payload
   pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
-  tscSetDnodeEpSet(pSql, &pTableMeta->vgroupInfo);
-  
+  tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet);
+
   tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit,
       pSql->epSet.numOfEps);
   return TSDB_CODE_SUCCESS;
@@ -567,11 +599,11 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     } else {
       pVgroupInfo = &pTableMeta->vgroupInfo;
     }
-
     tscSetDnodeEpSet(pSql, pVgroupInfo);
+
     if (pVgroupInfo != NULL) {
       pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId);
-    }
+    } 
 
     STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg;
     pTableIdInfo->tid = htonl(pTableMeta->sid);
@@ -588,8 +620,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups);
 
     SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index);
-    
-    // set the vgroup info
+
+    // set the vgroup info 
     tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo);
     pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId);
     
@@ -1136,11 +1168,11 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
       pShowMsg->payloadLen = htons(pPattern->n);
     }
   } else {
-    SSQLToken *pIpAddr = &pShowInfo->prefix;
-    assert(pIpAddr->n > 0 && pIpAddr->type > 0);
+    SSQLToken *pEpAddr = &pShowInfo->prefix;
+    assert(pEpAddr->n > 0 && pEpAddr->type > 0);
 
-    strncpy(pShowMsg->payload, pIpAddr->z, pIpAddr->n);
-    pShowMsg->payloadLen = htons(pIpAddr->n);
+    strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
+    pShowMsg->payloadLen = htons(pEpAddr->n);
   }
 
   pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen;
@@ -1323,7 +1355,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) {
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-  tscSetDnodeEpSet(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo);
+  tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1847,13 +1879,14 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
 
     memcpy(pInfo->vgroupList, pVgroupInfo, size);
     for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
+      //just init, no need to lock
       SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
-
       pVgroups->vgId = htonl(pVgroups->vgId);
       assert(pVgroups->numOfEps >= 1);
 
       for (int32_t k = 0; k < pVgroups->numOfEps; ++k) {
         pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port);
+
       }
 
       pMsg += size;
@@ -1946,8 +1979,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   assert(len <= sizeof(pObj->db));
   tstrncpy(pObj->db, temp, sizeof(pObj->db));
   
-  if (pConnect->epSet.numOfEps > 0) 
-    tscSetMgmtEpSet(&pConnect->epSet);
+  if (pConnect->epSet.numOfEps > 0) {
+    tscEpSetHtons(&pConnect->epSet);
+    tscUpdateMgmtEpSet(&pConnect->epSet);
+  } 
 
   strcpy(pObj->sversion, pConnect->serverVersion);
   pObj->writeAuth = pConnect->writeAuth;

src/client/src/tscSql.c

@@ -15,7 +15,7 @@
 
 #include "hash.h"
 #include "os.h"
-#include "qast.h"
+#include "qAst.h"
 #include "tcache.h"
 #include "tnote.h"
 #include "trpc.h"
@@ -63,7 +63,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   if (ip) {
     if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL;
-    if (port) tscMgmtEpSet.port[0] = port;
+    if (port) tscMgmtEpSet.epSet.port[0] = port;
   } 
  
   void *pDnodeConn = NULL;
@@ -724,6 +724,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
   return len;
 }
 
+static void asyncCallback(void *param, TAOS_RES *tres, int code) {
+  assert(param != NULL);
+  SSqlObj *pSql = ((SSqlObj *)param);
+  pSql->res.code = code;
+  sem_post(&pSql->rspSem);
+}
+
 int taos_validate_sql(TAOS *taos, const char *sql) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
@@ -732,7 +739,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
   }
 
   SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
-  
+  pSql->pTscObj = taos;
+  pSql->signature = pSql;
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
   
@@ -766,10 +774,17 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
     pCmd->pTableList = NULL;
   }
 
-  pRes->code = (uint8_t)tsParseSql(pSql, false);
-  int code = pRes->code;
-
-  tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+  pSql->fp = asyncCallback;
+  pSql->fetchFp = asyncCallback;
+  pSql->param = pSql;
+  int code = tsParseSql(pSql, true);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    sem_wait(&pSql->rspSem);
+    code = pSql->res.code;
+  }
+  if (code != TSDB_CODE_SUCCESS) {
+    tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj);
+  }
   taos_free_result(pSql);
 
   return code;
@@ -865,6 +880,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   }
 
   SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  pSql->pTscObj = taos;
+  pSql->signature = pSql;
   SSqlRes *pRes = &pSql->res;
 
   pRes->numOfTotal = 0;  // the number of getting table meta from server

src/client/src/tscStream.c

@@ -122,7 +122,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
       pQueryInfo->window.ekey = pStream->etime;
     }
   } else {
-    pQueryInfo->window.skey = pStream->stime - pStream->interval;
+    pQueryInfo->window.skey = pStream->stime;
     int64_t etime = taosGetTimestamp(pStream->precision);
     // delay to wait all data in last time window
     if (pStream->precision == TSDB_TIME_PRECISION_MICRO) {
@@ -232,6 +232,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
       (*pStream->fp)(pStream->param, res, row);
     }
 
+    if (!pStream->isProject) {
+      pStream->stime += pStream->slidingTime;
+    }
     // actually only one row is returned. this following is not necessary
     taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream);
   } else {  // numOfRows == 0, all data has been retrieved
@@ -432,6 +435,7 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
   } else {             // timewindow based aggregation stream
     if (stime == 0) {  // no data in meter till now
       stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval;
+      stime -= pStream->interval;
       tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
     } else {
       int64_t newStime = (stime / pStream->interval) * pStream->interval;

src/client/src/tscSub.c

@@ -34,6 +34,7 @@ typedef struct SSubscriptionProgress {
 typedef struct SSub {
   void *                  signature;
   char                    topic[32];
+  sem_t                   sem;
   int64_t                 lastSyncTime;
   int64_t                 lastConsumeTime;
   TAOS *                  taos;
@@ -83,84 +84,108 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
 
 static void asyncCallback(void *param, TAOS_RES *tres, int code) {
   assert(param != NULL);
-  SSqlObj *pSql = ((SSqlObj *)param);
-  
-  pSql->res.code = code;
-  sem_post(&pSql->rspSem);
+  SSub *pSub = ((SSub *)param);
+  pSub->pSql->res.code = code;
+  sem_post(&pSub->sem);
 }
 
 
 static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) {
-  SSub* pSub = NULL;
-
-  TRY( 8 ) {
-    SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj));
-    CLEANUP_PUSH_FREE(true, pSql);
-    SSqlCmd *pCmd = &pSql->cmd;
-    SSqlRes *pRes = &pSql->res;
-
-    if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
-      THROW(TAOS_SYSTEM_ERROR(errno));
-    }
-    CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem);
+  int code = TSDB_CODE_SUCCESS, line = __LINE__;
+  SSqlObj* pSql = NULL;
 
-    pSql->signature = pSql;
-    pSql->param = pSql;
-    pSql->pTscObj = pObj;
-    pSql->maxRetry = TSDB_MAX_REPLICA;
-    pSql->fp = asyncCallback;
-
-    int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
-    if (code != TSDB_CODE_SUCCESS) {
-      THROW(code);
-    }
-    CLEANUP_PUSH_FREE(true, pCmd->payload);
+  SSub* pSub = calloc(1, sizeof(SSub));
+  if (pSub == NULL) {
+    line = __LINE__;
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto fail;
+  }
+  pSub->signature = pSub;
+  if (tsem_init(&pSub->sem, 0, 0) == -1) {
+    line = __LINE__;
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto fail;
+  }
+  tstrncpy(pSub->topic, topic, sizeof(pSub->topic));
+  pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
+  if (pSub->progress == NULL) {
+    line = __LINE__;
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto fail;
+  }
 
-    pRes->qhandle = 0;
-    pRes->numOfRows = 1;
+  pSql = calloc(1, sizeof(SSqlObj));
+  if (pSql == NULL) {
+    line = __LINE__;
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto fail;
+  }
+  pSql->signature = pSql;
+  pSql->pTscObj = pObj;
+  pSql->pSubscription = pSub;
+  pSub->pSql = pSql;
 
-    pSql->sqlstr = strdup_throw(sql);
-    CLEANUP_PUSH_FREE(true, pSql->sqlstr);
-    strtolower(pSql->sqlstr, pSql->sqlstr);
+  SSqlCmd* pCmd = &pSql->cmd;
+  SSqlRes* pRes = &pSql->res;
+  if (tsem_init(&pSql->rspSem, 0, 0) == -1) {
+    line = __LINE__;
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto fail;
+  }
 
-    code = tsParseSql(pSql, false);
-    if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
-      // wait for the callback function to post the semaphore
-      sem_wait(&pSql->rspSem);
-      code = pSql->res.code;
-    }
-    if (code != TSDB_CODE_SUCCESS) {
-      tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code));
-      THROW( code );
-    }
+  pSql->param = pSub;
+  pSql->maxRetry = TSDB_MAX_REPLICA;
+  pSql->fp = asyncCallback;
+  pSql->fetchFp = asyncCallback;
+  pSql->sqlstr = strdup(sql);
+  if (pSql->sqlstr == NULL) {
+    line = __LINE__;
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto fail;
+  }
+  strtolower(pSql->sqlstr, pSql->sqlstr);
+  pRes->qhandle = 0;
+  pRes->numOfRows = 1;
+
+  code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE);
+  if (code != TSDB_CODE_SUCCESS) {
+    line = __LINE__;
+    goto fail;
+  }
 
-    if (pSql->cmd.command != TSDB_SQL_SELECT) {
-      tscError("only 'select' statement is allowed in subscription: %s", pSub->topic);
-      THROW( -1 );  // TODO
-    }
+  code = tsParseSql(pSql, false);
+  if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+    sem_wait(&pSub->sem);
+    code = pSql->res.code;
+  }
+  if (code != TSDB_CODE_SUCCESS) {
+    line = __LINE__;
+    goto fail;
+  }
 
-    pSub = calloc_throw(1, sizeof(SSub));
-    CLEANUP_PUSH_FREE(true, pSub);
-    pSql->pSubscription = pSub;
-    pSub->pSql = pSql;
-    pSub->signature = pSub;
-    strncpy(pSub->topic, topic, sizeof(pSub->topic));
-    pSub->topic[sizeof(pSub->topic) - 1] = 0;
-    pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress));
-    if (pSub->progress == NULL) {
-      THROW(TSDB_CODE_TSC_OUT_OF_MEMORY);
-    }
+  if (pSql->cmd.command != TSDB_SQL_SELECT) {
+    line = __LINE__;
+    code = TSDB_CODE_TSC_INVALID_SQL;
+    goto fail;
+  }
 
-    CLEANUP_EXECUTE();
+  return pSub;
 
-  } CATCH( code ) {
-    tscError("failed to create subscription object: %s", tstrerror(code));
-    CLEANUP_EXECUTE();
+fail:
+  tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code));
+  if (pSql != NULL) {
+    tscFreeSqlObj(pSql);
+    pSql = NULL;
+  }
+  if (pSub != NULL) {
+    taosArrayDestroy(pSub->progress);
+    tsem_destroy(&pSub->sem);
+    free(pSub);
     pSub = NULL;
+  }
 
-  } END_TRY
-
-  return pSub;
+  terrno = code;
+  return NULL;
 }
 
 
@@ -405,9 +430,10 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
     tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0;
 
     pSql->fp = asyncCallback;
-    pSql->param = pSql;
+    pSql->fetchFp = asyncCallback;
+    pSql->param = pSub;
     tscDoQuery(pSql);
-    sem_wait(&pSql->rspSem);
+    sem_wait(&pSub->sem);
 
     if (pRes->code != TSDB_CODE_SUCCESS) {
       continue;
@@ -437,7 +463,9 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
   }
 
   if (keepProgress) {
-    tscSaveSubscriptionProgress(pSub);
+    if (pSub->progress != NULL) {
+      tscSaveSubscriptionProgress(pSub);
+    }
   } else {
     char path[256];
     sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
@@ -448,6 +476,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
 
   tscFreeSqlObj(pSub->pSql);
   taosArrayDestroy(pSub->progress);
+  tsem_destroy(&pSub->sem);
   memset(pSub, 0, sizeof(*pSub));
   free(pSub);
 }

src/client/src/tscSubquery.c

@@ -14,8 +14,8 @@
  */
 #include "os.h"
 
-#include "qtsbuf.h"
-#include "qast.h"
+#include "qAst.h"
+#include "qTsbuf.h"
 #include "tcompare.h"
 #include "tscLog.h"
 #include "tscSubquery.h"

src/client/src/tscSystem.c

@@ -41,7 +41,7 @@ int tscNumOfThreads;
 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
-void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
+//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet);
 
 void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
   taosGetDisk();

src/client/src/tscUtil.c

@@ -13,11 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "hash.h"
 #include "tscUtil.h"
+#include "hash.h"
+#include "os.h"
+#include "qAst.h"
 #include "taosmsg.h"
-#include "qast.h"
 #include "tcache.h"
 #include "tkey.h"
 #include "tmd5.h"
@@ -2146,16 +2146,19 @@ char* strdup_throw(const char* str) {
 }
 
 int tscSetMgmtEpSetFromCfg(const char *first, const char *second) {
-  tscMgmtEpSet.numOfEps = 0;
-  tscMgmtEpSet.inUse = 0;
+  // init mgmt ip set 
+  tscMgmtEpSet.version = 0;
+  SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet);
+  mgmtEpSet->numOfEps = 0;
+  mgmtEpSet->inUse = 0;
 
   if (first && first[0] != 0) {
     if (strlen(first) >= TSDB_EP_LEN) {
       terrno = TSDB_CODE_TSC_INVALID_FQDN;
       return -1;
     }
-    taosGetFqdnPortFromEp(first, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]);
-    tscMgmtEpSet.numOfEps++;
+    taosGetFqdnPortFromEp(first, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
+    mgmtEpSet->numOfEps++;
   }
 
   if (second && second[0] != 0) {
@@ -2163,11 +2166,11 @@ int tscSetMgmtEpSetFromCfg(const char *first, const char *second) {
       terrno = TSDB_CODE_TSC_INVALID_FQDN;
       return -1;
     }
-    taosGetFqdnPortFromEp(second, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]);
-    tscMgmtEpSet.numOfEps++;
+    taosGetFqdnPortFromEp(second, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
+    mgmtEpSet->numOfEps++;
   }
 
-  if ( tscMgmtEpSet.numOfEps == 0) {
+  if (mgmtEpSet->numOfEps == 0) {
     terrno = TSDB_CODE_TSC_INVALID_FQDN;
     return -1;
   }

src/inc/taos.h

@@ -67,8 +67,6 @@ DLL_EXPORT void  taos_init();
 DLL_EXPORT void  taos_cleanup();
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
-DLL_EXPORT TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen,
-                 const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);
 
 typedef struct TAOS_BIND {
@@ -90,7 +88,6 @@ TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt);
 int        taos_stmt_close(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
-DLL_EXPORT TAOS_RES *taos_query_c(TAOS *taos, const char *sql, uint32_t sqlLen);
 DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
 DLL_EXPORT int taos_result_precision(TAOS_RES *res);  // get the time precision of result
 DLL_EXPORT void taos_free_result(TAOS_RES *res);

src/inc/trpc.h

@@ -35,6 +35,11 @@ typedef struct SRpcEpSet {
   char      fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
 } SRpcEpSet;
 
+typedef struct SRpcCorEpSet {
+  int32_t version; 
+  SRpcEpSet epSet; 
+} SRpcCorEpSet;
+
 typedef struct SRpcConnInfo {
   uint32_t  clientIp;
   uint16_t  clientPort;

src/kit/taosdemo/taosdemo.c

@@ -361,7 +361,7 @@ int main(int argc, char *argv[]) {
   arguments.num_of_DPT = 100000;
   arguments.num_of_RPR = 1000;
   arguments.use_metric = true;
-  arguments.insert_only = true;
+  arguments.insert_only = false;
   // end change
 
   argp_parse(&argp, argc, argv, 0, 0, &arguments);
@@ -954,13 +954,13 @@ void *readMetric(void *sarg) {
 
     for (int i = 1; i <= m; i++) {
       if (i == 1) {
-        sprintf(tempS, "index = %d", i);
+        sprintf(tempS, "areaid = %d", i);
       } else {
-        sprintf(tempS, " or index = %d ", i);
+        sprintf(tempS, " or areaid = %d ", i);
       }
       strcat(condition, tempS);
 
-      sprintf(command, "select %s from m1 where %s", aggreFunc[j], condition);
+      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
 
       printf("Where condition: %s\n", condition);
       fprintf(fp, "%s\n", command);

src/query/inc/qast.h → src/query/inc/qAst.h

@@ -45,7 +45,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *);
  *
  */
 typedef struct tQueryInfo {
-  int32_t       colIndex; // index of column in schema
   uint8_t       optr;     // expression operator
   SSchema       sch;      // schema of tags
   char*         q;

src/query/inc/qExecutor.h

@@ -18,16 +18,16 @@
 #include "os.h"
 
 #include "hash.h"
-#include "qfill.h"
-#include "qresultBuf.h"
+#include "qFill.h"
+#include "qResultbuf.h"
+#include "qTsbuf.h"
 #include "qsqlparser.h"
-#include "qtsbuf.h"
+#include "query.h"
 #include "taosdef.h"
 #include "tarray.h"
 #include "tlockfree.h"
 #include "tsdb.h"
 #include "tsqlfunction.h"
-#include "query.h"
 
 struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
@@ -158,7 +158,7 @@ typedef struct SQueryRuntimeEnv {
   SResultInfo*         resultInfo;       // todo refactor to merge with SWindowResInfo
   SQuery*              pQuery;
   SQLFunctionCtx*      pCtx;
-  int16_t              numOfRowsPerPage;
+  int32_t              numOfRowsPerPage;
   int16_t              offset[TSDB_MAX_COLUMNS];
   uint16_t             scanFlag;         // denotes reversed scan of data or not
   SFillInfo*           pFillInfo;

src/query/inc/qextbuffer.h → src/query/inc/qExtbuffer.h

@@ -28,7 +28,7 @@ extern "C" {
 #include "tdataformat.h"
 #include "talgo.h"
 
-#define DEFAULT_PAGE_SIZE (1024L*64)  // 16k larger than the SHistoInfo
+#define DEFAULT_PAGE_SIZE (1024L*4)  // 16k larger than the SHistoInfo
 #define MAX_TMPFILE_PATH_LENGTH PATH_MAX
 #define INITIAL_ALLOCATION_BUFFER_SIZE 64
 

src/query/inc/qfill.h → src/query/inc/qFill.h

@@ -21,8 +21,8 @@ extern "C" {
 #endif
 
 #include "os.h"
+#include "qExtbuffer.h"
 #include "taosdef.h"
-#include "qextbuffer.h"
 
 typedef struct {
   STColumn col;             // column info

src/query/inc/qpercentile.h → src/query/inc/qPercentile.h

@@ -16,7 +16,7 @@
 #ifndef TDENGINE_QPERCENTILE_H
 #define TDENGINE_QPERCENTILE_H
 
-#include "qextbuffer.h"
+#include "qExtbuffer.h"
 
 typedef struct MinMaxEntry {
   union {

src/query/inc/qresultBuf.h → src/query/inc/qResultbuf.h

@@ -20,9 +20,9 @@
 extern "C" {
 #endif
 
-#include "os.h"
-#include "qextbuffer.h"
 #include "hash.h"
+#include "os.h"
+#include "qExtbuffer.h"
 
 typedef struct SArray* SIDList;
 
@@ -33,14 +33,20 @@ typedef struct SDiskbasedResultBuf {
   int32_t   fd;                  // data file fd
   int32_t   allocateId;          // allocated page id
   int32_t   incStep;             // minimum allocated pages
-  char*     pBuf;                // mmap buffer pointer
+  void*     pBuf;                // mmap buffer pointer
   char*     path;                // file path
-  
+  int32_t   pageSize;            // current used page size
+  int32_t   inMemPages;          // numOfPages that are allocated in memory
   SHashObj* idsTable;            // id hash table
   SIDList   list;                // for each id, there is a page id list
+
+  void*     iBuf;                // inmemory buf
+  void*     handle;              // for debug purpose
+  void*     emptyDummyIdList;    // dummy id list
 } SDiskbasedResultBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5)
+#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
+#define DEFAULT_INMEM_BUF_PAGES       10
 
 /**
  * create disk-based result buffer
@@ -49,7 +55,8 @@ typedef struct SDiskbasedResultBuf {
  * @param rowSize
  * @return
  */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle);
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
+    int32_t inMemPages, void* handle);
 
 /**
  *
@@ -81,8 +88,13 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
  * @param id
  * @return
  */
-#define GET_RES_BUF_PAGE_BY_ID(buf, id)  ((tFilePage*)((buf)->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE*(id)))
-
+static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
+  if (id < pResultBuf->inMemPages) {
+    return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize);
+  } else {
+    return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize);
+  }
+}
 /**
  * get the total buffer size in the format of disk file
  * @param pResultBuf

src/query/inc/qUtil.h

@@ -49,7 +49,7 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3
   assert(pResult != NULL && pRuntimeEnv != NULL);
 
   SQuery    *pQuery = pRuntimeEnv->pQuery;
-  tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
   int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
 
   return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
@@ -59,6 +59,4 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3
 __filter_func_t *getRangeFilterFuncArray(int32_t type);
 __filter_func_t *getValueFilterFuncArray(int32_t type);
 
-bool supportPrefilter(int32_t type);
-
 #endif  // TDENGINE_QUERYUTIL_H

src/query/inc/tsqlfunction.h

@@ -255,7 +255,15 @@ extern int32_t functionCompatList[]; // compatible check array list
 
 bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval);
 
-void resetResultInfo(SResultInfo *pResInfo);
+/**
+ * the numOfRes should be kept, since it may be used later
+ * and allow the ResultInfo to be re initialized
+ */
+#define RESET_RESULT_INFO(_r)  \
+  do {                         \
+    (_r)->initialized = false; \
+  } while (0)
+
 void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);
 
 static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) {

src/query/src/qast.c → src/query/src/qAst.c

@@ -16,17 +16,17 @@
 
 #include "os.h"
 
-#include "tname.h"
-#include "qast.h"
-#include "tsdb.h"
 #include "exception.h"
+#include "qAst.h"
+#include "qSyntaxtreefunction.h"
 #include "qsqlparser.h"
-#include "qsyntaxtreefunction.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tarray.h"
 #include "tbuffer.h"
 #include "tcompare.h"
+#include "tname.h"
+#include "tsdb.h"
 #include "tskiplist.h"
 #include "tsqlfunction.h"
 #include "tstoken.h"
@@ -678,7 +678,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
 
     tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
     // todo speed up by using hash
-    if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
+    if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
         addToResult = pQueryInfo->compare(name, pQueryInfo->q);
       } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) {
@@ -716,7 +716,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
     }
 
     tQueryInfo *pQueryInfo = pExpr->_node.info;
-    if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) {
+    if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) {
       tQueryIndexColumn(pSkipList, pQueryInfo, result);
     } else {
       tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn);

src/query/src/qextbuffer.c → src/query/src/qExtbuffer.c

@@ -12,16 +12,15 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "qExtbuffer.h"
 #include "os.h"
-#include "tulog.h"
-#include "qextbuffer.h"
+#include "queryLog.h"
 #include "taos.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tsqlfunction.h"
-#include "ttime.h"
+#include "tulog.h"
 #include "tutil.h"
-#include "queryLog.h"
 
 #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \
   (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes)

src/query/src/qfill.c → src/query/src/qFill.c

@@ -13,9 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qFill.h"
 #include "os.h"
-#include "qfill.h"
-#include "qextbuffer.h"
+#include "qExtbuffer.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tsqlfunction.h"

src/query/src/qFilterFunc.c → src/query/src/qFilterfunc.c

@@ -554,5 +554,3 @@ __filter_func_t* getValueFilterFuncArray(int32_t type) {
     default: return NULL;
   }
 }
-
-bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; }

src/query/src/qhistogram.c → src/query/src/qHistogram.c

@@ -14,7 +14,7 @@
  */
 #include "os.h"
 
-#include "qhistogram.h"
+#include "qHistogram.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tlosertree.h"

src/query/src/qpercentile.c → src/query/src/qPercentile.c

@@ -13,12 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qPercentile.h"
 #include "os.h"
-#include "tulog.h"
-#include "qpercentile.h"
+#include "queryLog.h"
 #include "taosdef.h"
 #include "taosmsg.h"
-#include "queryLog.h"
+#include "tulog.h"
 
 tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, int16_t slotIdx) {
   tExtMemBuffer *pBuffer = NULL;

src/query/src/qresultBuf.c → src/query/src/qResultbuf.c

-#include "qresultBuf.h"
+#include "qResultbuf.h"
 #include "hash.h"
-#include "qextbuffer.h"
-#include "taoserror.h"
+#include "qExtbuffer.h"
 #include "queryLog.h"
+#include "taoserror.h"
+
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize,
+    int32_t pagesize, int32_t inMemPages, void* handle) {
 
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) {
   *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
   SDiskbasedResultBuf* pResBuf = *pResultBuf;
   if (pResBuf == NULL) {
     return TSDB_CODE_COM_OUT_OF_MEMORY;  
   }
-  
-  pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize;
-  pResBuf->numOfPages = size;
 
-  pResBuf->totalBufSize = pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE;
+  pResBuf->pageSize   = pagesize;
+  pResBuf->numOfPages = inMemPages;  // all pages are in buffer in the first place
+  pResBuf->inMemPages = inMemPages;
+  assert(inMemPages <= numOfPages);
+
+  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
+
+  pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
   pResBuf->incStep = 4;
+  pResBuf->allocateId = -1;
+
+  pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize);
 
   // init id hash table
-  pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
-  pResBuf->list = taosArrayInit(size, POINTER_BYTES);
+  pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES);
 
-  char path[4096] = {0};
+  char path[PATH_MAX] = {0};
   getTmpfilePath("tsdb_qbuf", path);
   pResBuf->path = strdup(path);
 
-  pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666);
-  if (!FD_VALID(pResBuf->fd)) {
-    qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno));
+  pResBuf->fd = FD_INITIALIZER;
+  pResBuf->pBuf = NULL;
+  pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
+
+  qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
+      pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize);
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); }
+
+int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
+
+#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages)
+#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
+
+static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
+  pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666);
+  if (!FD_VALID(pResultBuf->fd)) {
+    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE);
+  assert(pResultBuf->numOfPages == pResultBuf->inMemPages);
+  pResultBuf->numOfPages += pResultBuf->incStep;
+
+  int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
   if (ret != TSDB_CODE_SUCCESS) {
-    qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno));
+    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0);
-  if (pResBuf->pBuf == MAP_FAILED) {
-    qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno));
+  pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
+  if (pResultBuf->pBuf == MAP_FAILED) {
+    qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  qDebug("QInfo:%p create tmp file for output result:%s, %" PRId64 "bytes", handle, pResBuf->path,
-      pResBuf->totalBufSize);
-  
+  pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); }
+static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) {
+  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize);
+  int32_t ret = TSDB_CODE_SUCCESS;
 
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
+  if (pResultBuf->pBuf == NULL) {
+    assert(pResultBuf->fd == FD_INITIALIZER);
 
-static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) {
-  assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE == pResultBuf->totalBufSize);
-
-  int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize);
-  pResultBuf->numOfPages += numOfPages;
-
-  /*
-   * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
-   * be insufficient
-   */
-  ret = ftruncate(pResultBuf->fd, pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE);
-  if (ret != 0) {
-    //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
-    //           strerror(errno));
-    return TSDB_CODE_QRY_NO_DISKSPACE;
-  }
+    if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+  } else {
+    ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
+    pResultBuf->numOfPages += incNumOfPages;
+
+    /*
+     * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
+     * be insufficient
+     */
+    ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
+    if (ret != TSDB_CODE_SUCCESS) {
+      //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
+      //           strerror(errno));
+      return TSDB_CODE_QRY_NO_DISKSPACE;
+    }
 
-  pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE;
-  pResultBuf->pBuf = mmap(NULL, pResultBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
+    pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
+    pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
 
-  if (pResultBuf->pBuf == MAP_FAILED) {
-    //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
-    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    if (pResultBuf->pBuf == MAP_FAILED) {
+      //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
+      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) {
-  return (pResultBuf->allocateId == pResultBuf->numOfPages - 1);
-}
+#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1)
 
 static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
   assert(pResultBuf != NULL);
@@ -121,20 +152,19 @@ static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int
 }
 
 tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
-  if (noMoreAvailablePages(pResultBuf)) {
+  if (NO_AVAILABLE_PAGES(pResultBuf)) {
     if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
       return NULL;
     }
   }
 
   // register new id in this group
-  *pageId = (pResultBuf->allocateId++);
+  *pageId = (++pResultBuf->allocateId);
   registerPageId(pResultBuf, groupId, *pageId);
 
-  tFilePage* page = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pageId);
-  
   // clear memory for the new page
-  memset(page, 0, DEFAULT_INTERN_BUF_PAGE_SIZE);
+  tFilePage* page = getResBufPage(pResultBuf, *pageId);
+  memset(page, 0, pResultBuf->pageSize);
   
   return page;
 }
@@ -144,7 +174,7 @@ int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf
 SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
   int32_t slot = getGroupIndex(pResultBuf, groupId);
   if (slot < 0) {
-    return taosArrayInit(1, sizeof(int32_t));
+    return pResultBuf->emptyDummyIdList;
   } else {
     return taosArrayGetP(pResultBuf->list, slot);
   }
@@ -156,13 +186,18 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
   }
 
   if (FD_VALID(pResultBuf->fd)) {
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
+        pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
+
     close(pResultBuf->fd);
+    munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
+    pResultBuf->pBuf = NULL;
+  } else {
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
+           pResultBuf->totalBufSize);
   }
 
-  qDebug("QInfo:%p disk-based output buffer closed, %" PRId64 " bytes, file:%s", handle, pResultBuf->totalBufSize, pResultBuf->path);
-  munmap(pResultBuf->pBuf, pResultBuf->totalBufSize);
   unlink(pResultBuf->path);
-  
   tfree(pResultBuf->path);
 
   size_t size = taosArrayGetSize(pResultBuf->list);
@@ -172,8 +207,10 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
   }
 
   taosArrayDestroy(pResultBuf->list);
+  taosArrayDestroy(pResultBuf->emptyDummyIdList);
   taosHashCleanup(pResultBuf->idsTable);
-  
+
+  tfree(pResultBuf->iBuf);
   tfree(pResultBuf);
 }
 

src/query/src/qsyntaxtreefunction.c → src/query/src/qSyntaxtreefunction.c

@@ -15,7 +15,7 @@
 
 #include "os.h"
 
-#include "qsyntaxtreefunction.h"
+#include "qSyntaxtreefunction.h"
 #include "taosdef.h"
 #include "tutil.h"
 

src/query/src/qtsbuf.c → src/query/src/qTsbuf.c

-#include "qtsbuf.h"
+#include "qTsbuf.h"
+#include "taoserror.h"
 #include "tscompression.h"
 #include "tutil.h"
-#include "taoserror.h"
 
 static int32_t getDataStartOffset();
 static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo);

src/query/src/qUtil.c

@@ -26,12 +26,10 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) {
   int32_t size = 0;
 
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE);
     size += pQuery->pSelectExpr[i].interBytes;
   }
 
   assert(size > 0);
-
   return size;
 }
 
@@ -243,7 +241,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
     size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     memset(s, 0, size);
     
-    resetResultInfo(pResultInfo);
+    RESET_RESULT_INFO(pResultInfo);
   }
   
   pWindowRes->numOfRows = 0;

src/query/tests/CMakeLists.txt

@@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
     AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
     ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
-    TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
-ENDIF()
+    TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov)
+ENDIF()
\ No newline at end of file

src/query/tests/astTest.cpp

@@ -3,8 +3,8 @@
 #include <cassert>
 #include <iostream>
 
+#include "qAst.h"
 #include "taosmsg.h"
-#include "qast.h"
 #include "tsdb.h"
 #include "tskiplist.h"
 

src/query/tests/histogramTest.cpp

@@ -9,7 +9,7 @@
 #include "tstoken.h"
 #include "tutil.h"
 
-#include "qhistogram.h"
+#include "qHistogram.h"
 
 /* test validate the names for table/database */
 TEST(testCase, histogram_binary_search) {

src/query/tests/resultBufferTest.cpp

@@ -2,15 +2,15 @@
 #include <cassert>
 #include <iostream>
 
+#include "qResultbuf.h"
 #include "taos.h"
-#include "qresultBuf.h"
 #include "tsdb.h"
 
 namespace {
 // simple test
 void simpleTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -22,8 +22,7 @@ void simpleTest() {
   ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
   
   SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
-  ASSERT_EQ(list.size, 1);
-  
+  ASSERT_EQ(taosArrayGetSize(list), 1);
   ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
   
   destroyResultBuf(pResultBuf, NULL);

src/query/tests/tsBufTest.cpp

@@ -5,10 +5,10 @@
 #include "taos.h"
 #include "tsdb.h"
 
+#include "qTsbuf.h"
 #include "tstoken.h"
 #include "ttime.h"
 #include "tutil.h"
-#include "qtsbuf.h"
 
 namespace {
 /**

src/rpc/src/rpcMain.c

@@ -446,7 +446,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) {
   // set the idle timer to monitor the activity
   taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer);
   rpcSendMsgToPeer(pConn, msg, msgLen);
-  pConn->secured = 1; // connection shall be secured
+
+  // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured
+  if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY)
+    pConn->secured = 1; // connection shall be secured
 
   if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg);
   pConn->pReqMsg = NULL;

src/tsdb/inc/tsdbMain.h

@@ -426,6 +426,7 @@ int         tsdbUpdateFileHeader(SFile* pFile, uint32_t version);
 int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
 void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
 void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
+void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
 
 // ------------------ tsdbRWHelper.c
 #define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state

src/tsdb/src/tsdbMemTable.c

@@ -31,7 +31,6 @@ static int         tsdbCommitMeta(STsdbRepo *pRepo);
 static void        tsdbEndCommit(STsdbRepo *pRepo);
 static int         tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
 static int  tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
-static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
 static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
 static void         tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
 
@@ -544,7 +543,7 @@ static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSK
   return 0;
 }
 
-static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
+void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
   *minKey = fileId * daysPerFile * tsMsPerDay[precision];
   *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
 }

src/tsdb/src/tsdbRead.c

@@ -21,7 +21,7 @@
 #include "tcompare.h"
 #include "exception.h"
 
-#include "../../../query/inc/qast.h"  // todo move to common module
+#include "../../query/inc/qAst.h"  // todo move to common module
 #include "tlosertree.h"
 #include "tsdb.h"
 #include "tsdbMain.h"
@@ -128,8 +128,7 @@ typedef struct STsdbQueryHandle {
 
 static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle);
 static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle);
-static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
-                                   SArray* sa);
+static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock);
 static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
 static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win,
                                  STsdbQueryHandle* pQueryHandle);
@@ -240,7 +239,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
 
   pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true);
 
-  tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo));
+  tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo);
 
   tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo);
   tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo);
@@ -331,7 +330,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle,
            pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo);
   } else {
-    tsdbDebug("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid);
+    tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
+        pHandle->qinfo);
   }
   
   if (!imemEmpty) {
@@ -343,7 +343,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle,
            pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo);
   } else {
-    tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid);
+    tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
+        pHandle->qinfo);
   }
   
   return true;
@@ -354,7 +355,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
   tSkipListDestroyIter(pCheckInfo->iiter);
 }
 
-SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
+SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) {
   SDataRow rmem = NULL, rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
@@ -371,20 +372,35 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
   }
 
   if (rmem != NULL && rimem != NULL) {
-    if (dataRowKey(rmem) < dataRowKey(rimem)) {
-      pCheckInfo->chosen = 0;
-      return rmem;
-    } else if (dataRowKey(rmem) == dataRowKey(rimem)) {
-      // data ts are duplicated, ignore the data in mem
+    TSKEY r1 = dataRowKey(rmem);
+    TSKEY r2 = dataRowKey(rimem);
+
+    if (r1 == r2) { // data ts are duplicated, ignore the data in mem
       tSkipListIterNext(pCheckInfo->iter);
       pCheckInfo->chosen = 1;
       return rimem;
     } else {
-      pCheckInfo->chosen = 1;
-      return rimem;
+      if (ASCENDING_TRAVERSE(order)) {
+        if (r1 < r2) {
+          pCheckInfo->chosen = 0;
+          return rmem;
+        } else {
+          pCheckInfo->chosen = 1;
+          return rimem;
+        }
+      } else {
+        if (r1 < r2) {
+          pCheckInfo->chosen = 1;
+          return rimem;
+        } else {
+          pCheckInfo->chosen = 0;
+          return rmem;
+        }
+      }
     }
   }
 
+  // at least one (rmem or rimem) is absent here
   if (rmem != NULL) {
     pCheckInfo->chosen = 0;
     return rmem;
@@ -398,7 +414,7 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) {
   return NULL;
 }
 
-static bool moveToNextRow(STableCheckInfo* pCheckInfo) {
+static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
   bool hasNext = false;
   if (pCheckInfo->chosen == 0) {
     if (pCheckInfo->iter != NULL) {
@@ -412,19 +428,17 @@ static bool moveToNextRow(STableCheckInfo* pCheckInfo) {
     if (pCheckInfo->iiter != NULL) {
       return tSkipListIterGet(pCheckInfo->iiter) != NULL;
     }
-  } else {
-    if (pCheckInfo->chosen == 1) {
-      if (pCheckInfo->iiter != NULL) {
-        hasNext = tSkipListIterNext(pCheckInfo->iiter);
-      }
+  } else { //pCheckInfo->chosen == 1
+    if (pCheckInfo->iiter != NULL) {
+      hasNext = tSkipListIterNext(pCheckInfo->iiter);
+    }
 
-      if (hasNext) {
-        return hasNext;
-      }
+    if (hasNext) {
+      return hasNext;
+    }
 
-      if (pCheckInfo->iter != NULL) {
-        return tSkipListIterGet(pCheckInfo->iter) != NULL;
-      }
+    if (pCheckInfo->iter != NULL) {
+      return tSkipListIterGet(pCheckInfo->iter) != NULL;
     }
   }
 
@@ -445,7 +459,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
     initTableMemIterator(pHandle, pCheckInfo);
   }
 
-  SDataRow row = getSDataRowInTableMem(pCheckInfo);
+  SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order);
   if (row == NULL) {
     return false;
   }
@@ -650,7 +664,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
   SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
 
   /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
-  SDataRow row = getSDataRowInTableMem(pCheckInfo);
+  SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
 
   TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
@@ -680,7 +694,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
     }
   
     doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
-    doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
+    doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
   } else {
     /*
      * no data in cache, only load data from file
@@ -696,6 +710,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
     cur->mixBlock = false;
     cur->blockCompleted = true;
     cur->lastKey = binfo.window.ekey + (ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1);
+    pCheckInfo->lastKey = cur->lastKey;
   }
 }
 
@@ -719,7 +734,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
         cur->pos = 0;
       }
       
-      doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
+      doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {  // the whole block is loaded in to buffer
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
     }
@@ -736,7 +751,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
         cur->pos = pBlock->numOfRows - 1;
       }
       
-      doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn);
+      doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
     }
@@ -892,12 +907,12 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
   
   pQueryHandle->cur.win.ekey = tsArray[end];
   pQueryHandle->cur.lastKey = tsArray[end] + step;
-  
+
   return numOfRows + num;
 }
 
 static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
-                              STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) {
+                              int32_t numOfCols, STable* pTable) {
   char* pData = NULL;
 
   // the schema version info is embeded in SDataRow
@@ -958,8 +973,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
 
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
-static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
-                                  SArray* sa) {
+static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
   
@@ -972,7 +986,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
   int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
 
-  STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
   STable* pTable = pCheckInfo->pTableObj;
 
   int32_t endPos = cur->pos;
@@ -1033,7 +1046,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
     do {
-      SDataRow row = getSDataRowInTableMem(pCheckInfo);
+      SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
       if (row == NULL) {
         break;
       }
@@ -1051,7 +1064,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
       if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-        copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable);
+        copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable);
         numOfRows += 1;
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = key;
@@ -1061,9 +1074,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         cur->lastKey  = key + step;
         cur->mixBlock = true;
 
-        moveToNextRow(pCheckInfo);
+        moveToNextRowInMem(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
-        moveToNextRow(pCheckInfo);
+        moveToNextRowInMem(pCheckInfo);
       } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
                   (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1072,7 +1085,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
         int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order);
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
-          moveToNextRow(pCheckInfo);
+          moveToNextRowInMem(pCheckInfo);
         }
         
         int32_t start = -1;
@@ -1376,7 +1389,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
    * }
    */
 
-  tsdbDebug("%p %d data blocks sort completed", pQueryHandle, cnt);
+  tsdbDebug("%p %d data blocks sort completed, %p", pQueryHandle, cnt, pQueryHandle->qinfo);
   cleanBlockOrderSupporter(&sup, numOfTables);
   free(pTree);
 
@@ -1391,8 +1404,21 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
 
   int32_t numOfBlocks = 0;
   int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
+  STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
+  STimeWindow win = TSWINDOW_INITIALIZER;
+
   while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) {
+    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey);
+
+    // current file are not overlapped with query time window, ignore remain files
+    if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
+    (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+      tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo)
+      pQueryHandle->pFileGroup = NULL;
+      break;
+    }
+
     if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
@@ -1750,11 +1776,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   win->skey = TSKEY_INITIAL_VAL;
 
   int64_t st = taosGetTimestampUs();
-  STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
   STable* pTable = pCheckInfo->pTableObj;
 
   do {
-    SDataRow row = getSDataRowInTableMem(pCheckInfo);
+    SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order);
     if (row == NULL) {
       break;
     }
@@ -1772,14 +1797,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     }
 
     win->ekey = key;
-    copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable);
+    copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable);
 
     if (++numOfRows >= maxRowsToRead) {
-      moveToNextRow(pCheckInfo);
+      moveToNextRowInMem(pCheckInfo);
       break;
     }
     
-  } while(moveToNextRow(pCheckInfo));
+  } while(moveToNextRowInMem(pCheckInfo));
 
   assert(numOfRows <= maxRowsToRead);
   
@@ -1869,7 +1894,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
       pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows;
     }
 
-    // todo opt perf
     SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i);
     if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) {
       pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst;
@@ -1961,43 +1985,20 @@ static void destroyHelper(void* param) {
   free(param);
 }
 
-#define TAG_INVALID_COLUMN_INDEX  -2
-static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) {
-  // filter on table name(TBNAME)
-  if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) {
-    return TSDB_TBNAME_COLUMN_INDEX;
-  }
-  
-  for(int32_t i = 0; i < schemaNCols(pTSchema); ++i) {
-    STColumn* pColumn = &pTSchema->columns[i];
-    if (pColumn->bytes == pSchema->bytes && pColumn->type  == pSchema->type  && pColumn->colId == pSchema->colId) {
-      return i;
-    }
-  }
-  
-  return -2;
-}
-
 void filterPrepare(void* expr, void* param) {
   tExprNode* pExpr = (tExprNode*)expr;
   if (pExpr->_node.info != NULL) {
     return;
   }
 
-  int32_t i = 0;
   pExpr->_node.info = calloc(1, sizeof(tQueryInfo));
   
-  STSchema* pTSSchema = (STSchema*) param;
-
+  STSchema*   pTSSchema = (STSchema*) param;
   tQueryInfo* pInfo = pExpr->_node.info;
   tVariant*   pCond = pExpr->_node.pRight->pVal;
   SSchema*    pSchema = pExpr->_node.pLeft->pSchema;
 
-  int32_t index = getTagColumnIndex(pTSSchema, pSchema);
-  assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX) || index == TAG_INVALID_COLUMN_INDEX);
-
   pInfo->sch      = *pSchema;
-  pInfo->colIndex = index;
   pInfo->optr     = pExpr->_node.optr;
   pInfo->compare  = getComparFunc(pSchema->type, pInfo->optr);
   pInfo->param    = pTSSchema;
@@ -2143,7 +2144,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
 
   char*  val = NULL;
 
-  if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) {
+  if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
     val = (char*) TABLE_NAME(pTable);
   } else {
     val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);

src/util/inc/tlockfree.h

@@ -75,7 +75,7 @@ void taosRUnLockLatch(SRWLatch *pLatch);
 
 // copy on read
 #define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \
-    int32_t old_ = atomic_load_32(x); \
+    int32_t old_ = atomic_add_fetch_32((x), 0); \
     if (old_ & 0x00000001) { \
       if (i_ % 1000 == 0) { \
         sched_yield(); \
@@ -84,7 +84,7 @@ void taosRUnLockLatch(SRWLatch *pLatch);
     }
 
 #define taosCorEndRead(x) \
-    if (atomic_load_32(x) == old_) { \
+    if (atomic_add_fetch_32((x), 0) == old_) { \
       break; \
     } \
   }

src/util/inc/tutil.h

@@ -35,12 +35,12 @@ extern "C" {
 #define WCHAR wchar_t
 
 #define tfree(x)         \
-  {                      \
+  do {                   \
     if (x) {             \
       free((void *)(x)); \
       x = 0;             \
     }                    \
-  }
+  } while(0);
 
 #define tstrncpy(dst, src, size)   \
   do {                             \

src/util/src/tutil.c

@@ -522,7 +522,7 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP
 void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
   const char* tdengineTmpFileNamePrefix = "tdengine-";
   
-  char tmpPath[PATH_MAX] = {0};
+  char tmpPath[PATH_MAX];
 
 #ifdef WINDOWS
   char *tmpDir = getenv("tmp");

src/util/tests/CMakeLists.txt

@@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
     AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
     ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
-    TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread)
-ENDIF()
+    TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread gcov)
+ENDIF()
\ No newline at end of file

tests/examples/c/apitest.c

+// sample code to verify all TDengine API
+// to compile: gcc -o apitest apitest.c -ltaos
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <taos.h>
+#include <unistd.h>
+
+
+static void prepare_data(TAOS* taos) {
+  taos_query(taos, "drop database if exists test;");
+  usleep(100000);
+  taos_query(taos, "create database test;");
+  usleep(100000);
+  taos_select_db(taos, "test");
+
+  taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);");
+
+  taos_query(taos, "create table t0 using meters tags(0);");
+  taos_query(taos, "create table t1 using meters tags(1);");
+  taos_query(taos, "create table t2 using meters tags(2);");
+  taos_query(taos, "create table t3 using meters tags(3);");
+  taos_query(taos, "create table t4 using meters tags(4);");
+  taos_query(taos, "create table t5 using meters tags(5);");
+  taos_query(taos, "create table t6 using meters tags(6);");
+  taos_query(taos, "create table t7 using meters tags(7);");
+  taos_query(taos, "create table t8 using meters tags(8);");
+  taos_query(taos, "create table t9 using meters tags(9);");
+
+  TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)"
+    " ('2020-01-01 00:01:00.000', 0)"
+    " ('2020-01-01 00:02:00.000', 0)"
+    " t1 values('2020-01-01 00:00:00.000', 0)"
+    " ('2020-01-01 00:01:00.000', 0)"
+    " ('2020-01-01 00:02:00.000', 0)"
+    " ('2020-01-01 00:03:00.000', 0)"
+    " t2 values('2020-01-01 00:00:00.000', 0)"
+    " ('2020-01-01 00:01:00.000', 0)"
+    " ('2020-01-01 00:01:01.000', 0)"
+    " ('2020-01-01 00:01:02.000', 0)"
+    " t3 values('2020-01-01 00:01:02.000', 0)"
+    " t4 values('2020-01-01 00:01:02.000', 0)"
+    " t5 values('2020-01-01 00:01:02.000', 0)"
+    " t6 values('2020-01-01 00:01:02.000', 0)"
+    " t7 values('2020-01-01 00:01:02.000', 0)"
+    " t8 values('2020-01-01 00:01:02.000', 0)"
+    " t9 values('2020-01-01 00:01:02.000', 0)");
+  int affected = taos_affected_rows(res);
+  if (affected != 18) {
+    printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected);
+  }
+  // super tables subscription
+  usleep(1000000);
+}
+
+
+static int print_result(TAOS_RES* res, int blockFetch) {
+  TAOS_ROW    row = NULL;
+  int         num_fields = taos_num_fields(res);
+  TAOS_FIELD* fields = taos_fetch_fields(res);
+  int         nRows = 0;
+
+  if (blockFetch) {
+    int rows = 0;
+    while ((rows = taos_fetch_block(res, &row))) {
+      for (int i = 0; i < rows; i++) {
+        char temp[256];
+        taos_print_row(temp, row + i, fields, num_fields);
+        puts(temp);
+      }
+      nRows += rows;
+    }
+  } else {
+    while ((row = taos_fetch_row(res))) {
+      char temp[256];
+      taos_print_row(temp, row, fields, num_fields);
+      puts(temp);
+      nRows++;
+    }
+  }
+
+  printf("%d rows consumed.\n", nRows);
+  return nRows;
+}
+
+
+static void check_row_count(int line, TAOS_RES* res, int expected) {
+  int actual = print_result(res, expected % 2);
+  if (actual != expected) {
+    printf("\033[31mline %d: row count mismatch, expected: %d, actual: %d\033[0m\n", line, expected, actual);
+  } else {
+    printf("line %d: %d rows consumed as expected\n", line, actual);
+  }
+}
+
+
+static void verify_query(TAOS* taos) {
+  prepare_data(taos);
+
+  int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9");
+  if (code != 0) {
+    printf("\033[31mfailed to load table info: 0x%08x\033[0m\n", code);
+  }
+
+  code = taos_validate_sql(taos, "select * from nonexisttable");
+  if (code == 0) {
+    printf("\033[31mimpossible, the table does not exists\033[0m\n");
+  }
+
+  code = taos_validate_sql(taos, "select * from meters");
+  if (code != 0) {
+    printf("\033[31mimpossible, the table does exists: 0x%08x\033[0m\n", code);
+  }
+
+  TAOS_RES* res = taos_query(taos, "select * from meters");
+  check_row_count(__LINE__, res, 18);
+  printf("result precision is: %d\n",  taos_result_precision(res));
+  int c = taos_field_count(res);
+  printf("field count is: %d\n",  c);
+  int* lengths = taos_fetch_lengths(res);
+  for (int i = 0; i < c; i++) {
+    printf("length of column %d is %d\n", i, lengths[i]);
+  }
+  taos_free_result(res);
+
+  res = taos_query(taos, "select * from t0");
+  check_row_count(__LINE__, res, 3);
+  taos_free_result(res);
+
+  res = taos_query(taos, "select * from nonexisttable");
+  code = taos_errno(res);
+  printf("code=%d, error msg=%s\n", code, taos_errstr(res));
+  taos_free_result(res);
+
+  res = taos_query(taos, "select * from meters");
+  taos_stop_query(res);
+}
+
+
+void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
+  int rows = print_result(res, *(int*)param);
+  printf("%d rows consumed in subscribe_callback\n", rows);
+}
+
+static void verify_subscribe(TAOS* taos) {
+  prepare_data(taos);
+
+  TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0);
+  TAOS_RES* res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 18);
+
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 0);
+
+  taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);");
+  taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);");
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 2);
+
+  taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);");
+  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);");
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 2);
+
+  taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);");
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 1);
+
+  // keep progress information and restart subscription
+  taos_unsubscribe(tsub, 1);
+  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);");
+  tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0);
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 24);
+
+  // keep progress information and continue previous subscription
+  taos_unsubscribe(tsub, 1);
+  tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0);
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 0);
+
+  // don't keep progress information and continue previous subscription
+  taos_unsubscribe(tsub, 0);
+  tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0);
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 24);
+
+  // single meter subscription
+
+  taos_unsubscribe(tsub, 0);
+  tsub = taos_subscribe(taos, 0, "test", "select * from t0;", NULL, NULL, 0);
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 5);
+
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 0);
+
+  taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);");
+  res = taos_consume(tsub);
+  check_row_count(__LINE__, res, 1);
+
+  taos_unsubscribe(tsub, 0);
+
+  int blockFetch = 0;
+  tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000);
+  usleep(2000000);
+  taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);");
+  usleep(2000000);
+  taos_unsubscribe(tsub, 0);
+}
+
+
+void verify_prepare(TAOS* taos) {
+  TAOS_RES* result = taos_query(taos, "drop database if exists test;");
+  usleep(100000);
+  taos_query(taos, "create database test;");
+
+  int code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  usleep(100000);
+  taos_select_db(taos, "test");
+
+  // create table
+  const char* sql = "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))";
+  result = taos_query(taos, sql);
+  code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  // insert 10 records
+  struct {
+      int64_t ts;
+      int8_t b;
+      int8_t v1;
+      int16_t v2;
+      int32_t v4;
+      int64_t v8;
+      float f4;
+      double f8;
+      char bin[40];
+      char blob[80];
+  } v = {0};
+
+  TAOS_STMT* stmt = taos_stmt_init(taos);
+  TAOS_BIND params[10];
+  params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
+  params[0].buffer_length = sizeof(v.ts);
+  params[0].buffer = &v.ts;
+  params[0].length = &params[0].buffer_length;
+  params[0].is_null = NULL;
+
+  params[1].buffer_type = TSDB_DATA_TYPE_BOOL;
+  params[1].buffer_length = sizeof(v.b);
+  params[1].buffer = &v.b;
+  params[1].length = &params[1].buffer_length;
+  params[1].is_null = NULL;
+
+  params[2].buffer_type = TSDB_DATA_TYPE_TINYINT;
+  params[2].buffer_length = sizeof(v.v1);
+  params[2].buffer = &v.v1;
+  params[2].length = &params[2].buffer_length;
+  params[2].is_null = NULL;
+
+  params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT;
+  params[3].buffer_length = sizeof(v.v2);
+  params[3].buffer = &v.v2;
+  params[3].length = &params[3].buffer_length;
+  params[3].is_null = NULL;
+
+  params[4].buffer_type = TSDB_DATA_TYPE_INT;
+  params[4].buffer_length = sizeof(v.v4);
+  params[4].buffer = &v.v4;
+  params[4].length = &params[4].buffer_length;
+  params[4].is_null = NULL;
+
+  params[5].buffer_type = TSDB_DATA_TYPE_BIGINT;
+  params[5].buffer_length = sizeof(v.v8);
+  params[5].buffer = &v.v8;
+  params[5].length = &params[5].buffer_length;
+  params[5].is_null = NULL;
+
+  params[6].buffer_type = TSDB_DATA_TYPE_FLOAT;
+  params[6].buffer_length = sizeof(v.f4);
+  params[6].buffer = &v.f4;
+  params[6].length = &params[6].buffer_length;
+  params[6].is_null = NULL;
+
+  params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE;
+  params[7].buffer_length = sizeof(v.f8);
+  params[7].buffer = &v.f8;
+  params[7].length = &params[7].buffer_length;
+  params[7].is_null = NULL;
+
+  params[8].buffer_type = TSDB_DATA_TYPE_BINARY;
+  params[8].buffer_length = sizeof(v.bin);
+  params[8].buffer = v.bin;
+  params[8].length = &params[8].buffer_length;
+  params[8].is_null = NULL;
+
+  strcpy(v.blob, "一二三四五六七八九十");
+  params[9].buffer_type = TSDB_DATA_TYPE_NCHAR;
+  params[9].buffer_length = strlen(v.blob);
+  params[9].buffer = v.blob;
+  params[9].length = &params[9].buffer_length;
+  params[9].is_null = NULL;
+
+  int is_null = 1;
+
+  sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)";
+  code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code);
+  }
+  v.ts = 1591060628000;
+  for (int i = 0; i < 10; ++i) {
+    v.ts += 1;
+    for (int j = 1; j < 10; ++j) {
+      params[j].is_null = ((i == j) ? &is_null : 0);
+    }
+    v.b = (int8_t)i % 2;
+    v.v1 = (int8_t)i;
+    v.v2 = (int16_t)(i * 2);
+    v.v4 = (int32_t)(i * 4);
+    v.v8 = (int64_t)(i * 8);
+    v.f4 = (float)(i * 40);
+    v.f8 = (double)(i * 80);
+    for (int j = 0; j < sizeof(v.bin) - 1; ++j) {
+      v.bin[j] = (char)(i + '0');
+    }
+
+    taos_stmt_bind_param(stmt, params);
+    taos_stmt_add_batch(stmt);
+  }
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute insert statement.\033[0m\n");
+    return;
+  }
+  taos_stmt_close(stmt);
+
+  // query the records
+  stmt = taos_stmt_init(taos);
+  taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0);
+  v.v1 = 5;
+  v.v2 = 15;
+  taos_stmt_bind_param(stmt, params + 2);
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute select statement.\033[0m\n");
+    return;
+  }
+
+  result = taos_stmt_use_result(stmt);
+
+  TAOS_ROW    row;
+  int         rows = 0;
+  int         num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  char        temp[256];
+
+  // fetch the records row by row
+  while ((row = taos_fetch_row(result))) {
+    rows++;
+    taos_print_row(temp, row, fields, num_fields);
+    printf("%s\n", temp);
+  }
+
+  taos_free_result(result);
+  taos_stmt_close(stmt);
+}
+
+void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows)
+{
+  if (numOfRows > 0) {
+    printf("%d rows async retrieved\n", numOfRows);
+    taos_fetch_rows_a(tres, retrieve_callback, param);
+  } else {
+    if (numOfRows < 0) {
+      printf("\033[31masync retrieve failed, code: %d\033[0m\n", numOfRows);
+    } else {
+      printf("async retrieve completed\n");
+    }
+    taos_free_result(tres);
+  }
+}
+
+void select_callback(void *param, TAOS_RES *tres, int code)
+{
+  if (code == 0 && tres) {
+    taos_fetch_rows_a(tres, retrieve_callback, param);
+  } else {
+    printf("\033[31masync select failed, code: %d\033[0m\n", code);
+  }
+}
+
+void verify_async(TAOS* taos) {
+  prepare_data(taos);
+  taos_query_a(taos, "select * from meters", select_callback, NULL);
+  usleep(1000000);
+}
+
+void stream_callback(void *param, TAOS_RES *res, TAOS_ROW row) {
+  int         num_fields = taos_num_fields(res);
+  TAOS_FIELD* fields = taos_fetch_fields(res);
+
+  printf("got one row from stream_callback\n");
+  char temp[256];
+  taos_print_row(temp, row, fields, num_fields);
+  puts(temp);
+}
+
+void verify_stream(TAOS* taos) {
+  prepare_data(taos);
+  TAOS_STREAM* strm = taos_open_stream(
+    taos,
+    "select count(*) from meters interval(1m)",
+    stream_callback,
+    0,
+    NULL,
+    NULL);
+  printf("waiting for stream data\n"); 
+  usleep(100000);
+  taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);");
+  usleep(200000000);
+  taos_close_stream(strm);
+}
+
+int main(int argc, char *argv[]) {
+  const char* host = "127.0.0.1";
+  const char* user = "root";
+  const char* passwd = "taosdata";
+
+  taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
+  taos_init();
+
+  TAOS* taos = taos_connect(host, user, passwd, "", 0);
+  if (taos == NULL) {
+    printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));
+    exit(1);
+  }
+
+  char* info = taos_get_server_info(taos);
+  printf("server info: %s\n", info);
+  info = taos_get_client_info(taos);
+  printf("client info: %s\n", info);
+
+  printf("************  verify query  *************\n");
+  verify_query(taos);
+
+  printf("*********  verify async query  **********\n");
+  verify_async(taos);
+
+  printf("*********** verify subscribe ************\n");
+  verify_subscribe(taos);
+
+  printf("************ verify prepare *************\n");
+  verify_prepare(taos);
+
+  printf("************ verify stream  *************\n");
+  verify_stream(taos);
+  printf("done\n");
+
+  taos_close(taos);
+  taos_cleanup();
+}
\ No newline at end of file

tests/examples/c/makefile

@@ -4,7 +4,6 @@
 ROOT=./
 TARGET=exe
 LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt
-#LFLAGS = '-Wl,-rpath,/home/zbm/project/td/debug/build/lib/' -L/home/zbm/project/td/debug/build/lib -ltaos -lpthread -lm -lrt
 CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX -msse4.2 -Wno-unused-function -D_M_X64 \
 	-I/usr/local/taos/include -std=gnu99
 
@@ -16,6 +15,7 @@ exe:
 	gcc $(CFLAGS) ./prepare.c -o $(ROOT)/prepare $(LFLAGS)
 	gcc $(CFLAGS) ./stream.c -o $(ROOT)/stream $(LFLAGS)
 	gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS)
+	gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS)
 
 clean:
 	rm $(ROOT)/asyncdemo

tests/examples/python/read_example.py

@@ -29,7 +29,7 @@ if __name__ == '__main__':
 
     # Create a database named db
     try:
-        c1.execute('create database db')
+        c1.execute('create database if not exists db ')
     except Exception as err:
         conn.close()
         raise(err)

tests/pytest/functions/function_bottom.py

@@ -28,19 +28,14 @@ class TDTestCase:
         self.ts = 1537146000000
         
     def run(self):
-        tdSql.prepare()
-
-        intData = []        
-        floatData = []
+        tdSql.prepare()    
 
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                        
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                            
 
         # bottom verifacation 
         tdSql.error("select bottom(ts, 10) from test")

tests/pytest/functions/function_count.py

@@ -30,17 +30,12 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
                         % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                
         
         # Count verifacation
         tdSql.query("select count(*) from test")        

tests/pytest/functions/function_diff.py

@@ -30,16 +30,15 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1))
+        
+        # diff verifacation 
         tdSql.query("select diff(col1) from test1")
         tdSql.checkRows(0)
-
+        
         tdSql.query("select diff(col2) from test1")
         tdSql.checkRows(0)
 
@@ -57,11 +56,8 @@ class TDTestCase:
 
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                        
-
-        # diff verifacation 
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                                
+        
         tdSql.error("select diff(ts) from test")
         tdSql.error("select diff(ts) from test1")
         tdSql.error("select diff(col1) from test")

tests/pytest/functions/function_first.py

@@ -30,9 +30,6 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
@@ -72,9 +69,7 @@ class TDTestCase:
 
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)  
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))            
 
         tdSql.query("select first(*) from test1")
         tdSql.checkRows(1)

tests/pytest/functions/function_last.py

@@ -30,15 +30,12 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1))
 
-        # first verifacation         
+        # last verifacation         
         tdSql.query("select last(*) from test1")
         tdSql.checkRows(1)
         tdSql.checkData(0, 1, None)
@@ -72,9 +69,7 @@ class TDTestCase:
 
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)  
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))            
 
         tdSql.query("select last(*) from test1")
         tdSql.checkRows(1)

tests/pytest/functions/function_last_row.py

@@ -30,15 +30,12 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1))
 
-        # first verifacation         
+        # last_row verifacation         
         tdSql.query("select last_row(*) from test1")
         tdSql.checkRows(1)
         tdSql.checkData(0, 1, None)
@@ -81,9 +78,7 @@ class TDTestCase:
 
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)  
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))              
 
         tdSql.query("select last_row(*) from test1")
         tdSql.checkRows(1)

tests/pytest/functions/function_leastsquares.py

@@ -30,17 +30,12 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                        
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                                   
 
         # leastsquares verifacation 
         tdSql.error("select leastsquares(ts, 1, 1) from test1")
@@ -72,8 +67,6 @@ class TDTestCase:
         tdSql.query("select leastsquares(col6, 1, 1) from test1")
         tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}')
 
-        
-
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/functions/function_max.py

@@ -42,7 +42,7 @@ class TDTestCase:
             intData.append(i + 1)            
             floatData.append(i + 0.1)                        
 
-        # min verifacation 
+        # max verifacation 
         tdSql.error("select max(ts) from test")
         tdSql.error("select max(ts) from test1")
         tdSql.error("select max(col7) from test")

tests/pytest/functions/function_operations.py

@@ -30,17 +30,12 @@ class TDTestCase:
     def run(self):
         tdSql.prepare()
 
-        intData = []        
-        floatData = []
-
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
         tdSql.execute("create table test1 using test tags('beijing')")
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                        
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                       
 
         # min verifacation 
         tdSql.error("select ts + col1 from test")

tests/pytest/functions/function_spread.py

@@ -28,10 +28,7 @@ class TDTestCase:
         self.ts = 1537146000000
         
     def run(self):
-        tdSql.prepare()
-
-        intData = []        
-        floatData = []
+        tdSql.prepare()        
 
         tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
                     col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''')
@@ -68,9 +65,7 @@ class TDTestCase:
 
         for i in range(self.rowNum):
             tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" 
-                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
-            intData.append(i + 1)            
-            floatData.append(i + 0.1)                        
+                        % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))                                  
        
         tdSql.error("select spread(col7) from test")     
         tdSql.error("select spread(col7) from test1")               

tests/pytest/functions/function_sum.py

@@ -57,8 +57,7 @@ class TDTestCase:
         tdSql.query("select sum(col4) from test")
         tdSql.checkData(0, 0, np.sum(intData))
         tdSql.query("select sum(col5) from test")
-        print("query result for folat sum function: %f" % tdSql.getData(0, 0))
-        print("calculation result for numpy sum function: %f" % np.sum(floatData))
+        tdSql.checkData(0, 0, np.sum(floatData)) 
         tdSql.query("select sum(col6) from test")
         tdSql.checkData(0, 0, np.sum(floatData))