Merge pull request #1458 from taosdata/liaohj_2

Liaohj 2

src/client/src/tscAsync.c

@@ -514,7 +514,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
     tscTansformSQLFunctionForSTableQuery(pQueryInfo);
     tscIncStreamExecutionCount(pSql->pStream);
   } else {
-    tscTrace("%p get tableMeta/metricMeta successfully", pSql);
+    tscTrace("%p get tableMeta successfully", pSql);
   }
 
   tscDoQuery(pSql);

src/client/src/tscSecondaryMerge.c

@@ -605,7 +605,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
-//  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pTableMetaInfo->pMetricMeta->numOfVnodes);
+  (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * 1);
   if (*pMemBuffer == NULL) {
     tscError("%p failed to allocate memory", pSql);
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@@ -636,10 +636,11 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
 
   pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
 
-//  for (int32_t i = 0; i < pTableMetaInfo->pMetricMeta->numOfVnodes; ++i) {
-//    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
-//    (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
-//  }
+  size_t numOfSubs = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
+  for (int32_t i = 0; i < numOfSubs; ++i) {
+    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
+    (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
+  }
 
   if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) {
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;

src/client/src/tscServer.c

@@ -509,31 +509,31 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 }
 
 int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
-  SSubmitMsg *pShellMsg;
-  char *           pMsg, *pStart;
-
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
   STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
   
-  pStart = pSql->cmd.payload + tsRpcHeadSize;
-  pMsg = pStart;
+  char* pMsg = pSql->cmd.payload + tsRpcHeadSize;
+  
+  // NOTE: shell message size should not include SMsgDesc
+  int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc);
   
   SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg;
-  pMsgDesc->numOfVnodes = htonl(1);       //set the number of vnodes
+  
+  pMsgDesc->numOfVnodes = htonl(1);       //todo set the right number of vnodes
   pMsg += sizeof(SMsgDesc);
   
-  pShellMsg = (SSubmitMsg *)pMsg;
+  SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg;
+  
   pShellMsg->header.vgId = htonl(pTableMeta->vgId);
-  pShellMsg->header.contLen = htonl(pSql->cmd.payloadLen);
+  pShellMsg->header.contLen = htonl(size);
   pShellMsg->length = pShellMsg->header.contLen;
   
   pShellMsg->numOfBlocks = htonl(pSql->cmd.numOfTablesInSubmit);  // number of meters to be inserted
 
-  // pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here
+  // pSql->cmd.payloadLen is set during copying data into paylaod
   pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT;
+  tscTrace("%p build submit msg, vgId:%d numOfVnodes:%d", pSql, pTableMeta->vgId, htons(pMsgDesc->numOfVnodes));
   
-//  tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip),
-//           htons(pShellMsg->vnode));
   return TSDB_CODE_SUCCESS;
 }
 

src/client/src/tscSubquery.c

@@ -1006,13 +1006,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   
   pRes->qhandle = 1;  // hack the qhandle check
   
-  const uint32_t nBufferSize = (1 << 16);  // 64KB
+  const uint32_t nBufferSize = (1u << 16);  // 64KB
   
   SQueryInfo *    pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
-  int32_t numOfSubQueries = 0;
-//  int32_t numOfSubQueries = pTableMetaInfo->pMetricMeta->numOfVnodes;
+  int32_t numOfSubQueries = taosArrayGetSize(pTableMetaInfo->vgroupIdList);
   assert(numOfSubQueries > 0);
   
   int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize);
@@ -1118,7 +1117,7 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
   tfree(trsupport);
 }
 
-static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
+static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows);
 
 static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t errCode) {
 // set no disk space error info
@@ -1140,7 +1139,7 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES
   
   pthread_mutex_unlock(&trsupport->queryMutex);
   
-  tscRetrieveFromVnodeCallBack(trsupport, tres, trsupport->pState->code);
+  tscRetrieveFromDnodeCallBack(trsupport, tres, trsupport->pState->code);
 }
 
 static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) {
@@ -1235,7 +1234,86 @@ static void tscHandleSubRetrievalError(SRetrieveSupport *trsupport, SSqlObj *pSq
   }
 }
 
-static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
+static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* pSql) {
+  int32_t           idx = trsupport->subqueryIndex;
+  SSqlObj *         pPObj = trsupport->pParentSqlObj;
+  tOrderDescriptor *pDesc = trsupport->pOrderDescriptor;
+  
+  SSubqueryState* pState = trsupport->pState;
+  SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
+  
+  // data in from current vnode is stored in cache and disk
+  uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
+//    tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
+//             pSvd->vnode, numOfRowsFromSubquery, idx);
+  
+  tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
+
+#ifdef _DEBUG_VIEW
+  printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
+    SSrcColumnInfo colInfo[256] = {0};
+    tscGetSrcColumnInfo(colInfo, pQueryInfo);
+    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
+                       trsupport->localBuffer->numOfElems, colInfo);
+#endif
+  
+  if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
+    tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
+             tsAvailTmpDirGB, tsMinimalTmpDirGB);
+    tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
+    return;
+  }
+  
+  // each result for a vnode is ordered as an independant list,
+  // then used as an input of loser tree for disk-based merge routine
+  int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
+                                  pQueryInfo->groupbyExpr.orderType);
+  if (ret != 0) { // set no disk space error info, and abort retry
+    return tscAbortFurtherRetryRetrieval(trsupport, pSql, TSDB_CODE_CLI_NO_DISKSPACE);
+  }
+  
+  // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
+  // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
+  // In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
+  int32_t numOfTotal = pState->numOfTotal;
+  
+  int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
+  if (finished < numOfTotal) {
+    tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
+    return tscFreeSubSqlObj(trsupport, pSql);
+  }
+  
+  // all sub-queries are returned, start to local merge process
+  pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
+  
+  tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
+           pState->numOfTotal, pState->numOfRetrievedRows);
+  
+  SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
+  tscClearInterpInfo(pPQueryInfo);
+  
+  tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
+                        &pPObj->cmd, &pPObj->res);
+  tscTrace("%p build loser tree completed", pPObj);
+  
+  pPObj->res.precision = pSql->res.precision;
+  pPObj->res.numOfRows = 0;
+  pPObj->res.row = 0;
+  
+  // only free once
+  tfree(trsupport->pState);
+  tscFreeSubSqlObj(trsupport, pSql);
+  
+  // set the command flag must be after the semaphore been correctly set.
+  pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
+  if (pPObj->res.code == TSDB_CODE_SUCCESS) {
+    (*pPObj->fp)(pPObj->param, pPObj, 0);
+  } else {
+    tscQueueAsyncRes(pPObj);
+  }
+}
+
+static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows) {
   SRetrieveSupport *trsupport = (SRetrieveSupport *)param;
   int32_t           idx = trsupport->subqueryIndex;
   SSqlObj *         pPObj = trsupport->pParentSqlObj;
@@ -1264,15 +1342,15 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
 //  SVnodeSidList *vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx);
-  SVnodeSidList *vnodeInfo = 0;
-  SVnodeDesc *   pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
+//  SVnodeSidList *vnodeInfo = 0;
+//  SVnodeDesc *   pSvd = &vnodeInfo->vpeerDesc[vnodeInfo->index];
   
   if (numOfRows > 0) {
     assert(pRes->numOfRows == numOfRows);
     int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
     
-    tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
-             pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
+//    tscTrace("%p sub:%p retrieve numOfRows:%d totalNumOfRows:%d from ip:%u,vid:%d,orderOfSub:%d", pPObj, pSql,
+//             pRes->numOfRows, pState->numOfRetrievedRows, pSvd->ip, pSvd->vnode, idx);
     
     if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) {
       tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId64 " , current:%" PRId64,
@@ -1298,85 +1376,16 @@ static void tscRetrieveFromVnodeCallBack(void *param, TAOS_RES *tres, int numOfR
     
     int32_t ret = saveToBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer, pRes->data,
                                pRes->numOfRows, pQueryInfo->groupbyExpr.orderType);
-    if (ret < 0) {
-      // set no disk space error info, and abort retry
+    if (ret < 0) { // set no disk space error info, and abort retry
       tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
-    } else {
+    } else if (pRes->completed) {
+      tscAllDataRetrievedFromDnode(trsupport, pSql);
+    } else { // continue fetch data from dnode
       pthread_mutex_unlock(&trsupport->queryMutex);
-      taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
-    }
-    
-  } else {  // all data has been retrieved to client
-    /* data in from current vnode is stored in cache and disk */
-    uint32_t numOfRowsFromVnode = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems;
-    tscTrace("%p sub:%p all data retrieved from ip:%u,vid:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, pSvd->ip,
-             pSvd->vnode, numOfRowsFromVnode, idx);
-    
-    tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity);
-
-#ifdef _DEBUG_VIEW
-    printf("%" PRIu64 " rows data flushed to disk:\n", trsupport->localBuffer->numOfElems);
-    SSrcColumnInfo colInfo[256] = {0};
-    tscGetSrcColumnInfo(colInfo, pQueryInfo);
-    tColModelDisplayEx(pDesc->pColumnModel, trsupport->localBuffer->data, trsupport->localBuffer->numOfElems,
-                       trsupport->localBuffer->numOfElems, colInfo);
-#endif
-    
-    if (tsTotalTmpDirGB != 0 && tsAvailTmpDirGB < tsMinimalTmpDirGB) {
-      tscError("%p sub:%p client disk space remain %.3f GB, need at least %.3f GB, stop query", pPObj, pSql,
-               tsAvailTmpDirGB, tsMinimalTmpDirGB);
-      tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
-      return;
-    }
-    
-    // each result for a vnode is ordered as an independant list,
-    // then used as an input of loser tree for disk-based merge routine
-    int32_t ret = tscFlushTmpBuffer(trsupport->pExtMemBuffer[idx], pDesc, trsupport->localBuffer,
-                                    pQueryInfo->groupbyExpr.orderType);
-    if (ret != 0) {
-      /* set no disk space error info, and abort retry */
-      return tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_CLI_NO_DISKSPACE);
-    }
-    
-    // keep this value local variable, since the pState variable may be released by other threads, if atomic_add opertion
-    // increases the finished value up to pState->numOfTotal value, which means all subqueries are completed.
-    // In this case, the comparsion between finished value and released pState->numOfTotal is not safe.
-    int32_t numOfTotal = pState->numOfTotal;
-    
-    int32_t finished = atomic_add_fetch_32(&pState->numOfCompleted, 1);
-    if (finished < numOfTotal) {
-      tscTrace("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pPObj, pSql, trsupport->subqueryIndex, finished);
-      return tscFreeSubSqlObj(trsupport, pSql);
-    }
-    
-    // all sub-queries are returned, start to local merge process
-    pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage;
-    
-    tscTrace("%p retrieve from %d vnodes completed.final NumOfRows:%d,start to build loser tree", pPObj,
-             pState->numOfTotal, pState->numOfRetrievedRows);
-    
-    SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0);
-    tscClearInterpInfo(pPQueryInfo);
-    
-    tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel,
-                          &pPObj->cmd, &pPObj->res);
-    tscTrace("%p build loser tree completed", pPObj);
-    
-    pPObj->res.precision = pSql->res.precision;
-    pPObj->res.numOfRows = 0;
-    pPObj->res.row = 0;
-    
-    // only free once
-    tfree(trsupport->pState);
-    tscFreeSubSqlObj(trsupport, pSql);
-    
-    // set the command flag must be after the semaphore been correctly set.
-    pPObj->cmd.command = TSDB_SQL_RETRIEVE_METRIC;
-    if (pPObj->res.code == TSDB_CODE_SUCCESS) {
-      (*pPObj->fp)(pPObj->param, pPObj, 0);
-    } else {
-      tscQueueAsyncRes(pPObj);
+      taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
     }
+  } else { // all data has been retrieved to client
+    tscAllDataRetrievedFromDnode(trsupport, pSql);
   }
 }
 
@@ -1436,7 +1445,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
   
   /*
    * if a query on a vnode is failed, all retrieve operations from vnode that occurs later
-   * than this one are actually not necessary, we simply call the tscRetrieveFromVnodeCallBack
+   * than this one are actually not necessary, we simply call the tscRetrieveFromDnodeCallBack
    * function to abort current and remain retrieve process.
    *
    * NOTE: threadsafe is required.
@@ -1474,7 +1483,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
                trsupport->subqueryIndex, pState->code);
     }
     
-    tscRetrieveFromVnodeCallBack(param, tres, pState->code);
+    tscRetrieveFromDnodeCallBack(param, tres, pState->code);
   } else {  // success, proceed to retrieve data from dnode
     if (vnodeInfo != NULL) {
       tscTrace("%p sub:%p query complete,ip:%u,vid:%d,orderOfSub:%d,retrieve data", trsupport->pParentSqlObj, pSql,
@@ -1485,7 +1494,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) {
                trsupport->subqueryIndex);
     }
     
-    taos_fetch_rows_a(tres, tscRetrieveFromVnodeCallBack, param);
+    taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param);
   }
 }
 

src/client/src/tscUtil.c

@@ -608,7 +608,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
    * the payloadLen should be actual message body size
    * the old value of payloadLen is the allocated payload size
    */
-  pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize - sizeof(SMsgDesc);
+  pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize;
 
   assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + 100 && pCmd->payloadLen > 0);
   return TSDB_CODE_SUCCESS;

src/client/tests/timeParseTest.cpp

+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "tstoken.h"
+#include "ttime.h"
+#include "tutil.h"
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+extern void deltaToUtcInitOnce();
+/* test parse time function */
+TEST(testCase, parse_time) {
+  
+  taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
+  deltaToUtcInitOnce();
+  
+  char t1[] = "2018-1-1 1:1:1.952798";
+  char t13[] = "1970-1-1 0:0:0";
+
+  int64_t time = 0, time1 = 0;
+
+  taosParseTime(t1, &time, strlen(t1), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 1514739661952);
+
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, timezone * MILLISECOND_PER_SECOND);
+
+  char t2[] = "2018-1-1T1:1:1.952Z";
+  taosParseTime(t2, &time, strlen(t2), TSDB_TIME_PRECISION_MILLI);
+
+  EXPECT_EQ(time, 1514739661952 + 28800000);
+
+  char t3[] = "2018-1-1 1:01:01.952";
+  taosParseTime(t3, &time, strlen(t3), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 1514739661952);
+
+  char t4[] = "2018-1-1 1:01:01.9";
+  char t5[] = "2018-1-1 1:01:1.900";
+  char t6[] = "2018-01-01 1:1:1.90";
+  char t7[] = "2018-01-01 01:01:01.9";
+  char t8[] = "2018-01-01 01:01:01.9007865";
+
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t5, &time1, strlen(t5), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t6, &time1, strlen(t6), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  taosParseTime(t4, &time, strlen(t4), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t7, &time1, strlen(t7), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  taosParseTime(t5, &time, strlen(t5), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t8, &time1, strlen(t8), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  char t9[] = "2017-4-3 1:1:2.980";
+  char t10[] = "2017-4-3T2:1:2.98+9:00";
+  taosParseTime(t9, &time, strlen(t9), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  char t11[] = "2017-4-3T2:1:2.98+09:00";
+  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t10, &time1, strlen(t10), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  char t12[] = "2017-4-3T2:1:2.98+0900";
+  taosParseTime(t11, &time, strlen(t11), TSDB_TIME_PRECISION_MILLI);
+  taosParseTime(t12, &time1, strlen(t12), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, time1);
+
+  taos_options(TSDB_OPTION_TIMEZONE, "UTC");  
+  deltaToUtcInitOnce();
+  
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 0);
+
+  taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
+  deltaToUtcInitOnce();
+  
+  char t14[] = "1970-1-1T0:0:0Z";
+  taosParseTime(t14, &time, strlen(t14), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 0);
+
+  char t40[] = "1970-1-1 0:0:0.999999999";
+  taosParseTime(t40, &time, strlen(t40), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 999 + timezone * MILLISECOND_PER_SECOND);
+
+  char t41[] = "1997-1-1 0:0:0.999999999";
+  taosParseTime(t41, &time, strlen(t41), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 852048000999);
+
+  int64_t k = timezone;
+  char    t42[] = "1997-1-1T0:0:0.999999999Z";
+  taosParseTime(t42, &time, strlen(t42), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 852048000999 - timezone * MILLISECOND_PER_SECOND);
+
+  ////////////////////////////////////////////////////////////////////
+  // illegal timestamp format
+  char t15[] = "2017-12-33 0:0:0";
+  EXPECT_EQ(taosParseTime(t15, &time, strlen(t15), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t16[] = "2017-12-31 99:0:0";
+  EXPECT_EQ(taosParseTime(t16, &time, strlen(t16), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t17[] = "2017-12-31T9:0:0";
+  EXPECT_EQ(taosParseTime(t17, &time, strlen(t17), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t18[] = "2017-12-31T9:0:0.Z";
+  EXPECT_EQ(taosParseTime(t18, &time, strlen(t18), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t19[] = "2017-12-31 9:0:0.-1";
+  EXPECT_EQ(taosParseTime(t19, &time, strlen(t19), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t20[] = "2017-12-31 9:0:0.1+12:99";
+  EXPECT_EQ(taosParseTime(t20, &time, strlen(t20), TSDB_TIME_PRECISION_MILLI), 0);
+  EXPECT_EQ(time, 1514682000100);
+
+  char t21[] = "2017-12-31T9:0:0.1+12:99";
+  EXPECT_EQ(taosParseTime(t21, &time, strlen(t21), TSDB_TIME_PRECISION_MILLI), -1);
+
+  char t22[] = "2017-12-31 9:0:0.1+13:1";
+  EXPECT_EQ(taosParseTime(t22, &time, strlen(t22), TSDB_TIME_PRECISION_MILLI), 0);
+
+  char t23[] = "2017-12-31T9:0:0.1+13:1";
+  EXPECT_EQ(taosParseTime(t23, &time, strlen(t23), TSDB_TIME_PRECISION_MILLI), 0);
+
+
+  //======================== add some case ============================//
+  
+  char b1[] = "9999-12-31 23:59:59.999";
+  taosParseTime(b1, &time, strlen(b1), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 253402271999999);
+
+
+  char b2[] = "2020-01-01 01:01:01.321";
+  taosParseTime(b2, &time, strlen(b2), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 1577811661321);
+
+  taos_options(TSDB_OPTION_TIMEZONE, "America/New_York");  
+  deltaToUtcInitOnce();
+  
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, 18000 * MILLISECOND_PER_SECOND);
+
+  taos_options(TSDB_OPTION_TIMEZONE, "Asia/Tokyo");  
+  deltaToUtcInitOnce();
+  
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, -32400 * MILLISECOND_PER_SECOND);
+
+  taos_options(TSDB_OPTION_TIMEZONE, "Asia/Shanghai");
+  deltaToUtcInitOnce();
+  
+  taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI);
+  EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
+}
+
+

src/inc/taosmsg.h

@@ -188,15 +188,6 @@ extern char *taosMsg[];
 
 #pragma pack(push, 1)
 
-// typedef struct {
-//  int32_t  vnode;
-//  int32_t  sid;
-//  int32_t  sversion;
-//  uint64_t uid;
-//  int16_t  numOfRows;
-//  char     payLoad[];
-//} SShellSubmitBlock;
-
 typedef struct {
   int32_t numOfVnodes;
 } SMsgDesc;

src/query/CMakeLists.txt

@@ -12,4 +12,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(query ${SRC})
   TARGET_LINK_LIBRARIES(query tsdb tutil m rt)
-ENDIF ()
\ No newline at end of file
+ENDIF ()
+
+ADD_SUBDIRECTORY(tests)
\ No newline at end of file

src/query/inc/qtsbuf.h

@@ -48,10 +48,10 @@ typedef struct STSElem {
 } STSElem;
 
 typedef struct STSCursor {
-  int32_t vnodeIndex;
-  int32_t blockIndex;
-  int32_t tsIndex;
-  int32_t order;
+  int32_t  vnodeIndex;
+  int32_t  blockIndex;
+  int32_t  tsIndex;
+  uint32_t order;
 } STSCursor;
 
 typedef struct STSBlock {

src/query/inc/queryExecutor.h

@@ -32,12 +32,6 @@ typedef struct SData {
   char    data[];
 } SData;
 
-enum {
-//  ST_QUERY_KILLED = 0,     // query killed
-  ST_QUERY_PAUSED = 1,     // query paused, due to full of the response buffer
-  ST_QUERY_COMPLETED = 2,  // query completed
-};
-
 struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
 typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
@@ -60,18 +54,20 @@ typedef struct SWindowStatus {
 } SWindowStatus;
 
 typedef struct SWindowResult {
-  uint16_t      numOfRows;
+  uint16_t      numOfRows;   // number of rows of current  time window
   SPosInfo      pos;         // Position of current result in disk-based output buffer
   SResultInfo*  resultInfo;  // For each result column, there is a resultInfo
   STimeWindow   window;      // The time window that current result covers.
-  SWindowStatus status;
+  SWindowStatus status;      // this result status: closed or opened
 } SWindowResult;
 
 typedef struct SResultRec {
-  int64_t total;
-  int64_t size;
-  int64_t capacity;
-  int32_t threshold;   // the threshold size, when the number of rows in result buffer, return to client
+  int64_t total;     // total generated result size in rows
+  int64_t size;      // current result set size in rows
+  int64_t capacity;  // capacity of current result output buffer
+  
+  // result size threshold in rows. If the result buffer is larger than this, pause query and return to client
+  int32_t threshold;
 } SResultRec;
 
 typedef struct SWindowResInfo {
@@ -99,7 +95,6 @@ typedef struct SSingleColumnFilterInfo {
   void*              pData;
 } SSingleColumnFilterInfo;
 
-/* intermediate pos during multimeter query involves interval */
 typedef struct STableQueryInfo {
   int64_t     lastKey;
   STimeWindow win;
@@ -107,7 +102,7 @@ typedef struct STableQueryInfo {
   int16_t     queryRangeSet;  // denote if the query range is set, only available for interval query
   int64_t     tag;
   STSCursor   cur;
-  int32_t     sid;  // for retrieve the page id list
+  int32_t     tid;  // for retrieve the page id list
 
   SWindowResInfo windowResInfo;
 } STableQueryInfo;
@@ -116,7 +111,6 @@ typedef struct STableDataInfo {
   int32_t          numOfBlocks;
   int32_t          start;     // start block index
   int32_t          tableIndex;
-  void*            pMeterObj;
   int32_t          groupIdx;  // group id in table list
   STableQueryInfo* pTableQInfo;
 } STableDataInfo;

src/query/tests/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
+FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
+
+IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
+    MESSAGE(STATUS "gTest library found, build unit test")
+
+    INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+    ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
+    TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
+ENDIF()
\ No newline at end of file

src/query/tests/histogramTest.cpp

+#include <gtest/gtest.h>
+#include <sys/time.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "tsdb.h"
+
+#include "tstoken.h"
+#include "tutil.h"
+
+#include "qhistogram.h"
+
+/* test validate the names for table/database */
+TEST(testCase, histogram_binary_search) {
+  SHistogramInfo* pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
+
+  pHisto->numOfEntries = 10;
+  for (int32_t i = 0; i < 10; ++i) {
+    pHisto->elems[i].num = 1;
+    pHisto->elems[i].val = i;
+  }
+
+  int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
+  assert(idx == 1);
+
+  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
+  assert(idx == 9);
+
+  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
+  assert(idx == 10);
+
+  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
+  assert(idx == 0);
+
+  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
+  assert(idx == 4);
+
+  free(pHisto);
+}
+
+TEST(testCase, histogram_add) {
+  SHistogramInfo* pHisto = NULL;
+
+  /**
+   * use arrayList, elapsed time is:
+   * before:
+   * 10,000,000  45sec, bin:1000 (-O0) / 17sec. bin:1000, (-O3)
+   *
+   * after:
+   *
+   */
+  struct timeval systemTime;
+  gettimeofday(&systemTime, NULL);
+  int64_t st =
+      (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
+  for (int32_t i = 0; i < 10000; ++i) {
+    tHistogramAdd(&pHisto, i);
+    //        tHistogramPrint(pHisto);
+  }
+  //
+  gettimeofday(&systemTime, NULL);
+  int64_t et =
+      (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
+  printf("total elapsed time: %ld\n", et - st);
+
+  printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
+  tHistogramPrint(pHisto);
+
+  printf("%ld\n", tHistogramSum(pHisto, 1.5));
+  printf("%ld\n", tHistogramSum(pHisto, 2));
+  printf("%ld\n", tHistogramSum(pHisto, 3));
+  printf("%ld\n", tHistogramSum(pHisto, 4));
+  printf("%ld\n", tHistogramSum(pHisto, 5));
+  printf("%ld\n", tHistogramSum(pHisto, 6));
+
+  for (int32_t i = 399; i < 400; ++i) {
+    printf("val:%d, %ld\n", i, tHistogramSum(pHisto, i));
+  }
+
+  double ratio[] = {0 / 100, 20.0 / 100, 88.0 / 100, 100 / 100};
+  double* res = tHistogramUniform(pHisto, ratio, 4);
+  for (int32_t i = 0; i < 4; ++i) {
+    printf("%f\n", res[i]);
+  }
+
+  SHistogramInfo* pHisto1 = NULL;
+  for (int32_t i = (90000 - 1); i >= 80000; --i) {
+    tHistogramAdd(&pHisto1, i);
+  }
+  tHistogramPrint(pHisto1);
+
+  SHistogramInfo* pRes = tHistogramMerge(pHisto1, pHisto, MAX_HISTOGRAM_BIN);
+  assert(pRes->numOfElems == pHisto->numOfElems + pHisto1->numOfElems);
+  tHistogramPrint(pRes);
+
+  tHistogramDestroy(&pHisto);
+  tHistogramDestroy(&pHisto1);
+  tHistogramDestroy(&pRes);
+  free(res);
+}
+
+TEST(testCase, heapsort) {
+  //    int32_t num = 20;
+  //
+  //    SHeapEntry* pEntry = tHeapCreate(num);
+  //
+  //    for(int32_t i=0; i<num; ++i) {
+  //        pEntry[i].val = num - 1 - i;
+  //    }
+  //
+  //    tHeapSort(pEntry, num);
+  //
+  //    for(int32_t i=0; i<num; ++i) {
+  //        printf("%lf, ", pEntry[i].val);
+  //    }
+  //
+  //    printf("\n");
+  //
+  //    free(pEntry);
+}

src/query/tests/patternMatchTest.cpp

+#include <gtest/gtest.h>
+#include <sys/time.h>
+#include <cassert>
+#include <iostream>
+
+#include "tsqlfunction.h"
+
+TEST(testCase, patternMatchTest) {
+  SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
+
+  const char* str = "abcdef";
+  int32_t ret = patternMatch("a%b%", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "tm01";
+  ret = patternMatch("tm__", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "tkm1";
+  ret = patternMatch("t%m1", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "tkm1";
+  ret = patternMatch("%m1", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "";
+  ret = patternMatch("%_", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
+
+  str = "1";
+  ret = patternMatch("%__", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
+
+  str = "";
+  ret = patternMatch("%", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = " ";
+  ret = patternMatch("_", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "!";
+  ret = patternMatch("%_", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "abcdefg";
+  ret = patternMatch("abc%fg", str, strlen(str), &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("abc%fg", str, 7, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("abc%f_", str, 6, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("abc%f_", str, 1, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("ab", str, 2, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("a%", str, 2, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_MATCH);
+
+  str = "abcdefgabcdeju";
+  ret = patternMatch("a__", str, 2, &info);
+  EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH);
+}

src/query/tests/resultBufferTest.cpp

+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "qresultBuf.h"
+#include "tsdb.h"
+
+namespace {
+// simple test
+void simpleTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64);
+  
+  int32_t pageId = 0;
+  int32_t groupId = 0;
+  
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  
+  ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
+  ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
+  
+  SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(list.size, 1);
+  
+  ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
+  
+  destroyResultBuf(pResultBuf);
+}
+} // namespace
+
+TEST(testCase, resultBufferTest) {
+  simpleTest();
+}

src/query/tests/tsBufTest.cpp

+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "tsdb.h"
+
+#include "tstoken.h"
+#include "ttime.h"
+#include "tutil.h"
+#include "qtsbuf.h"
+
+namespace {
+/**
+ *
+ * @param num  total number
+ * @param step gap between two consecutive ts
+ * @return
+ */
+int64_t* createTsList(int32_t num, int64_t start, int32_t step) {
+  int64_t* pList = (int64_t*)malloc(num * sizeof(int64_t));
+
+  for (int64_t i = 0; i < num; ++i) {
+    pList[i] = start + i * step;
+  }
+
+  return pList;
+}
+
+// simple test
+void simpleTest() {
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  // write 10 ts points
+  int32_t num = 10;
+  int64_t tag = 1;
+
+  int64_t* list = createTsList(10, 10000000, 30);
+  tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
+  EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
+
+  EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num);
+  EXPECT_EQ(pTSBuf->block.tag, tag);
+  EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+
+  tsBufFlush(pTSBuf);
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.numOfElem, num);
+
+  tsBufDestory(pTSBuf);
+}
+
+// one large list of ts, the ts list need to be split into several small blocks
+void largeTSTest() {
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  // write 10 ts points
+  int32_t num = 1000000;
+  int64_t tag = 1;
+
+  int64_t* list = createTsList(num, 10000000, 30);
+  tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t));
+
+  // the data has been flush to disk, no data in cache
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.tag, tag);
+  EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+  EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
+
+  tsBufFlush(pTSBuf);
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.numOfElem, num);
+
+  tsBufDestory(pTSBuf);
+}
+
+void multiTagsTest() {
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  int32_t num = 10000;
+  int64_t tag = 1;
+  int64_t start = 10000000;
+  int32_t numOfTags = 50;
+  int32_t step = 30;
+
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    int64_t* list = createTsList(num, start, step);
+    tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+
+  EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
+  EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
+
+  EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
+  EXPECT_EQ(pTSBuf->numOfVnodes, 1);
+
+  tsBufFlush(pTSBuf);
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.numOfElem, num);
+
+  tsBufDestory(pTSBuf);
+}
+
+void multiVnodeTagsTest() {
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  int32_t num = 10000;
+  int64_t start = 10000000;
+  int32_t numOfTags = 50;
+  int32_t step = 30;
+
+  // 2000 vnodes
+  for (int32_t j = 0; j < 20; ++j) {
+    // vnodeId:0
+    start = 10000000;
+    for (int32_t i = 0; i < numOfTags; ++i) {
+      int64_t* list = createTsList(num, start, step);
+      tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
+      free(list);
+
+      start += step * num;
+    }
+
+    EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+  }
+
+  EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
+  EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
+  EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
+
+  EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
+
+  EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
+
+  tsBufFlush(pTSBuf);
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.numOfElem, num);
+
+  tsBufDestory(pTSBuf);
+}
+
+void loadDataTest() {
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  int32_t num = 10000;
+  int64_t oldStart = 10000000;
+  int32_t numOfTags = 50;
+  int32_t step = 30;
+  int32_t numOfVnode = 200;
+
+  // 10000 vnodes
+  for (int32_t j = 0; j < numOfVnode; ++j) {
+    // vnodeId:0
+    int64_t start = 10000000;
+    for (int32_t i = 0; i < numOfTags; ++i) {
+      int64_t* list = createTsList(num, start, step);
+      tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
+      printf("%d - %lld\n", i, list[0]);
+
+      free(list);
+      start += step * num;
+    }
+
+    EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+  }
+
+  EXPECT_EQ(pTSBuf->tsOrder, TSQL_SO_ASC);
+
+  EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
+  EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
+
+  EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t));
+
+  EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1);
+
+  tsBufFlush(pTSBuf);
+  EXPECT_EQ(pTSBuf->tsData.len, 0);
+  EXPECT_EQ(pTSBuf->block.numOfElem, num);
+
+  // create from exists file
+  STSBuf* pNewBuf = tsBufCreateFromFile(pTSBuf->path, false);
+  EXPECT_EQ(pNewBuf->tsOrder, pTSBuf->tsOrder);
+  EXPECT_EQ(pNewBuf->numOfVnodes, numOfVnode);
+  EXPECT_EQ(pNewBuf->fileSize, pTSBuf->fileSize);
+
+  EXPECT_EQ(pNewBuf->pData[0].info.offset, pTSBuf->pData[0].info.offset);
+  EXPECT_EQ(pNewBuf->pData[0].info.numOfBlocks, pTSBuf->pData[0].info.numOfBlocks);
+  EXPECT_EQ(pNewBuf->pData[0].info.compLen, pTSBuf->pData[0].info.compLen);
+
+  EXPECT_STREQ(pNewBuf->path, pTSBuf->path);
+
+  tsBufResetPos(pNewBuf);
+
+  int64_t s = taosGetTimestampUs();
+  printf("start:%lld\n", s);
+
+  int32_t x = 0;
+  while (tsBufNextPos(pNewBuf)) {
+    STSElem elem = tsBufGetElem(pNewBuf);
+    if (++x == 100000000) {
+      break;
+    }
+
+    //    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+  }
+
+  int64_t e = taosGetTimestampUs();
+  printf("end:%lld, elapsed:%lld, total obj:%d\n", e, e - s, x);
+}
+
+void randomIncTsTest() {}
+
+void TSTraverse() {
+  // 10000 vnodes
+  int32_t num = 200000;
+  int64_t oldStart = 10000000;
+  int32_t numOfTags = 3;
+  int32_t step = 30;
+  int32_t numOfVnode = 2;
+
+  STSBuf* pTSBuf = tsBufCreate(true);
+
+  for (int32_t j = 0; j < numOfVnode; ++j) {
+    // vnodeId:0
+    int64_t start = 10000000;
+    for (int32_t i = 0; i < numOfTags; ++i) {
+      int64_t* list = createTsList(num, start, step);
+      tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
+      printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
+
+      free(list);
+      start += step * num;
+
+      list = createTsList(num, start, step);
+      tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t));
+      printf("%d - %d - %lld, %lld\n", j, i, list[0], list[num - 1]);
+      free(list);
+
+      start += step * num;
+    }
+
+    EXPECT_EQ(pTSBuf->numOfVnodes, j + 1);
+  }
+
+  tsBufResetPos(pTSBuf);
+
+  ////////////////////////////////////////////////////////////////////////////////////////
+  // reverse traverse
+  int64_t s = taosGetTimestampUs();
+  printf("start:%lld\n", s);
+
+  pTSBuf->cur.order = TSQL_SO_DESC;
+
+  // complete reverse traverse
+  int32_t x = 0;
+  while (tsBufNextPos(pTSBuf)) {
+    STSElem elem = tsBufGetElem(pTSBuf);
+    //    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+  }
+
+  // specify the data block with vnode and tags value
+  tsBufResetPos(pTSBuf);
+  pTSBuf->cur.order = TSQL_SO_DESC;
+
+  int32_t startVnode = 1;
+  int32_t startTag = 2;
+
+  tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
+
+  int32_t totalOutput = 10;
+  while (1) {
+    STSElem elem = tsBufGetElem(pTSBuf);
+    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+
+    if (!tsBufNextPos(pTSBuf)) {
+      break;
+    }
+
+    if (--totalOutput <= 0) {
+      totalOutput = 10;
+
+      tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
+
+      if (startTag == 0) {
+        startVnode -= 1;
+        startTag = 3;
+      }
+
+      if (startVnode < 0) {
+        break;
+      }
+    }
+  }
+
+  /////////////////////////////////////////////////////////////////////////////////
+  // traverse
+  pTSBuf->cur.order = TSQL_SO_ASC;
+  tsBufResetPos(pTSBuf);
+
+  // complete forwards traverse
+  while (tsBufNextPos(pTSBuf)) {
+    STSElem elem = tsBufGetElem(pTSBuf);
+    //    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+  }
+
+  // specify the data block with vnode and tags value
+  tsBufResetPos(pTSBuf);
+  pTSBuf->cur.order = TSQL_SO_ASC;
+
+  startVnode = 1;
+  startTag = 2;
+
+  tsBufGetElemStartPos(pTSBuf, startVnode, startTag);
+
+  totalOutput = 10;
+  while (1) {
+    STSElem elem = tsBufGetElem(pTSBuf);
+    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+
+    if (!tsBufNextPos(pTSBuf)) {
+      break;
+    }
+
+    if (--totalOutput <= 0) {
+      totalOutput = 10;
+
+      tsBufGetElemStartPos(pTSBuf, startVnode, --startTag);
+
+      if (startTag < 0) {
+        startVnode -= 1;
+        startTag = 3;
+      }
+
+      if (startVnode < 0) {
+        break;
+      }
+    }
+  }
+}
+
+void performanceTest() {}
+
+void emptyTagTest() {}
+
+void invalidFileTest() {
+  const char* cmd = "touch /tmp/test";
+
+  // create empty file
+  system(cmd);
+
+  STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true);
+  EXPECT_TRUE(pNewBuf == NULL);
+
+  pNewBuf = tsBufCreateFromFile("/tmp/911", true);
+  EXPECT_TRUE(pNewBuf == NULL);
+}
+
+void mergeDiffVnodeBufferTest() {
+  STSBuf* pTSBuf1 = tsBufCreate(true);
+  STSBuf* pTSBuf2 = tsBufCreate(true);
+
+  int32_t step = 30;
+  int32_t num = 1000;
+  int32_t numOfTags = 10;
+
+  // vnodeId:0
+  int64_t start = 10000000;
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    int64_t* list = createTsList(num, start, step);
+    tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t));
+    tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t));
+
+    free(list);
+
+    start += step * num;
+  }
+
+  tsBufFlush(pTSBuf2);
+
+  tsBufMerge(pTSBuf1, pTSBuf2, 9);
+  EXPECT_EQ(pTSBuf1->numOfVnodes, 2);
+  EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
+
+  tsBufDisplay(pTSBuf1);
+
+  tsBufDestory(pTSBuf2);
+  tsBufDestory(pTSBuf1);
+}
+
+void mergeIdenticalVnodeBufferTest() {
+  STSBuf* pTSBuf1 = tsBufCreate(true);
+  STSBuf* pTSBuf2 = tsBufCreate(true);
+
+  int32_t step = 30;
+  int32_t num = 1000;
+  int32_t numOfTags = 10;
+
+  // vnodeId:0
+  int64_t start = 10000000;
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    int64_t* list = createTsList(num, start, step);
+
+    tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+
+  for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
+    int64_t* list = createTsList(num, start, step);
+
+    tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+
+  tsBufFlush(pTSBuf2);
+
+  tsBufMerge(pTSBuf1, pTSBuf2, 12);
+  EXPECT_EQ(pTSBuf1->numOfVnodes, 1);
+  EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
+
+  tsBufResetPos(pTSBuf1);
+  while (tsBufNextPos(pTSBuf1)) {
+    STSElem elem = tsBufGetElem(pTSBuf1);
+    EXPECT_EQ(elem.vnode, 12);
+
+    printf("%d-%lld-%lld\n", elem.vnode, elem.tag, elem.ts);
+  }
+
+  tsBufDestory(pTSBuf1);
+  tsBufDestory(pTSBuf2);
+}
+}  // namespace
+
+TEST(testCase, tsBufTest) {
+  simpleTest();
+  largeTSTest();
+  multiTagsTest();
+  multiVnodeTagsTest();
+  loadDataTest();
+  invalidFileTest();
+  //  randomIncTsTest();
+  TSTraverse();
+  mergeDiffVnodeBufferTest();
+  mergeIdenticalVnodeBufferTest();
+}

src/util/CMakeLists.txt

@@ -29,6 +29,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
       MESSAGE(STATUS "Failed to find iconv, use default encoding method")
     ENDIF ()
   ENDIF ()
+
+  ADD_SUBDIRECTORY(tests)
 ELSEIF (TD_WINDOWS_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
@@ -113,4 +115,3 @@ ENDIF()
 #ENDIF ()
 
 
-

src/util/tests/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
+FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
+
+IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
+    MESSAGE(STATUS "gTest library found, build unit test")
+
+    INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+    ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
+    TARGET_LINK_LIBRARIES(utilTest tutil gtest pthread)
+ENDIF()
\ No newline at end of file

src/util/tests/cacheTest.cpp

+#include <iostream>
+#include <gtest/gtest.h>
+#include <sys/time.h>
+
+#include "taos.h"
+//#include "tsdb.h"
+
+//#include "testCommon.h"
+#include "tstoken.h"
+#include "tutil.h"
+#include "tcache.h"
+#include "ttimer.h"
+#include "ttime.h"
+
+namespace {
+int32_t tsMaxMgmtConnections = 10000;
+int32_t tsMaxMeterConnections = 200;
+}
+// test cache
+TEST(testCase, client_cache_test) {
+  const int32_t REFRESH_TIME_IN_SEC = 2;
+  void* tscTmr = taosTmrInit (tsMaxMgmtConnections*2, 200, 6000, "TSC");
+  SCacheObj* tscCacheHandle = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
+
+  char* key1 = "test1";
+  char* data1 = "test11";
+
+  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1);
+  sleep(REFRESH_TIME_IN_SEC+1);
+
+  printf("obj is still valid: %s\n", cachedObj);
+
+  char* data2 = "test22";
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
+
+  /* the object is cleared by cache clean operation */
+  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20);
+  printf("after updated: %s\n", cachedObj);
+
+  printf("start to remove data from cache\n");
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
+  printf("end of removing data from cache\n");
+
+  getchar();
+
+  char* key3 = "test2";
+  char* data3 = "kkkkkkk";
+
+  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1);
+  printf("%s\n", cachedObj2);
+
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
+
+  sleep(3);
+  char* d = (char*) taosCacheAcquireByName(tscCacheHandle, key3);
+//    assert(d == NULL);
+
+  char* key5 = "test5";
+  char* data5 = "data5kkkkk";
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20);
+
+  char* data6= "new Data after updated";
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
+
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20);
+  printf("%s\n", cachedObj2);
+
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
+
+  char* data7 = "add call update procedure";
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20);
+  printf("%s\n=======================================\n\n", cachedObj2);
+
+  char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5);
+
+  taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
+  taosCacheRelease(tscCacheHandle, (void**) &cc, false);
+
+  char* data8 = "ttft";
+  char* key6 = "key6";
+
+  char* ft = (char*) taosCachePut(tscCacheHandle, key6, data8, strlen(data8), 20);
+  taosCacheRelease(tscCacheHandle, (void**) &ft, false);
+
+  /**
+   * 140ns
+   */
+  uint64_t startTime = taosGetTimestampUs();
+  printf("Cache Performance Test\nstart time:%lld\n", startTime);
+  for(int32_t i=0; i<1000; ++i) {
+    char* dd = (char*) taosCacheAcquireByName(tscCacheHandle, key6);
+    if (dd != NULL) {
+//      printf("get the data\n");
+    } else {
+      printf("data has been released\n");
+    }
+
+    taosCacheRelease(tscCacheHandle, (void**) &dd, false);
+  }
+
+  uint64_t endTime = taosGetTimestampUs();
+  int64_t el = endTime - startTime;
+
+  printf("End of Test, %lld\nTotal Elapsed Time:%lld us.avg:%f us\n", endTime, el, el/1000.0);
+
+  taosCacheCleanup(tscCacheHandle);
+}
+
+TEST(testCase, cache_resize_test) {
+  const int32_t REFRESH_TIME_IN_SEC = 2;
+  void* tscTmr = taosTmrInit (1000*2, 200, 6000, "TSC");
+
+  auto* pCache = taosCacheInit(tscTmr, REFRESH_TIME_IN_SEC);
+
+  char key[256] = {0};
+  char data[1024] = "abcdefghijk";
+  int32_t len = strlen(data);
+
+  uint64_t startTime = taosGetTimestampUs();
+  int32_t num = 10000;
+
+  for(int32_t i = 0; i < num; ++i) {
+    int32_t len = sprintf(key, "abc_%7d", i);
+    taosCachePut(pCache, key, data, len, 3600);
+  }
+  uint64_t endTime = taosGetTimestampUs();
+
+  printf("add 10,000,000 object cost:%lld us, avg:%f us\n", endTime - startTime, (endTime-startTime)/(double)num);
+
+  startTime = taosGetTimestampUs();
+  for(int32_t i = 0; i < num; ++i) {
+    int32_t len = sprintf(key, "abc_%7d", i);
+    void* k = taosCacheAcquireByName(pCache, key);
+    assert(k != 0);
+  }
+  endTime = taosGetTimestampUs();
+  printf("retrieve 10,000,000 object cost:%lld us,avg:%f\n", endTime - startTime, (endTime - startTime)/(double)num);
+
+  taosCacheCleanup(pCache);
+  taosMsleep(20000);
+  getchar();
+}
\ No newline at end of file

src/util/tests/hashTest.cpp

+#include <gtest/gtest.h>
+#include <limits.h>
+#include <taosdef.h>
+#include <iostream>
+
+#include "hash.h"
+#include "taos.h"
+#include "ttime.h"
+
+namespace {
+// the simple test code for basic operations
+void simpleTest() {
+  auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  ASSERT_EQ(taosHashGetSize(hashTable), 0);
+  
+  // put 400 elements in the hash table
+  for(int32_t i = -200; i < 200; ++i) {
+    taosHashPut(hashTable, (const char*) &i, sizeof(int32_t), (char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 400);
+  
+  for(int32_t i = 0; i < 200; ++i) {
+    char* p = (char*) taosHashGet(hashTable, (const char*) &i, sizeof(int32_t));
+    ASSERT_TRUE(p != nullptr);
+    ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
+  }
+  
+  for(int32_t i = 1000; i < 2000; ++i) {
+    taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 400);
+  
+  for(int32_t i = 0; i < 100; ++i) {
+    taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 300);
+  
+  for(int32_t i = 100; i < 150; ++i) {
+    taosHashRemove(hashTable, (const char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 250);
+  taosHashCleanup(hashTable);
+}
+
+void stringKeyTest() {
+  auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
+  ASSERT_EQ(taosHashGetSize(hashTable), 0);
+  
+  char key[128] = {0};
+  
+  // put 200 elements in the hash table
+  for(int32_t i = 0; i < 1000; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+    taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 1000);
+  
+  for(int32_t i = 0; i < 1000; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+    
+    char* p = (char*) taosHashGet(hashTable, key, len);
+    ASSERT_TRUE(p != nullptr);
+    
+    ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
+  }
+  
+  for(int32_t i = 500; i < 1000; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+    
+    taosHashRemove(hashTable, key, len);
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 500);
+  
+  for(int32_t i = 0; i < 499; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+  
+    taosHashRemove(hashTable, key, len);
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), 1);
+  
+  taosHashCleanup(hashTable);
+}
+
+void functionTest() {
+
+}
+
+/**
+ * evaluate the performance issue, by add 10million elements in to hash table in
+ * a single threads situation
+ */
+void noLockPerformanceTest() {
+  auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false);
+  ASSERT_EQ(taosHashGetSize(hashTable), 0);
+  
+  char key[128] = {0};
+  int32_t num = 5000000;
+  
+  int64_t st = taosGetTimestampUs();
+  
+  // put 10M elements in the hash table
+  for(int32_t i = 0; i < num; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+    taosHashPut(hashTable, key, len, (char*) &i, sizeof(int32_t));
+  }
+  
+  ASSERT_EQ(taosHashGetSize(hashTable), num);
+  
+  int64_t et = taosGetTimestampUs();
+  printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
+  
+  st = taosGetTimestampUs();
+  for(int32_t i = 0; i < num; ++i) {
+    int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10);
+    char* p = (char*) taosHashGet(hashTable, key, len);
+    ASSERT_TRUE(p != nullptr);
+    
+    ASSERT_EQ(*reinterpret_cast<int32_t*>(p), i);
+  }
+  
+  et = taosGetTimestampUs();
+  printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st)/(double) num);
+  
+  printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable));
+  taosHashCleanup(hashTable);
+}
+
+void multithreadsTest() {
+  //todo
+}
+
+// check the function robustness
+void invalidOperationTest() {
+
+}
+
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(testCase, hashTest) {
+  simpleTest();
+  stringKeyTest();
+  noLockPerformanceTest();
+  multithreadsTest();
+}
\ No newline at end of file

src/util/tests/stringTest.cpp

+#include <gtest/gtest.h>
+#include <limits.h>
+#include <taosdef.h>
+#include <iostream>
+
+#include "taos.h"
+#include "tutil.h"
+
+TEST(testCase, string_dequote_test) {
+  char    t1[] = "'abc'";
+  int32_t len = strdequote(t1);
+
+  EXPECT_EQ(3, len);
+  EXPECT_STRCASEEQ(t1, "abc");
+
+  char t2[] = "\"abc\"";
+  len = strdequote(t2);
+
+  EXPECT_EQ(3, len);
+  EXPECT_STRCASEEQ(t1, "abc");
+
+  char t21[] = " abc ";
+  strtrim(t21);
+
+  EXPECT_STREQ("abc", t21);
+  EXPECT_EQ(3, strlen(t21));
+}
+
+TEST(testCase, string_replace_test) {
+  char  t3[] = "abc01abc02abc";
+  char* ret = strreplace(t3, "abc", "7");
+
+  EXPECT_EQ(strlen(ret), 7);
+  EXPECT_STREQ("7017027", ret);
+  free(ret);
+
+  char t4[] = "a01a02b03c04d05";
+  ret = strreplace(t4, "0", "9999999999");
+
+  EXPECT_EQ(strlen(ret), 5 * 10 + 10);
+  EXPECT_STREQ("a99999999991a99999999992b99999999993c99999999994d99999999995", ret);
+  free(ret);
+
+  char t5[] = "abc";
+  ret = strreplace(t5, "abc", "12345678901234567890");
+
+  EXPECT_EQ(strlen(ret), 20);
+  EXPECT_STREQ("12345678901234567890", ret);
+  free(ret);
+
+  char t6[] = "abc";
+  ret = strreplace(t6, "def", "abc");
+
+  EXPECT_EQ(strlen(ret), 3);
+  EXPECT_STREQ("abc", ret);
+  free(ret);
+
+  char t7[] = "abcde000000000000001234";
+  ret = strreplace(t7, "ab", "0000000");
+
+  EXPECT_EQ(strlen(ret), 28);
+  EXPECT_STREQ("0000000cde000000000000001234", ret);
+  free(ret);
+
+  char t8[] = "abc\ndef";
+  char t[] = {10, 0};
+
+  char    f1[] = "\\n";
+  int32_t fx = strlen(f1);
+  ret = strreplace(t8, "\n", "\\n");
+
+  EXPECT_EQ(strlen(ret), 8);
+  EXPECT_STREQ("abc\\ndef", ret);
+  free(ret);
+
+  char t9[] = "abc\\ndef";
+  ret = strreplace(t9, "\\n", "\n");
+
+  EXPECT_EQ(strlen(ret), 7);
+  EXPECT_STREQ("abc\ndef", ret);
+  free(ret);
+
+  char t10[] = "abcdef";
+  ret = strreplace(t10, "", "0");
+
+  EXPECT_EQ(strlen(ret), 6);
+  EXPECT_STREQ("abcdef", ret);
+  free(ret);
+}
+
+TEST(testCase, string_tolower_test) {
+  char t[1024] = {1};
+  memset(t, 1, tListLen(t));
+
+  const char* a1 = "ABC";
+  strtolower(t, a1);
+  EXPECT_STREQ(t, "abc");
+
+  memset(t, 1, tListLen(t));
+  const char* a2 = "ABC\'ABC\'D";
+  strtolower(t, a2);
+  EXPECT_STREQ(t, "abc\'ABC\'d");
+
+  memset(t, 1, tListLen(t));
+  const char* a3 = "";
+  strtolower(t, a3);
+  EXPECT_STREQ(t, "");
+
+  memset(t, 1, tListLen(t));
+  const char* a4 = "\"AbcDEF\"";
+  strtolower(t, a4);
+  EXPECT_STREQ(t, a4);
+
+  memset(t, 1, tListLen(t));
+  const char* a5 = "1234\"AbcDEF\"456";
+  strtolower(t, a5);
+  EXPECT_STREQ(t, a5);
+
+  memset(t, 1, tListLen(t));
+  const char* a6 = "1234";
+  strtolower(t, a6);
+  EXPECT_STREQ(t, a6);
+}
+
+TEST(testCase, string_strnchr_test) {
+  char t[1024] = {0};
+  memset(t, 1, tListLen(t));
+
+  char a1[] = "AB.C";
+  EXPECT_TRUE(strnchr(a1, '.', strlen(a1), true) != NULL);
+
+  char a2[] = "abc.";
+  EXPECT_TRUE(strnchr(a2, '.', strlen(a2), true) != NULL);
+
+  char a8[] = "abc.";
+  EXPECT_TRUE(strnchr(a8, '.', 1, true) == NULL);
+
+  char a3[] = ".abc";
+  EXPECT_TRUE(strnchr(a3, '.', strlen(a3), true) != NULL);
+
+  char a4[] = "'.abc'";
+  EXPECT_TRUE(strnchr(a4, '.', strlen(a4), true) == NULL);
+
+  char a5[] = "'.abc.'abc";
+  EXPECT_TRUE(strnchr(a5, '.', strlen(a5), true) == NULL);
+
+  char a6[] = "0123456789.";
+  EXPECT_TRUE(strnchr(a6, '.', strlen(a6), true) != NULL);
+
+  char a7[] = "0123456789.";
+  EXPECT_TRUE(strnchr(a7, '.', 3, true) == NULL);
+
+  char a9[] = "0123456789.";
+  EXPECT_TRUE(strnchr(a9, '.', 0, true) == NULL);
+
+  char a10[] = "0123456789'.'";
+  EXPECT_TRUE(strnchr(a10, '.', strlen(a10), true) == NULL);
+}
+
+TEST(testCase, cache_resize_test) {
+  char a11[] = "abc'.'";
+  EXPECT_TRUE(strnchr(a11, '.', strlen(a11), false) != NULL);
+
+  char a12[] = "abc'-'";
+  EXPECT_TRUE(strnchr(a12, '-', strlen(a12), false) != NULL);
+
+  char a15[] = "abc'-'";
+  EXPECT_TRUE(strnchr(a15, '-', strlen(a15), true) == NULL);
+
+  char a13[] = "'-'";
+  EXPECT_TRUE(strnchr(a13, '-', strlen(a13), false) != NULL);
+
+  char a14[] = "'-'";
+  EXPECT_TRUE(strnchr(a14, '-', strlen(a14), true) == NULL);
+
+  char a16[] = "'-'.";
+  EXPECT_TRUE(strnchr(a16, '.', strlen(a16), true) != NULL);
+}
\ No newline at end of file

src/vnode/tsdb/src/tsdbMeta.c

@@ -349,10 +349,10 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
   } else {
     // add non-super table to the array
     pMeta->tables[pTable->tableId.tid] = pTable;
-    if (pTable->type == TSDB_CHILD_TABLE) {
-      // add STABLE to the index
+    if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index
       tsdbAddTableIntoIndex(pMeta, pTable);
     }
+    
     pMeta->nTables++;
   }
 

src/vnode/tsdb/src/tsdbRead.c

@@ -73,7 +73,7 @@ typedef struct SQueryFilesInfo {
   char dbFilePathPrefix[PATH_MAX];
 } SQueryFilesInfo;
 
-typedef struct STableQueryInfo {
+typedef struct STableCheckInfo {
   STableId    tableId;
   TSKEY       lastKey;
   STable *    pTableObj;
@@ -81,7 +81,8 @@ typedef struct STableQueryInfo {
   int32_t     numOfBlocks;
   int32_t     start;
   SCompBlock *pBlock;
-} STableQueryInfo;
+  SSkipListIterator* iter;
+} STableCheckInfo;
 
 typedef struct {
   SCompBlock *compBlock;
@@ -90,14 +91,19 @@ typedef struct {
 
 typedef struct STableDataBlockInfoEx {
   SCompBlockFields pBlock;
-  STableQueryInfo* pMeterDataInfo;
+  STableCheckInfo* pMeterDataInfo;
   int32_t          blockIndex;
   int32_t          groupIdx; /* number of group is less than the total number of meters */
 } STableDataBlockInfoEx;
 
+enum {
+  SINGLE_TABLE_MODEL = 1,
+  MULTI_TABLE_MODEL = 2,
+};
+
 typedef struct STsdbQueryHandle {
   struct STsdbRepo* pTsdb;
-  
+  int8_t          model;  // access model, single table model or multi-table model
   SQueryHandlePos cur;    // current position
   SQueryHandlePos start;  // the start position, used for secondary/third iteration
   int32_t         unzipBufSize;
@@ -120,14 +126,13 @@ typedef struct STsdbQueryHandle {
   bool        locateStart;
   int32_t     realNumOfRows;
   bool        loadDataAfterSeek;  // load data after seek.
-  SArray*     pTableQueryInfo;
+  SArray*     pTableCheckInfo;
   int32_t     activeIndex;
   
   int32_t     tableIndex;
   bool        isFirstSlot;
   void *      qinfo;              // query info handle, for debug purpose
   
-  SSkipListIterator*     memIter;
   STableDataBlockInfoEx *pDataBlockInfoEx;
 } STsdbQueryHandle;
 
@@ -271,19 +276,21 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
   size_t size = taosArrayGetSize(idList);
   assert(size >= 1);
 
-  pQueryHandle->pTableQueryInfo = taosArrayInit(size, sizeof(STableQueryInfo));
+  pQueryHandle->pTableCheckInfo = taosArrayInit(size, sizeof(STableCheckInfo));
   for(int32_t i = 0; i < size; ++i) {
     STableId id = *(STableId*) taosArrayGet(idList, i);
     
-    STableQueryInfo info = {
+    STableCheckInfo info = {
       .lastKey = pQueryHandle->window.skey,
       .tableId = id,
       .pTableObj = tsdbGetTableByUid(tsdbGetMeta(tsdb), id.uid),  //todo this may be failed
     };
     
-    taosArrayPush(pQueryHandle->pTableQueryInfo, &info);
+    taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
   }
   
+  pQueryHandle->model = (size > 1)? MULTI_TABLE_MODEL:SINGLE_TABLE_MODEL;
+  
   pQueryHandle->activeIndex = 0;
   
   // malloc buffer in order to load data from file
@@ -314,11 +321,13 @@ tsdb_query_handle_t *tsdbQueryByTableId(tsdb_repo_t* tsdb, STsdbQueryCond *pCond
   return (tsdb_query_handle_t)pQueryHandle;
 }
 
-bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
-  STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
-  STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
+static bool hasMoreDataInCacheForSingleModel(STsdbQueryHandle* pHandle) {
+  assert(pHandle->activeIndex == 0 && taosArrayGetSize(pHandle->pTableCheckInfo) == 1);
   
-  STable *pTable = pTableQInfo->pTableObj;
+  STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
+  
+  STable *pTable = pTableCheckInfo->pTableObj;
+  assert(pTable != NULL);
   
   // no data in cache, abort
   if (pTable->mem == NULL && pTable->imem == NULL) {
@@ -326,13 +335,49 @@ bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
   }
   
   // all data in mem are checked already.
-  if (pTableQInfo->lastKey > pTable->mem->keyLast) {
+  if (pTableCheckInfo->lastKey > pTable->mem->keyLast) {
     return false;
   }
   
   return true;
 }
 
+static bool hasMoreDataInCacheForMultiModel(STsdbQueryHandle* pHandle) {
+  size_t numOfTables = taosArrayGetSize(pHandle->pTableCheckInfo);
+  assert(numOfTables > 0);
+  
+  while(pHandle->activeIndex < numOfTables) {
+    STableCheckInfo* pTableCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
+  
+    STable *pTable = pTableCheckInfo->pTableObj;
+    if (pTable->mem == NULL && pTable->imem == NULL) {
+      pHandle->activeIndex += 1;  // try next table if exits
+      continue;
+    }
+  
+    // all data in mem are checked already.
+    if (pTableCheckInfo->lastKey > pTable->mem->keyLast) {
+      pHandle->activeIndex += 1;  // try next table if exits
+      continue;
+    }
+    
+    return true;
+  }
+  
+  // all tables has checked already
+  return false;
+}
+
+// handle data in cache situation
+bool tsdbNextDataBlock(tsdb_query_handle_t *pQueryHandle) {
+  STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
+  if (pHandle->model == SINGLE_TABLE_MODEL) {
+    return hasMoreDataInCacheForSingleModel(pHandle);
+  } else {
+    return hasMoreDataInCacheForMultiModel(pHandle);
+  }
+}
+
 static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
     TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pHandle) {
   int numOfRows = 0;
@@ -370,7 +415,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
 SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
   
-  STableQueryInfo* pTableQInfo = taosArrayGet(pHandle->pTableQueryInfo, pHandle->activeIndex);
+  STableCheckInfo* pTableQInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
   STable *pTable = pTableQInfo->pTableObj;
   
   TSKEY skey = 0, ekey = 0;
@@ -379,11 +424,11 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle) {
   if (pTable->mem != NULL) {
     
     // create mem table iterator if it is not created yet
-    if (pHandle->memIter == NULL) {
-      pHandle->memIter = tSkipListCreateIter(pTable->mem->pData);
+    if (pTableQInfo->iter == NULL) {
+      pTableQInfo->iter = tSkipListCreateIter(pTable->mem->pData);
     }
     
-    rows = tsdbReadRowsFromCache(pHandle->memIter, INT64_MAX, 2, &skey, &ekey, pHandle);
+    rows = tsdbReadRowsFromCache(pTableQInfo->iter, INT64_MAX, 2, &skey, &ekey, pHandle);
   }
   
   SDataBlockInfo blockInfo = {

src/vnode/tsdb/tests/CMakeLists.txt

@@ -3,9 +3,4 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 add_executable(tsdbTests ${SOURCE_LIST})
 target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb)
 
-add_test(
-    NAME
-      unit   
-    COMMAND
-      ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests
-)
\ No newline at end of file
+add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests)
\ No newline at end of file