Merge branch 'hotfix/leak' of https://github.com/taosdata/TDengine into hotfix/leak

src/client/CMakeLists.txt

@@ -35,6 +35,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   MESSAGE(STATUS "build version ${VERSION_INFO}")
   SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1)
 
+  ADD_SUBDIRECTORY(tests)
+
 ELSEIF (TD_WINDOWS_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32)

src/client/src/tscParseInsert.c

@@ -1351,6 +1351,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
 static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlocks *pTableDataBlocks) {
   int32_t  code = TSDB_CODE_SUCCESS;
   SSqlCmd *pCmd = &pSql->cmd;
+  pSql->res.numOfRows = 0;
 
   assert(pCmd->numOfClause == 1);
   STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta;
@@ -1394,6 +1395,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
     fclose(fp);
 
     pParentSql->res.code = code;
+    tscQueueAsyncRes(pParentSql);
     return;
   }
 
@@ -1458,8 +1460,11 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) {
   free(line);
 
   if (count > 0) {
-    if ((code = doPackSendDataBlock(pSql, count, pTableDataBlock)) != TSDB_CODE_SUCCESS) {
+    code = doPackSendDataBlock(pSql, count, pTableDataBlock);
+    if (code != TSDB_CODE_SUCCESS) {
       pParentSql->res.code = code;
+      tscQueueAsyncRes(pParentSql);
+      return;
     }
 
   } else {

src/client/tests/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
+FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
+
+IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
+    MESSAGE(STATUS "gTest library found, build unit test")
+
+    INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+    ADD_EXECUTABLE(cliTest ${SOURCE_LIST})
+    TARGET_LINK_LIBRARIES(cliTest taos tutil common gtest pthread)
+ENDIF()
\ No newline at end of file

src/common/src/ttimezone.c

@@ -20,6 +20,7 @@
 #include "tconfig.h"
 #include "tutil.h"
 
+// TODO refactor to set the tz value through parameter
 void tsSetTimeZone() {
   SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
   uPrint("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]);

src/dnode/src/dnodeMPeer.c

@@ -113,6 +113,7 @@ void dnodeFreeMnodePqueue() {
 void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, false);
+    rpcFreeCont(pMsg->pCont);
     return;
   }
 

src/dnode/src/dnodeMRead.c

@@ -116,6 +116,7 @@ void dnodeFreeMnodeRqueue() {
 void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMReadQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, true);
+    rpcFreeCont(pMsg->pCont);
     return;
   }
 

src/dnode/src/dnodeMWrite.c

@@ -115,6 +115,7 @@ void dnodeFreeMnodeWqueue() {
 void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, true);
+    rpcFreeCont(pMsg->pCont);
     return;
   }
 

src/inc/taosdef.h

@@ -354,7 +354,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_DBS_HASH_SIZE      100
 #define TSDB_DEFAULT_VGROUPS_HASH_SIZE  100
 #define TSDB_DEFAULT_STABLES_HASH_SIZE  100
-#define TSDB_DEFAULT_CTABLES_HASH_SIZE  10000
+#define TSDB_DEFAULT_CTABLES_HASH_SIZE  20000
 
 #define TSDB_PORT_DNODESHELL 0 
 #define TSDB_PORT_DNODEDNODE 5 

src/kit/taosdemo/taosdemo.c

@@ -499,7 +499,7 @@ int main(int argc, char *argv[]) {
     /* Create all the tables; */
     printf("Creating %d table(s)......\n", ntables);
     for (int i = 0; i < ntables; i++) {
-      snprintf(command, BUFFER_SIZE, "create table %s.%s%d (ts timestamp%s;", db_name, tb_prefix, i, cols);
+      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", db_name, tb_prefix, i, cols);
       queryDB(taos, command);
     }
 
@@ -509,7 +509,7 @@ int main(int argc, char *argv[]) {
   } else {
     /* Create metric table */
     printf("Creating meters super table...\n");
-    snprintf(command, BUFFER_SIZE, "create table %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols);
+    snprintf(command, BUFFER_SIZE, "create table if not exists %s.meters (ts timestamp%s tags (areaid int, loc binary(10))", db_name, cols);
     queryDB(taos, command);
     printf("meters created!\n");
 
@@ -523,9 +523,9 @@ int main(int argc, char *argv[]) {
         j = i % 10;
       }
     if (j % 2 == 0) {
-      snprintf(command, BUFFER_SIZE, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "shanghai");
+      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "shanghai");
     } else {
-      snprintf(command, BUFFER_SIZE, "create table %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "beijing");
+      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", db_name, tb_prefix, i, db_name, j, "beijing");
     }
       queryDB(taos, command);
     }
@@ -847,10 +847,10 @@ void *syncWrite(void *sarg) {
       pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID);
       int k;
       for (k = 0; k < winfo->nrecords_per_request;) {
-        int rand_num = trand() % 100;
+        int rand_num = rand() % 100;
         int len = -1;
         if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
-          long d = tmp_time - trand() % 1000000 + rand_num;
+          long d = tmp_time - rand() % 1000000 + rand_num;
           len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
         } else {
           len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
@@ -942,10 +942,10 @@ void callBack(void *param, TAOS_RES *res, int code) {
   pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name);
 
   for (int i = 0; i < tb_info->nrecords_per_request; i++) {
-    int rand_num = trand() % 100;
+    int rand_num = rand() % 100;
     if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate)
     {
-      long d = tmp_time - trand() % 1000000 + rand_num;
+      long d = tmp_time - rand() % 1000000 + rand_num;
       generateData(data, datatype, ncols_per_record, d, len_of_binary);
     } else 
     {
@@ -994,20 +994,20 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times
 
   for (int i = 0; i < num_of_cols; i++) {
     if (strcasecmp(data_type[i % c], "tinyint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(trand() % 128));
+      pstr += sprintf(pstr, ", %d", (int)(rand() % 128));
     } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(trand() % 32767));
+      pstr += sprintf(pstr, ", %d", (int)(rand() % 32767));
     } else if (strcasecmp(data_type[i % c], "int") == 0) {
-      pstr += sprintf(pstr, ", %d", (int)(trand() % 10)); 
+      pstr += sprintf(pstr, ", %d", (int)(rand() % 10)); 
     } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
-      pstr += sprintf(pstr, ", %" PRId64, trand() % 2147483648);
+      pstr += sprintf(pstr, ", %" PRId64, rand() % 2147483648);
     } else if (strcasecmp(data_type[i % c], "float") == 0) {
-      pstr += sprintf(pstr, ", %10.4f", (float)(trand() / 1000.0));
+      pstr += sprintf(pstr, ", %10.4f", (float)(rand() / 1000.0));
     } else if (strcasecmp(data_type[i % c], "double") == 0) {
-      double t = (double)(trand() / 1000000.0);
+      double t = (double)(rand() / 1000000.0);
       pstr += sprintf(pstr, ", %20.8f", t);
     } else if (strcasecmp(data_type[i % c], "bool") == 0) {
-      bool b = trand() & 1;
+      bool b = rand() & 1;
       pstr += sprintf(pstr, ", %s", b ? "true" : "false");
     } else if (strcasecmp(data_type[i % c], "binary") == 0) {
       char s[len_of_binary];
@@ -1033,7 +1033,7 @@ void rand_string(char *str, int size) {
     --size;
     int n;
     for (n = 0; n < size; n++) {
-      int key = trand() % (int)(sizeof charset - 1);
+      int key = rand() % (int)(sizeof charset - 1);
       str[n] = charset[key];
     }
     str[n] = 0;

src/mnode/src/mnodeDb.c

@@ -968,6 +968,17 @@ static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) {
   return mnodeAlterDb(pMsg->pDb, pAlter, pMsg);
 }
 
+static int32_t mnodeDropDbCb(SMnodeMsg *pMsg, int32_t code) {
+  SDbObj *pDb = pMsg->pDb;
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("db:%s, failed to drop from sdb, reason:%s", pDb->name, tstrerror(code));
+  } else {
+    mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
+  }
+
+  return code;
+}
+
 static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
   
@@ -978,12 +989,12 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
     .type  = SDB_OPER_GLOBAL,
     .table = tsDbSdb,
     .pObj  = pDb,
-    .pMsg  = pMsg
+    .pMsg  = pMsg,
+    .cb    = mnodeDropDbCb
   };
 
   int32_t code = sdbDeleteRow(&oper);
   if (code == TSDB_CODE_SUCCESS) {
-    mLPrint("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
     code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
 

src/mnode/src/mnodeSdb.c

@@ -250,7 +250,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   sdbTrace("forward request confirmed, version:%" PRIu64 ", result:%s", (int64_t)param, tstrerror(code));
 }
 
-static int32_t sdbForwardToPeer(SWalHead *pHead) {
+ static int32_t sdbForwardToPeer(SWalHead *pHead) {
   if (tsSdbObj.sync == NULL) return TSDB_CODE_SUCCESS;
 
   int32_t code = syncForwardToPeer(tsSdbObj.sync, pHead, (void*)pHead->version, TAOS_QTYPE_RPC);
@@ -782,7 +782,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   pTable->restoredFp   = pDesc->restoredFp;
 
   _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
-  if (pTable->keyType == SDB_KEY_STRING) {
+  if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
     hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   }
   pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true);

src/mnode/src/mnodeTable.c

@@ -854,13 +854,15 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
 
 static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
   SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable;
-  if (pTable != NULL) {
-    mLPrint("app:%p:%p, stable:%s, is dropped from sdb, result:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
-            tstrerror(code));
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+  } else {
+    mLPrint("app:%p:%p, stable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
   }
 
   return code;
 }
+
 static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
   if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR;
 
@@ -899,12 +901,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
   };
 
   int32_t code = sdbDeleteRow(&oper);
-  if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
-    return code;
-  } else {
+  if (code == TSDB_CODE_SUCCESS) {
     return TSDB_CODE_MND_ACTION_IN_PROGRESS;
   }
+  return code;
 }
 
 static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) {

src/query/src/qExecutor.c

@@ -767,6 +767,9 @@ static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) {
 
 static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size,
                     SArray *pDataBlock) {
+  if (pDataBlock == NULL) {
+    return NULL;
+  }
   char *dataBlock = NULL;
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
@@ -819,7 +822,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
 }
 
 /**
- *
+ * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions
  * @param pRuntimeEnv
  * @param forwardStep
  * @param tsCols
@@ -854,6 +857,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
 
     STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
     if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win) != TSDB_CODE_SUCCESS) {
+      tfree(sasArray);
       return;
     }
 
@@ -1061,15 +1065,17 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo* item = pQuery->current;
 
-  TSKEY  *tsCols = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData;
-  bool    groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr);
+  SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0);
+
+  TSKEY  *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL;
+  bool    groupbyColumnValue = isGroupbyNormalCol(pQuery->pGroupbyExpr);
   SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport));
 
   int16_t type = 0;
   int16_t bytes = 0;
 
   char *groupbyColumnData = NULL;
-  if (groupbyStateValue) {
+  if (groupbyColumnValue) {
     groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock);
   }
 
@@ -1157,7 +1163,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       pWindowResInfo->curIndex = index;
     } else {  // other queries
       // decide which group this rows belongs to according to current state value
-      if (groupbyStateValue) {
+      if (groupbyColumnValue) {
         char *val = groupbyColumnData + bytes * offset;
 
         int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes);
@@ -1183,7 +1189,12 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
     }
   }
 
+  assert(offset >= 0);
+  if (tsCols != NULL) {
     item->lastKey = tsCols[offset] + step;
+  } else {
+    item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step;
+  }
 
   // todo refactor: extract method
   for(int32_t i = 0; i < pQuery->numOfOutput; ++i) {
@@ -1349,10 +1360,13 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) {
         // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ
       }
     }
-
+    if (p != NULL) {
       p->tagInfo.pTagCtxList = pTagCtx;
       p->tagInfo.numOfTagCols = num;
       p->tagInfo.tagsLen = tagLen;
+    } else {
+      tfree(pTagCtx); 
+    }
   }
 }
 
@@ -3497,7 +3511,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde
       continue;
     }
 
-    assert(result[i].numOfRows >= 0 && pQInfo->offset <= 1);
+    assert(pQInfo->offset <= 1);
 
     int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset;
     int32_t oldOffset = pQInfo->offset;
@@ -5295,9 +5309,9 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
       bytes = s.bytes;
     } else{
       int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols);
-      assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags || j == TSDB_TBNAME_COLUMN_INDEX);
+      assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags);
 
-      if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) {
+      if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) {
         SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j];
         type = pCol->type;
         bytes = pCol->bytes;
@@ -5339,8 +5353,6 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo *
       assert(ret == TSDB_CODE_SUCCESS);
     }
   }
-
-  tfree(pExprMsg);
   *pExprInfo = pExprs;
 
   return TSDB_CODE_SUCCESS;
@@ -5591,10 +5603,13 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   pQInfo->signature = pQInfo;
 
   pQInfo->tableGroupInfo = *pTableGroupInfo;
-  size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
+  size_t numOfGroups = 0;
+  if (pTableGroupInfo->pGroupList != NULL) {
+    numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
 
     pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
     pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables;
+  } 
 
   int tableIndex = 0;
   STimeWindow window = pQueryMsg->window;
@@ -5693,7 +5708,8 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
     pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder);
 
     tsBufResetPos(pTSBuf);
-    tsBufNextPos(pTSBuf);
+    bool ret = tsBufNextPos(pTSBuf);
+    UNUSED(ret);
   }
 
   // only the successful complete requries the sem_post/over = 1 operations.
@@ -5839,18 +5855,23 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
 
     // make sure file exist
     if (FD_VALID(fd)) {
-      size_t s = lseek(fd, 0, SEEK_END);
-      qTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s);
-
-      lseek(fd, 0, SEEK_SET);
-      read(fd, data, s);
+      int32_t s = lseek(fd, 0, SEEK_END);
+      UNUSED(s);
+      qTrace("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s);
+      s = lseek(fd, 0, SEEK_SET);
+      if (s >= 0) {
+        size_t sz = read(fd, data, s);
+        UNUSED(sz);
+      }
       close(fd);
-
       unlink(pQuery->sdata[0]->data);
     } else {
-      // todo return the error code to client
+      // todo return the error code to client and handle invalid fd
       qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo,
              pQuery->sdata[0]->data, strerror(errno));
+      if (fd != -1) {
+        close(fd); 
+      }
     }
 
     // all data returned, set query over
@@ -5903,7 +5924,6 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
   }
 
   if ((code = createQFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) {
-    free(pExprMsg);
     goto _over;
   }
 
@@ -5975,6 +5995,7 @@ _over:
   } 
   free(pTagColumnInfo);
   free(pExprs);
+  free(pExprMsg);
   taosArrayDestroy(pTableIdList);
 
   //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;

src/query/src/qpercentile.c

@@ -880,9 +880,12 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
 
           for (uint32_t jx = 0; jx < pFlushInfo->numOfPages; ++jx) {
             size_t sz = fread(pPage, pMemBuffer->pageSize, 1, pMemBuffer->file);
-            UNUSED(sz);
+            if (sz != pMemBuffer->pageSize) {
+              uError("MemBucket:%p, read tmp file %s failed", pMemBucket, pMemBuffer->path);
+            } else {
               tMemBucketPut(pMemBucket, pPage->data, pPage->num);
             }
+          }
 
           fclose(pMemBuffer->file);
           if (unlink(pMemBuffer->path) != 0) {

src/rpc/src/rpcMain.c

@@ -331,6 +331,7 @@ void rpcFreeCont(void *cont) {
   if ( cont ) {
     char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext);
     free(temp);
+    // tTrace("free mem: %p", temp);
   }
 }
 
@@ -540,6 +541,7 @@ static void rpcFreeMsg(void *msg) {
   if ( msg ) {
     char *temp = (char *)msg - sizeof(SRpcReqContext);
     free(temp);
+    // tTrace("free mem: %p", temp);
   }
 }
 

src/rpc/src/rpcTcp.c

@@ -418,6 +418,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
   if ( NULL == buffer) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
+  } else {
+    // tTrace("malloc mem: %p", buffer);
   }
 
   msg = buffer + tsRpcOverhead;

src/rpc/src/rpcUdp.c

@@ -211,6 +211,8 @@ static void *taosRecvUdpData(void *param) {
     if (NULL == tmsg) {
       tError("%s failed to allocate memory, size:%ld", pConn->label, dataLen);
       continue;
+    } else {
+      // tTrace("malloc mem: %p", tmsg);
     }
 
     tmsg += tsRpcOverhead;  // overhead for SRpcReqContext

src/tsdb/src/tsdbMeta.c

@@ -553,10 +553,18 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) {
   return 0;
 }
 
-void tsdbRefTable(STable *pTable) { T_REF_INC(pTable); }
+void tsdbRefTable(STable *pTable) {
+  int16_t ref = T_REF_INC(pTable);
+  tsdbTrace("ref table:%s, uid:%"PRIu64", tid:%d, ref:%d", TABLE_CHAR_NAME(pTable), pTable->tableId.uid, pTable->tableId.tid, ref);
+}
 
 void tsdbUnRefTable(STable *pTable) {
-  if (T_REF_DEC(pTable) == 0) {
+  int16_t ref = T_REF_DEC(pTable);
+  tsdbTrace("unref table:%s, uid:%"PRIu64", tid:%d, ref:%d", TABLE_CHAR_NAME(pTable), pTable->tableId.uid, pTable->tableId.tid, ref);
+
+  if (ref == 0) {
+    tsdbTrace("destroy table:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), pTable->tableId.uid, pTable->tableId.tid);
+
     if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) {
       tsdbUnRefTable(pTable->pSuper);
     }

src/util/src/tskiplist.c

@@ -38,7 +38,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
   const uint32_t factor = 4;
 
   int32_t n = 1;
-  while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) {
+  while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
     n++;
   }
 

src/util/src/ttime.c

@@ -56,11 +56,9 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
     year -= 1;
   }
 
-  //int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) +
-	//	  year*365 - 719499)*24 + hour)*60 + min)*60 + sec);
-  int64_t res;
-  res  = 367*((int64_t)mon)/12;
-  res += year/4 - year/100 + year/400 + day + year*365 - 719499;
+  int64_t res = 367*((int64_t)mon)/12;
+
+  res += ((int64_t)(year/4 - year/100 + year/400 + day + year*365) - 719499);  // this value may be less than 0
   res  = res*24;
   res  = ((res + hour) * 60 + min) * 60 + sec;  
 

tests/script/general/parser/auto_create_tb.sim

@@ -153,13 +153,13 @@ print $rows $data00 $data10 $data20
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb3 then
+if $data00 != tb1 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb1 then
+if $data20 != tb3 then
   return -1
 endi
 
@@ -221,13 +221,13 @@ sql show tables
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb3 then
+if $data00 != tb1 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb1 then
+if $data20 != tb3 then
   return -1
 endi
 

tests/script/general/parser/projection_limit_offset.sim

@@ -80,6 +80,7 @@ print $rows
 
 sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0;
 if $rows != 4008 then
+  print expect 4008, actual:$rows
   return -1
 endi
 

tests/test/c/CMakeLists.txt

@@ -7,15 +7,21 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  add_executable(insertPerTable insertPerTable.c)
-  target_link_libraries(insertPerTable taos_static pthread)
+  #add_executable(insertPerTable insertPerTable.c)
+  #target_link_libraries(insertPerTable taos_static pthread)
 
-  add_executable(insertPerRow insertPerRow.c)
-  target_link_libraries(insertPerRow taos_static pthread)
+  #add_executable(insertPerRow insertPerRow.c)
+  #target_link_libraries(insertPerRow taos_static pthread)
 
-  add_executable(importOneRow importOneRow.c)
-  target_link_libraries(importOneRow taos_static pthread)
+  #add_executable(importOneRow importOneRow.c)
+  #target_link_libraries(importOneRow taos_static pthread)
 
-  add_executable(importPerTable importPerTable.c)
-  target_link_libraries(importPerTable taos_static pthread)
+  #add_executable(importPerTable importPerTable.c)
+  #target_link_libraries(importPerTable taos_static pthread)
+
+  #add_executable(hashPerformance hashPerformance.c)
+  #target_link_libraries(hashPerformance taos_static tutil common pthread)
+
+  add_executable(createTablePerformance createTablePerformance.c)
+  target_link_libraries(createTablePerformance taos_static tutil common pthread)
 ENDIF()

tests/test/c/createTablePerformance.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tulog.h"
+#include "ttime.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "hash.h"
+
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+char    dbName[32] = "db";
+char    stableName[64] = "st";
+int32_t numOfThreads = 30;
+int32_t numOfTables = 100000;
+int32_t maxTables = 5000;
+int32_t numOfColumns = 2;
+
+typedef struct {
+  int32_t   tableBeginIndex;
+  int32_t   tableEndIndex;
+  int32_t   threadIndex;
+  char      dbName[32];
+  char      stableName[64];
+  float     createTableSpeed;
+  pthread_t thread;
+} SThreadInfo;
+
+void  shellParseArgument(int argc, char *argv[]);
+void *threadFunc(void *param);
+void  createDbAndSTable();
+
+int main(int argc, char *argv[]) {
+  shellParseArgument(argc, argv);
+  taos_init();
+  createDbAndSTable();
+
+  pPrint("%d threads are spawned to create table", numOfThreads);
+  
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo));
+
+  int32_t numOfTablesPerThread = numOfTables / numOfThreads;
+  numOfTables = numOfTablesPerThread * numOfThreads;
+  for (int i = 0; i < numOfThreads; ++i) {
+    pInfo[i].tableBeginIndex = i * numOfTablesPerThread;
+    pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread;
+    pInfo[i].threadIndex = i;
+    strcpy(pInfo[i].dbName, dbName);
+    strcpy(pInfo[i].stableName, stableName);
+    pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
+  }
+
+  taosMsleep(300);
+  for (int i = 0; i < numOfThreads; i++) {
+    pthread_join(pInfo[i].thread, NULL);
+  }
+
+  float createTableSpeed = 0;
+  for (int i = 0; i < numOfThreads; ++i) {
+    createTableSpeed += pInfo[i].createTableSpeed;
+  }
+
+  pPrint("%s total speed:%.1f tables/second, threads:%d %s", GREEN, createTableSpeed, numOfThreads, NC);
+
+  pthread_attr_destroy(&thattr);
+  free(pInfo);
+}
+
+void createDbAndSTable() {
+  pPrint("start to create db and stable");
+  char qstr[64000];
+  
+  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  if (con == NULL) {
+    pError("failed to connect to DB, reason:%s", taos_errstr(con));
+    exit(1);
+  }
+
+  sprintf(qstr, "create database if not exists %s maxtables %d", dbName, maxTables);
+  TAOS_RES *pSql = taos_query(con, qstr);
+  int32_t code = taos_errno(pSql);
+  if (code != 0) {
+    pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con));
+    exit(0);
+  }
+  taos_free_result(pSql);
+
+  sprintf(qstr, "use %s", dbName);
+  pSql = taos_query(con, qstr);
+  code = taos_errno(pSql);
+  if (code != 0) {
+    pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+    exit(0);
+  }
+  taos_free_result(pSql);
+
+  int len = sprintf(qstr, "create table if not exists %s(ts timestamp", stableName);
+  for (int32_t f = 0; f < numOfColumns - 1; ++f) {
+    len += sprintf(qstr + len, ", f%d double", f);
+  }
+  sprintf(qstr + len, ") tags(t int)");
+
+  pSql = taos_query(con, qstr);
+  code = taos_errno(pSql);
+  if (code != 0) {
+    pError("failed to create stable, code:%d reason:%s", taos_errno(con), taos_errstr(con));
+    exit(0);
+  }
+  taos_free_result(pSql);
+
+  taos_close(con);
+}
+
+void *threadFunc(void *param) {
+  SThreadInfo *pInfo = (SThreadInfo *)param;
+  char qstr[65000];
+  int  code;
+
+  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  if (con == NULL) {
+    pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con));
+    exit(1);
+  }
+
+  sprintf(qstr, "use %s", pInfo->dbName);
+  TAOS_RES *pSql = taos_query(con, qstr);
+  taos_free_result(pSql);
+
+  int64_t startMs = taosGetTimestampMs();
+
+  for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
+    sprintf(qstr, "create table if not exists %s%d using %s tags(%d)", stableName, t, stableName, t);
+    TAOS_RES *pSql = taos_query(con, qstr);
+    code = taos_errno(pSql);
+    if (code != 0) {
+      pError("failed to create table %s%d, reason:%s", stableName, t, taos_errstr(con));
+    }
+    taos_free_result(pSql);
+  }
+
+  float createTableSpeed = 0;
+  for (int i = 0; i < numOfThreads; ++i) {
+    createTableSpeed += pInfo[i].createTableSpeed;
+  }
+
+  int64_t endMs = taosGetTimestampMs();
+  int32_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex;
+  float   seconds = (endMs - startMs) / 1000.0;
+  float   speed = totalTables / seconds;
+  pInfo->createTableSpeed = speed;
+
+  pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed);
+  taos_close(con);
+
+  return 0;
+}
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance while create table\n");
+
+  printf("%s%s\n", indent, "-c");
+  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
+  printf("%s%s\n", indent, "-d");
+  printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
+  printf("%s%s\n", indent, "-s");
+  printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName);
+  printf("%s%s\n", indent, "-t");
+  printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads);
+  printf("%s%s\n", indent, "-n");
+  printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", numOfTables);
+  printf("%s%s\n", indent, "-columns");
+  printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns);
+  printf("%s%s\n", indent, "-tables");
+  printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", maxTables);
+  
+  exit(EXIT_SUCCESS);
+}
+
+void shellParseArgument(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-d") == 0) {
+      strcpy(dbName, argv[++i]);
+    } else if (strcmp(argv[i], "-c") == 0) {
+      strcpy(configDir, argv[++i]);
+    } else if (strcmp(argv[i], "-s") == 0) {
+      strcpy(stableName, argv[++i]);
+    } else if (strcmp(argv[i], "-t") == 0) {
+      numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      numOfTables = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-tables") == 0) {
+      maxTables = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-columns") == 0) {
+      numOfColumns = atoi(argv[++i]);
+    } else {
+    }
+  }
+
+  pPrint("%s dbName:%s %s", GREEN, dbName, NC);
+  pPrint("%s stableName:%s %s", GREEN, stableName, NC);
+  pPrint("%s configDir:%s %s", GREEN, configDir, NC);
+  pPrint("%s numOfTables:%d %s", GREEN, numOfTables, NC);
+  pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC);
+  pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC);
+  pPrint("%s dbPara maxTables:%d %s", GREEN, maxTables, NC);
+  
+  pPrint("%s start create table performace test %s", GREEN, NC);
+}

tests/test/c/hashPerformance.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tulog.h"
+#include "ttime.h"
+#include "tutil.h"
+#include "hash.h"
+
+#define MAX_RANDOM_POINTS 20000
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+int32_t capacity = 100000;
+int32_t q1Times = 1;
+int32_t q2Times = 1;
+int32_t keyNum = 100000;
+int32_t printInterval = 10000;
+
+typedef struct HashTestRow {
+  int32_t size;
+  void *  ptr;
+} HashTestRow;
+
+void shellParseArgument(int argc, char *argv[]);
+
+void testHashPerformance() {
+  int64_t    initialMs = taosGetTimestampMs();
+  _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  void *     hashHandle = taosHashInit(capacity, hashFp, true);
+
+  int64_t startMs = taosGetTimestampMs();
+  float   seconds = (startMs - initialMs) / 1000.0;
+  pPrint("initial time %.2f sec", seconds);
+
+  for (int32_t t = 1; t <= keyNum; ++t) {
+    HashTestRow row = {0};
+    char key[100] = {0};
+    int32_t keySize = sprintf(key, "0.db.st%d", t);
+
+    for (int32_t q = 0; q < q1Times; q++) {
+      taosHashGet(hashHandle, &key, keySize);
+    }
+
+    taosHashPut(hashHandle, key, keySize, &row, sizeof(HashTestRow));
+
+    for (int32_t q = 0; q < q2Times; q++) {
+      taosHashGet(hashHandle, &key, keySize);
+    }
+
+    if (t % printInterval == 0) {
+      int64_t endMs = taosGetTimestampMs();
+      int64_t hashSize = taosHashGetSize(hashHandle);
+      float seconds = (endMs - startMs) / 1000.0;
+      float speed = printInterval / seconds;
+      pPrint("time:%.2f sec, speed:%.1f rows/second, hashSize:%ld", seconds, speed, hashSize);
+      startMs = endMs;
+    }
+  }
+
+  int64_t endMs = taosGetTimestampMs();
+  int64_t hashSize = taosHashGetSize(hashHandle);
+  seconds = (endMs - initialMs) / 1000.0;
+  float speed = hashSize / seconds;
+
+  pPrint("total time:%.2f sec, avg speed:%.1f rows/second, hashSize:%ld", seconds, speed, hashSize);
+  taosHashCleanup(hashHandle);
+}
+
+int main(int argc, char *argv[]) {
+  shellParseArgument(argc, argv);
+  testHashPerformance();
+}
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance of cache\n");
+
+  printf("%s%s\n", indent, "-k");
+  printf("%s%s%s%d\n", indent, indent, "key num, default is ", keyNum);
+  printf("%s%s\n", indent, "-p");
+  printf("%s%s%s%d\n", indent, indent, "print interval while put into hash, default is ", printInterval);
+  printf("%s%s\n", indent, "-c");
+  printf("%s%s%s%d\n", indent, indent, "the initial capacity of hash ", capacity);
+  printf("%s%s\n", indent, "-q1");
+  printf("%s%s%s%d\n", indent, indent, "query times before put into hash", q1Times);
+  printf("%s%s\n", indent, "-q2");
+  printf("%s%s%s%d\n", indent, indent, "query times after put into hash", q2Times);
+  
+  exit(EXIT_SUCCESS);
+}
+
+void shellParseArgument(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-k") == 0) {
+      keyNum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-p") == 0) {
+      printInterval = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-c") == 0) {
+      capacity = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q1") == 0) {
+      q1Times = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q2") == 0) {
+      q2Times = atoi(argv[++i]);
+    } else {
+    }
+  }
+
+  pPrint("%s capacity:%d %s", GREEN, capacity, NC);
+  pPrint("%s printInterval:%d %s", GREEN, printInterval, NC);
+  pPrint("%s q1Times:%d %s", GREEN, q1Times, NC);
+  pPrint("%s q2Times:%d %s", GREEN, q2Times, NC);
+  pPrint("%s keyNum:%d %s", GREEN, keyNum, NC);
+}